Hoist slot address calculation above SetCtrl and outline RecordInsertMissCold in raw_hash_set. 1. Hoist `SlotAddress` / `slot_array` calculations above `SetCtrl*` and `if (common.has_infoz())` in `PrepareInsertLarge*`, `PrepareInsertSmallNonSoo`, and `GrowEmptySooTableToNextCapacityForceSamplingAndPrepareInsert`. 2. Outline cold `infoz` recording into an `ABSL_ATTRIBUTE_NOINLINE ABSL_ATTRIBUTE_COLD` helper `RecordInsertMissCold(common, probe_length, hash)`. PiperOrigin-RevId: 986725883 Change-Id: I1fb22721bc738ba41d0c1e5a90aadb0fb944ccad
diff --git a/absl/container/internal/raw_hash_set.cc b/absl/container/internal/raw_hash_set.cc index 3b0cf42..97fd8ad 100644 --- a/absl/container/internal/raw_hash_set.cc +++ b/absl/container/internal/raw_hash_set.cc
@@ -1026,6 +1026,14 @@ /*distance_from_desired=*/0); } +// Outlines cold infoz recording so callers do not inline infoz extraction +// or spill registers across SetCtrl. `common` is first and `hash` is third to +// match argument order in callers. +ABSL_ATTRIBUTE_NOINLINE ABSL_ATTRIBUTE_COLD void RecordInsertMissCold( + CommonFields& common, size_t probe_length, size_t hash) { + common.infoz().RecordInsertMiss(hash, probe_length); +} + ABSL_ATTRIBUTE_NOINLINE void ReportGrowthToInfoz(CommonFields& common, HashtablezInfoHandle infoz, size_t hash, @@ -1819,11 +1827,14 @@ if (common.capacity() == 1) { if (common.empty()) { IncrementSmallSizeNonSoo(common, policy); + // Compute before `RecordInsertMissCold` to avoid reloading `control_`. + void* res = common.slot_array(/*capacity=*/1); + // Call NOINLINE function to move infoz instructions out of line. if (common.has_infoz()) { - common.infoz().RecordInsertMiss(get_hash(common.seed().seed()), - /*distance_from_desired=*/0); + RecordInsertMissCold(common, /*probe_length=*/0, + get_hash(common.seed().seed())); } - return common.slot_array(/*capacity=*/1); + return res; } else { return Grow1To3AndPrepareInsert(common, policy, get_hash); } @@ -1949,9 +1960,14 @@ FindInfo target = find_first_non_full(common, hash); PrepareInsertCommon(common); common.OverwriteControlAsFull(common.control()[target.offset]); + // Compute before `SetCtrl*` to avoid reloading `control_`. + void* res = + SlotAddress(common.slot_array(cap), target.offset, policy.slot_size); SetCtrlInLargeTable(common, target.offset, H2(hash), policy.slot_size); - common.infoz().RecordInsertMiss(hash, target.probe_length); - return SlotAddress(common.slot_array(cap), target.offset, policy.slot_size); + if (common.has_infoz()) { + RecordInsertMissCold(common, target.probe_length, hash); + } + return res; } // Resizes empty non-allocated SOO table to NextCapacity(SooCapacity()), @@ -1971,11 +1987,15 @@ PrepareInsertCommon(common); common.OverwriteEmptyAsFull(); const size_t new_hash = get_hash(common.seed().seed()); + // Compute before `SetCtrl*` to avoid reloading `control_`. + void* res = SlotAddress(common.slot_array(kNewCapacity), SooSlotIndex(), + policy.slot_size); SetCtrlInSingleGroupTable(common, SooSlotIndex(), H2(new_hash), policy.slot_size); - common.infoz().RecordInsertMiss(new_hash, /*distance_from_desired=*/0); - return SlotAddress(common.slot_array(kNewCapacity), SooSlotIndex(), - policy.slot_size); + if (common.has_infoz()) { + RecordInsertMissCold(common, /*probe_length=*/0, new_hash); + } + return res; } // Returns the number of elements to block for the given capacity and reserved @@ -2340,10 +2360,15 @@ ABSL_ASSUME(cap > kMaxSmallCapacity); target_group.offset += mask_empty.LowestBitSet(); target_group.offset &= cap; + // Compute before `SetCtrl` to avoid reloading `control_`. + void* res = SlotAddress(common.slot_array(cap), target_group.offset, + policy.slot_size); SetCtrl(common, target_group.offset, H2(hash), policy.slot_size); - common.infoz().RecordInsertMiss(hash, target_group.probe_length); - return SlotAddress(common.slot_array(cap), target_group.offset, - policy.slot_size); + // Call NOINLINE function to move infoz instructions out of line. + if (common.has_infoz()) { + RecordInsertMissCold(common, target_group.probe_length, hash); + } + return res; } } // namespace