Add branch prediction hints to the H2 match loops in `find_large` and `find_or_prepare_insert_large`. In the previous range-for loop, clang would by default predict that the loop would have multiple iterations. We change the code to instead predict one iteration of the loop (which is the common case for find-hit). Even with iFDO enabled (in which case the ABSL_PREDICT_TRUE/ABSL_PREDICT_FALSE aren't as important), splitting the loop entry condition from the loop continuation condition allows for better optimization because the probability of loop entry is very different from loop continuation. PiperOrigin-RevId: 998814452 Change-Id: Ic4dfa2ad21a0fdb12f1350f71d3e36a6f7cd7cfa
diff --git a/absl/container/internal/raw_hash_set.h b/absl/container/internal/raw_hash_set.h index e1de6e0..1792ec2 100644 --- a/absl/container/internal/raw_hash_set.h +++ b/absl/container/internal/raw_hash_set.h
@@ -3620,10 +3620,13 @@ #ifndef ABSL_HAVE_MEMORY_SANITIZER absl::PrefetchToLocalCache(slot_array + seq.offset()); #endif - for (uint32_t i : g.Match(h2)) { - const size_t offset = seq.offset(i); - if (ABSL_PREDICT_TRUE(equal_to(key, slot_array + offset))) - return iterator_at_ptr(ctrl + offset, slot_array + offset); + // Predict one iteration (common case for find-hit). + if (auto match = g.Match(h2); ABSL_PREDICT_TRUE(match)) { + do { + const size_t offset = seq.offset(*match); + if (ABSL_PREDICT_TRUE(equal_to(key, slot_array + offset))) + return iterator_at_ptr(ctrl + offset, slot_array + offset); + } while (ABSL_PREDICT_FALSE(++match)); } if (ABSL_PREDICT_TRUE(g.MaskEmpty())) return end(); seq.next(); @@ -3913,11 +3916,12 @@ #ifndef ABSL_HAVE_MEMORY_SANITIZER absl::PrefetchToLocalCache(slot_array + seq.offset()); #endif - for (uint32_t i : g.Match(h2)) { - slot_type* slot = slot_array + seq.offset(i); - if (ABSL_PREDICT_TRUE(equal_to(key, slot))) { - return {slot, false}; - } + // Predict one iteration (common case for insert-hit). + if (auto match = g.Match(h2); ABSL_PREDICT_TRUE(match)) { + do { + slot_type* slot = slot_array + seq.offset(*match); + if (ABSL_PREDICT_TRUE(equal_to(key, slot))) return {slot, false}; + } while (ABSL_PREDICT_FALSE(++match)); } auto mask_empty = g.MaskEmpty(); if (ABSL_PREDICT_TRUE(mask_empty)) {