Make controls bytes to end right before slots by moving necessary alignment at the beginning of the allocation. It is a preparation change for removing `slot_array`. It will be computed from `control`. This change would simplify and make this computation cheaper. PiperOrigin-RevId: 943535792 Change-Id: I2b67e137340d5691e9c9a5a24b009cb6d99972df
diff --git a/absl/container/internal/raw_hash_set.cc b/absl/container/internal/raw_hash_set.cc index 1275748..6c94b53 100644 --- a/absl/container/internal/raw_hash_set.cc +++ b/absl/container/internal/raw_hash_set.cc
@@ -447,6 +447,23 @@ IterateOverFullSlotsImpl(c, slot_size, cb); } +HashtablezInfoHandle CommonFields::infoz_ptr() const { + // growth_info is stored before control bytes. + ABSL_SWISSTABLE_ASSERT(has_infoz()); + HashtablezInfoHandle res; + void* src = reinterpret_cast<char*>(control()) - + MetadataBeforeControlSize(/*has_infoz=*/true, capacity()); + std::memcpy(&res, src, sizeof(HashtablezInfoHandle)); + return res; +} + +void CommonFields::set_infoz(HashtablezInfoHandle infoz) { + ABSL_SWISSTABLE_ASSERT(has_infoz()); + void* dst = reinterpret_cast<char*>(control()) - + MetadataBeforeControlSize(/*has_infoz=*/true, capacity()); + std::memcpy(dst, &infoz, sizeof(HashtablezInfoHandle)); +} + namespace { void ResetGrowthLeft(GrowthInfoAccessor growth_info, size_t capacity, @@ -1507,7 +1524,7 @@ "kGuaranteedFitToBuffer is true."); // We reuse GrowthInfo memory as well. return AlignToNextItem( - control_ - ControlOffset(/*has_infoz=*/false, + control_ - MetadataBeforeControlSize(/*has_infoz=*/false, NextCapacity(kMaxLocalBufferOldCapacity))); }
diff --git a/absl/container/internal/raw_hash_set.h b/absl/container/internal/raw_hash_set.h index ac93efb..6d0e4ef 100644 --- a/absl/container/internal/raw_hash_set.h +++ b/absl/container/internal/raw_hash_set.h
@@ -1131,9 +1131,9 @@ : sizeof(uint64_t); } -// Computes the offset from the start of the backing allocation of control. +// Computes the size of the metadata before the control bytes. // infoz and growth_info are stored at the beginning of the backing array. -constexpr size_t ControlOffset(bool has_infoz, size_t capacity) { +constexpr size_t MetadataBeforeControlSize(bool has_infoz, size_t capacity) { if (ABSL_PREDICT_FALSE(has_infoz)) { // We always allocate 8 bytes of growth info for sampled tables to allow // branchless access to infoz pointer. @@ -1154,7 +1154,7 @@ explicit RawHashSetLayout(size_t capacity, size_t slot_size, size_t slot_align, bool has_infoz, size_t blocked_element_count) - : control_offset_(ControlOffset(has_infoz, capacity)), + : control_offset_(MetadataBeforeControlSize(has_infoz, capacity)), generation_offset_(control_offset_ + NumControlBytes(capacity)), slot_offset_( AlignUpTo(generation_offset_ + NumGenerationBytes(), slot_align)), @@ -1164,6 +1164,10 @@ ABSL_SWISSTABLE_ASSERT( slot_size <= ((std::numeric_limits<size_t>::max)() - slot_offset_) / capacity); + slot_array_padding_ = + slot_offset_ - generation_offset_ - NumGenerationBytes(); + control_offset_ += slot_array_padding_; + generation_offset_ += slot_array_padding_; } // Returns precomputed offset from the start of the backing allocation of @@ -1187,6 +1191,7 @@ size_t generation_offset_; size_t slot_offset_; size_t alloc_size_; + size_t slot_array_padding_; }; struct HashtableFreeFunctionsAccess; @@ -1384,22 +1389,12 @@ inline_data_.set_has_infoz(); } - HashtablezInfoHandle* infoz_ptr() const { - // growth_info is stored before control bytes. - ABSL_SWISSTABLE_ASSERT( - reinterpret_cast<uintptr_t>(control()) % alignof(size_t) == 0); - ABSL_SWISSTABLE_ASSERT(has_infoz()); - return reinterpret_cast<HashtablezInfoHandle*>( - control() - ControlOffset(/*has_infoz=*/true, capacity())); - } + HashtablezInfoHandle infoz_ptr() const; HashtablezInfoHandle infoz() { - return has_infoz() ? *infoz_ptr() : HashtablezInfoHandle(); + return has_infoz() ? infoz_ptr() : HashtablezInfoHandle(); } - void set_infoz(HashtablezInfoHandle infoz) { - ABSL_SWISSTABLE_ASSERT(has_infoz()); - *infoz_ptr() = infoz; - } + void set_infoz(HashtablezInfoHandle infoz); bool should_rehash_for_bug_detection_on_insert() const { if constexpr (!SwisstableGenerationsEnabled()) { @@ -2419,7 +2414,7 @@ slot_(slot) { // This assumption helps the compiler know that any non-end iterator is // not equal to any end iterator. - ABSL_ASSUME(slot != nullptr); + ABSL_ASSUME(slot != nullptr); // NOLINT } // For end() iterators. explicit iterator(const GenerationType* generation_ptr)
diff --git a/absl/container/internal/raw_hash_set_test.cc b/absl/container/internal/raw_hash_set_test.cc index 37bc030..d4160ce 100644 --- a/absl/container/internal/raw_hash_set_test.cc +++ b/absl/container/internal/raw_hash_set_test.cc
@@ -138,7 +138,9 @@ constexpr size_t kAlignment = 4; RawHashSetLayout layout(1, kSlotSize, kAlignment, /*has_infoz=*/false, /*blocked_element_count=*/0); - EXPECT_EQ(layout.control_offset(), 0); + EXPECT_EQ(layout.control_offset(), NumGenerationBytes() == 0 + ? 0 + : kAlignment - NumGenerationBytes()); EXPECT_EQ(layout.slot_offset(), NumGenerationBytes() == 0 ? 0 : kAlignment); EXPECT_EQ(layout.alloc_size(), layout.slot_offset() + kSlotSize); } @@ -157,7 +159,7 @@ ASSERT_LE(capacity, GrowthInfoLowerBound::kMaxGrowthLeftLowerBound); RawHashSetLayout layout(capacity, slot_size, slot_align, has_infoz, blocked_element_count); - EXPECT_EQ(layout.control_offset(), 1); // 1 byte for growth_info + EXPECT_EQ(layout.control_offset(), 1 + padding); // 1 byte for growth_info size_t expected_slot_offset = capacity + NumClonedBytes() + 1 + /*growth*/ 1 + NumGenerationBytes(); EXPECT_LT(padding, slot_align); @@ -204,12 +206,14 @@ RawHashSetLayout layout(kCapacity, /*slot_size=*/kSlotSize, /*slot_align=*/kAlignment, /*has_infoz=*/true, /*blocked_element_count=*/0); + size_t padding = NumGenerationBytes() == 0 ? 1 : 0; + padding += sizeof(HashtablezInfoHandle) == 4 ? 0 : 4; EXPECT_EQ(layout.control_offset(), // growth_info is always 8 bytes for sampled tables. - 8 + sizeof(HashtablezInfoHandle)); + 8 + sizeof(HashtablezInfoHandle) + padding); size_t expected_slot_offset = layout.control_offset() + kCapacity + NumClonedBytes() + 1 + - /*padding+generation*/ (sizeof(HashtablezInfoHandle) == 4 ? 1 : 5); + NumGenerationBytes(); EXPECT_EQ(expected_slot_offset % kAlignment, 0); EXPECT_EQ(layout.slot_offset(), expected_slot_offset); EXPECT_EQ(layout.alloc_size(), @@ -228,10 +232,11 @@ ASSERT_GT(capacity, GrowthInfoLowerBound::kMaxGrowthLeftLowerBound); RawHashSetLayout layout(capacity, slot_size, slot_align, has_infoz, blocked_element_count); + size_t padding = NumGenerationBytes() == 0 ? 1 : 0; EXPECT_EQ(layout.control_offset(), - has_infoz ? 8 + sizeof(HashtablezInfoHandle) : 8); + padding + (has_infoz ? 8 + sizeof(HashtablezInfoHandle) : 8)); size_t expected_slot_offset = layout.control_offset() + capacity + - NumClonedBytes() + 1 + /*padding+generation*/ 1; + NumClonedBytes() + 1 + NumGenerationBytes(); EXPECT_EQ(expected_slot_offset % slot_align, 0); EXPECT_EQ(layout.slot_offset(), expected_slot_offset); EXPECT_EQ( @@ -1259,7 +1264,7 @@ constexpr size_t kNonSooSize = sizeof(HeapOrSoo) + 8; using NonSooIntTableSlotType = SizedValue<kNonSooSize>; -static_assert(sizeof(NonSooIntTableSlotType) >= kNonSooSize, "too small"); +static_assert(sizeof(NonSooIntTableSlotType) > MaxSooSlotSize(), "too small"); using NonSooIntTable = ValueTable<NonSooIntTableSlotType>; using NonSooIntTableTrivialDestroy = ValueTable<NonSooIntTableSlotType, /*kTransferable=*/false, /*kSoo=*/false,