Mark Abseil 64-bit large-input hash as noinline and tail-call internal helpers. ThinLTO and FDO inlining of CombineLargeContiguousImplOn64BitLengthGt32 bloats hot callers (such as hash table lookups and growth) with duplicated dispatch and vector hashing code. - Mark CombineLargeContiguousImplOn64BitLengthGt32 as noinline so ThinLTO and FDO keep the >32-byte slow path out-of-line rather than duplicating it into callers. - Forward between the 64-bit >32-byte internal helpers with musttail, align their pointer parameters to const unsigned char*, and mark LowLevelHash33To64 as always_inline. PiperOrigin-RevId: 1026863058
diff --git a/absl/hash/internal/hash.cc b/absl/hash/internal/hash.cc index df75c25..aa04def 100644 --- a/absl/hash/internal/hash.cc +++ b/absl/hash/internal/hash.cc
@@ -195,7 +195,8 @@ #endif // ABSL_AES_INTERNAL_HAVE_X86_SIMD -uint64_t LowLevelHash33To64(uint64_t seed, const uint8_t* ptr, size_t len) { +ABSL_ATTRIBUTE_ALWAYS_INLINE inline uint64_t LowLevelHash33To64( + uint64_t seed, const uint8_t* ptr, size_t len) { assert(len > 32); assert(len <= 64); Vector128 state = Set128(seed, len); @@ -216,9 +217,8 @@ } [[maybe_unused]] ABSL_ATTRIBUTE_NOINLINE uint64_t -LowLevelHashLenGt64(uint64_t seed, const void* data, size_t len) { +LowLevelHashLenGt64(uint64_t seed, const unsigned char* ptr, size_t len) { assert(len > 64); - const uint8_t* ptr = static_cast<const uint8_t*>(data); const uint8_t* last_32_ptr = ptr + len - 32; // If we have more than 64 bytes, we're going to handle chunks of 64 @@ -286,7 +286,8 @@ return cs0 ^ cs1; } -uint64_t LowLevelHash33To64(uint64_t seed, const uint8_t* ptr, size_t len) { +ABSL_ATTRIBUTE_ALWAYS_INLINE inline uint64_t LowLevelHash33To64( + uint64_t seed, const uint8_t* ptr, size_t len) { assert(len > 32); assert(len <= 64); uint64_t current_state = seed ^ kStaticRandomData[0] ^ len; @@ -295,9 +296,8 @@ } [[maybe_unused]] ABSL_ATTRIBUTE_NOINLINE uint64_t -LowLevelHashLenGt64(uint64_t seed, const void* data, size_t len) { +LowLevelHashLenGt64(uint64_t seed, const unsigned char* ptr, size_t len) { assert(len > 64); - const uint8_t* ptr = static_cast<const uint8_t*>(data); uint64_t current_state = seed ^ kStaticRandomData[0] ^ len; const uint8_t* last_32_ptr = ptr + len - 32; // If we have more than 64 bytes, we're going to handle chunks of 64 @@ -344,13 +344,14 @@ } #endif // ABSL_AES_INTERNAL_HAVE_X86_SIMD -[[maybe_unused]] uint64_t LowLevelHashLenGt32(uint64_t seed, const void* data, +[[maybe_unused]] uint64_t LowLevelHashLenGt32(uint64_t seed, + const unsigned char* data, size_t len) { assert(len > 32); if (ABSL_PREDICT_FALSE(len > 64)) { - return LowLevelHashLenGt64(seed, data, len); + ABSL_INTERNAL_MUSTTAIL return LowLevelHashLenGt64(seed, data, len); } - return LowLevelHash33To64(seed, static_cast<const uint8_t*>(data), len); + return LowLevelHash33To64(seed, data, len); } ABSL_ATTRIBUTE_ALWAYS_INLINE inline uint64_t HashBlockOn32Bit( @@ -382,7 +383,7 @@ ABSL_ATTRIBUTE_ALWAYS_INLINE inline uint64_t HashBlockOn64Bit( uint64_t state, const unsigned char* data, size_t len) { #ifdef ABSL_HAVE_INTRINSIC_INT128 - return LowLevelHashLenGt32(state, data, len); + ABSL_INTERNAL_MUSTTAIL return LowLevelHashLenGt32(state, data, len); #else return hash_internal::CityHash64WithSeed(reinterpret_cast<const char*>(data), len, state); @@ -419,15 +420,16 @@ return SplitAndCombineOn32Bit(state, first, len); } -uint64_t CombineLargeContiguousImplOn64BitLengthGt32(uint64_t state, - const unsigned char* first, - size_t len) { +ABSL_ATTRIBUTE_NOINLINE uint64_t +CombineLargeContiguousImplOn64BitLengthGt32(uint64_t state, + const unsigned char* first, + size_t len) { assert(len > 32); assert(sizeof(size_t) == 8); // NOLINT(misc-static-assert) if (ABSL_PREDICT_TRUE(len <= PiecewiseChunkSize())) { - return HashBlockOn64Bit(state, first, len); + ABSL_INTERNAL_MUSTTAIL return HashBlockOn64Bit(state, first, len); } - return SplitAndCombineOn64Bit(state, first, len); + ABSL_INTERNAL_MUSTTAIL return SplitAndCombineOn64Bit(state, first, len); } ABSL_CONST_INIT const void* const MixingHashState::kSeed = &kSeed;
diff --git a/absl/hash/internal/hash.h b/absl/hash/internal/hash.h index 191fef7..e25788b 100644 --- a/absl/hash/internal/hash.h +++ b/absl/hash/internal/hash.h
@@ -1091,7 +1091,7 @@ CombineLargeContiguousImplOn32BitLengthGt8(uint64_t state, const unsigned char* first, size_t len); -ABSL_ATTRIBUTE_PURE_FUNCTION uint64_t +ABSL_ATTRIBUTE_PURE_FUNCTION ABSL_ATTRIBUTE_NOINLINE uint64_t CombineLargeContiguousImplOn64BitLengthGt32(uint64_t state, const unsigned char* first, size_t len);