Disable VPCLMUL for Non-temporal copies. The VPCLMUL functionality relies on inline assembly, and LLVM's new x86-fence-nontemporal-stores pass inserts an sfence before every inline assembly block when non-temporal stores are issued before them, which dramatically hurts non-temporal crc_memcpy performance here. So, we mark the stateless stream helpers static and have ExtendAndCopy on PclmulStreamType::VPCLMUL dispatch non-temporal copies to the PclmulStreamType::PCLMUL specialization. PiperOrigin-RevId: 1017480997
diff --git a/absl/crc/internal/crc_x86_arm_combined.cc b/absl/crc/internal/crc_x86_arm_combined.cc index 6f16154..edc2691 100644 --- a/absl/crc/internal/crc_x86_arm_combined.cc +++ b/absl/crc/internal/crc_x86_arm_combined.cc
@@ -61,7 +61,7 @@ CRC32AcceleratedX86ARMCombined() {} ~CRC32AcceleratedX86ARMCombined() override {} void ExtendByZeroes(uint32_t* crc, size_t length) const override; - uint32_t ComputeZeroConstant(size_t length) const; + static uint32_t ComputeZeroConstant(size_t length); private: CRC32AcceleratedX86ARMCombined(const CRC32AcceleratedX86ARMCombined&) = @@ -164,8 +164,7 @@ // manually disable bounds checking. [[clang::no_sanitize("array-bounds")]] #endif -uint32_t CRC32AcceleratedX86ARMCombined::ComputeZeroConstant( - size_t length) const { +uint32_t CRC32AcceleratedX86ARMCombined::ComputeZeroConstant(size_t length) { length >>= kNumDroppedBits; int index = absl::countr_zero(length); @@ -277,7 +276,7 @@ // PCLMULQDQ Instruction" // https://www.intel.com/content/dam/www/public/us/en/documents/white-papers/fast-crc-computation-generic-polynomials-pclmulqdq-paper.pdf // We are applying it to CRC32C polynomial. - ABSL_ATTRIBUTE_ALWAYS_INLINE V128 Pclmul16Bytes(V128* partialCRC) const { + ABSL_ATTRIBUTE_ALWAYS_INLINE static V128 Pclmul16Bytes(V128* partialCRC) { V128 loopMultiplicands = V128_Load(reinterpret_cast<const V128*>(kFoldAcross512Bits)); V128 tmp = V128_PMulHi(*partialCRC, loopMultiplicands); @@ -287,8 +286,8 @@ #if defined(ABSL_CRC_INTERNAL_HAVE_ARM_SIMD) template <bool kUseEor3 = false> - ABSL_ATTRIBUTE_ALWAYS_INLINE void Xor16BytesNeonPclmul( - V128 data, V128 tmp, V128* partialCRC) const { + ABSL_ATTRIBUTE_ALWAYS_INLINE static void Xor16BytesNeonPclmul( + V128 data, V128 tmp, V128* partialCRC) { *partialCRC = V128_Xor3<kUseEor3>(tmp, *partialCRC, data); } @@ -296,8 +295,8 @@ // into a single value, that represents crc checksum of all the processed // bytes. template <bool kUseEor3 = false> - ABSL_ATTRIBUTE_ALWAYS_INLINE uint64_t - FinalizeNeonPclmulStream(V128* partialCRC) const { + ABSL_ATTRIBUTE_ALWAYS_INLINE static uint64_t FinalizeNeonPclmulStream( + V128* partialCRC) { V128 partialCRC1 = partialCRC[0]; V128 partialCRC2 = partialCRC[1]; V128 partialCRC3 = partialCRC[2]; @@ -331,21 +330,22 @@ return crc; } - ABSL_ATTRIBUTE_ALWAYS_INLINE void Xor16BytesPclmul(V128, V128, V128*) const {} + ABSL_ATTRIBUTE_ALWAYS_INLINE static void Xor16BytesPclmul(V128, V128, + V128*) {} - ABSL_ATTRIBUTE_ALWAYS_INLINE uint64_t FinalizePclmulStream(V128*) const { + ABSL_ATTRIBUTE_ALWAYS_INLINE static uint64_t FinalizePclmulStream(V128*) { return 0; } #else - ABSL_ATTRIBUTE_ALWAYS_INLINE void Xor16BytesPclmul(V128 data, V128 tmp, - V128* partialCRC) const { + ABSL_ATTRIBUTE_ALWAYS_INLINE static void Xor16BytesPclmul( + V128 data, V128 tmp, V128* partialCRC) { *partialCRC = V128_Xor(V128_Xor(tmp, *partialCRC), data); } // Reduce partialCRC produced by Pclmul16Bytes and Xor16BytesPclmul into a // single value, that represents crc checksum of all the processed bytes. - ABSL_ATTRIBUTE_ALWAYS_INLINE uint64_t - FinalizePclmulStream(V128* partialCRC) const { + ABSL_ATTRIBUTE_ALWAYS_INLINE static uint64_t FinalizePclmulStream( + V128* partialCRC) { V128 partialCRC1 = partialCRC[0]; V128 partialCRC2 = partialCRC[1]; V128 partialCRC3 = partialCRC[2]; @@ -383,26 +383,26 @@ } template <bool kUseEor3 = false> - ABSL_INTERNAL_ATTRIBUTE_AVX ABSL_ATTRIBUTE_ALWAYS_INLINE void - Xor16BytesNeonPclmul(V128, V128, V128*) const {} + ABSL_INTERNAL_ATTRIBUTE_AVX ABSL_ATTRIBUTE_ALWAYS_INLINE static void + Xor16BytesNeonPclmul(V128, V128, V128*) {} template <bool kUseEor3 = false> - ABSL_INTERNAL_ATTRIBUTE_AVX ABSL_ATTRIBUTE_ALWAYS_INLINE uint64_t - FinalizeNeonPclmulStream(V128*) const { + ABSL_INTERNAL_ATTRIBUTE_AVX ABSL_ATTRIBUTE_ALWAYS_INLINE static uint64_t + FinalizeNeonPclmulStream(V128*) { return 0; } #endif // Update crc with 8 bytes of data from p. - ABSL_ATTRIBUTE_ALWAYS_INLINE uint64_t Process8BytesCRC(const uint8_t* p, - uint64_t crc) const { + ABSL_ATTRIBUTE_ALWAYS_INLINE static uint64_t Process8BytesCRC( + const uint8_t* p, uint64_t crc) { return CRC32_u64(static_cast<uint32_t>(crc), absl::little_endian::Load64(p)); } // Update crc with 16 bytes of data from p. - ABSL_ATTRIBUTE_ALWAYS_INLINE uint64_t Process16BytesCRC(const uint8_t* p, - uint64_t crc) const { + ABSL_ATTRIBUTE_ALWAYS_INLINE static uint64_t Process16BytesCRC( + const uint8_t* p, uint64_t crc) { crc = Process8BytesCRC(p, crc); crc = Process8BytesCRC(p + 8, crc); return crc; @@ -418,8 +418,8 @@ // with new and default flags or use inline asm. // The code below is the same as FinalizePclmulStream, but with // PCLMUL and XOR operating on 2 values in a vector at the same time. - ABSL_INTERNAL_ATTRIBUTE_AVX ABSL_ATTRIBUTE_ALWAYS_INLINE uint64_t - FinalizeVpclmulStream(V256* partialCRC) const { + ABSL_INTERNAL_ATTRIBUTE_AVX ABSL_ATTRIBUTE_ALWAYS_INLINE static uint64_t + FinalizeVpclmulStream(V256* partialCRC) { uint64_t crc = 0; uint64_t low64, high64; __asm__( @@ -449,8 +449,8 @@ return crc; } - ABSL_INTERNAL_ATTRIBUTE_AVX ABSL_ATTRIBUTE_ALWAYS_INLINE V256 - Pclmul32BytesVpclmul(V256* partialCRC) const { + ABSL_INTERNAL_ATTRIBUTE_AVX ABSL_ATTRIBUTE_ALWAYS_INLINE static V256 + Pclmul32BytesVpclmul(V256* partialCRC) { V256 loopMultiplicands = V256_Broadcast128(reinterpret_cast<const V128*>(kFoldAcross512Bits)); V256 tmp; @@ -462,8 +462,8 @@ return tmp; } - ABSL_INTERNAL_ATTRIBUTE_AVX ABSL_ATTRIBUTE_ALWAYS_INLINE void - Xor32BytesVpclmul(const uint8_t* p, V256 tmp, V256* partialCRC) const { + ABSL_INTERNAL_ATTRIBUTE_AVX ABSL_ATTRIBUTE_ALWAYS_INLINE static void + Xor32BytesVpclmul(const uint8_t* p, V256 tmp, V256* partialCRC) { V256 v = V256_LoadU(p); __asm__ volatile( "vpxor %2, %0, %0 \n" @@ -473,16 +473,16 @@ } #else template <typename T = V256> - ABSL_ATTRIBUTE_ALWAYS_INLINE T Pclmul32BytesVpclmul(T*) const { + ABSL_ATTRIBUTE_ALWAYS_INLINE static T Pclmul32BytesVpclmul(T*) { static_assert(sizeof(T) == 0, "Vector PCLMUL not supported"); return T{}; } template <typename T = V256> - ABSL_ATTRIBUTE_ALWAYS_INLINE void Xor32BytesVpclmul(const uint8_t*, T, - T*) const { + ABSL_ATTRIBUTE_ALWAYS_INLINE static void Xor32BytesVpclmul(const uint8_t*, T, + T*) { static_assert(sizeof(T) == 0, "Vector PCLMUL not supported"); } - ABSL_ATTRIBUTE_ALWAYS_INLINE uint64_t FinalizeVpclmulStream(V256*) const { + ABSL_ATTRIBUTE_ALWAYS_INLINE static uint64_t FinalizeVpclmulStream(V256*) { return 0; } #endif // defined(ABSL_CRC_INTERNAL_HAVE_X86_SIMD) && (defined(__AVX__) || @@ -539,7 +539,18 @@ const void* __restrict src, size_t length, bool non_temporal) const override { if (non_temporal) { - Extend<CopyType::kNonTemporal>(crc, src, length, dst); + if constexpr (pclmul_stream_type == PclmulStreamType::VPCLMUL) { + // On x86, VPCLMUL uses inline assembly, which causes LLVM's + // X86FenceNonTemporalStoresPass to insert an `sfence` before every + // inline assembly block in functions containing non-temporal stores, so + // non-temporal copies fall back to 128-bit PCLMUL intrinsics. + CRC32AcceleratedX86ARMCombinedMultipleStreams< + num_crc_streams, num_pclmul_streams, + PclmulStreamType::PCLMUL>::template Extend<CopyType::kNonTemporal>( + crc, src, length, dst); + } else { + Extend<CopyType::kNonTemporal>(crc, src, length, dst); + } } else { Extend<CopyType::kTemporal>(crc, src, length, dst); } @@ -549,9 +560,9 @@ // `ExtendCrc32cInline()` in `absl::ExtendCrc32c()` and // `CrcMemcpy::CrcAndCopy()`. template <CopyType copy_type> - ABSL_ATTRIBUTE_HOT void Extend(uint32_t* crc, const void* bytes, - const size_t length, - void* __restrict dst = nullptr) const { + ABSL_ATTRIBUTE_ALWAYS_INLINE static void Extend( + uint32_t* crc, const void* bytes, const size_t length, + void* __restrict dst = nullptr) { assert(length >= kSmallCutoff); if constexpr (pclmul_stream_type != PclmulStreamType::NONE) { if (length >= kMediumCutoff) { @@ -563,9 +574,9 @@ } template <bool copy> - ABSL_ATTRIBUTE_HOT void ExtendWithoutPclmul( + ABSL_ATTRIBUTE_HOT static void ExtendWithoutPclmul( uint32_t* crc, const void* bytes, const size_t length, - void* __restrict dst = nullptr) const { + void* __restrict dst = nullptr) { static_assert(num_crc_streams >= 1 && num_crc_streams <= kMaxStreams, "Invalid number of crc streams"); static_assert(num_pclmul_streams >= 0 && num_pclmul_streams <= kMaxStreams, @@ -662,10 +673,17 @@ *crc = l; } + // Always inline when the translation unit already supports AVX (or does not + // force AVX via `gnu::target`), so shared specializations like + // `<3, 3, PCLMUL>` stay inlined into `ExtendAndCopy`. template <CopyType copy_type> - ABSL_INTERNAL_ATTRIBUTE_AVX ABSL_ATTRIBUTE_HOT void ExtendWithPclmul( - uint32_t* crc, const void* bytes, const size_t length, - void* __restrict dst = nullptr) const { +#if defined(__AVX__) || !defined(ABSL_INTERNAL_CAN_FORCE_AVX) + ABSL_INTERNAL_ATTRIBUTE_AVX ABSL_ATTRIBUTE_ALWAYS_INLINE static void +#else + ABSL_INTERNAL_ATTRIBUTE_AVX ABSL_ATTRIBUTE_HOT static void +#endif + ExtendWithPclmul(uint32_t* crc, const void* bytes, const size_t length, + void* __restrict dst = nullptr) { static_assert(num_crc_streams >= 1 && num_crc_streams <= kMaxStreams, "Invalid number of crc streams"); static_assert(num_pclmul_streams >= 0 && num_pclmul_streams <= kMaxStreams, @@ -992,8 +1010,8 @@ using PclmulProduct = std::conditional_t<kIsVpclmul && kVpclmulBytes == 32, V256, V128>; - ABSL_INTERNAL_ATTRIBUTE_AVX ABSL_ATTRIBUTE_ALWAYS_INLINE void - InitPclmulStream(V256 v0, V256 v1, V128* partialCRC) const { + ABSL_INTERNAL_ATTRIBUTE_AVX ABSL_ATTRIBUTE_ALWAYS_INLINE static void + InitPclmulStream(V256 v0, V256 v1, V128* partialCRC) { if constexpr (pclmul_stream_type == PclmulStreamType::NONE) { return; } else if constexpr (kIsVpclmul) { @@ -1014,9 +1032,9 @@ // storing the low product in `partialCRC` and returning the high product, // which must be passed to `XorPclmulStream` to complete the fold. template <size_t kVpclmulBytes> - ABSL_INTERNAL_ATTRIBUTE_AVX ABSL_ATTRIBUTE_ALWAYS_INLINE - PclmulProduct<kVpclmulBytes> - PclmulStream(V128* partialCRC) const { + ABSL_INTERNAL_ATTRIBUTE_AVX ABSL_ATTRIBUTE_ALWAYS_INLINE static PclmulProduct< + kVpclmulBytes> + PclmulStream(V128* partialCRC) { if constexpr (pclmul_stream_type == PclmulStreamType::NONE || (kIsVpclmul && kVpclmulBytes == 0)) { return {}; @@ -1031,9 +1049,9 @@ // the high product returned by `PclmulStream`, the low product in // `partialCRC`, and the loaded data, and advances `*stream`. template <size_t kVpclmulBytes> - ABSL_INTERNAL_ATTRIBUTE_AVX ABSL_ATTRIBUTE_ALWAYS_INLINE void XorPclmulStream( - const uint8_t** stream, const PclmulProduct<kVpclmulBytes>& high, - V128* partialCRC) const { + ABSL_INTERNAL_ATTRIBUTE_AVX ABSL_ATTRIBUTE_ALWAYS_INLINE static void + XorPclmulStream(const uint8_t** stream, + const PclmulProduct<kVpclmulBytes>& high, V128* partialCRC) { if constexpr (pclmul_stream_type == PclmulStreamType::NONE || (kIsVpclmul && kVpclmulBytes == 0)) { return; @@ -1077,10 +1095,10 @@ // always folded. template <size_t kNumActiveCrcStreams, size_t kNumActivePclmulStreams, size_t kVpclmulBytes> - ABSL_INTERNAL_ATTRIBUTE_AVX ABSL_ATTRIBUTE_ALWAYS_INLINE void + ABSL_INTERNAL_ATTRIBUTE_AVX ABSL_ATTRIBUTE_ALWAYS_INLINE static void ProcessAllStreams16(const uint8_t** crc_streams, uint64_t* l64_crc, const uint8_t** pclmul_streams, V128 (*partialCRC)[4], - size_t lane) const { + size_t lane) { for (size_t j = 0; j < kMaxStreams; ++j) { if (j < kNumActiveCrcStreams) { l64_crc[j] = Process8BytesCRC(crc_streams[j], l64_crc[j]); @@ -1101,8 +1119,8 @@ } } - ABSL_INTERNAL_ATTRIBUTE_AVX ABSL_ATTRIBUTE_ALWAYS_INLINE uint64_t - FinalizePclmulStream(V128* partialCRC) const { + ABSL_INTERNAL_ATTRIBUTE_AVX ABSL_ATTRIBUTE_ALWAYS_INLINE static uint64_t + FinalizePclmulStream(V128* partialCRC) { if constexpr (pclmul_stream_type == PclmulStreamType::NONE) { return 0; } else if constexpr (kIsVpclmul) {