Use AVX-512 VPTERNLOGD for 3-way XORs in the x86 PCLMUL and VPCLMUL CRC32C streams. Every fold of a PCLMUL stream XORs three values: the two carry-less products and the next block of input. Add `PCLMUL_EOR3` and `VPCLMUL_EOR3` stream types that do this with a single AVX-512 `vpternlogd` instead of two XORs. `PCLMUL_EOR3` also reads the input block straight from memory instead of loading it first. Copies already hold the input in registers, so `PCLMUL_EOR3` copies keep using `PCLMUL`. Like `VPCLMUL`, the new stream types use inline assembly, so their non-temporal copies keep using `PCLMUL` as well. Use `PCLMUL_EOR3` on Skylake, Cascade Lake, and Ice Lake Xeons, and `VPCLMUL_EOR3` on Sapphire Rapids, Emerald Rapids, and Granite Rapids. PiperOrigin-RevId: 1021627029
diff --git a/absl/crc/internal/crc_memcpy_test.cc b/absl/crc/internal/crc_memcpy_test.cc index 021f5e5..5f4f3ec 100644 --- a/absl/crc/internal/crc_memcpy_test.cc +++ b/absl/crc/internal/crc_memcpy_test.cc
@@ -313,6 +313,32 @@ EXPECT_EQ(g_num_unfenced_stores, 0); } +// Checks that Crc32CAndCopy() writes every requested byte and no surrounding +// bytes across all cache-line alignments and stream-tail remainders. +TEST(CrcMemcpyTest, WritesExactDestinationRange) { + constexpr size_t kGuard = 64, kAlign = 64, kBase = 4096, kMaxExtra = 256; + std::vector<char> src(kBase + kMaxExtra); + for (size_t i = 0; i < src.size(); ++i) { + src[i] = static_cast<char>(i * 131); + } + const std::vector<char> guard(kGuard + kAlign, static_cast<char>(0xa5)); + std::vector<char> buf(kGuard + kAlign + src.size() + kGuard); + for (bool non_temporal : {false, true}) { + for (size_t size = kBase; size <= kBase + kMaxExtra; ++size) { + for (size_t offset = 0; offset < kAlign; ++offset) { + char* dst = buf.data() + kGuard + offset; + std::memset(buf.data(), 0xa5, buf.size()); + ASSERT_EQ(absl::crc_internal::Crc32CAndCopy( + dst, src.data(), size, absl::crc32c_t{0}, non_temporal), + absl::ComputeCrc32c({src.data(), size})); + ASSERT_EQ(std::memcmp(dst, src.data(), size), 0); + ASSERT_EQ(std::memcmp(buf.data(), guard.data(), dst - buf.data()), 0); + ASSERT_EQ(std::memcmp(dst + size, guard.data(), kGuard), 0); + } + } + } +} + // Non-temporal stores are weakly ordered, so Crc32CAndCopy() has to fence them // before it returns, or the copy is not guaranteed to be visible to the thread // that reads it next. This checks that it always does, across the sizes and
diff --git a/absl/crc/internal/crc_x86_arm_combined.cc b/absl/crc/internal/crc_x86_arm_combined.cc index 5fe704f..d4d0bcd 100644 --- a/absl/crc/internal/crc_x86_arm_combined.cc +++ b/absl/crc/internal/crc_x86_arm_combined.cc
@@ -219,6 +219,9 @@ NONE, PCLMUL, VPCLMUL, + // PCLMUL and VPCLMUL that fold with a single AVX-512 `vpternlogd` 3-way XOR. + PCLMUL_EOR3, + VPCLMUL_EOR3, NEON_PCLMUL, NEON_PCLMUL_EOR3, }; @@ -462,14 +465,31 @@ return tmp; } + template <bool kUseEor3> ABSL_INTERNAL_ATTRIBUTE_AVX ABSL_ATTRIBUTE_ALWAYS_INLINE static void Xor32BytesVpclmul(const uint8_t* p, V256 tmp, V256* partialCRC) { V256 v = V256_LoadU(p); - __asm__ volatile( - "vpxor %2, %0, %0 \n" - "vpxor %1, %0, %0 \n" - : "+x"(*partialCRC) - : "x"(v), "x"(tmp)); + if constexpr (kUseEor3) { + __asm__ volatile("vpternlogd $0x96, %2, %1, %0" + : "+x"(*partialCRC) + : "x"(v), "x"(tmp)); + } else { + __asm__ volatile( + "vpxor %2, %0, %0 \n" + "vpxor %1, %0, %0 \n" + : "+x"(*partialCRC) + : "x"(v), "x"(tmp)); + } + } + + // Same as Xor16BytesPclmul on the 16 bytes at `p`, but with a single AVX-512 + // `vpternlogd` (0x96 is the truth table of a 3-way XOR) that reads the bytes + // straight from memory. + ABSL_ATTRIBUTE_ALWAYS_INLINE static void Xor16BytesPclmulEor3( + const uint8_t* p, V128 tmp, V128* partialCRC) { + __asm__("vpternlogd $0x96, %2, %1, %0" + : "+x"(*partialCRC) + : "x"(tmp), "m"(*reinterpret_cast<const __m128i_u*>(p))); } #else template <typename T = V256> @@ -477,11 +497,16 @@ static_assert(sizeof(T) == 0, "Vector PCLMUL not supported"); return T{}; } - template <typename T = V256> + template <bool kUseEor3, typename T = V256> ABSL_ATTRIBUTE_ALWAYS_INLINE static void Xor32BytesVpclmul(const uint8_t*, T, T*) { static_assert(sizeof(T) == 0, "Vector PCLMUL not supported"); } + template <typename T = V128> + ABSL_ATTRIBUTE_ALWAYS_INLINE static void Xor16BytesPclmulEor3(const uint8_t*, + T, T*) { + static_assert(sizeof(T) == 0, "VPTERNLOGD not supported"); + } ABSL_ATTRIBUTE_ALWAYS_INLINE static uint64_t FinalizeVpclmulStream(V256*) { return 0; } @@ -540,11 +565,13 @@ size_t length, bool non_temporal) const override { if (non_temporal) { - if constexpr (pclmul_stream_type == PclmulStreamType::VPCLMUL) { - // On x86, VPCLMUL uses inline assembly, which causes LLVM's - // X86FenceNonTemporalStoresPass to insert an `sfence` before every - // inline assembly block in functions containing non-temporal stores, so - // non-temporal copies fall back to 128-bit PCLMUL intrinsics. + if constexpr (kIsVpclmul || + pclmul_stream_type == PclmulStreamType::PCLMUL_EOR3) { + // On x86, VPCLMUL and PCLMUL_EOR3 use inline assembly, which causes + // LLVM's X86FenceNonTemporalStoresPass to insert an `sfence` before + // every inline assembly block in functions containing non-temporal + // stores, so non-temporal copies fall back to 128-bit PCLMUL + // intrinsics. CRC32AcceleratedX86ARMCombinedMultipleStreams< num_crc_streams, num_pclmul_streams, PclmulStreamType::PCLMUL>::template Extend<CopyType::kNonTemporal>( @@ -552,6 +579,12 @@ } else { Extend<CopyType::kNonTemporal>(crc, src, length, dst); } + } else if constexpr (pclmul_stream_type == PclmulStreamType::PCLMUL_EOR3) { + // PCLMUL_EOR3 reads the folded bytes from memory, but copies already hold + // them in registers, so temporal copies fall back to PCLMUL as well. + CRC32AcceleratedX86ARMCombinedMultipleStreams< + num_crc_streams, num_pclmul_streams, PclmulStreamType::PCLMUL>:: + template Extend<CopyType::kTemporal>(crc, src, length, dst); } else { Extend<CopyType::kTemporal>(crc, src, length, dst); } @@ -706,15 +739,17 @@ uintptr_t dst_addr = reinterpret_cast<uintptr_t>(d); size_t bytes_to_align = (kAlignment - (dst_addr & (kAlignment - 1))) & (kAlignment - 1); - std::memcpy(d, p, bytes_to_align); - d += bytes_to_align; - while (bytes_to_align >= 8) { - ABSL_INTERNAL_STEP8(l, p); - bytes_to_align -= 8; - } - while (bytes_to_align > 0) { - ABSL_INTERNAL_STEP1(l, p); - bytes_to_align -= 1; + if (bytes_to_align != 0) { + std::memcpy(d, p, bytes_to_align); + d += bytes_to_align; + while (bytes_to_align >= 8) { + ABSL_INTERNAL_STEP8(l, p); + bytes_to_align -= 8; + } + while (bytes_to_align > 0) { + ABSL_INTERNAL_STEP1(l, p); + bytes_to_align -= 1; + } } } else { // Point x at first 8-byte aligned byte in string. @@ -927,15 +962,11 @@ if constexpr (copy_type != CopyType::kNone) { d = reinterpret_cast<char*>( dst_pclmul_streams[kNumActivePclmulStreams - 1]); - size_t remaining_to_copy = static_cast<size_t>(e - p); - std::memcpy(d, p, remaining_to_copy); } } else { p = crc_streams[kNumActiveCrcStreams - 1]; if constexpr (copy_type != CopyType::kNone) { d = reinterpret_cast<char*>(dst_crc_streams[kNumActiveCrcStreams - 1]); - size_t remaining_to_copy = static_cast<size_t>(e - p); - std::memcpy(d, p, remaining_to_copy); } } @@ -963,15 +994,26 @@ *crc = l; - if constexpr (copy_type == CopyType::kNonTemporal) { - // According to the Intel 64 Software Developer's Manual - // (https://www.intel.com/content/www/us/en/developer/articles/technical/intel-sdm.html), - // non-temporal stores are weakly ordered due to bypassing caches, so - // they have to be fenced before the copy is visible to other threads. - // Regular stores are ordered and need no fence. Note that the fence is - // placed after all stores, to avoid unnecessary serialization and a - // performance hit. - NonTemporalStoreFence(); + if constexpr (copy_type != CopyType::kNone) { + const uint8_t* copy_src = e - remaining_bytes; + while (e - copy_src >= 64) { + Copy64Bytes<copy_type>(reinterpret_cast<uint8_t*>(d), copy_src); + d += 64; + copy_src += 64; + } + if constexpr (copy_type == CopyType::kNonTemporal) { + // According to the Intel 64 Software Developer's Manual + // (https://www.intel.com/content/www/us/en/developer/articles/technical/intel-sdm.html), + // non-temporal stores are weakly ordered due to bypassing caches, so + // they have to be fenced before the copy is visible to other threads. + // Regular stores are ordered and need no fence. Note that the fence is + // placed after all non-temporal stores, to avoid unnecessary + // serialization and a performance hit. + NonTemporalStoreFence(); + } + if (copy_src != e) { + std::memcpy(d, copy_src, static_cast<size_t>(e - copy_src)); + } } } @@ -988,8 +1030,7 @@ return {num_crc_streams, num_pclmul_streams}; } // A single 256-bit VPCLMUL stream keeps up with temporal copies on its own. - if (pclmul_stream_type == PclmulStreamType::VPCLMUL && - copy_type == CopyType::kTemporal) { + if (kIsVpclmul && copy_type == CopyType::kTemporal) { return {0, 1}; } // Non-temporal copies benefit from a second crc stream, notably on AMD @@ -1002,7 +1043,8 @@ // Whether PCLMUL streams are folded with 256-bit `vpclmulqdq`. static constexpr bool kIsVpclmul = - (pclmul_stream_type == PclmulStreamType::VPCLMUL); + (pclmul_stream_type == PclmulStreamType::VPCLMUL || + pclmul_stream_type == PclmulStreamType::VPCLMUL_EOR3); // The high product of a fold, which is returned by `PclmulStream` and // consumed by `XorPclmulStream`. Folding 32 bytes with 256-bit `vpclmulqdq` @@ -1057,8 +1099,12 @@ (kIsVpclmul && kVpclmulBytes == 0)) { return; } else if constexpr (kIsVpclmul && kVpclmulBytes == 32) { - Xor32BytesVpclmul(*stream, high, reinterpret_cast<V256*>(partialCRC)); + Xor32BytesVpclmul<(pclmul_stream_type == PclmulStreamType::VPCLMUL_EOR3)>( + *stream, high, reinterpret_cast<V256*>(partialCRC)); *stream += 32; + } else if constexpr (pclmul_stream_type == PclmulStreamType::PCLMUL_EOR3) { + Xor16BytesPclmulEor3(*stream, high, partialCRC); + *stream += 16; } else { V128 data = V128_LoadU(reinterpret_cast<const V128*>(*stream)); if constexpr (pclmul_stream_type == PclmulStreamType::NEON_PCLMUL || @@ -1182,7 +1228,7 @@ (defined(__AVX__) || defined(ABSL_INTERNAL_CAN_FORCE_AVX)) && \ (!defined(_MSC_VER) || defined(__clang__)) return new CRC32AcceleratedX86ARMCombinedMultipleStreams< - 3, 3, PclmulStreamType::VPCLMUL>(); + 3, 3, PclmulStreamType::VPCLMUL_EOR3>(); #else [[fallthrough]]; #endif @@ -1192,8 +1238,16 @@ 3, 2, PclmulStreamType::PCLMUL>(); case CpuType::kIntelCascadelakeXeon: case CpuType::kIntelSkylakeXeon: - case CpuType::kIntelSkylake: case CpuType::kIntelIcelake: +#if defined(ABSL_CRC_INTERNAL_HAVE_X86_SIMD) && \ + (defined(__AVX__) || defined(ABSL_INTERNAL_CAN_FORCE_AVX)) && \ + (!defined(_MSC_VER) || defined(__clang__)) + return new CRC32AcceleratedX86ARMCombinedMultipleStreams< + 3, 3, PclmulStreamType::PCLMUL_EOR3>(); +#else + [[fallthrough]]; +#endif + case CpuType::kIntelSkylake: return new CRC32AcceleratedX86ARMCombinedMultipleStreams< 3, 3, PclmulStreamType::PCLMUL>(); // PCLMULQDQ is slow, don't use it.