Merge CRC+memcpy functionality into Extend function. This improves performance through improved instruction level parallelism. We mostly do the copying separately, but in lockstep with the crc streams. We don't reuse the loads between the crc calculation and the copy, since the loads are to different register types, except for VPCLMUL streams. PiperOrigin-RevId: 964840262 Change-Id: I4cfc80a78cc9e2f288df666db6a5f063cc9a183b
diff --git a/absl/crc/crc32c.cc b/absl/crc/crc32c.cc index 9b1ef7e..8244e32 100644 --- a/absl/crc/crc32c.cc +++ b/absl/crc/crc32c.cc
@@ -73,7 +73,7 @@ crc32c_t MemcpyCrc32c(void* dest, const void* src, size_t count, crc32c_t initial_crc) { return static_cast<crc32c_t>( - crc_internal::Crc32CAndCopy(dest, src, count, initial_crc, false)); + crc_internal::Crc32CAndCopy(dest, src, count, initial_crc)); } // Remove a Suffix of given size from a buffer
diff --git a/absl/crc/internal/crc.cc b/absl/crc/internal/crc.cc index ec6a031..8019248 100644 --- a/absl/crc/internal/crc.cc +++ b/absl/crc/internal/crc.cc
@@ -43,6 +43,7 @@ #include <cstddef> #include <cstdint> +#include <cstring> #include <iterator> #include "absl/base/internal/endian.h" @@ -290,8 +291,7 @@ // starting at `ptr` and twelve zero bytes, so that four CRCs can be // built incrementally and combined at the end. const auto step_swath = [this](uint32_t crc_in, const std::uint8_t* ptr) { - return absl::little_endian::Load32(ptr) ^ - this->table_[3][crc_in & 0xff] ^ + return absl::little_endian::Load32(ptr) ^ this->table_[3][crc_in & 0xff] ^ this->table_[2][(crc_in >> 8) & 0xff] ^ this->table_[1][(crc_in >> 16) & 0xff] ^ this->table_[0][crc_in >> 24]; @@ -425,6 +425,12 @@ CRC::~CRC() {} CRC::CRC() {} +void CRC::ExtendAndCopy(uint32_t* crc, void* __restrict dst, + const void* __restrict src, size_t length) const { + std::memcpy(dst, src, length); + Extend(crc, dst, length); +} + // The "constructor" for a CRC32C with a standard polynomial. CRC* CRC::Crc32c() { static CRC* singleton = CRCImpl::NewInternal();
diff --git a/absl/crc/internal/crc.h b/absl/crc/internal/crc.h index 4efdd03..5efde0a 100644 --- a/absl/crc/internal/crc.h +++ b/absl/crc/internal/crc.h
@@ -46,6 +46,11 @@ virtual void Extend(uint32_t* crc, const void* bytes, size_t length) const = 0; + // Copy 'length' bytes from 'src' to 'dst' and extend the CRC with the copied + // bytes. + virtual void ExtendAndCopy(uint32_t* crc, void* __restrict dst, + const void* __restrict src, size_t length) const; + // Equivalent to Extend(crc, bytes, length) where "bytes" // points to an array of "length" zero bytes. virtual void ExtendByZeroes(uint32_t* crc, size_t length) const = 0;
diff --git a/absl/crc/internal/crc32_x86_arm_combined_simd.h b/absl/crc/internal/crc32_x86_arm_combined_simd.h index 9c287b5..e2eaf86 100644 --- a/absl/crc/internal/crc32_x86_arm_combined_simd.h +++ b/absl/crc/internal/crc32_x86_arm_combined_simd.h
@@ -16,7 +16,9 @@ #define ABSL_CRC_INTERNAL_CRC32_X86_ARM_COMBINED_SIMD_H_ #include <array> +#include <cstddef> #include <cstdint> +#include <cstring> #include "absl/base/config.h" @@ -70,7 +72,9 @@ using V256 = __m256i; #else // Placeholder for V256 when AVX is not available. -using V256 = std::array<uint64_t, 4>; +struct alignas(32) V256 { + std::array<uint64_t, 4> val; +}; #endif // Starting with the initial value in |crc|, accumulates a CRC32 value for @@ -133,17 +137,34 @@ // Add packed 64-bit integers in |l| and |r|. V128 V128_Add64(const V128 l, const V128 r); +// Performs a store fence on architectures that require it. +void StoreFence(); + #if defined(__AVX__) inline V256 V256_LoadU(const V256* src); inline V256 V256_Broadcast128(const V128* src); +inline void V256_StoreU(V256* dst, V256 data); +inline void V256_StoreNonTemporal(V256* dst, V256 data); #else template <typename T = V256> T V256_LoadU(const T* src); template <typename T = V256> T V256_Broadcast128(const V128* src); + +template <typename T = V256> +void V256_StoreU(T* dst, T data); + +template <typename T = V256> +void V256_StoreNonTemporal(T* dst, T data); #endif +inline void V256_LoadPairU(const void* src, V256* v0, V256* v1); +inline void V256_StorePairU(void* dst, V256 v0, V256 v1); +inline void V256_StorePairNonTemporal(void* dst, V256 v0, V256 v1); +inline void V256_CopyPairU(void* dst, const void* src); +inline void V256_CopyPairNonTemporal(void* dst, const void* src); + #endif #if defined(ABSL_CRC_INTERNAL_HAVE_X86_SIMD) @@ -213,6 +234,8 @@ return _mm_add_epi64(l, r); } +inline void StoreFence() { _mm_sfence(); } + #elif defined(ABSL_CRC_INTERNAL_HAVE_ARM_SIMD) inline uint32_t CRC32_u8(uint32_t crc, uint8_t v) { return __crc32cb(crc, v); } @@ -299,12 +322,11 @@ return V128_Xor(V128_Xor(a, b), c); } -inline V128 V128_From64WithZeroFill(const uint64_t r){ +inline V128 V128_From64WithZeroFill(const uint64_t r) { constexpr uint64x2_t kZero = {0, 0}; return vsetq_lane_u64(r, kZero, 0); } - template <int imm> inline int V128_Extract32(const V128 l) { return vgetq_lane_s32(vreinterpretq_s32_u64(l), imm); @@ -321,6 +343,8 @@ inline V128 V128_Add64(const V128 l, const V128 r) { return vaddq_u64(l, r); } +inline void StoreFence() {} + #endif #if defined(__AVX__) && defined(ABSL_CRC_INTERNAL_HAVE_X86_SIMD) @@ -330,12 +354,21 @@ return _mm256_castps_si256( _mm256_broadcast_ps(reinterpret_cast<const __m128*>(src))); } + +inline void V256_StoreU(V256* dst, V256 data) { + _mm256_storeu_si256(dst, data); +} + +inline void V256_StoreNonTemporal(V256* dst, V256 data) { + _mm256_stream_si256(dst, data); +} #elif defined(ABSL_CRC_INTERNAL_HAVE_X86_SIMD) || \ defined(ABSL_CRC_INTERNAL_HAVE_ARM_SIMD) template <typename T> inline T V256_LoadU(const T* src) { - (void)src; - return T{}; + T res; + std::memcpy(&res, src, sizeof(T)); + return res; } template <typename T> @@ -343,8 +376,54 @@ (void)src; return T{}; } + +template <typename T> +inline void V256_StoreU(T* dst, T data) { + std::memcpy(dst, &data, sizeof(T)); +} + +template <typename T> +inline void V256_StoreNonTemporal(T* dst, T data) { + std::memcpy(dst, &data, sizeof(T)); +} #endif +#if defined(ABSL_CRC_INTERNAL_HAVE_ARM_SIMD) || \ + defined(ABSL_CRC_INTERNAL_HAVE_X86_SIMD) + +inline void V256_LoadPairU(const void* src, V256* v0, V256* v1) { + const char* src_ptr = reinterpret_cast<const char*>(src); + *v0 = V256_LoadU(reinterpret_cast<const V256*>(src_ptr)); + *v1 = V256_LoadU(reinterpret_cast<const V256*>(src_ptr + sizeof(V256))); +} + +inline void V256_StorePairU(void* dst, V256 v0, V256 v1) { + V256* dst_ptr = reinterpret_cast<V256*>(dst); + V256_StoreU(dst_ptr, v0); + V256_StoreU(dst_ptr + 1, v1); +} + +inline void V256_StorePairNonTemporal(void* dst, V256 v0, V256 v1) { + V256* dst_ptr = reinterpret_cast<V256*>(dst); + V256_StoreNonTemporal(dst_ptr, v0); + V256_StoreNonTemporal(dst_ptr + 1, v1); +} + +inline void V256_CopyPairU(void* dst, const void* src) { + V256 v0, v1; + V256_LoadPairU(src, &v0, &v1); + V256_StorePairU(dst, v0, v1); +} + +inline void V256_CopyPairNonTemporal(void* dst, const void* src) { + V256 v0, v1; + V256_LoadPairU(src, &v0, &v1); + V256_StorePairNonTemporal(dst, v0, v1); +} + +#endif // defined(ABSL_CRC_INTERNAL_HAVE_ARM_SIMD) || + // defined(ABSL_CRC_INTERNAL_HAVE_X86_SIMD) + } // namespace crc_internal ABSL_NAMESPACE_END } // namespace absl
diff --git a/absl/crc/internal/crc_memcpy.h b/absl/crc/internal/crc_memcpy.h index a0fed65..72d7ce4 100644 --- a/absl/crc/internal/crc_memcpy.h +++ b/absl/crc/internal/crc_memcpy.h
@@ -16,10 +16,12 @@ #define ABSL_CRC_INTERNAL_CRC_MEMCPY_H_ #include <cstddef> +#include <cstdint> #include <memory> #include "absl/base/config.h" #include "absl/crc/crc32c.h" +#include "absl/crc/internal/crc.h" #include "absl/crc/internal/crc32_x86_arm_combined_simd.h" // Defined if the class AcceleratedCrcMemcpyEngine exists. @@ -50,11 +52,12 @@ public: static crc32c_t CrcAndCopy(void* __restrict dst, const void* __restrict src, std::size_t length, - crc32c_t initial_crc = crc32c_t{0}, - bool non_temporal = false) { - static const ArchSpecificEngines engines = GetArchSpecificEngines(); - auto* engine = non_temporal ? engines.non_temporal : engines.temporal; - return engine->Compute(dst, src, length, initial_crc); + crc32c_t initial_crc = crc32c_t{0}) { + uint32_t crc = static_cast<uint32_t>(initial_crc); + crc ^= 0xffffffffU; + CRC::Crc32c()->ExtendAndCopy(&crc, dst, src, length); + crc ^= 0xffffffffU; + return crc32c_t{crc}; } // For testing only: get an architecture-specific engine for tests. @@ -110,9 +113,8 @@ // the generic fallback version. inline crc32c_t Crc32CAndCopy(void* __restrict dst, const void* __restrict src, std::size_t length, - crc32c_t initial_crc = crc32c_t{0}, - bool non_temporal = false) { - return CrcMemcpy::CrcAndCopy(dst, src, length, initial_crc, non_temporal); + crc32c_t initial_crc = crc32c_t{0}) { + return CrcMemcpy::CrcAndCopy(dst, src, length, initial_crc); } } // namespace crc_internal
diff --git a/absl/crc/internal/crc_x86_arm_combined.cc b/absl/crc/internal/crc_x86_arm_combined.cc index edffd36..250732a 100644 --- a/absl/crc/internal/crc_x86_arm_combined.cc +++ b/absl/crc/internal/crc_x86_arm_combined.cc
@@ -16,6 +16,7 @@ #include <cstddef> #include <cstdint> +#include <cstring> #include <memory> #include <vector> @@ -228,6 +229,12 @@ class CRC32AcceleratedX86ARMCombinedMultipleStreamsBase : public CRC32AcceleratedX86ARMCombined { protected: + ABSL_ATTRIBUTE_ALWAYS_INLINE static void Copy64Bytes(char* dst, + const uint8_t* src) { + PrefetchToLocalCache(reinterpret_cast<const char*>(src) + kPrefetchHorizon); + V256_CopyPairU(dst, src); + } + // Update partialCRC with crc of 64 byte block. Calling FinalizePclmulStream // would produce a single crc checksum, but it is expensive. PCLMULQDQ has a // high latency, so we run 4 128-bit partial checksums that can be reduced to @@ -239,7 +246,7 @@ #if defined(ABSL_CRC_INTERNAL_HAVE_ARM_SIMD) template <bool kUseEor3 = false> ABSL_ATTRIBUTE_ALWAYS_INLINE void Process64BytesNeonPclmul( - const uint8_t* p, V128* partialCRC) const { + V256 v0, V256 v1, V128* partialCRC) const { V128 loopMultiplicands = V128_Load(reinterpret_cast<const V128*>(kFoldAcross512Bits)); @@ -252,10 +259,12 @@ V128 tmp2 = V128_PMulHi(partialCRC2, loopMultiplicands); V128 tmp3 = V128_PMulHi(partialCRC3, loopMultiplicands); V128 tmp4 = V128_PMulHi(partialCRC4, loopMultiplicands); - V128 data1 = V128_LoadU(reinterpret_cast<const V128*>(p + 16 * 0)); - V128 data2 = V128_LoadU(reinterpret_cast<const V128*>(p + 16 * 1)); - V128 data3 = V128_LoadU(reinterpret_cast<const V128*>(p + 16 * 2)); - V128 data4 = V128_LoadU(reinterpret_cast<const V128*>(p + 16 * 3)); + const V128* data_ptr0 = reinterpret_cast<const V128*>(&v0); + const V128* data_ptr1 = reinterpret_cast<const V128*>(&v1); + V128 data1 = data_ptr0[0]; + V128 data2 = data_ptr0[1]; + V128 data3 = data_ptr1[0]; + V128 data4 = data_ptr1[1]; partialCRC1 = V128_PMulLow(partialCRC1, loopMultiplicands); partialCRC2 = V128_PMulLow(partialCRC2, loopMultiplicands); partialCRC3 = V128_PMulLow(partialCRC3, loopMultiplicands); @@ -308,7 +317,8 @@ return crc; } - ABSL_ATTRIBUTE_ALWAYS_INLINE void Process64BytesPclmul(const uint8_t*, + template <bool copy_data = false> + ABSL_ATTRIBUTE_ALWAYS_INLINE void Process64BytesPclmul(V256, V256, V128*) const {} ABSL_ATTRIBUTE_ALWAYS_INLINE uint64_t FinalizePclmulStream(V128*) const { @@ -316,7 +326,7 @@ } #else ABSL_ATTRIBUTE_ALWAYS_INLINE void Process64BytesPclmul( - const uint8_t* p, V128* partialCRC) const { + V256 v0, V256 v1, V128* partialCRC) const { V128 loopMultiplicands = V128_Load(reinterpret_cast<const V128*>(kFoldAcross512Bits)); @@ -329,10 +339,12 @@ V128 tmp2 = V128_PMulHi(partialCRC2, loopMultiplicands); V128 tmp3 = V128_PMulHi(partialCRC3, loopMultiplicands); V128 tmp4 = V128_PMulHi(partialCRC4, loopMultiplicands); - V128 data1 = V128_LoadU(reinterpret_cast<const V128*>(p + 16 * 0)); - V128 data2 = V128_LoadU(reinterpret_cast<const V128*>(p + 16 * 1)); - V128 data3 = V128_LoadU(reinterpret_cast<const V128*>(p + 16 * 2)); - V128 data4 = V128_LoadU(reinterpret_cast<const V128*>(p + 16 * 3)); + const V128* data_ptr0 = reinterpret_cast<const V128*>(&v0); + const V128* data_ptr1 = reinterpret_cast<const V128*>(&v1); + V128 data1 = data_ptr0[0]; + V128 data2 = data_ptr0[1]; + V128 data3 = data_ptr1[0]; + V128 data4 = data_ptr1[1]; partialCRC1 = V128_PMulLow(partialCRC1, loopMultiplicands); partialCRC2 = V128_PMulLow(partialCRC2, loopMultiplicands); partialCRC3 = V128_PMulLow(partialCRC3, loopMultiplicands); @@ -392,7 +404,7 @@ } template <bool kUseEor3 = false> - ABSL_ATTRIBUTE_ALWAYS_INLINE void Process64BytesNeonPclmul(const uint8_t*, + ABSL_ATTRIBUTE_ALWAYS_INLINE void Process64BytesNeonPclmul(V256, V256, V128*) const {} template <bool kUseEor3 = false> @@ -493,7 +505,7 @@ } ABSL_ATTRIBUTE_ALWAYS_INLINE void Process64BytesVpclmul( - const uint8_t* p, V256* vpartialCRC, V256 loopMultiplicands) const { + V256 v0, V256 v1, V256* vpartialCRC, V256 loopMultiplicands) const { __asm__ volatile( "vpclmulqdq $0x11, %3, %0, %%ymm0 \n" "vpclmulqdq $0x11, %3, %1, %%ymm1 \n" @@ -501,16 +513,15 @@ "vpclmulqdq $0x00, %3, %1, %1 \n" "vpxor %%ymm0, %0, %0 \n" "vpxor %%ymm1, %1, %1 \n" - "vpxor (%2), %0, %0 \n" - "vpxor 32(%2), %1, %1 \n" + "vpxor %2, %0, %0 \n" + "vpxor %4, %1, %1 \n" : "+x"(vpartialCRC[0]), "+x"(vpartialCRC[1]) - : "r"(p), "x"(loopMultiplicands) + : "x"(v0), "x"(loopMultiplicands), "x"(v1) : "ymm0", "ymm1"); } #else template <typename T = V256> - ABSL_ATTRIBUTE_ALWAYS_INLINE void Process64BytesVpclmul(const uint8_t*, T*, - T) const { + ABSL_ATTRIBUTE_ALWAYS_INLINE void Process64BytesVpclmul(T, T, T*, T) const { static_assert(sizeof(T) == 0, "Vector PCLMUL not supported"); } ABSL_ATTRIBUTE_ALWAYS_INLINE uint64_t FinalizeVpclmulStream(V256*) const { @@ -551,24 +562,44 @@ PclmulStreamType pclmul_stream_type> class CRC32AcceleratedX86ARMCombinedMultipleStreams : public CRC32AcceleratedX86ARMCombinedMultipleStreamsBase { - ABSL_ATTRIBUTE_HOT + public: void Extend(uint32_t* crc, const void* bytes, const size_t length) const override { + Extend<false>(crc, bytes, length); + } + + void ExtendAndCopy(uint32_t* crc, void* __restrict dst, + const void* __restrict src, size_t length) const override { + Extend<true>(crc, src, length, dst); + StoreFence(); + } + + template <bool copy = false> + ABSL_ATTRIBUTE_HOT void Extend(uint32_t* crc, const void* bytes, + const size_t length, + void* __restrict dst = nullptr) const { static_assert(num_crc_streams >= 1 && num_crc_streams <= kMaxStreams, "Invalid number of crc streams"); static_assert(num_pclmul_streams >= 0 && num_pclmul_streams <= kMaxStreams, "Invalid number of pclmul streams"); const uint8_t* p = static_cast<const uint8_t*>(bytes); const uint8_t* e = p + length; + char* d = static_cast<char*>(dst); uint32_t l = *crc; uint64_t l64; // For small blocks just run simple loop, because cost of combining multiple // streams is significant. if (num_crc_streams > 1 && (length < kSmallCutoff)) { + if constexpr (copy) { + std::memcpy(d, p, length); + } // fallthrough; Use the same strategy as we do for processing the // remaining bytes after any other strategy. } else if (length < kMediumCutoff) { + if constexpr (copy) { + std::memcpy(d, p, length); + } // For medium blocks we run 3 crc streams and combine them as described in // Intel paper above. Running 4th stream doesn't help, because crc // instruction has latency 3 and throughput 1. @@ -628,11 +659,29 @@ // so on some cpus it makes sense to execute both of them for different // streams. - // Point x at first 8-byte aligned byte in string. - const uint8_t* x = RoundUp<8>(p); - // Process bytes until p is 8-byte aligned, if that isn't past the end. - while (p != x) { - ABSL_INTERNAL_STEP1(l, p); + if constexpr (copy) { + // Align destination pointer for V256 non-temporal stores. + constexpr size_t kAlignment = 64; + uintptr_t dst_addr = reinterpret_cast<uintptr_t>(d); + size_t bytes_to_align = + (kAlignment - (dst_addr & (kAlignment - 1))) & (kAlignment - 1); + std::memcpy(d, p, bytes_to_align); + d += bytes_to_align; + while (bytes_to_align >= 8) { + ABSL_INTERNAL_STEP8(l, p); + bytes_to_align -= 8; + } + while (bytes_to_align > 0) { + ABSL_INTERNAL_STEP1(l, p); + bytes_to_align -= 1; + } + } else { + // Point x at first 8-byte aligned byte in string. + const uint8_t* x = RoundUp<8>(p); + // Process bytes until p is 8-byte aligned, if that isn't past the end. + while (p != x) { + ABSL_INTERNAL_STEP1(l, p); + } } size_t bs = static_cast<size_t>(e - p) / @@ -649,11 +698,41 @@ stream_start += bs * 64; } + uint8_t* dst_crc_streams[kMaxStreams]; + uint8_t* dst_pclmul_streams[kMaxStreams]; + if constexpr (copy) { + uint8_t* dst_stream_start = reinterpret_cast<uint8_t*>(d); + for (size_t i = 0; i < num_crc_streams; i++) { + dst_crc_streams[i] = dst_stream_start; + dst_stream_start += bs * 64; + } + for (size_t i = 0; i < num_pclmul_streams; i++) { + dst_pclmul_streams[i] = dst_stream_start; + dst_stream_start += bs * 64; + } + } + // Per stream crc sums. uint64_t l64_crc[kMaxStreams] = {l}; uint64_t l64_pclmul[kMaxStreams] = {0}; - // Peel first iteration, because PCLMULQDQ stream, needs setup. + V256 pclmul_v0_0[kMaxStreams]; + V256 pclmul_v1_0[kMaxStreams]; + for (size_t j = 0; j < num_pclmul_streams; j++) { + V256_LoadPairU(pclmul_streams[j], &pclmul_v0_0[j], &pclmul_v1_0[j]); + } + + if constexpr (copy) { + for (size_t j = 0; j < num_crc_streams; j++) { + V256_CopyPairNonTemporal(dst_crc_streams[j], crc_streams[j]); + dst_crc_streams[j] += 64; + } + for (size_t j = 0; j < num_pclmul_streams; j++) { + V256_StorePairNonTemporal(dst_pclmul_streams[j], pclmul_v0_0[j], + pclmul_v1_0[j]); + dst_pclmul_streams[j] += 64; + } + } if (num_crc_streams == 1) { l64_crc[0] = Process64BytesCRC(crc_streams[0], l64_crc[0]); crc_streams[0] += 16 * 4; @@ -672,20 +751,18 @@ // Align to 32 bytes for vpclmul implementation. alignas(32) V128 partialCRC[kMaxStreams][4]; for (size_t i = 0; i < num_pclmul_streams; i++) { - InitPclmulStream(&pclmul_streams[i], partialCRC[i]); + InitPclmulStream(pclmul_v0_0[i], pclmul_v1_0[i], partialCRC[i]); + pclmul_streams[i] += 64; } for (size_t i = 1; i < bs; i++) { // Prefetch data for next iterations. for (size_t j = 0; j < num_crc_streams; j++) { - PrefetchToLocalCache( - reinterpret_cast<const char*>(crc_streams[j] + kPrefetchHorizon)); + PrefetchToLocalCache(crc_streams[j] + kPrefetchHorizon); } for (size_t j = 0; j < num_pclmul_streams; j++) { - PrefetchToLocalCache(reinterpret_cast<const char*>(pclmul_streams[j] + - kPrefetchHorizon)); + PrefetchToLocalCache(pclmul_streams[j] + kPrefetchHorizon); } - // We process each stream in 64 byte blocks. This can be written as // for (int i = 0; i < num_pclmul_streams; i++) { // Process64BytesPclmul(pclmul_streams[i], partialCRC[i]); @@ -697,6 +774,24 @@ // } // But unrolling and interleaving PCLMULQDQ and CRC blocks manually // gives ~2% performance boost. + if constexpr (copy) { + for (size_t j = 0; j < num_crc_streams; j++) { + V256_CopyPairNonTemporal(dst_crc_streams[j], crc_streams[j]); + dst_crc_streams[j] += 64; + } + } + + for (size_t j = 0; j < num_pclmul_streams; j++) { + if constexpr (copy) { + ProcessPclmulStream<true>(pclmul_streams[j], dst_pclmul_streams[j], + partialCRC[j]); + dst_pclmul_streams[j] += 64; + } else { + ProcessPclmulStream<false>(pclmul_streams[j], nullptr, + partialCRC[j]); + } + pclmul_streams[j] += 64; + } if (num_crc_streams == 1) { l64_crc[0] = Process64BytesCRC(crc_streams[0], l64_crc[0]); crc_streams[0] += 16 * 4; @@ -711,9 +806,6 @@ crc_streams[1] += 16 * 4; crc_streams[2] += 16 * 4; } - for (size_t j = 0; j < num_pclmul_streams; j++) { - ProcessPclmulStream(&pclmul_streams[j], partialCRC[j]); - } } // PCLMULQDQ based streams require special final step; @@ -735,12 +827,24 @@ l64 ^= l64_pclmul[i]; } - // Update p. + // Update p and d. if constexpr (num_pclmul_streams > 0) { p = pclmul_streams[num_pclmul_streams - 1]; + if constexpr (copy) { + d = reinterpret_cast<char*>( + dst_pclmul_streams[num_pclmul_streams - 1]); + } } else { p = crc_streams[num_crc_streams - 1]; + if constexpr (copy) { + d = reinterpret_cast<char*>(dst_crc_streams[num_crc_streams - 1]); + } } + if constexpr (copy) { + size_t remaining_to_copy = static_cast<size_t>(e - p); + std::memcpy(d, p, remaining_to_copy); + } + l = static_cast<uint32_t>(l64); } @@ -768,44 +872,45 @@ } private: - ABSL_ATTRIBUTE_ALWAYS_INLINE void InitPclmulStream( - const uint8_t** pclmul_stream, V128* partialCRC) const { + ABSL_ATTRIBUTE_ALWAYS_INLINE void InitPclmulStream(V256 v0, V256 v1, + V128* partialCRC) const { if constexpr (pclmul_stream_type == PclmulStreamType::VPCLMUL) { V256* vpartialCRC = reinterpret_cast<V256*>(partialCRC); - vpartialCRC[0] = - V256_LoadU(reinterpret_cast<const V256*>(*pclmul_stream + 32 * 0)); - vpartialCRC[1] = - V256_LoadU(reinterpret_cast<const V256*>(*pclmul_stream + 32 * 1)); + vpartialCRC[0] = v0; + vpartialCRC[1] = v1; } else { - partialCRC[0] = - V128_LoadU(reinterpret_cast<const V128*>(*pclmul_stream + 16 * 0)); - partialCRC[1] = - V128_LoadU(reinterpret_cast<const V128*>(*pclmul_stream + 16 * 1)); - partialCRC[2] = - V128_LoadU(reinterpret_cast<const V128*>(*pclmul_stream + 16 * 2)); - partialCRC[3] = - V128_LoadU(reinterpret_cast<const V128*>(*pclmul_stream + 16 * 3)); + const V128* data_ptr0 = reinterpret_cast<const V128*>(&v0); + const V128* data_ptr1 = reinterpret_cast<const V128*>(&v1); + partialCRC[0] = data_ptr0[0]; + partialCRC[1] = data_ptr0[1]; + partialCRC[2] = data_ptr1[0]; + partialCRC[3] = data_ptr1[1]; } - *pclmul_stream += 16 * 4; } + template <bool copy = false> ABSL_ATTRIBUTE_ALWAYS_INLINE void ProcessPclmulStream( - const uint8_t** pclmul_stream, V128* partialCRC) const { + const uint8_t* stream, uint8_t* dst_stream, V128* partialCRC) const { + const uint8_t* src_ptr = stream; + V256 v0, v1; + V256_LoadPairU(src_ptr, &v0, &v1); + if constexpr (copy) { + V256_StorePairNonTemporal(dst_stream, v0, v1); + } if constexpr (pclmul_stream_type == PclmulStreamType::VPCLMUL) { V256 loopMultiplicands = V256_Broadcast128(reinterpret_cast<const V128*>(kFoldAcross512Bits)); - Process64BytesVpclmul(*pclmul_stream, reinterpret_cast<V256*>(partialCRC), + Process64BytesVpclmul(v0, v1, reinterpret_cast<V256*>(partialCRC), loopMultiplicands); } else if constexpr (pclmul_stream_type == PclmulStreamType::NEON_PCLMUL || pclmul_stream_type == PclmulStreamType::NEON_PCLMUL_EOR3) { constexpr bool kUseEor3 = (pclmul_stream_type == PclmulStreamType::NEON_PCLMUL_EOR3); - Process64BytesNeonPclmul<kUseEor3>(*pclmul_stream, partialCRC); + Process64BytesNeonPclmul<kUseEor3>(v0, v1, partialCRC); } else { - Process64BytesPclmul(*pclmul_stream, partialCRC); + Process64BytesPclmul(v0, v1, partialCRC); } - *pclmul_stream += 16 * 4; } ABSL_ATTRIBUTE_ALWAYS_INLINE uint64_t