Rollback Merge CRC+memcpy functionality into Extend function PiperOrigin-RevId: 965170146 Change-Id: I88422ad30e170e4acb6511cba4afa467c9d4532b
diff --git a/absl/crc/crc32c.cc b/absl/crc/crc32c.cc index 8244e32..9b1ef7e 100644 --- a/absl/crc/crc32c.cc +++ b/absl/crc/crc32c.cc
@@ -73,7 +73,7 @@ crc32c_t MemcpyCrc32c(void* dest, const void* src, size_t count, crc32c_t initial_crc) { return static_cast<crc32c_t>( - crc_internal::Crc32CAndCopy(dest, src, count, initial_crc)); + crc_internal::Crc32CAndCopy(dest, src, count, initial_crc, false)); } // Remove a Suffix of given size from a buffer
diff --git a/absl/crc/internal/crc.cc b/absl/crc/internal/crc.cc index 8019248..ec6a031 100644 --- a/absl/crc/internal/crc.cc +++ b/absl/crc/internal/crc.cc
@@ -43,7 +43,6 @@ #include <cstddef> #include <cstdint> -#include <cstring> #include <iterator> #include "absl/base/internal/endian.h" @@ -291,7 +290,8 @@ // starting at `ptr` and twelve zero bytes, so that four CRCs can be // built incrementally and combined at the end. const auto step_swath = [this](uint32_t crc_in, const std::uint8_t* ptr) { - return absl::little_endian::Load32(ptr) ^ this->table_[3][crc_in & 0xff] ^ + return absl::little_endian::Load32(ptr) ^ + this->table_[3][crc_in & 0xff] ^ this->table_[2][(crc_in >> 8) & 0xff] ^ this->table_[1][(crc_in >> 16) & 0xff] ^ this->table_[0][crc_in >> 24]; @@ -425,12 +425,6 @@ CRC::~CRC() {} CRC::CRC() {} -void CRC::ExtendAndCopy(uint32_t* crc, void* __restrict dst, - const void* __restrict src, size_t length) const { - std::memcpy(dst, src, length); - Extend(crc, dst, length); -} - // The "constructor" for a CRC32C with a standard polynomial. CRC* CRC::Crc32c() { static CRC* singleton = CRCImpl::NewInternal();
diff --git a/absl/crc/internal/crc.h b/absl/crc/internal/crc.h index 5efde0a..4efdd03 100644 --- a/absl/crc/internal/crc.h +++ b/absl/crc/internal/crc.h
@@ -46,11 +46,6 @@ virtual void Extend(uint32_t* crc, const void* bytes, size_t length) const = 0; - // Copy 'length' bytes from 'src' to 'dst' and extend the CRC with the copied - // bytes. - virtual void ExtendAndCopy(uint32_t* crc, void* __restrict dst, - const void* __restrict src, size_t length) const; - // Equivalent to Extend(crc, bytes, length) where "bytes" // points to an array of "length" zero bytes. virtual void ExtendByZeroes(uint32_t* crc, size_t length) const = 0;
diff --git a/absl/crc/internal/crc32_x86_arm_combined_simd.h b/absl/crc/internal/crc32_x86_arm_combined_simd.h index e2eaf86..9c287b5 100644 --- a/absl/crc/internal/crc32_x86_arm_combined_simd.h +++ b/absl/crc/internal/crc32_x86_arm_combined_simd.h
@@ -16,9 +16,7 @@ #define ABSL_CRC_INTERNAL_CRC32_X86_ARM_COMBINED_SIMD_H_ #include <array> -#include <cstddef> #include <cstdint> -#include <cstring> #include "absl/base/config.h" @@ -72,9 +70,7 @@ using V256 = __m256i; #else // Placeholder for V256 when AVX is not available. -struct alignas(32) V256 { - std::array<uint64_t, 4> val; -}; +using V256 = std::array<uint64_t, 4>; #endif // Starting with the initial value in |crc|, accumulates a CRC32 value for @@ -137,34 +133,17 @@ // Add packed 64-bit integers in |l| and |r|. V128 V128_Add64(const V128 l, const V128 r); -// Performs a store fence on architectures that require it. -void StoreFence(); - #if defined(__AVX__) inline V256 V256_LoadU(const V256* src); inline V256 V256_Broadcast128(const V128* src); -inline void V256_StoreU(V256* dst, V256 data); -inline void V256_StoreNonTemporal(V256* dst, V256 data); #else template <typename T = V256> T V256_LoadU(const T* src); template <typename T = V256> T V256_Broadcast128(const V128* src); - -template <typename T = V256> -void V256_StoreU(T* dst, T data); - -template <typename T = V256> -void V256_StoreNonTemporal(T* dst, T data); #endif -inline void V256_LoadPairU(const void* src, V256* v0, V256* v1); -inline void V256_StorePairU(void* dst, V256 v0, V256 v1); -inline void V256_StorePairNonTemporal(void* dst, V256 v0, V256 v1); -inline void V256_CopyPairU(void* dst, const void* src); -inline void V256_CopyPairNonTemporal(void* dst, const void* src); - #endif #if defined(ABSL_CRC_INTERNAL_HAVE_X86_SIMD) @@ -234,8 +213,6 @@ return _mm_add_epi64(l, r); } -inline void StoreFence() { _mm_sfence(); } - #elif defined(ABSL_CRC_INTERNAL_HAVE_ARM_SIMD) inline uint32_t CRC32_u8(uint32_t crc, uint8_t v) { return __crc32cb(crc, v); } @@ -322,11 +299,12 @@ return V128_Xor(V128_Xor(a, b), c); } -inline V128 V128_From64WithZeroFill(const uint64_t r) { +inline V128 V128_From64WithZeroFill(const uint64_t r){ constexpr uint64x2_t kZero = {0, 0}; return vsetq_lane_u64(r, kZero, 0); } + template <int imm> inline int V128_Extract32(const V128 l) { return vgetq_lane_s32(vreinterpretq_s32_u64(l), imm); @@ -343,8 +321,6 @@ inline V128 V128_Add64(const V128 l, const V128 r) { return vaddq_u64(l, r); } -inline void StoreFence() {} - #endif #if defined(__AVX__) && defined(ABSL_CRC_INTERNAL_HAVE_X86_SIMD) @@ -354,21 +330,12 @@ return _mm256_castps_si256( _mm256_broadcast_ps(reinterpret_cast<const __m128*>(src))); } - -inline void V256_StoreU(V256* dst, V256 data) { - _mm256_storeu_si256(dst, data); -} - -inline void V256_StoreNonTemporal(V256* dst, V256 data) { - _mm256_stream_si256(dst, data); -} #elif defined(ABSL_CRC_INTERNAL_HAVE_X86_SIMD) || \ defined(ABSL_CRC_INTERNAL_HAVE_ARM_SIMD) template <typename T> inline T V256_LoadU(const T* src) { - T res; - std::memcpy(&res, src, sizeof(T)); - return res; + (void)src; + return T{}; } template <typename T> @@ -376,54 +343,8 @@ (void)src; return T{}; } - -template <typename T> -inline void V256_StoreU(T* dst, T data) { - std::memcpy(dst, &data, sizeof(T)); -} - -template <typename T> -inline void V256_StoreNonTemporal(T* dst, T data) { - std::memcpy(dst, &data, sizeof(T)); -} #endif -#if defined(ABSL_CRC_INTERNAL_HAVE_ARM_SIMD) || \ - defined(ABSL_CRC_INTERNAL_HAVE_X86_SIMD) - -inline void V256_LoadPairU(const void* src, V256* v0, V256* v1) { - const char* src_ptr = reinterpret_cast<const char*>(src); - *v0 = V256_LoadU(reinterpret_cast<const V256*>(src_ptr)); - *v1 = V256_LoadU(reinterpret_cast<const V256*>(src_ptr + sizeof(V256))); -} - -inline void V256_StorePairU(void* dst, V256 v0, V256 v1) { - V256* dst_ptr = reinterpret_cast<V256*>(dst); - V256_StoreU(dst_ptr, v0); - V256_StoreU(dst_ptr + 1, v1); -} - -inline void V256_StorePairNonTemporal(void* dst, V256 v0, V256 v1) { - V256* dst_ptr = reinterpret_cast<V256*>(dst); - V256_StoreNonTemporal(dst_ptr, v0); - V256_StoreNonTemporal(dst_ptr + 1, v1); -} - -inline void V256_CopyPairU(void* dst, const void* src) { - V256 v0, v1; - V256_LoadPairU(src, &v0, &v1); - V256_StorePairU(dst, v0, v1); -} - -inline void V256_CopyPairNonTemporal(void* dst, const void* src) { - V256 v0, v1; - V256_LoadPairU(src, &v0, &v1); - V256_StorePairNonTemporal(dst, v0, v1); -} - -#endif // defined(ABSL_CRC_INTERNAL_HAVE_ARM_SIMD) || - // defined(ABSL_CRC_INTERNAL_HAVE_X86_SIMD) - } // namespace crc_internal ABSL_NAMESPACE_END } // namespace absl
diff --git a/absl/crc/internal/crc_memcpy.h b/absl/crc/internal/crc_memcpy.h index 72d7ce4..a0fed65 100644 --- a/absl/crc/internal/crc_memcpy.h +++ b/absl/crc/internal/crc_memcpy.h
@@ -16,12 +16,10 @@ #define ABSL_CRC_INTERNAL_CRC_MEMCPY_H_ #include <cstddef> -#include <cstdint> #include <memory> #include "absl/base/config.h" #include "absl/crc/crc32c.h" -#include "absl/crc/internal/crc.h" #include "absl/crc/internal/crc32_x86_arm_combined_simd.h" // Defined if the class AcceleratedCrcMemcpyEngine exists. @@ -52,12 +50,11 @@ public: static crc32c_t CrcAndCopy(void* __restrict dst, const void* __restrict src, std::size_t length, - crc32c_t initial_crc = crc32c_t{0}) { - uint32_t crc = static_cast<uint32_t>(initial_crc); - crc ^= 0xffffffffU; - CRC::Crc32c()->ExtendAndCopy(&crc, dst, src, length); - crc ^= 0xffffffffU; - return crc32c_t{crc}; + crc32c_t initial_crc = crc32c_t{0}, + bool non_temporal = false) { + static const ArchSpecificEngines engines = GetArchSpecificEngines(); + auto* engine = non_temporal ? engines.non_temporal : engines.temporal; + return engine->Compute(dst, src, length, initial_crc); } // For testing only: get an architecture-specific engine for tests. @@ -113,8 +110,9 @@ // the generic fallback version. inline crc32c_t Crc32CAndCopy(void* __restrict dst, const void* __restrict src, std::size_t length, - crc32c_t initial_crc = crc32c_t{0}) { - return CrcMemcpy::CrcAndCopy(dst, src, length, initial_crc); + crc32c_t initial_crc = crc32c_t{0}, + bool non_temporal = false) { + return CrcMemcpy::CrcAndCopy(dst, src, length, initial_crc, non_temporal); } } // namespace crc_internal
diff --git a/absl/crc/internal/crc_x86_arm_combined.cc b/absl/crc/internal/crc_x86_arm_combined.cc index 250732a..edffd36 100644 --- a/absl/crc/internal/crc_x86_arm_combined.cc +++ b/absl/crc/internal/crc_x86_arm_combined.cc
@@ -16,7 +16,6 @@ #include <cstddef> #include <cstdint> -#include <cstring> #include <memory> #include <vector> @@ -229,12 +228,6 @@ class CRC32AcceleratedX86ARMCombinedMultipleStreamsBase : public CRC32AcceleratedX86ARMCombined { protected: - ABSL_ATTRIBUTE_ALWAYS_INLINE static void Copy64Bytes(char* dst, - const uint8_t* src) { - PrefetchToLocalCache(reinterpret_cast<const char*>(src) + kPrefetchHorizon); - V256_CopyPairU(dst, src); - } - // Update partialCRC with crc of 64 byte block. Calling FinalizePclmulStream // would produce a single crc checksum, but it is expensive. PCLMULQDQ has a // high latency, so we run 4 128-bit partial checksums that can be reduced to @@ -246,7 +239,7 @@ #if defined(ABSL_CRC_INTERNAL_HAVE_ARM_SIMD) template <bool kUseEor3 = false> ABSL_ATTRIBUTE_ALWAYS_INLINE void Process64BytesNeonPclmul( - V256 v0, V256 v1, V128* partialCRC) const { + const uint8_t* p, V128* partialCRC) const { V128 loopMultiplicands = V128_Load(reinterpret_cast<const V128*>(kFoldAcross512Bits)); @@ -259,12 +252,10 @@ V128 tmp2 = V128_PMulHi(partialCRC2, loopMultiplicands); V128 tmp3 = V128_PMulHi(partialCRC3, loopMultiplicands); V128 tmp4 = V128_PMulHi(partialCRC4, loopMultiplicands); - const V128* data_ptr0 = reinterpret_cast<const V128*>(&v0); - const V128* data_ptr1 = reinterpret_cast<const V128*>(&v1); - V128 data1 = data_ptr0[0]; - V128 data2 = data_ptr0[1]; - V128 data3 = data_ptr1[0]; - V128 data4 = data_ptr1[1]; + V128 data1 = V128_LoadU(reinterpret_cast<const V128*>(p + 16 * 0)); + V128 data2 = V128_LoadU(reinterpret_cast<const V128*>(p + 16 * 1)); + V128 data3 = V128_LoadU(reinterpret_cast<const V128*>(p + 16 * 2)); + V128 data4 = V128_LoadU(reinterpret_cast<const V128*>(p + 16 * 3)); partialCRC1 = V128_PMulLow(partialCRC1, loopMultiplicands); partialCRC2 = V128_PMulLow(partialCRC2, loopMultiplicands); partialCRC3 = V128_PMulLow(partialCRC3, loopMultiplicands); @@ -317,8 +308,7 @@ return crc; } - template <bool copy_data = false> - ABSL_ATTRIBUTE_ALWAYS_INLINE void Process64BytesPclmul(V256, V256, + ABSL_ATTRIBUTE_ALWAYS_INLINE void Process64BytesPclmul(const uint8_t*, V128*) const {} ABSL_ATTRIBUTE_ALWAYS_INLINE uint64_t FinalizePclmulStream(V128*) const { @@ -326,7 +316,7 @@ } #else ABSL_ATTRIBUTE_ALWAYS_INLINE void Process64BytesPclmul( - V256 v0, V256 v1, V128* partialCRC) const { + const uint8_t* p, V128* partialCRC) const { V128 loopMultiplicands = V128_Load(reinterpret_cast<const V128*>(kFoldAcross512Bits)); @@ -339,12 +329,10 @@ V128 tmp2 = V128_PMulHi(partialCRC2, loopMultiplicands); V128 tmp3 = V128_PMulHi(partialCRC3, loopMultiplicands); V128 tmp4 = V128_PMulHi(partialCRC4, loopMultiplicands); - const V128* data_ptr0 = reinterpret_cast<const V128*>(&v0); - const V128* data_ptr1 = reinterpret_cast<const V128*>(&v1); - V128 data1 = data_ptr0[0]; - V128 data2 = data_ptr0[1]; - V128 data3 = data_ptr1[0]; - V128 data4 = data_ptr1[1]; + V128 data1 = V128_LoadU(reinterpret_cast<const V128*>(p + 16 * 0)); + V128 data2 = V128_LoadU(reinterpret_cast<const V128*>(p + 16 * 1)); + V128 data3 = V128_LoadU(reinterpret_cast<const V128*>(p + 16 * 2)); + V128 data4 = V128_LoadU(reinterpret_cast<const V128*>(p + 16 * 3)); partialCRC1 = V128_PMulLow(partialCRC1, loopMultiplicands); partialCRC2 = V128_PMulLow(partialCRC2, loopMultiplicands); partialCRC3 = V128_PMulLow(partialCRC3, loopMultiplicands); @@ -404,7 +392,7 @@ } template <bool kUseEor3 = false> - ABSL_ATTRIBUTE_ALWAYS_INLINE void Process64BytesNeonPclmul(V256, V256, + ABSL_ATTRIBUTE_ALWAYS_INLINE void Process64BytesNeonPclmul(const uint8_t*, V128*) const {} template <bool kUseEor3 = false> @@ -505,7 +493,7 @@ } ABSL_ATTRIBUTE_ALWAYS_INLINE void Process64BytesVpclmul( - V256 v0, V256 v1, V256* vpartialCRC, V256 loopMultiplicands) const { + const uint8_t* p, V256* vpartialCRC, V256 loopMultiplicands) const { __asm__ volatile( "vpclmulqdq $0x11, %3, %0, %%ymm0 \n" "vpclmulqdq $0x11, %3, %1, %%ymm1 \n" @@ -513,15 +501,16 @@ "vpclmulqdq $0x00, %3, %1, %1 \n" "vpxor %%ymm0, %0, %0 \n" "vpxor %%ymm1, %1, %1 \n" - "vpxor %2, %0, %0 \n" - "vpxor %4, %1, %1 \n" + "vpxor (%2), %0, %0 \n" + "vpxor 32(%2), %1, %1 \n" : "+x"(vpartialCRC[0]), "+x"(vpartialCRC[1]) - : "x"(v0), "x"(loopMultiplicands), "x"(v1) + : "r"(p), "x"(loopMultiplicands) : "ymm0", "ymm1"); } #else template <typename T = V256> - ABSL_ATTRIBUTE_ALWAYS_INLINE void Process64BytesVpclmul(T, T, T*, T) const { + ABSL_ATTRIBUTE_ALWAYS_INLINE void Process64BytesVpclmul(const uint8_t*, T*, + T) const { static_assert(sizeof(T) == 0, "Vector PCLMUL not supported"); } ABSL_ATTRIBUTE_ALWAYS_INLINE uint64_t FinalizeVpclmulStream(V256*) const { @@ -562,44 +551,24 @@ PclmulStreamType pclmul_stream_type> class CRC32AcceleratedX86ARMCombinedMultipleStreams : public CRC32AcceleratedX86ARMCombinedMultipleStreamsBase { - public: + ABSL_ATTRIBUTE_HOT void Extend(uint32_t* crc, const void* bytes, const size_t length) const override { - Extend<false>(crc, bytes, length); - } - - void ExtendAndCopy(uint32_t* crc, void* __restrict dst, - const void* __restrict src, size_t length) const override { - Extend<true>(crc, src, length, dst); - StoreFence(); - } - - template <bool copy = false> - ABSL_ATTRIBUTE_HOT void Extend(uint32_t* crc, const void* bytes, - const size_t length, - void* __restrict dst = nullptr) const { static_assert(num_crc_streams >= 1 && num_crc_streams <= kMaxStreams, "Invalid number of crc streams"); static_assert(num_pclmul_streams >= 0 && num_pclmul_streams <= kMaxStreams, "Invalid number of pclmul streams"); const uint8_t* p = static_cast<const uint8_t*>(bytes); const uint8_t* e = p + length; - char* d = static_cast<char*>(dst); uint32_t l = *crc; uint64_t l64; // For small blocks just run simple loop, because cost of combining multiple // streams is significant. if (num_crc_streams > 1 && (length < kSmallCutoff)) { - if constexpr (copy) { - std::memcpy(d, p, length); - } // fallthrough; Use the same strategy as we do for processing the // remaining bytes after any other strategy. } else if (length < kMediumCutoff) { - if constexpr (copy) { - std::memcpy(d, p, length); - } // For medium blocks we run 3 crc streams and combine them as described in // Intel paper above. Running 4th stream doesn't help, because crc // instruction has latency 3 and throughput 1. @@ -659,29 +628,11 @@ // so on some cpus it makes sense to execute both of them for different // streams. - if constexpr (copy) { - // Align destination pointer for V256 non-temporal stores. - constexpr size_t kAlignment = 64; - uintptr_t dst_addr = reinterpret_cast<uintptr_t>(d); - size_t bytes_to_align = - (kAlignment - (dst_addr & (kAlignment - 1))) & (kAlignment - 1); - std::memcpy(d, p, bytes_to_align); - d += bytes_to_align; - while (bytes_to_align >= 8) { - ABSL_INTERNAL_STEP8(l, p); - bytes_to_align -= 8; - } - while (bytes_to_align > 0) { - ABSL_INTERNAL_STEP1(l, p); - bytes_to_align -= 1; - } - } else { - // Point x at first 8-byte aligned byte in string. - const uint8_t* x = RoundUp<8>(p); - // Process bytes until p is 8-byte aligned, if that isn't past the end. - while (p != x) { - ABSL_INTERNAL_STEP1(l, p); - } + // Point x at first 8-byte aligned byte in string. + const uint8_t* x = RoundUp<8>(p); + // Process bytes until p is 8-byte aligned, if that isn't past the end. + while (p != x) { + ABSL_INTERNAL_STEP1(l, p); } size_t bs = static_cast<size_t>(e - p) / @@ -698,41 +649,11 @@ stream_start += bs * 64; } - uint8_t* dst_crc_streams[kMaxStreams]; - uint8_t* dst_pclmul_streams[kMaxStreams]; - if constexpr (copy) { - uint8_t* dst_stream_start = reinterpret_cast<uint8_t*>(d); - for (size_t i = 0; i < num_crc_streams; i++) { - dst_crc_streams[i] = dst_stream_start; - dst_stream_start += bs * 64; - } - for (size_t i = 0; i < num_pclmul_streams; i++) { - dst_pclmul_streams[i] = dst_stream_start; - dst_stream_start += bs * 64; - } - } - // Per stream crc sums. uint64_t l64_crc[kMaxStreams] = {l}; uint64_t l64_pclmul[kMaxStreams] = {0}; - V256 pclmul_v0_0[kMaxStreams]; - V256 pclmul_v1_0[kMaxStreams]; - for (size_t j = 0; j < num_pclmul_streams; j++) { - V256_LoadPairU(pclmul_streams[j], &pclmul_v0_0[j], &pclmul_v1_0[j]); - } - - if constexpr (copy) { - for (size_t j = 0; j < num_crc_streams; j++) { - V256_CopyPairNonTemporal(dst_crc_streams[j], crc_streams[j]); - dst_crc_streams[j] += 64; - } - for (size_t j = 0; j < num_pclmul_streams; j++) { - V256_StorePairNonTemporal(dst_pclmul_streams[j], pclmul_v0_0[j], - pclmul_v1_0[j]); - dst_pclmul_streams[j] += 64; - } - } + // Peel first iteration, because PCLMULQDQ stream, needs setup. if (num_crc_streams == 1) { l64_crc[0] = Process64BytesCRC(crc_streams[0], l64_crc[0]); crc_streams[0] += 16 * 4; @@ -751,18 +672,20 @@ // Align to 32 bytes for vpclmul implementation. alignas(32) V128 partialCRC[kMaxStreams][4]; for (size_t i = 0; i < num_pclmul_streams; i++) { - InitPclmulStream(pclmul_v0_0[i], pclmul_v1_0[i], partialCRC[i]); - pclmul_streams[i] += 64; + InitPclmulStream(&pclmul_streams[i], partialCRC[i]); } for (size_t i = 1; i < bs; i++) { // Prefetch data for next iterations. for (size_t j = 0; j < num_crc_streams; j++) { - PrefetchToLocalCache(crc_streams[j] + kPrefetchHorizon); + PrefetchToLocalCache( + reinterpret_cast<const char*>(crc_streams[j] + kPrefetchHorizon)); } for (size_t j = 0; j < num_pclmul_streams; j++) { - PrefetchToLocalCache(pclmul_streams[j] + kPrefetchHorizon); + PrefetchToLocalCache(reinterpret_cast<const char*>(pclmul_streams[j] + + kPrefetchHorizon)); } + // We process each stream in 64 byte blocks. This can be written as // for (int i = 0; i < num_pclmul_streams; i++) { // Process64BytesPclmul(pclmul_streams[i], partialCRC[i]); @@ -774,24 +697,6 @@ // } // But unrolling and interleaving PCLMULQDQ and CRC blocks manually // gives ~2% performance boost. - if constexpr (copy) { - for (size_t j = 0; j < num_crc_streams; j++) { - V256_CopyPairNonTemporal(dst_crc_streams[j], crc_streams[j]); - dst_crc_streams[j] += 64; - } - } - - for (size_t j = 0; j < num_pclmul_streams; j++) { - if constexpr (copy) { - ProcessPclmulStream<true>(pclmul_streams[j], dst_pclmul_streams[j], - partialCRC[j]); - dst_pclmul_streams[j] += 64; - } else { - ProcessPclmulStream<false>(pclmul_streams[j], nullptr, - partialCRC[j]); - } - pclmul_streams[j] += 64; - } if (num_crc_streams == 1) { l64_crc[0] = Process64BytesCRC(crc_streams[0], l64_crc[0]); crc_streams[0] += 16 * 4; @@ -806,6 +711,9 @@ crc_streams[1] += 16 * 4; crc_streams[2] += 16 * 4; } + for (size_t j = 0; j < num_pclmul_streams; j++) { + ProcessPclmulStream(&pclmul_streams[j], partialCRC[j]); + } } // PCLMULQDQ based streams require special final step; @@ -827,24 +735,12 @@ l64 ^= l64_pclmul[i]; } - // Update p and d. + // Update p. if constexpr (num_pclmul_streams > 0) { p = pclmul_streams[num_pclmul_streams - 1]; - if constexpr (copy) { - d = reinterpret_cast<char*>( - dst_pclmul_streams[num_pclmul_streams - 1]); - } } else { p = crc_streams[num_crc_streams - 1]; - if constexpr (copy) { - d = reinterpret_cast<char*>(dst_crc_streams[num_crc_streams - 1]); - } } - if constexpr (copy) { - size_t remaining_to_copy = static_cast<size_t>(e - p); - std::memcpy(d, p, remaining_to_copy); - } - l = static_cast<uint32_t>(l64); } @@ -872,45 +768,44 @@ } private: - ABSL_ATTRIBUTE_ALWAYS_INLINE void InitPclmulStream(V256 v0, V256 v1, - V128* partialCRC) const { + ABSL_ATTRIBUTE_ALWAYS_INLINE void InitPclmulStream( + const uint8_t** pclmul_stream, V128* partialCRC) const { if constexpr (pclmul_stream_type == PclmulStreamType::VPCLMUL) { V256* vpartialCRC = reinterpret_cast<V256*>(partialCRC); - vpartialCRC[0] = v0; - vpartialCRC[1] = v1; + vpartialCRC[0] = + V256_LoadU(reinterpret_cast<const V256*>(*pclmul_stream + 32 * 0)); + vpartialCRC[1] = + V256_LoadU(reinterpret_cast<const V256*>(*pclmul_stream + 32 * 1)); } else { - const V128* data_ptr0 = reinterpret_cast<const V128*>(&v0); - const V128* data_ptr1 = reinterpret_cast<const V128*>(&v1); - partialCRC[0] = data_ptr0[0]; - partialCRC[1] = data_ptr0[1]; - partialCRC[2] = data_ptr1[0]; - partialCRC[3] = data_ptr1[1]; + partialCRC[0] = + V128_LoadU(reinterpret_cast<const V128*>(*pclmul_stream + 16 * 0)); + partialCRC[1] = + V128_LoadU(reinterpret_cast<const V128*>(*pclmul_stream + 16 * 1)); + partialCRC[2] = + V128_LoadU(reinterpret_cast<const V128*>(*pclmul_stream + 16 * 2)); + partialCRC[3] = + V128_LoadU(reinterpret_cast<const V128*>(*pclmul_stream + 16 * 3)); } + *pclmul_stream += 16 * 4; } - template <bool copy = false> ABSL_ATTRIBUTE_ALWAYS_INLINE void ProcessPclmulStream( - const uint8_t* stream, uint8_t* dst_stream, V128* partialCRC) const { - const uint8_t* src_ptr = stream; - V256 v0, v1; - V256_LoadPairU(src_ptr, &v0, &v1); - if constexpr (copy) { - V256_StorePairNonTemporal(dst_stream, v0, v1); - } + const uint8_t** pclmul_stream, V128* partialCRC) const { if constexpr (pclmul_stream_type == PclmulStreamType::VPCLMUL) { V256 loopMultiplicands = V256_Broadcast128(reinterpret_cast<const V128*>(kFoldAcross512Bits)); - Process64BytesVpclmul(v0, v1, reinterpret_cast<V256*>(partialCRC), + Process64BytesVpclmul(*pclmul_stream, reinterpret_cast<V256*>(partialCRC), loopMultiplicands); } else if constexpr (pclmul_stream_type == PclmulStreamType::NEON_PCLMUL || pclmul_stream_type == PclmulStreamType::NEON_PCLMUL_EOR3) { constexpr bool kUseEor3 = (pclmul_stream_type == PclmulStreamType::NEON_PCLMUL_EOR3); - Process64BytesNeonPclmul<kUseEor3>(v0, v1, partialCRC); + Process64BytesNeonPclmul<kUseEor3>(*pclmul_stream, partialCRC); } else { - Process64BytesPclmul(v0, v1, partialCRC); + Process64BytesPclmul(*pclmul_stream, partialCRC); } + *pclmul_stream += 16 * 4; } ABSL_ATTRIBUTE_ALWAYS_INLINE uint64_t