Disable VPCLMUL for Non-temporal copies.

The VPCLMUL functionality relies on inline assembly, and LLVM's new x86-fence-nontemporal-stores pass inserts an sfence before every inline assembly block when non-temporal stores are issued before them, which dramatically hurts non-temporal crc_memcpy performance here.

So, we mark the stateless stream helpers static and have ExtendAndCopy on PclmulStreamType::VPCLMUL dispatch non-temporal copies to the PclmulStreamType::PCLMUL specialization.

PiperOrigin-RevId: 1017480997
diff --git a/absl/crc/internal/crc_x86_arm_combined.cc b/absl/crc/internal/crc_x86_arm_combined.cc
index 6f16154..edc2691 100644
--- a/absl/crc/internal/crc_x86_arm_combined.cc
+++ b/absl/crc/internal/crc_x86_arm_combined.cc
@@ -61,7 +61,7 @@
   CRC32AcceleratedX86ARMCombined() {}
   ~CRC32AcceleratedX86ARMCombined() override {}
   void ExtendByZeroes(uint32_t* crc, size_t length) const override;
-  uint32_t ComputeZeroConstant(size_t length) const;
+  static uint32_t ComputeZeroConstant(size_t length);
 
  private:
   CRC32AcceleratedX86ARMCombined(const CRC32AcceleratedX86ARMCombined&) =
@@ -164,8 +164,7 @@
 // manually disable bounds checking.
 [[clang::no_sanitize("array-bounds")]]
 #endif
-uint32_t CRC32AcceleratedX86ARMCombined::ComputeZeroConstant(
-    size_t length) const {
+uint32_t CRC32AcceleratedX86ARMCombined::ComputeZeroConstant(size_t length) {
   length >>= kNumDroppedBits;
 
   int index = absl::countr_zero(length);
@@ -277,7 +276,7 @@
   // PCLMULQDQ Instruction"
   // https://www.intel.com/content/dam/www/public/us/en/documents/white-papers/fast-crc-computation-generic-polynomials-pclmulqdq-paper.pdf
   // We are applying it to CRC32C polynomial.
-  ABSL_ATTRIBUTE_ALWAYS_INLINE V128 Pclmul16Bytes(V128* partialCRC) const {
+  ABSL_ATTRIBUTE_ALWAYS_INLINE static V128 Pclmul16Bytes(V128* partialCRC) {
     V128 loopMultiplicands =
         V128_Load(reinterpret_cast<const V128*>(kFoldAcross512Bits));
     V128 tmp = V128_PMulHi(*partialCRC, loopMultiplicands);
@@ -287,8 +286,8 @@
 
 #if defined(ABSL_CRC_INTERNAL_HAVE_ARM_SIMD)
   template <bool kUseEor3 = false>
-  ABSL_ATTRIBUTE_ALWAYS_INLINE void Xor16BytesNeonPclmul(
-      V128 data, V128 tmp, V128* partialCRC) const {
+  ABSL_ATTRIBUTE_ALWAYS_INLINE static void Xor16BytesNeonPclmul(
+      V128 data, V128 tmp, V128* partialCRC) {
     *partialCRC = V128_Xor3<kUseEor3>(tmp, *partialCRC, data);
   }
 
@@ -296,8 +295,8 @@
   // into a single value, that represents crc checksum of all the processed
   // bytes.
   template <bool kUseEor3 = false>
-  ABSL_ATTRIBUTE_ALWAYS_INLINE uint64_t
-  FinalizeNeonPclmulStream(V128* partialCRC) const {
+  ABSL_ATTRIBUTE_ALWAYS_INLINE static uint64_t FinalizeNeonPclmulStream(
+      V128* partialCRC) {
     V128 partialCRC1 = partialCRC[0];
     V128 partialCRC2 = partialCRC[1];
     V128 partialCRC3 = partialCRC[2];
@@ -331,21 +330,22 @@
     return crc;
   }
 
-  ABSL_ATTRIBUTE_ALWAYS_INLINE void Xor16BytesPclmul(V128, V128, V128*) const {}
+  ABSL_ATTRIBUTE_ALWAYS_INLINE static void Xor16BytesPclmul(V128, V128,
+                                                            V128*) {}
 
-  ABSL_ATTRIBUTE_ALWAYS_INLINE uint64_t FinalizePclmulStream(V128*) const {
+  ABSL_ATTRIBUTE_ALWAYS_INLINE static uint64_t FinalizePclmulStream(V128*) {
     return 0;
   }
 #else
-  ABSL_ATTRIBUTE_ALWAYS_INLINE void Xor16BytesPclmul(V128 data, V128 tmp,
-                                                     V128* partialCRC) const {
+  ABSL_ATTRIBUTE_ALWAYS_INLINE static void Xor16BytesPclmul(
+      V128 data, V128 tmp, V128* partialCRC) {
     *partialCRC = V128_Xor(V128_Xor(tmp, *partialCRC), data);
   }
 
   // Reduce partialCRC produced by Pclmul16Bytes and Xor16BytesPclmul into a
   // single value, that represents crc checksum of all the processed bytes.
-  ABSL_ATTRIBUTE_ALWAYS_INLINE uint64_t
-  FinalizePclmulStream(V128* partialCRC) const {
+  ABSL_ATTRIBUTE_ALWAYS_INLINE static uint64_t FinalizePclmulStream(
+      V128* partialCRC) {
     V128 partialCRC1 = partialCRC[0];
     V128 partialCRC2 = partialCRC[1];
     V128 partialCRC3 = partialCRC[2];
@@ -383,26 +383,26 @@
   }
 
   template <bool kUseEor3 = false>
-  ABSL_INTERNAL_ATTRIBUTE_AVX ABSL_ATTRIBUTE_ALWAYS_INLINE void
-  Xor16BytesNeonPclmul(V128, V128, V128*) const {}
+  ABSL_INTERNAL_ATTRIBUTE_AVX ABSL_ATTRIBUTE_ALWAYS_INLINE static void
+  Xor16BytesNeonPclmul(V128, V128, V128*) {}
 
   template <bool kUseEor3 = false>
-  ABSL_INTERNAL_ATTRIBUTE_AVX ABSL_ATTRIBUTE_ALWAYS_INLINE uint64_t
-  FinalizeNeonPclmulStream(V128*) const {
+  ABSL_INTERNAL_ATTRIBUTE_AVX ABSL_ATTRIBUTE_ALWAYS_INLINE static uint64_t
+  FinalizeNeonPclmulStream(V128*) {
     return 0;
   }
 #endif
 
   // Update crc with 8 bytes of data from p.
-  ABSL_ATTRIBUTE_ALWAYS_INLINE uint64_t Process8BytesCRC(const uint8_t* p,
-                                                         uint64_t crc) const {
+  ABSL_ATTRIBUTE_ALWAYS_INLINE static uint64_t Process8BytesCRC(
+      const uint8_t* p, uint64_t crc) {
     return CRC32_u64(static_cast<uint32_t>(crc),
                      absl::little_endian::Load64(p));
   }
 
   // Update crc with 16 bytes of data from p.
-  ABSL_ATTRIBUTE_ALWAYS_INLINE uint64_t Process16BytesCRC(const uint8_t* p,
-                                                          uint64_t crc) const {
+  ABSL_ATTRIBUTE_ALWAYS_INLINE static uint64_t Process16BytesCRC(
+      const uint8_t* p, uint64_t crc) {
     crc = Process8BytesCRC(p, crc);
     crc = Process8BytesCRC(p + 8, crc);
     return crc;
@@ -418,8 +418,8 @@
   // with new and default flags or use inline asm.
   // The code below is the same as FinalizePclmulStream, but with
   // PCLMUL and XOR operating on 2 values in a vector at the same time.
-  ABSL_INTERNAL_ATTRIBUTE_AVX ABSL_ATTRIBUTE_ALWAYS_INLINE uint64_t
-  FinalizeVpclmulStream(V256* partialCRC) const {
+  ABSL_INTERNAL_ATTRIBUTE_AVX ABSL_ATTRIBUTE_ALWAYS_INLINE static uint64_t
+  FinalizeVpclmulStream(V256* partialCRC) {
     uint64_t crc = 0;
     uint64_t low64, high64;
     __asm__(
@@ -449,8 +449,8 @@
     return crc;
   }
 
-  ABSL_INTERNAL_ATTRIBUTE_AVX ABSL_ATTRIBUTE_ALWAYS_INLINE V256
-  Pclmul32BytesVpclmul(V256* partialCRC) const {
+  ABSL_INTERNAL_ATTRIBUTE_AVX ABSL_ATTRIBUTE_ALWAYS_INLINE static V256
+  Pclmul32BytesVpclmul(V256* partialCRC) {
     V256 loopMultiplicands =
         V256_Broadcast128(reinterpret_cast<const V128*>(kFoldAcross512Bits));
     V256 tmp;
@@ -462,8 +462,8 @@
     return tmp;
   }
 
-  ABSL_INTERNAL_ATTRIBUTE_AVX ABSL_ATTRIBUTE_ALWAYS_INLINE void
-  Xor32BytesVpclmul(const uint8_t* p, V256 tmp, V256* partialCRC) const {
+  ABSL_INTERNAL_ATTRIBUTE_AVX ABSL_ATTRIBUTE_ALWAYS_INLINE static void
+  Xor32BytesVpclmul(const uint8_t* p, V256 tmp, V256* partialCRC) {
     V256 v = V256_LoadU(p);
     __asm__ volatile(
         "vpxor %2, %0, %0 \n"
@@ -473,16 +473,16 @@
   }
 #else
   template <typename T = V256>
-  ABSL_ATTRIBUTE_ALWAYS_INLINE T Pclmul32BytesVpclmul(T*) const {
+  ABSL_ATTRIBUTE_ALWAYS_INLINE static T Pclmul32BytesVpclmul(T*) {
     static_assert(sizeof(T) == 0, "Vector PCLMUL not supported");
     return T{};
   }
   template <typename T = V256>
-  ABSL_ATTRIBUTE_ALWAYS_INLINE void Xor32BytesVpclmul(const uint8_t*, T,
-                                                      T*) const {
+  ABSL_ATTRIBUTE_ALWAYS_INLINE static void Xor32BytesVpclmul(const uint8_t*, T,
+                                                             T*) {
     static_assert(sizeof(T) == 0, "Vector PCLMUL not supported");
   }
-  ABSL_ATTRIBUTE_ALWAYS_INLINE uint64_t FinalizeVpclmulStream(V256*) const {
+  ABSL_ATTRIBUTE_ALWAYS_INLINE static uint64_t FinalizeVpclmulStream(V256*) {
     return 0;
   }
 #endif  // defined(ABSL_CRC_INTERNAL_HAVE_X86_SIMD) && (defined(__AVX__) ||
@@ -539,7 +539,18 @@
                      const void* __restrict src, size_t length,
                      bool non_temporal) const override {
     if (non_temporal) {
-      Extend<CopyType::kNonTemporal>(crc, src, length, dst);
+      if constexpr (pclmul_stream_type == PclmulStreamType::VPCLMUL) {
+        // On x86, VPCLMUL uses inline assembly, which causes LLVM's
+        // X86FenceNonTemporalStoresPass to insert an `sfence` before every
+        // inline assembly block in functions containing non-temporal stores, so
+        // non-temporal copies fall back to 128-bit PCLMUL intrinsics.
+        CRC32AcceleratedX86ARMCombinedMultipleStreams<
+            num_crc_streams, num_pclmul_streams,
+            PclmulStreamType::PCLMUL>::template Extend<CopyType::kNonTemporal>(
+            crc, src, length, dst);
+      } else {
+        Extend<CopyType::kNonTemporal>(crc, src, length, dst);
+      }
     } else {
       Extend<CopyType::kTemporal>(crc, src, length, dst);
     }
@@ -549,9 +560,9 @@
   // `ExtendCrc32cInline()` in `absl::ExtendCrc32c()` and
   // `CrcMemcpy::CrcAndCopy()`.
   template <CopyType copy_type>
-  ABSL_ATTRIBUTE_HOT void Extend(uint32_t* crc, const void* bytes,
-                                 const size_t length,
-                                 void* __restrict dst = nullptr) const {
+  ABSL_ATTRIBUTE_ALWAYS_INLINE static void Extend(
+      uint32_t* crc, const void* bytes, const size_t length,
+      void* __restrict dst = nullptr) {
     assert(length >= kSmallCutoff);
     if constexpr (pclmul_stream_type != PclmulStreamType::NONE) {
       if (length >= kMediumCutoff) {
@@ -563,9 +574,9 @@
   }
 
   template <bool copy>
-  ABSL_ATTRIBUTE_HOT void ExtendWithoutPclmul(
+  ABSL_ATTRIBUTE_HOT static void ExtendWithoutPclmul(
       uint32_t* crc, const void* bytes, const size_t length,
-      void* __restrict dst = nullptr) const {
+      void* __restrict dst = nullptr) {
     static_assert(num_crc_streams >= 1 && num_crc_streams <= kMaxStreams,
                   "Invalid number of crc streams");
     static_assert(num_pclmul_streams >= 0 && num_pclmul_streams <= kMaxStreams,
@@ -662,10 +673,17 @@
     *crc = l;
   }
 
+  // Always inline when the translation unit already supports AVX (or does not
+  // force AVX via `gnu::target`), so shared specializations like
+  // `<3, 3, PCLMUL>` stay inlined into `ExtendAndCopy`.
   template <CopyType copy_type>
-  ABSL_INTERNAL_ATTRIBUTE_AVX ABSL_ATTRIBUTE_HOT void ExtendWithPclmul(
-      uint32_t* crc, const void* bytes, const size_t length,
-      void* __restrict dst = nullptr) const {
+#if defined(__AVX__) || !defined(ABSL_INTERNAL_CAN_FORCE_AVX)
+  ABSL_INTERNAL_ATTRIBUTE_AVX ABSL_ATTRIBUTE_ALWAYS_INLINE static void
+#else
+  ABSL_INTERNAL_ATTRIBUTE_AVX ABSL_ATTRIBUTE_HOT static void
+#endif
+  ExtendWithPclmul(uint32_t* crc, const void* bytes, const size_t length,
+                   void* __restrict dst = nullptr) {
     static_assert(num_crc_streams >= 1 && num_crc_streams <= kMaxStreams,
                   "Invalid number of crc streams");
     static_assert(num_pclmul_streams >= 0 && num_pclmul_streams <= kMaxStreams,
@@ -992,8 +1010,8 @@
   using PclmulProduct =
       std::conditional_t<kIsVpclmul && kVpclmulBytes == 32, V256, V128>;
 
-  ABSL_INTERNAL_ATTRIBUTE_AVX ABSL_ATTRIBUTE_ALWAYS_INLINE void
-  InitPclmulStream(V256 v0, V256 v1, V128* partialCRC) const {
+  ABSL_INTERNAL_ATTRIBUTE_AVX ABSL_ATTRIBUTE_ALWAYS_INLINE static void
+  InitPclmulStream(V256 v0, V256 v1, V128* partialCRC) {
     if constexpr (pclmul_stream_type == PclmulStreamType::NONE) {
       return;
     } else if constexpr (kIsVpclmul) {
@@ -1014,9 +1032,9 @@
   // storing the low product in `partialCRC` and returning the high product,
   // which must be passed to `XorPclmulStream` to complete the fold.
   template <size_t kVpclmulBytes>
-  ABSL_INTERNAL_ATTRIBUTE_AVX ABSL_ATTRIBUTE_ALWAYS_INLINE
-      PclmulProduct<kVpclmulBytes>
-      PclmulStream(V128* partialCRC) const {
+  ABSL_INTERNAL_ATTRIBUTE_AVX ABSL_ATTRIBUTE_ALWAYS_INLINE static PclmulProduct<
+      kVpclmulBytes>
+  PclmulStream(V128* partialCRC) {
     if constexpr (pclmul_stream_type == PclmulStreamType::NONE ||
                   (kIsVpclmul && kVpclmulBytes == 0)) {
       return {};
@@ -1031,9 +1049,9 @@
   // the high product returned by `PclmulStream`, the low product in
   // `partialCRC`, and the loaded data, and advances `*stream`.
   template <size_t kVpclmulBytes>
-  ABSL_INTERNAL_ATTRIBUTE_AVX ABSL_ATTRIBUTE_ALWAYS_INLINE void XorPclmulStream(
-      const uint8_t** stream, const PclmulProduct<kVpclmulBytes>& high,
-      V128* partialCRC) const {
+  ABSL_INTERNAL_ATTRIBUTE_AVX ABSL_ATTRIBUTE_ALWAYS_INLINE static void
+  XorPclmulStream(const uint8_t** stream,
+                  const PclmulProduct<kVpclmulBytes>& high, V128* partialCRC) {
     if constexpr (pclmul_stream_type == PclmulStreamType::NONE ||
                   (kIsVpclmul && kVpclmulBytes == 0)) {
       return;
@@ -1077,10 +1095,10 @@
   // always folded.
   template <size_t kNumActiveCrcStreams, size_t kNumActivePclmulStreams,
             size_t kVpclmulBytes>
-  ABSL_INTERNAL_ATTRIBUTE_AVX ABSL_ATTRIBUTE_ALWAYS_INLINE void
+  ABSL_INTERNAL_ATTRIBUTE_AVX ABSL_ATTRIBUTE_ALWAYS_INLINE static void
   ProcessAllStreams16(const uint8_t** crc_streams, uint64_t* l64_crc,
                       const uint8_t** pclmul_streams, V128 (*partialCRC)[4],
-                      size_t lane) const {
+                      size_t lane) {
     for (size_t j = 0; j < kMaxStreams; ++j) {
       if (j < kNumActiveCrcStreams) {
         l64_crc[j] = Process8BytesCRC(crc_streams[j], l64_crc[j]);
@@ -1101,8 +1119,8 @@
     }
   }
 
-  ABSL_INTERNAL_ATTRIBUTE_AVX ABSL_ATTRIBUTE_ALWAYS_INLINE uint64_t
-  FinalizePclmulStream(V128* partialCRC) const {
+  ABSL_INTERNAL_ATTRIBUTE_AVX ABSL_ATTRIBUTE_ALWAYS_INLINE static uint64_t
+  FinalizePclmulStream(V128* partialCRC) {
     if constexpr (pclmul_stream_type == PclmulStreamType::NONE) {
       return 0;
     } else if constexpr (kIsVpclmul) {