Use AVX-512 VPTERNLOGD for 3-way XORs in the x86 PCLMUL and VPCLMUL CRC32C streams.

Every fold of a PCLMUL stream XORs three values: the two carry-less products
and the next block of input. Add `PCLMUL_EOR3` and `VPCLMUL_EOR3` stream types
that do this with a single AVX-512 `vpternlogd` instead of two XORs.

`PCLMUL_EOR3` also reads the input block straight from memory instead of
loading it first. Copies already hold the input in registers, so `PCLMUL_EOR3`
copies keep using `PCLMUL`. Like `VPCLMUL`, the new stream types use inline
assembly, so their non-temporal copies keep using `PCLMUL` as well.

Use `PCLMUL_EOR3` on Skylake, Cascade Lake, and Ice Lake Xeons, and
`VPCLMUL_EOR3` on Sapphire Rapids, Emerald Rapids, and Granite Rapids.

PiperOrigin-RevId: 1021627029
diff --git a/absl/crc/internal/crc_memcpy_test.cc b/absl/crc/internal/crc_memcpy_test.cc
index 021f5e5..5f4f3ec 100644
--- a/absl/crc/internal/crc_memcpy_test.cc
+++ b/absl/crc/internal/crc_memcpy_test.cc
@@ -313,6 +313,32 @@
   EXPECT_EQ(g_num_unfenced_stores, 0);
 }
 
+// Checks that Crc32CAndCopy() writes every requested byte and no surrounding
+// bytes across all cache-line alignments and stream-tail remainders.
+TEST(CrcMemcpyTest, WritesExactDestinationRange) {
+  constexpr size_t kGuard = 64, kAlign = 64, kBase = 4096, kMaxExtra = 256;
+  std::vector<char> src(kBase + kMaxExtra);
+  for (size_t i = 0; i < src.size(); ++i) {
+    src[i] = static_cast<char>(i * 131);
+  }
+  const std::vector<char> guard(kGuard + kAlign, static_cast<char>(0xa5));
+  std::vector<char> buf(kGuard + kAlign + src.size() + kGuard);
+  for (bool non_temporal : {false, true}) {
+    for (size_t size = kBase; size <= kBase + kMaxExtra; ++size) {
+      for (size_t offset = 0; offset < kAlign; ++offset) {
+        char* dst = buf.data() + kGuard + offset;
+        std::memset(buf.data(), 0xa5, buf.size());
+        ASSERT_EQ(absl::crc_internal::Crc32CAndCopy(
+                      dst, src.data(), size, absl::crc32c_t{0}, non_temporal),
+                  absl::ComputeCrc32c({src.data(), size}));
+        ASSERT_EQ(std::memcmp(dst, src.data(), size), 0);
+        ASSERT_EQ(std::memcmp(buf.data(), guard.data(), dst - buf.data()), 0);
+        ASSERT_EQ(std::memcmp(dst + size, guard.data(), kGuard), 0);
+      }
+    }
+  }
+}
+
 // Non-temporal stores are weakly ordered, so Crc32CAndCopy() has to fence them
 // before it returns, or the copy is not guaranteed to be visible to the thread
 // that reads it next. This checks that it always does, across the sizes and
diff --git a/absl/crc/internal/crc_x86_arm_combined.cc b/absl/crc/internal/crc_x86_arm_combined.cc
index 5fe704f..d4d0bcd 100644
--- a/absl/crc/internal/crc_x86_arm_combined.cc
+++ b/absl/crc/internal/crc_x86_arm_combined.cc
@@ -219,6 +219,9 @@
   NONE,
   PCLMUL,
   VPCLMUL,
+  // PCLMUL and VPCLMUL that fold with a single AVX-512 `vpternlogd` 3-way XOR.
+  PCLMUL_EOR3,
+  VPCLMUL_EOR3,
   NEON_PCLMUL,
   NEON_PCLMUL_EOR3,
 };
@@ -462,14 +465,31 @@
     return tmp;
   }
 
+  template <bool kUseEor3>
   ABSL_INTERNAL_ATTRIBUTE_AVX ABSL_ATTRIBUTE_ALWAYS_INLINE static void
   Xor32BytesVpclmul(const uint8_t* p, V256 tmp, V256* partialCRC) {
     V256 v = V256_LoadU(p);
-    __asm__ volatile(
-        "vpxor %2, %0, %0 \n"
-        "vpxor %1, %0, %0 \n"
-        : "+x"(*partialCRC)
-        : "x"(v), "x"(tmp));
+    if constexpr (kUseEor3) {
+      __asm__ volatile("vpternlogd $0x96, %2, %1, %0"
+                       : "+x"(*partialCRC)
+                       : "x"(v), "x"(tmp));
+    } else {
+      __asm__ volatile(
+          "vpxor %2, %0, %0 \n"
+          "vpxor %1, %0, %0 \n"
+          : "+x"(*partialCRC)
+          : "x"(v), "x"(tmp));
+    }
+  }
+
+  // Same as Xor16BytesPclmul on the 16 bytes at `p`, but with a single AVX-512
+  // `vpternlogd` (0x96 is the truth table of a 3-way XOR) that reads the bytes
+  // straight from memory.
+  ABSL_ATTRIBUTE_ALWAYS_INLINE static void Xor16BytesPclmulEor3(
+      const uint8_t* p, V128 tmp, V128* partialCRC) {
+    __asm__("vpternlogd $0x96, %2, %1, %0"
+            : "+x"(*partialCRC)
+            : "x"(tmp), "m"(*reinterpret_cast<const __m128i_u*>(p)));
   }
 #else
   template <typename T = V256>
@@ -477,11 +497,16 @@
     static_assert(sizeof(T) == 0, "Vector PCLMUL not supported");
     return T{};
   }
-  template <typename T = V256>
+  template <bool kUseEor3, typename T = V256>
   ABSL_ATTRIBUTE_ALWAYS_INLINE static void Xor32BytesVpclmul(const uint8_t*, T,
                                                              T*) {
     static_assert(sizeof(T) == 0, "Vector PCLMUL not supported");
   }
+  template <typename T = V128>
+  ABSL_ATTRIBUTE_ALWAYS_INLINE static void Xor16BytesPclmulEor3(const uint8_t*,
+                                                                T, T*) {
+    static_assert(sizeof(T) == 0, "VPTERNLOGD not supported");
+  }
   ABSL_ATTRIBUTE_ALWAYS_INLINE static uint64_t FinalizeVpclmulStream(V256*) {
     return 0;
   }
@@ -540,11 +565,13 @@
                                         size_t length,
                                         bool non_temporal) const override {
     if (non_temporal) {
-      if constexpr (pclmul_stream_type == PclmulStreamType::VPCLMUL) {
-        // On x86, VPCLMUL uses inline assembly, which causes LLVM's
-        // X86FenceNonTemporalStoresPass to insert an `sfence` before every
-        // inline assembly block in functions containing non-temporal stores, so
-        // non-temporal copies fall back to 128-bit PCLMUL intrinsics.
+      if constexpr (kIsVpclmul ||
+                    pclmul_stream_type == PclmulStreamType::PCLMUL_EOR3) {
+        // On x86, VPCLMUL and PCLMUL_EOR3 use inline assembly, which causes
+        // LLVM's X86FenceNonTemporalStoresPass to insert an `sfence` before
+        // every inline assembly block in functions containing non-temporal
+        // stores, so non-temporal copies fall back to 128-bit PCLMUL
+        // intrinsics.
         CRC32AcceleratedX86ARMCombinedMultipleStreams<
             num_crc_streams, num_pclmul_streams,
             PclmulStreamType::PCLMUL>::template Extend<CopyType::kNonTemporal>(
@@ -552,6 +579,12 @@
       } else {
         Extend<CopyType::kNonTemporal>(crc, src, length, dst);
       }
+    } else if constexpr (pclmul_stream_type == PclmulStreamType::PCLMUL_EOR3) {
+      // PCLMUL_EOR3 reads the folded bytes from memory, but copies already hold
+      // them in registers, so temporal copies fall back to PCLMUL as well.
+      CRC32AcceleratedX86ARMCombinedMultipleStreams<
+          num_crc_streams, num_pclmul_streams, PclmulStreamType::PCLMUL>::
+          template Extend<CopyType::kTemporal>(crc, src, length, dst);
     } else {
       Extend<CopyType::kTemporal>(crc, src, length, dst);
     }
@@ -706,15 +739,17 @@
       uintptr_t dst_addr = reinterpret_cast<uintptr_t>(d);
       size_t bytes_to_align =
           (kAlignment - (dst_addr & (kAlignment - 1))) & (kAlignment - 1);
-      std::memcpy(d, p, bytes_to_align);
-      d += bytes_to_align;
-      while (bytes_to_align >= 8) {
-        ABSL_INTERNAL_STEP8(l, p);
-        bytes_to_align -= 8;
-      }
-      while (bytes_to_align > 0) {
-        ABSL_INTERNAL_STEP1(l, p);
-        bytes_to_align -= 1;
+      if (bytes_to_align != 0) {
+        std::memcpy(d, p, bytes_to_align);
+        d += bytes_to_align;
+        while (bytes_to_align >= 8) {
+          ABSL_INTERNAL_STEP8(l, p);
+          bytes_to_align -= 8;
+        }
+        while (bytes_to_align > 0) {
+          ABSL_INTERNAL_STEP1(l, p);
+          bytes_to_align -= 1;
+        }
       }
     } else {
       // Point x at first 8-byte aligned byte in string.
@@ -927,15 +962,11 @@
       if constexpr (copy_type != CopyType::kNone) {
         d = reinterpret_cast<char*>(
             dst_pclmul_streams[kNumActivePclmulStreams - 1]);
-        size_t remaining_to_copy = static_cast<size_t>(e - p);
-        std::memcpy(d, p, remaining_to_copy);
       }
     } else {
       p = crc_streams[kNumActiveCrcStreams - 1];
       if constexpr (copy_type != CopyType::kNone) {
         d = reinterpret_cast<char*>(dst_crc_streams[kNumActiveCrcStreams - 1]);
-        size_t remaining_to_copy = static_cast<size_t>(e - p);
-        std::memcpy(d, p, remaining_to_copy);
       }
     }
 
@@ -963,15 +994,26 @@
 
     *crc = l;
 
-    if constexpr (copy_type == CopyType::kNonTemporal) {
-      // According to the Intel 64 Software Developer's Manual
-      // (https://www.intel.com/content/www/us/en/developer/articles/technical/intel-sdm.html),
-      // non-temporal stores are weakly ordered due to bypassing caches, so
-      // they have to be fenced before the copy is visible to other threads.
-      // Regular stores are ordered and need no fence. Note that the fence is
-      // placed after all stores, to avoid unnecessary serialization and a
-      // performance hit.
-      NonTemporalStoreFence();
+    if constexpr (copy_type != CopyType::kNone) {
+      const uint8_t* copy_src = e - remaining_bytes;
+      while (e - copy_src >= 64) {
+        Copy64Bytes<copy_type>(reinterpret_cast<uint8_t*>(d), copy_src);
+        d += 64;
+        copy_src += 64;
+      }
+      if constexpr (copy_type == CopyType::kNonTemporal) {
+        // According to the Intel 64 Software Developer's Manual
+        // (https://www.intel.com/content/www/us/en/developer/articles/technical/intel-sdm.html),
+        // non-temporal stores are weakly ordered due to bypassing caches, so
+        // they have to be fenced before the copy is visible to other threads.
+        // Regular stores are ordered and need no fence. Note that the fence is
+        // placed after all non-temporal stores, to avoid unnecessary
+        // serialization and a performance hit.
+        NonTemporalStoreFence();
+      }
+      if (copy_src != e) {
+        std::memcpy(d, copy_src, static_cast<size_t>(e - copy_src));
+      }
     }
   }
 
@@ -988,8 +1030,7 @@
       return {num_crc_streams, num_pclmul_streams};
     }
     // A single 256-bit VPCLMUL stream keeps up with temporal copies on its own.
-    if (pclmul_stream_type == PclmulStreamType::VPCLMUL &&
-        copy_type == CopyType::kTemporal) {
+    if (kIsVpclmul && copy_type == CopyType::kTemporal) {
       return {0, 1};
     }
     // Non-temporal copies benefit from a second crc stream, notably on AMD
@@ -1002,7 +1043,8 @@
 
   // Whether PCLMUL streams are folded with 256-bit `vpclmulqdq`.
   static constexpr bool kIsVpclmul =
-      (pclmul_stream_type == PclmulStreamType::VPCLMUL);
+      (pclmul_stream_type == PclmulStreamType::VPCLMUL ||
+       pclmul_stream_type == PclmulStreamType::VPCLMUL_EOR3);
 
   // The high product of a fold, which is returned by `PclmulStream` and
   // consumed by `XorPclmulStream`. Folding 32 bytes with 256-bit `vpclmulqdq`
@@ -1057,8 +1099,12 @@
                   (kIsVpclmul && kVpclmulBytes == 0)) {
       return;
     } else if constexpr (kIsVpclmul && kVpclmulBytes == 32) {
-      Xor32BytesVpclmul(*stream, high, reinterpret_cast<V256*>(partialCRC));
+      Xor32BytesVpclmul<(pclmul_stream_type == PclmulStreamType::VPCLMUL_EOR3)>(
+          *stream, high, reinterpret_cast<V256*>(partialCRC));
       *stream += 32;
+    } else if constexpr (pclmul_stream_type == PclmulStreamType::PCLMUL_EOR3) {
+      Xor16BytesPclmulEor3(*stream, high, partialCRC);
+      *stream += 16;
     } else {
       V128 data = V128_LoadU(reinterpret_cast<const V128*>(*stream));
       if constexpr (pclmul_stream_type == PclmulStreamType::NEON_PCLMUL ||
@@ -1182,7 +1228,7 @@
     (defined(__AVX__) || defined(ABSL_INTERNAL_CAN_FORCE_AVX)) && \
     (!defined(_MSC_VER) || defined(__clang__))
       return new CRC32AcceleratedX86ARMCombinedMultipleStreams<
-          3, 3, PclmulStreamType::VPCLMUL>();
+          3, 3, PclmulStreamType::VPCLMUL_EOR3>();
 #else
       [[fallthrough]];
 #endif
@@ -1192,8 +1238,16 @@
           3, 2, PclmulStreamType::PCLMUL>();
     case CpuType::kIntelCascadelakeXeon:
     case CpuType::kIntelSkylakeXeon:
-    case CpuType::kIntelSkylake:
     case CpuType::kIntelIcelake:
+#if defined(ABSL_CRC_INTERNAL_HAVE_X86_SIMD) &&                   \
+    (defined(__AVX__) || defined(ABSL_INTERNAL_CAN_FORCE_AVX)) && \
+    (!defined(_MSC_VER) || defined(__clang__))
+      return new CRC32AcceleratedX86ARMCombinedMultipleStreams<
+          3, 3, PclmulStreamType::PCLMUL_EOR3>();
+#else
+      [[fallthrough]];
+#endif
+    case CpuType::kIntelSkylake:
       return new CRC32AcceleratedX86ARMCombinedMultipleStreams<
           3, 3, PclmulStreamType::PCLMUL>();
     // PCLMULQDQ is slow, don't use it.