Use Copy64Bytes for full tail cache lines in CRC32C copies. In `ExtendWithPclmul`, copying the entire tail after the multi-stream loop with `std::memcpy` wrote remaining full 64-byte cache lines with temporal stores even in non-temporal mode and emitted mid-function calls before the tail CRC loop. In addition, the head alignment block called `std::memcpy` unconditionally even when the destination was already 64-byte aligned. Skip the head alignment copy when the destination is already aligned, copy remaining full 64-byte tail cache lines with `Copy64Bytes<copy_type>` after the tail CRC loads and before `NonTemporalStoreFence()`, and place the final sub-64-byte `std::memcpy` at the end of `ExtendWithPclmul` so it compiles to a tail call. PiperOrigin-RevId: 1017924640
diff --git a/absl/crc/internal/crc_memcpy_test.cc b/absl/crc/internal/crc_memcpy_test.cc index 021f5e5..5f4f3ec 100644 --- a/absl/crc/internal/crc_memcpy_test.cc +++ b/absl/crc/internal/crc_memcpy_test.cc
@@ -313,6 +313,32 @@ EXPECT_EQ(g_num_unfenced_stores, 0); } +// Checks that Crc32CAndCopy() writes every requested byte and no surrounding +// bytes across all cache-line alignments and stream-tail remainders. +TEST(CrcMemcpyTest, WritesExactDestinationRange) { + constexpr size_t kGuard = 64, kAlign = 64, kBase = 4096, kMaxExtra = 256; + std::vector<char> src(kBase + kMaxExtra); + for (size_t i = 0; i < src.size(); ++i) { + src[i] = static_cast<char>(i * 131); + } + const std::vector<char> guard(kGuard + kAlign, static_cast<char>(0xa5)); + std::vector<char> buf(kGuard + kAlign + src.size() + kGuard); + for (bool non_temporal : {false, true}) { + for (size_t size = kBase; size <= kBase + kMaxExtra; ++size) { + for (size_t offset = 0; offset < kAlign; ++offset) { + char* dst = buf.data() + kGuard + offset; + std::memset(buf.data(), 0xa5, buf.size()); + ASSERT_EQ(absl::crc_internal::Crc32CAndCopy( + dst, src.data(), size, absl::crc32c_t{0}, non_temporal), + absl::ComputeCrc32c({src.data(), size})); + ASSERT_EQ(std::memcmp(dst, src.data(), size), 0); + ASSERT_EQ(std::memcmp(buf.data(), guard.data(), dst - buf.data()), 0); + ASSERT_EQ(std::memcmp(dst + size, guard.data(), kGuard), 0); + } + } + } +} + // Non-temporal stores are weakly ordered, so Crc32CAndCopy() has to fence them // before it returns, or the copy is not guaranteed to be visible to the thread // that reads it next. This checks that it always does, across the sizes and
diff --git a/absl/crc/internal/crc_x86_arm_combined.cc b/absl/crc/internal/crc_x86_arm_combined.cc index 5fe704f..bf7d3c6 100644 --- a/absl/crc/internal/crc_x86_arm_combined.cc +++ b/absl/crc/internal/crc_x86_arm_combined.cc
@@ -706,15 +706,17 @@ uintptr_t dst_addr = reinterpret_cast<uintptr_t>(d); size_t bytes_to_align = (kAlignment - (dst_addr & (kAlignment - 1))) & (kAlignment - 1); - std::memcpy(d, p, bytes_to_align); - d += bytes_to_align; - while (bytes_to_align >= 8) { - ABSL_INTERNAL_STEP8(l, p); - bytes_to_align -= 8; - } - while (bytes_to_align > 0) { - ABSL_INTERNAL_STEP1(l, p); - bytes_to_align -= 1; + if (bytes_to_align != 0) { + std::memcpy(d, p, bytes_to_align); + d += bytes_to_align; + while (bytes_to_align >= 8) { + ABSL_INTERNAL_STEP8(l, p); + bytes_to_align -= 8; + } + while (bytes_to_align > 0) { + ABSL_INTERNAL_STEP1(l, p); + bytes_to_align -= 1; + } } } else { // Point x at first 8-byte aligned byte in string. @@ -927,15 +929,11 @@ if constexpr (copy_type != CopyType::kNone) { d = reinterpret_cast<char*>( dst_pclmul_streams[kNumActivePclmulStreams - 1]); - size_t remaining_to_copy = static_cast<size_t>(e - p); - std::memcpy(d, p, remaining_to_copy); } } else { p = crc_streams[kNumActiveCrcStreams - 1]; if constexpr (copy_type != CopyType::kNone) { d = reinterpret_cast<char*>(dst_crc_streams[kNumActiveCrcStreams - 1]); - size_t remaining_to_copy = static_cast<size_t>(e - p); - std::memcpy(d, p, remaining_to_copy); } } @@ -963,15 +961,26 @@ *crc = l; - if constexpr (copy_type == CopyType::kNonTemporal) { - // According to the Intel 64 Software Developer's Manual - // (https://www.intel.com/content/www/us/en/developer/articles/technical/intel-sdm.html), - // non-temporal stores are weakly ordered due to bypassing caches, so - // they have to be fenced before the copy is visible to other threads. - // Regular stores are ordered and need no fence. Note that the fence is - // placed after all stores, to avoid unnecessary serialization and a - // performance hit. - NonTemporalStoreFence(); + if constexpr (copy_type != CopyType::kNone) { + const uint8_t* copy_src = e - remaining_bytes; + while (e - copy_src >= 64) { + Copy64Bytes<copy_type>(reinterpret_cast<uint8_t*>(d), copy_src); + d += 64; + copy_src += 64; + } + if constexpr (copy_type == CopyType::kNonTemporal) { + // According to the Intel 64 Software Developer's Manual + // (https://www.intel.com/content/www/us/en/developer/articles/technical/intel-sdm.html), + // non-temporal stores are weakly ordered due to bypassing caches, so + // they have to be fenced before the copy is visible to other threads. + // Regular stores are ordered and need no fence. Note that the fence is + // placed after all non-temporal stores, to avoid unnecessary + // serialization and a performance hit. + NonTemporalStoreFence(); + } + if (copy_src != e) { + std::memcpy(d, copy_src, static_cast<size_t>(e - copy_src)); + } } }