Copy short heads and tails inline in CRC32C copies. In `ExtendWithPclmul`, copying the entire tail after the multi-stream loop with `std::memcpy` wrote remaining full 64-byte cache lines with temporal stores even in non-temporal mode. In addition, the head alignment copy and the tail copy used variable-length `std::memcpy` calls, which compile to out-of-line calls that measurably slow down shorter copies. This improves performance slightly in most configurations, and significantly in a few where the lower register pressure from being a leaf function saves a few stack spills and fills. PiperOrigin-RevId: 1022855644
diff --git a/absl/crc/internal/crc_x86_arm_combined.cc b/absl/crc/internal/crc_x86_arm_combined.cc index 3a2a2d3..99df29d 100644 --- a/absl/crc/internal/crc_x86_arm_combined.cc +++ b/absl/crc/internal/crc_x86_arm_combined.cc
@@ -27,6 +27,7 @@ #include "absl/base/config.h" #include "absl/base/internal/cpu_detect.h" #include "absl/base/internal/endian.h" +#include "absl/base/macros.h" #include "absl/base/optimization.h" #include "absl/base/prefetch.h" #include "absl/crc/internal/crc32_x86_arm_combined_simd.h" @@ -270,6 +271,45 @@ } } + // Copies n < 64 bytes from `src` to `dst`. Fixed-size `std::memcpy()` calls + // compile to inline loads and stores, while a variable-length one compiles to + // a call that measurably slows down short head and tail copies and forces a + // stack frame in callers. + ABSL_ATTRIBUTE_ALWAYS_INLINE static void CopyLessThan64Bytes( + char* dst, const uint8_t* src, size_t n) { + ABSL_ASSERT(n < 64); + if (n >= 32) { + CopyInTwoChunksInline<32>(dst, src, n); + } else if (n >= 16) { + CopyInTwoChunksInline<16>(dst, src, n); + } else if (n >= 8) { + CopyInTwoChunksInline<8>(dst, src, n); + } else if (n >= 4) { + CopyInTwoChunksInline<4>(dst, src, n); + } else if (n >= 2) { + CopyInTwoChunksInline<2>(dst, src, n); + } else if (n == 1) { + *dst = static_cast<char>(*src); + } + } + + // Copies `n` bytes from `src` to `dst`, with two possibly overlapping copies + // (the first kBytesPerCopy bytes of src and the last kBytesPerCopy bytes of + // src). Since the copies are of a fixed size, we must have kBytesPerCopy <= n + // <= 2 * kBytesPerCopy. This is faster than `std::memcpy()` for small, + // variable-sized copies, because it compiles to 2 simple loads and stores, + // instead of branching on the value of `n` and it is easily inlined. + template <size_t kBytesPerCopy> + ABSL_ATTRIBUTE_ALWAYS_INLINE static void CopyInTwoChunksInline(char* dst, + const uint8_t* src, + size_t n) { + ABSL_ASSERT(n <= 2 * kBytesPerCopy); + ABSL_ASSERT(n >= kBytesPerCopy); + std::memcpy(dst, src, kBytesPerCopy); + std::memcpy(dst + n - kBytesPerCopy, src + n - kBytesPerCopy, + kBytesPerCopy); + } + // Update a single 128-bit element of partialCRC with crc of 16 byte block. // Calling FinalizePclmulStream would produce a single crc checksum, but it is // expensive. PCLMULQDQ has a high latency, so we run 4 128-bit partial @@ -705,7 +745,7 @@ uintptr_t dst_addr = reinterpret_cast<uintptr_t>(d); size_t bytes_to_align = (kAlignment - (dst_addr & (kAlignment - 1))) & (kAlignment - 1); - std::memcpy(d, p, bytes_to_align); + CopyLessThan64Bytes(d, p, bytes_to_align); d += bytes_to_align; while (bytes_to_align >= 8) { ABSL_INTERNAL_STEP8(l, p); @@ -926,15 +966,11 @@ if constexpr (copy_type != CopyType::kNone) { d = reinterpret_cast<char*>( dst_pclmul_streams[kNumActivePclmulStreams - 1]); - size_t remaining_to_copy = static_cast<size_t>(e - p); - std::memcpy(d, p, remaining_to_copy); } } else { p = crc_streams[kNumActiveCrcStreams - 1]; if constexpr (copy_type != CopyType::kNone) { d = reinterpret_cast<char*>(dst_crc_streams[kNumActiveCrcStreams - 1]); - size_t remaining_to_copy = static_cast<size_t>(e - p); - std::memcpy(d, p, remaining_to_copy); } } @@ -962,15 +998,24 @@ *crc = l; - if constexpr (copy_type == CopyType::kNonTemporal) { - // According to the Intel 64 Software Developer's Manual - // (https://www.intel.com/content/www/us/en/developer/articles/technical/intel-sdm.html), - // non-temporal stores are weakly ordered due to bypassing caches, so - // they have to be fenced before the copy is visible to other threads. - // Regular stores are ordered and need no fence. Note that the fence is - // placed after all stores, to avoid unnecessary serialization and a - // performance hit. - NonTemporalStoreFence(); + if constexpr (copy_type != CopyType::kNone) { + const uint8_t* copy_src = e - remaining_bytes; + while (e - copy_src >= 64) { + Copy64Bytes<copy_type>(reinterpret_cast<uint8_t*>(d), copy_src); + d += 64; + copy_src += 64; + } + CopyLessThan64Bytes(d, copy_src, static_cast<size_t>(e - copy_src)); + if constexpr (copy_type == CopyType::kNonTemporal) { + // According to the Intel 64 Software Developer's Manual + // (https://www.intel.com/content/www/us/en/developer/articles/technical/intel-sdm.html), + // non-temporal stores are weakly ordered due to bypassing caches, so + // they have to be fenced before the copy is visible to other threads. + // Regular stores are ordered and need no fence. Note that the fence is + // placed after all stores, to avoid unnecessary serialization and a + // performance hit. + NonTemporalStoreFence(); + } } }