Tune the cutoffs for CRC32 implementation selection. 1) For inputs smaller than kSmallCutoff (256 bytes), we don't have any platform-specific code, even in the platform-specific functions, so we move to just calling the common ExtendCrc32cInline function instead (which we already did for inputs smaller than 64 bytes). This saves us an extra function call and may allow for more inlining inn large workloads. 2) After recent performance improvements for large inputs, the range where the "large" implementation is better than the "medium" implementation has grown, and changing kMediumCutoff from 2048 to 1024 helps. 0-6% faster on all tested platforms PiperOrigin-RevId: 1005732103
diff --git a/absl/crc/crc32c.cc b/absl/crc/crc32c.cc index 9020563..5b4672d 100644 --- a/absl/crc/crc32c.cc +++ b/absl/crc/crc32c.cc
@@ -44,8 +44,8 @@ return static_cast<crc32c_t>(crc ^ kCRC32Xor); } -// Called by `absl::ExtendCrc32c()` on strings with size > 64 or when hardware -// CRC32C support is missing. +// Called by `absl::ExtendCrc32c()` on strings with size >= `kSmallCutoff` or +// when hardware CRC32C support is missing. crc32c_t ExtendCrc32cInternal(crc32c_t initial_crc, absl::string_view buf_to_add) { uint32_t crc = static_cast<uint32_t>(initial_crc) ^ kCRC32Xor;
diff --git a/absl/crc/crc32c.h b/absl/crc/crc32c.h index edeacd0..f14b42d 100644 --- a/absl/crc/crc32c.h +++ b/absl/crc/crc32c.h
@@ -30,6 +30,7 @@ #include <ostream> #include "absl/base/config.h" +#include "absl/crc/internal/crc32_x86_arm_combined_simd.h" #include "absl/crc/internal/crc32c_inline.h" #include "absl/strings/str_format.h" #include "absl/strings/string_view.h" @@ -98,8 +99,8 @@ // This operation has a runtime cost of O(`buf_to_add.size()`) inline crc32c_t ExtendCrc32c(crc32c_t initial_crc, absl::string_view buf_to_add) { - // Approximately 75% of calls have size <= 64. - if (buf_to_add.size() <= 64) { + // Short inputs are faster to process inline with a simple serial loop. + if (buf_to_add.size() < crc_internal::kSmallCutoff) { uint32_t crc = static_cast<uint32_t>(initial_crc); if (crc_internal::ExtendCrc32cInline(&crc, buf_to_add.data(), buf_to_add.size())) {
diff --git a/absl/crc/internal/crc32_x86_arm_combined_simd.h b/absl/crc/internal/crc32_x86_arm_combined_simd.h index f1bb85b..168971f 100644 --- a/absl/crc/internal/crc32_x86_arm_combined_simd.h +++ b/absl/crc/internal/crc32_x86_arm_combined_simd.h
@@ -78,6 +78,12 @@ namespace absl { ABSL_NAMESPACE_BEGIN namespace crc_internal { +// Constants for switching between algorithms. +// Chosen by comparing speed at different powers of 2. +// Inputs shorter than kSmallCutoff are processed with a simple serial loop of +// CRC32 instructions instead of dispatching to platform-specific SIMD code. +constexpr size_t kSmallCutoff = 256; +constexpr size_t kMediumCutoff = 1024; #if defined(ABSL_CRC_INTERNAL_HAVE_ARM_SIMD) using V128 = uint64x2_t;
diff --git a/absl/crc/internal/crc_memcpy.h b/absl/crc/internal/crc_memcpy.h index 04eafce..141fbd9 100644 --- a/absl/crc/internal/crc_memcpy.h +++ b/absl/crc/internal/crc_memcpy.h
@@ -17,12 +17,14 @@ #include <cstddef> #include <cstdint> +#include <cstring> #include <memory> #include "absl/base/config.h" #include "absl/crc/crc32c.h" #include "absl/crc/internal/crc.h" #include "absl/crc/internal/crc32_x86_arm_combined_simd.h" +#include "absl/crc/internal/crc32c_inline.h" // Defined if the class AcceleratedCrcMemcpyEngine exists. // TODO(b/299127771): Consider relaxing the pclmul requirement once the other @@ -55,6 +57,12 @@ crc32c_t initial_crc = crc32c_t{0}, bool non_temporal = false) { uint32_t crc = static_cast<uint32_t>(initial_crc); + // Short inputs are faster to process inline with a simple serial loop. + if (length < kSmallCutoff && + ExtendCrc32cInline(&crc, static_cast<const char*>(src), length)) { + std::memcpy(dst, src, length); + return crc32c_t{crc}; + } crc ^= 0xffffffffU; CRC::Crc32c()->ExtendAndCopy(&crc, dst, src, length, non_temporal); crc ^= 0xffffffffU;
diff --git a/absl/crc/internal/crc_x86_arm_combined.cc b/absl/crc/internal/crc_x86_arm_combined.cc index c62fa0c..2d68ceb 100644 --- a/absl/crc/internal/crc_x86_arm_combined.cc +++ b/absl/crc/internal/crc_x86_arm_combined.cc
@@ -14,6 +14,7 @@ // Hardware accelerated CRC32 computation on Intel and ARM architecture. +#include <cassert> #include <cstddef> #include <cstdint> #include <cstring> @@ -69,11 +70,6 @@ const CRC32AcceleratedX86ARMCombined&) = delete; }; -// Constants for switching between algorithms. -// Chosen by comparing speed at different powers of 2. -constexpr size_t kSmallCutoff = 256; -constexpr size_t kMediumCutoff = 2048; - #define ABSL_INTERNAL_STEP1(crc, p) \ do { \ crc = CRC32_u8(static_cast<uint32_t>(crc), *p++); \ @@ -543,10 +539,14 @@ } } + // Requires `length >= kSmallCutoff`. Shorter inputs are handled by + // `ExtendCrc32cInline()` in `absl::ExtendCrc32c()` and + // `CrcMemcpy::CrcAndCopy()`. template <CopyType copy_type> ABSL_ATTRIBUTE_HOT void Extend(uint32_t* crc, const void* bytes, const size_t length, void* __restrict dst = nullptr) const { + assert(length >= kSmallCutoff); if constexpr (pclmul_stream_type != PclmulStreamType::NONE) { if (length >= kMediumCutoff) { ExtendWithPclmul<copy_type>(crc, bytes, length, dst); @@ -573,12 +573,10 @@ std::memcpy(d, p, length); } - // For small blocks just run simple loop, because cost of combining multiple - // streams is significant. - if (num_crc_streams == 1 || (length < kSmallCutoff)) { - // fallthrough; Use the same strategy as we do for processing the - // remaining bytes after any other strategy. - } else { + // A single CRC stream just runs the simple loop used to process the + // remaining bytes after the multi-stream strategy below. Inputs shorter + // than kSmallCutoff never reach this function (see `Extend()`). + if constexpr (num_crc_streams > 1) { l64 = l; const size_t blockSize = 32; size_t bs = static_cast<size_t>(e - p) / kGroupsSmall / blockSize;