Improve CRC32C default CPU dispatch using AVX2 and ARM64 detection Unrecognized x86_64 and ARM64 CPUs in `TryNewCRC32AcceleratedX86ARMCombined` previously fell back to a single-stream CRC32C implementation even when hardware carry-less multiplication and SHA3 (`EOR3`) were available, and ARM64 builds without `-march=armv8-a+crc+crypto` disabled hardware CRC32C entirely. Add `SupportsAvx2()` and `SupportsArmSHA3()` runtime feature checks to `cpu_detect`, enable ARM64 CRC32/PMULL intrinsics via `.arch_extension` pragmas guarded by runtime detection, and dispatch multi-stream `PCLMUL` / `NEON_PCLMUL_EOR3` kernels by default when supported. PiperOrigin-RevId: 995516057
diff --git a/absl/base/internal/cpu_detect.cc b/absl/base/internal/cpu_detect.cc index 43b6c4b..c6f0ba6 100644 --- a/absl/base/internal/cpu_detect.cc +++ b/absl/base/internal/cpu_detect.cc
@@ -35,6 +35,21 @@ #if defined(_WIN32) || defined(_WIN64) #include <intrin.h> +#if defined(__aarch64__) || defined(_M_ARM64) +#ifndef WIN32_LEAN_AND_MEAN +#define WIN32_LEAN_AND_MEAN +#endif +#include <windows.h> +#ifndef PF_ARM_V8_CRYPTO_INSTRUCTIONS_AVAILABLE +#define PF_ARM_V8_CRYPTO_INSTRUCTIONS_AVAILABLE 30 +#endif +#ifndef PF_ARM_V8_CRC32_INSTRUCTIONS_AVAILABLE +#define PF_ARM_V8_CRC32_INSTRUCTIONS_AVAILABLE 31 +#endif +#ifndef PF_ARM_SHA3_INSTRUCTIONS_AVAILABLE +#define PF_ARM_SHA3_INSTRUCTIONS_AVAILABLE 64 +#endif +#endif // defined(__aarch64__) || defined(_M_ARM64) #endif #if defined(__x86_64__) || defined(_M_X64) @@ -257,6 +272,33 @@ bool SupportsArmCRC32PMULL() { return false; } +bool SupportsArmSHA3() { return false; } + +bool SupportsAvx2() { +#if ABSL_HAVE_BUILTIN(__builtin_cpu_supports) && !defined(_WIN32) && \ + !defined(_WIN64) + return __builtin_cpu_supports("avx2") != 0; +#elif defined(_WIN32) || defined(_WIN64) + int cpu_info[4]; + __cpuid(cpu_info, 0); + if (cpu_info[0] < 7) { + return false; + } + __cpuid(cpu_info, 1); + constexpr int kOsxsaveAndAvx = (1 << 27) | (1 << 28); + if ((cpu_info[2] & kOsxsaveAndAvx) != kOsxsaveAndAvx) { + return false; + } + if ((_xgetbv(0) & 0x6) != 0x6) { + return false; + } + __cpuidex(cpu_info, 7, 0); + return (cpu_info[1] & (1 << 5)) != 0; +#else + return false; +#endif +} + bool SupportsBmi2() { int cpu_info[4]; __cpuid(cpu_info, 0); @@ -333,13 +375,27 @@ bool SupportsArmCRC32PMULL() { #if defined(HWCAP_CRC32) && defined(HWCAP_PMULL) - uint64_t hwcaps = getauxval(AT_HWCAP); - return (hwcaps & HWCAP_CRC32) && (hwcaps & HWCAP_PMULL); + static const bool supported = []() { + uint64_t hwcaps = getauxval(AT_HWCAP); + return (hwcaps & HWCAP_CRC32) && (hwcaps & HWCAP_PMULL); + }(); + return supported; #else return false; #endif } +bool SupportsArmSHA3() { +#if defined(HWCAP_SHA3) + static const bool supported = (getauxval(AT_HWCAP) & HWCAP_SHA3) != 0; + return supported; +#else + return false; +#endif +} + +bool SupportsAvx2() { return false; } + bool SupportsBmi2() { return false; } #elif defined(__aarch64__) && defined(__APPLE__) @@ -388,6 +444,42 @@ return supported; } +bool SupportsArmSHA3() { + static const bool supported = []() { +#if defined(CAP_BIT_FEAT_SHA3) + const std::optional<uint64_t> caps = + ReadSysctlByName<uint64_t>("hw.optional.arm.caps"); + if (caps.has_value()) { + return (*caps & (uint64_t{1} << CAP_BIT_FEAT_SHA3)) != 0; + } +#endif + const std::optional<int> feat_sha3 = + ReadSysctlByName<int>("hw.optional.armv8_2_sha3"); + return feat_sha3.value_or(0) != 0; + }(); + return supported; +} + +bool SupportsAvx2() { return false; } + +bool SupportsBmi2() { return false; } + +#elif (defined(__aarch64__) || defined(_M_ARM64)) && \ + (defined(_WIN32) || defined(_WIN64)) + +CpuType GetCpuType() { return CpuType::kUnknown; } + +bool SupportsArmCRC32PMULL() { + return IsProcessorFeaturePresent(PF_ARM_V8_CRC32_INSTRUCTIONS_AVAILABLE) && + IsProcessorFeaturePresent(PF_ARM_V8_CRYPTO_INSTRUCTIONS_AVAILABLE); +} + +bool SupportsArmSHA3() { + return IsProcessorFeaturePresent(PF_ARM_SHA3_INSTRUCTIONS_AVAILABLE); +} + +bool SupportsAvx2() { return false; } + bool SupportsBmi2() { return false; } #else @@ -396,6 +488,10 @@ bool SupportsArmCRC32PMULL() { return false; } +bool SupportsArmSHA3() { return false; } + +bool SupportsAvx2() { return false; } + bool SupportsBmi2() { return false; } #endif
diff --git a/absl/base/internal/cpu_detect.h b/absl/base/internal/cpu_detect.h index 06eb72a..472f491 100644 --- a/absl/base/internal/cpu_detect.h +++ b/absl/base/internal/cpu_detect.h
@@ -67,6 +67,12 @@ // tuning. bool SupportsArmCRC32PMULL(); +// Returns whether the host CPU supports ARM SHA3 instructions (e.g., EOR3). +bool SupportsArmSHA3(); + +// Returns whether the host CPU and OS support AVX2 instructions. +bool SupportsAvx2(); + // Returns whether the host CPU supports BMI2 instructions. bool SupportsBmi2();
diff --git a/absl/base/internal/cpu_detect_test.cc b/absl/base/internal/cpu_detect_test.cc index 80b8b87..1bfc831 100644 --- a/absl/base/internal/cpu_detect_test.cc +++ b/absl/base/internal/cpu_detect_test.cc
@@ -22,6 +22,26 @@ namespace base_internal { namespace { +TEST(CpuDetectTest, SupportsArmSHA3) { +#if defined(__aarch64__) || defined(_M_ARM64) + (void)SupportsArmSHA3(); +#else + EXPECT_FALSE(SupportsArmSHA3()); +#endif +} + +TEST(CpuDetectTest, SupportsAvx2) { +#if defined(__x86_64__) || defined(_M_X64) +#if ABSL_HAVE_BUILTIN(__builtin_cpu_supports) && defined(__linux__) + EXPECT_EQ(SupportsAvx2(), __builtin_cpu_supports("avx2") != 0); +#else + (void)SupportsAvx2(); +#endif +#else + EXPECT_FALSE(SupportsAvx2()); +#endif +} + TEST(CpuDetectTest, SupportsBmi2) { #if defined(__x86_64__) || defined(_M_X64) #if ABSL_HAVE_BUILTIN(__builtin_cpu_supports) && defined(__linux__)
diff --git a/absl/crc/internal/crc32_x86_arm_combined_simd.h b/absl/crc/internal/crc32_x86_arm_combined_simd.h index 3615c36..dc9d441 100644 --- a/absl/crc/internal/crc32_x86_arm_combined_simd.h +++ b/absl/crc/internal/crc32_x86_arm_combined_simd.h
@@ -49,10 +49,21 @@ #include <intrin.h> #define ABSL_CRC_INTERNAL_HAVE_X86_SIMD -#elif defined(__aarch64__) && defined(__LITTLE_ENDIAN__) && \ - defined(__ARM_FEATURE_CRC32) && defined(ABSL_INTERNAL_HAVE_ARM_NEON) && \ - defined(__ARM_FEATURE_CRYPTO) +#elif defined(__aarch64__) && defined(__LITTLE_ENDIAN__) && \ + defined(ABSL_INTERNAL_HAVE_ARM_NEON) && \ + ((defined(__ARM_FEATURE_CRC32) && defined(__ARM_FEATURE_CRYPTO)) || \ + defined(__clang__) || defined(__GNUC__)) +// Enable ARM64 CRC32 + PMULL when either: +// 1. Both extensions are enabled by the compiler target +// (-march=...+crc+crypto), providing standard ACLE intrinsics +// (__crc32c*, vmull_p64), or +// 2. Building with Clang or GCC, where GNU inline assembly with +// .arch_extension directives can emit CRC32 and PMULL instructions on +// baseline -march=armv8-a targets (e.g., Android, Windows clang-cl, and +// default open-source Linux builds) for runtime dispatch via +// SupportsArmCRC32PMULL(). Visual C++ (MSVC) does not support GNU inline +// assembly on ARM64. #include <arm_acle.h> #include <arm_neon.h> #define ABSL_CRC_INTERNAL_HAVE_ARM_SIMD @@ -75,6 +86,16 @@ #define ABSL_INTERNAL_ATTRIBUTE_AVX #endif +#if defined(ABSL_CRC_INTERNAL_HAVE_X86_SIMD) && \ + (defined(__AVX__) || defined(ABSL_INTERNAL_CAN_FORCE_AVX)) +#define ABSL_CRC_INTERNAL_HAVE_X86_V256 1 +#endif + +#if defined(ABSL_CRC_INTERNAL_HAVE_X86_V256) && \ + (!defined(_MSC_VER) || defined(__clang__)) +#define ABSL_CRC_INTERNAL_HAVE_VPCLMUL 1 +#endif + namespace absl { ABSL_NAMESPACE_BEGIN namespace crc_internal { @@ -100,8 +121,7 @@ using V128 = std::array<uint64_t, 2>; #endif -#if (defined(__AVX__) || defined(ABSL_INTERNAL_CAN_FORCE_AVX)) && \ - defined(ABSL_CRC_INTERNAL_HAVE_X86_SIMD) +#if defined(ABSL_CRC_INTERNAL_HAVE_X86_V256) using V256 = __m256i; #else // Placeholder for V256 when AVX is not available. @@ -178,8 +198,7 @@ // DebugReportNonTemporalStore() and DebugReportStoreFence() in crc.h. void NonTemporalStoreFence(); -#if (defined(__AVX__) || defined(ABSL_INTERNAL_CAN_FORCE_AVX)) && \ - defined(ABSL_CRC_INTERNAL_HAVE_X86_SIMD) +#if defined(ABSL_CRC_INTERNAL_HAVE_X86_V256) ABSL_INTERNAL_ATTRIBUTE_AVX inline V256 V256_LoadU(const void* src); ABSL_INTERNAL_ATTRIBUTE_AVX inline V256 V256_Broadcast128(const V128* src); ABSL_INTERNAL_ATTRIBUTE_AVX inline void V256_StoreU(void* dst, V256 data); @@ -279,6 +298,7 @@ } #elif defined(ABSL_CRC_INTERNAL_HAVE_ARM_SIMD) +#ifdef __ARM_FEATURE_CRC32 inline uint32_t CRC32_u8(uint32_t crc, uint8_t v) { return __crc32cb(crc, v); } inline uint32_t CRC32_u16(uint32_t crc, uint16_t v) { @@ -292,6 +312,51 @@ inline uint32_t CRC32_u64(uint32_t crc, uint64_t v) { return __crc32cd(crc, v); } +#else +inline uint32_t CRC32_u8(uint32_t crc, uint8_t v) { + uint32_t res; + __asm__( + ".arch_extension crc \n\t" + "crc32cb %w0, %w1, %w2 \n\t" + ".arch_extension nocrc \n\t" + : "=r"(res) + : "r"(crc), "r"(static_cast<uint32_t>(v))); + return res; +} + +inline uint32_t CRC32_u16(uint32_t crc, uint16_t v) { + uint32_t res; + __asm__( + ".arch_extension crc \n\t" + "crc32ch %w0, %w1, %w2 \n\t" + ".arch_extension nocrc \n\t" + : "=r"(res) + : "r"(crc), "r"(static_cast<uint32_t>(v))); + return res; +} + +inline uint32_t CRC32_u32(uint32_t crc, uint32_t v) { + uint32_t res; + __asm__( + ".arch_extension crc \n\t" + "crc32cw %w0, %w1, %w2 \n\t" + ".arch_extension nocrc \n\t" + : "=r"(res) + : "r"(crc), "r"(v)); + return res; +} + +inline uint32_t CRC32_u64(uint32_t crc, uint64_t v) { + uint32_t res; + __asm__( + ".arch_extension crc \n\t" + "crc32cx %w0, %w1, %2 \n\t" + ".arch_extension nocrc \n\t" + : "=r"(res) + : "r"(crc), "r"(v)); + return res; +} +#endif inline V128 V128_Load(const V128* src) { return vld1q_u64(reinterpret_cast<const uint64_t*>(src)); @@ -311,9 +376,16 @@ // using intrinsics instead of inline assembly. inline V128 V128_PMulHi(const V128 l, const V128 r) { uint64x2_t res; - __asm__ __volatile__("pmull2 %0.1q, %1.2d, %2.2d \n\t" - : "=w"(res) - : "w"(l), "w"(r)); + __asm__ __volatile__( +#ifndef __ARM_FEATURE_CRYPTO + ".arch_extension crypto \n\t" +#endif + "pmull2 %0.1q, %1.2d, %2.2d \n\t" +#ifndef __ARM_FEATURE_CRYPTO + ".arch_extension nocrypto \n\t" +#endif + : "=w"(res) + : "w"(l), "w"(r)); return res; } @@ -321,22 +393,37 @@ // loop multiplicands from GPR to Neon registers every loop iteration. inline V128 V128_PMulLow(const V128 l, const V128 r) { uint64x2_t res; - __asm__ __volatile__("pmull %0.1q, %1.1d, %2.1d \n\t" - : "=w"(res) - : "w"(l), "w"(r)); + __asm__ __volatile__( +#ifndef __ARM_FEATURE_CRYPTO + ".arch_extension crypto \n\t" +#endif + "pmull %0.1q, %1.1d, %2.1d \n\t" +#ifndef __ARM_FEATURE_CRYPTO + ".arch_extension nocrypto \n\t" +#endif + : "=w"(res) + : "w"(l), "w"(r)); return res; } inline V128 V128_PMul01(const V128 l, const V128 r) { +#ifdef __ARM_FEATURE_CRYPTO return reinterpret_cast<V128>(vmull_p64( reinterpret_cast<poly64_t>(vget_high_p64(vreinterpretq_p64_u64(l))), reinterpret_cast<poly64_t>(vget_low_p64(vreinterpretq_p64_u64(r))))); +#else + return V128_PMulLow(vextq_u64(l, l, 1), r); +#endif } inline V128 V128_PMul10(const V128 l, const V128 r) { +#ifdef __ARM_FEATURE_CRYPTO return reinterpret_cast<V128>(vmull_p64( reinterpret_cast<poly64_t>(vget_low_p64(vreinterpretq_p64_u64(l))), reinterpret_cast<poly64_t>(vget_high_p64(vreinterpretq_p64_u64(r))))); +#else + return V128_PMulLow(l, vextq_u64(r, r, 1)); +#endif } inline V128 V128_Xor(const V128 l, const V128 r) { return veorq_u64(l, r); } @@ -394,8 +481,7 @@ DebugReportStoreFence(); } -#if (defined(__AVX__) || defined(ABSL_INTERNAL_CAN_FORCE_AVX)) && \ - defined(ABSL_CRC_INTERNAL_HAVE_X86_SIMD) +#if defined(ABSL_CRC_INTERNAL_HAVE_X86_V256) ABSL_INTERNAL_ATTRIBUTE_AVX inline V256 V256_LoadU(const void* src) { return _mm256_loadu_si256(reinterpret_cast<const __m256i*>(src)); } @@ -441,8 +527,7 @@ std::memcpy(dst, &data, sizeof(T)); DebugReportNonTemporalStore(); } -#endif // (defined(__AVX__) || defined(ABSL_INTERNAL_CAN_FORCE_AVX)) && - // defined(ABSL_CRC_INTERNAL_HAVE_X86_SIMD) +#endif // defined(ABSL_CRC_INTERNAL_HAVE_X86_V256) ABSL_INTERNAL_ATTRIBUTE_AVX inline void V256_LoadPairU(const void* src, V256* v0, V256* v1) {
diff --git a/absl/crc/internal/crc32c_inline.h b/absl/crc/internal/crc32c_inline.h index e117c86..1792b4b 100644 --- a/absl/crc/internal/crc32c_inline.h +++ b/absl/crc/internal/crc32c_inline.h
@@ -19,6 +19,7 @@ #include <cstdint> #include "absl/base/config.h" +#include "absl/base/internal/cpu_detect.h" #include "absl/base/internal/endian.h" #include "absl/crc/internal/crc32_x86_arm_combined_simd.h" @@ -32,6 +33,11 @@ inline bool ExtendCrc32cInline(uint32_t* crc, const char* p, size_t n) { #if defined(ABSL_CRC_INTERNAL_HAVE_ARM_SIMD) || \ defined(ABSL_CRC_INTERNAL_HAVE_X86_SIMD) +#if defined(ABSL_CRC_INTERNAL_HAVE_ARM_SIMD) && !defined(__ARM_FEATURE_CRC32) + if (!absl::base_internal::SupportsArmCRC32PMULL()) { + return false; + } +#endif constexpr uint32_t kCrc32Xor = 0xffffffffU; *crc ^= kCrc32Xor; if (n & 1) {
diff --git a/absl/crc/internal/crc_x86_arm_combined.cc b/absl/crc/internal/crc_x86_arm_combined.cc index 0d1dfdc..accb050 100644 --- a/absl/crc/internal/crc_x86_arm_combined.cc +++ b/absl/crc/internal/crc_x86_arm_combined.cc
@@ -47,6 +47,9 @@ using ::absl::base_internal::GetCpuType; #if defined(__aarch64__) using ::absl::base_internal::SupportsArmCRC32PMULL; +using ::absl::base_internal::SupportsArmSHA3; +#else +using ::absl::base_internal::SupportsAvx2; #endif #if defined(ABSL_INTERNAL_CAN_USE_SIMD_CRC32C) @@ -408,9 +411,7 @@ return crc; } -#if defined(ABSL_CRC_INTERNAL_HAVE_X86_SIMD) && \ - (defined(__AVX__) || defined(ABSL_INTERNAL_CAN_FORCE_AVX)) && \ - (!defined(_MSC_VER) || defined(__clang__)) +#if defined(ABSL_CRC_INTERNAL_HAVE_VPCLMUL) // This is only used if we have vector version of PCLMULQDQ. // We don't have it on arm, and it isn't supported by default // compiler targets on x86. If we want to use it, we need to either use @@ -485,9 +486,7 @@ ABSL_ATTRIBUTE_ALWAYS_INLINE uint64_t FinalizeVpclmulStream(V256*) const { return 0; } -#endif // defined(ABSL_CRC_INTERNAL_HAVE_X86_SIMD) && (defined(__AVX__) || - // defined(ABSL_INTERNAL_CAN_FORCE_AVX)) && (!defined(_MSC_VER) || - // defined(__clang__)) +#endif // defined(ABSL_CRC_INTERNAL_HAVE_VPCLMUL) // Constants generated by './scripts/gen-crc-consts.py x86_pclmul // crc32_lsb_0x82f63b78' from the Linux kernel. @@ -1149,11 +1148,7 @@ case CpuType::kAmdSiena: case CpuType::kAmdTurin: case CpuType::kAmdVenice: -#if defined(ABSL_CRC_INTERNAL_HAVE_X86_SIMD) && \ - (defined(__AVX__) || defined(ABSL_INTERNAL_CAN_FORCE_AVX)) && \ - (!defined(_MSC_VER) || defined(__clang__)) - // We don't have vector pclmul on arm, but this still needs to - // compile. +#if defined(ABSL_CRC_INTERNAL_HAVE_VPCLMUL) return new CRC32AcceleratedX86ARMCombinedMultipleStreams< 2, 3, PclmulStreamType::VPCLMUL>(); #else @@ -1163,9 +1158,7 @@ case CpuType::kIntelSapphirerapids: case CpuType::kIntelEmeraldrapids: case CpuType::kIntelGraniterapids: -#if defined(ABSL_CRC_INTERNAL_HAVE_X86_SIMD) && \ - (defined(__AVX__) || defined(ABSL_INTERNAL_CAN_FORCE_AVX)) && \ - (!defined(_MSC_VER) || defined(__clang__)) +#if defined(ABSL_CRC_INTERNAL_HAVE_VPCLMUL) return new CRC32AcceleratedX86ARMCombinedMultipleStreams< 3, 3, PclmulStreamType::VPCLMUL>(); #else @@ -1210,9 +1203,17 @@ // before trying to use an accelerated implementation. if (SupportsArmCRC32PMULL()) { #if defined(__APPLE__) + if (SupportsArmSHA3()) { + return new CRC32AcceleratedX86ARMCombinedMultipleStreams< + 3, 3, PclmulStreamType::NEON_PCLMUL_EOR3>(); + } return new CRC32AcceleratedX86ARMCombinedMultipleStreams< 3, 3, PclmulStreamType::NEON_PCLMUL>(); #else + if (SupportsArmSHA3()) { + return new CRC32AcceleratedX86ARMCombinedMultipleStreams< + 1, 1, PclmulStreamType::NEON_PCLMUL_EOR3>(); + } return new CRC32AcceleratedX86ARMCombinedMultipleStreams< 1, 1, PclmulStreamType::NEON_PCLMUL>(); #endif @@ -1221,6 +1222,14 @@ } #else default: + // Use AVX2 support as a heuristic for modern x86_64 CPUs with fast + // PCLMULQDQ (Haswell and newer) when the microarchitecture is + // unrecognized (e.g., client CPUs, open-source builds, or other OSes), + // not because this path uses AVX2 instructions. + if (SupportsAvx2()) { + return new CRC32AcceleratedX86ARMCombinedMultipleStreams< + 3, 1, PclmulStreamType::PCLMUL>(); + } // Something else, play it safe and assume slow PCLMULQDQ. return new CRC32AcceleratedX86ARMCombinedMultipleStreams< 3, 0, PclmulStreamType::NONE>();