| /* |
| * Copyright 2016 Google Inc. |
| * |
| * Use of this source code is governed by a BSD-style license that can be |
| * found in the LICENSE file. |
| * |
| * This code detects what features the CPU we are currently running on has using cpuid |
| * (a built-in x86 instruction). The canonical source for the magic numbers/bits is |
| * Intel® Architecture Instruction Set Extensions Programming Reference (specifically |
| * 1.4 DETECTION OF FUTURE INSTRUCTIONS AND FEATURES) and AMD64 Architecture Programmer's Manual |
| * Volume 3: General Purpose and System Programming Instructions (D.2 CPUID Feature Flags Related |
| * to Instruction Support) |
| * |
| * https://www.sandpile.org/x86/cpuid.htm also visualizes this and is easier to reference. |
| * |
| * Intel® 64 and IA-32 Architectures Software Developer's Manual gives more details |
| * for some of the more intricate detection of features (e.g. AVX). |
| * |
| * See the Team Drive Skia > CPU Backend > Reference Manuals |
| */ |
| |
| #include "src/core/SkCpu.h" |
| |
| #include "include/private/base/SkFeatures.h" |
| #include "include/private/base/SkOnce.h" |
| |
| #if defined(SK_CPU_X86) |
| #if defined(_MSC_VER) |
| #include <intrin.h> |
| #else |
| #include <cpuid.h> |
| #endif |
| #elif defined(SK_CPU_LOONGARCH) |
| #include <sys/auxv.h> |
| #endif |
| |
| namespace { |
| #if defined(SK_CPU_X86) |
| // Vendor String https://www.sandpile.org/x86/cpuid.htm#leaf_0000_0000h |
| constexpr uint32_t kVendorLeaf = 0; |
| // Family/Model/Stepping/Feature Flags https://www.sandpile.org/x86/cpuid.htm#leaf_0000_0001h |
| constexpr uint32_t kFMSFLeaf = 1; |
| // Feature Flags https://www.sandpile.org/x86/cpuid.htm#leaf_0000_0007h |
| constexpr uint32_t kFlagsLeaf = 7; |
| constexpr uint32_t kFlagsSubleaf = 0; |
| |
| #if defined(_MSC_VER) |
| // https://learn.microsoft.com/en-us/cpp/intrinsics/cpuid-cpuidex |
| // The output will be written to the input arrays. |
| void cpu_vendor(uint32_t abcd[4]) { __cpuid((int*)abcd, kVendorLeaf); } |
| void cpu_features(uint32_t abcd[4]) { __cpuid((int*)abcd, kFMSFLeaf); } |
| void cpu_flags(uint32_t abcd[4]) { __cpuidex((int*)abcd, kFlagsLeaf, kFlagsSubleaf); } |
| |
| uint64_t xgetbv() { |
| // https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html |
| constexpr uint32_t xcr = 0; |
| return _xgetbv(xcr); |
| } |
| #else |
| #if !defined(__cpuid_count) // Old Mac Clang doesn't have this defined. |
| #define __cpuid_count(eax, ecx, a, b, c, d) \ |
| __asm__("cpuid" : "=a"(a), "=b"(b), "=c"(c), "=d"(d) : "0"(eax), "2"(ecx)) |
| #endif |
| // https://www.felixcloutier.com/x86/cpuid |
| // The output will be written to the input arrays. |
| void cpu_vendor(uint32_t abcd[4]) { |
| __cpuid(kVendorLeaf, abcd[0], abcd[1], abcd[2], abcd[3]); |
| } |
| void cpu_features(uint32_t abcd[4]) { |
| __cpuid(kFMSFLeaf, abcd[0], abcd[1], abcd[2], abcd[3]); |
| } |
| void cpu_flags(uint32_t abcd[4]) { |
| // __cpuid_count is just like __cpuid except it also takes a subleaf option (which |
| // we set to 0, as that's where the info we care about is). |
| __cpuid_count(kFlagsLeaf, kFlagsSubleaf, abcd[0], abcd[1], abcd[2], abcd[3]); |
| } |
| |
| uint64_t xgetbv() { |
| // https://www.felixcloutier.com/x86/xgetbv |
| // "Execute XGETBV with ECX = 0 to discover the value of XCR0 ... |
| constexpr uint32_t xcr = 0; |
| uint32_t eax, edx; |
| __asm__ __volatile__ ( "xgetbv" : "=a"(eax), "=d"(edx) : "c"(xcr)); |
| return (uint64_t)(edx) << 32 | eax; |
| } |
| #endif |
| |
| // The comments on each of the below lines correspond to constants defined in <cpuid.h> |
| |
| // cpuid(1) -> EDX |
| constexpr uint32_t kSSE = (1u << 25); // bit_SSE |
| constexpr uint32_t kSSE2 = (1u << 26); // bit_SSE2 |
| |
| // cpuid(1) -> ECX |
| constexpr uint32_t kSSE3 = (1u << 0); // bit_SSE3 |
| constexpr uint32_t kSSSE3 = (1u << 9); // bit_SSSE3 |
| constexpr uint32_t kSSE41 = (1u << 19); // bit_SSE4_1 |
| constexpr uint32_t kSSE42 = (1u << 20); // bit_SSE4_2 |
| constexpr uint32_t kFMA = (1u << 12); // bit_FMA |
| constexpr uint32_t kAVX = (1u << 28); // bit_AVX |
| constexpr uint32_t kF16C = (1u << 29); // bit_F16C |
| constexpr uint32_t kXSAVE = (1u << 26); // bit_XSAVE |
| constexpr uint32_t kOSXSAVE = (1u << 27); // bit_OSXSAVE |
| |
| // cpuid(7,0) -> EBX |
| constexpr uint32_t kBMI1 = (1u << 3); // bit_BMI |
| constexpr uint32_t kAVX2 = (1u << 5); // bit_AVX2 |
| constexpr uint32_t kBMI2 = (1u << 8); // bit_BMI2 |
| constexpr uint32_t kERMS = (1u << 9); // bit_ENH_MOVSB |
| constexpr uint32_t kAVX512F = (1u << 16); // bit_AVX512F |
| constexpr uint32_t kAVX512DQ = (1u << 17); // bit_AVX512DQ |
| constexpr uint32_t kAVX512IFMA = (1u << 21); // bit_AVX512IFMA |
| constexpr uint32_t kAVX512PF = (1u << 26); // bit_AVX512PF |
| constexpr uint32_t kAVX512ER = (1u << 27); // bit_AVX512ER |
| constexpr uint32_t kAVX512CD = (1u << 28); // bit_AVX512CD |
| constexpr uint32_t kAVX512BW = (1u << 30); // bit_AVX512BW |
| constexpr uint32_t kAVX512VL = (1u << 31); // bit_AVX512VL |
| |
| // cpuid(7,0) -> ECX |
| constexpr uint32_t kAVX512VBMI2 = (1u << 6); // bit_AVX512VBMI2 |
| |
| // xgetbv(0) -> XCR (eXtended Control Register) |
| constexpr uint64_t kXCR0_XMM_YMM_STATE = 0b00000110; // Bits 1 and 2 |
| constexpr uint64_t kXCR0_ZMM_STATE = 0b11100000; // Bits 5, 6, and 7 |
| |
| // Combine 4 ASCII characters together in little-endian order. |
| constexpr uint32_t ASCII_LE(const char str[4]) { |
| auto a = str[0], b = str[1], c = str[2], d = str[3]; |
| return (((uint32_t)d << 24) | ((uint32_t)c << 16) | ((uint32_t)b << 8) | (uint32_t)a); |
| } |
| |
| uint32_t read_cpu_features() { |
| uint32_t features = 0; |
| uint32_t abcd[4] = {0,0,0,0}; |
| |
| #define EAX abcd[0] |
| #define EBX abcd[1] |
| #define ECX abcd[2] |
| #define EDX abcd[3] |
| |
| cpu_vendor(abcd); |
| // The vendor string in EBX, EDX, ECX (yes, in that order). |
| // For AMD, this is "AuthenticAMD" encoded as ASCII (little-endian) |
| // Intel happens to be "GenuineIntel" https://www.sandpile.org/x86/cpuid.htm#leaf_0000_0000h |
| const bool isAMD = (EBX == ASCII_LE("Auth")) && |
| (EDX == ASCII_LE("enti")) && |
| (ECX == ASCII_LE("cAMD")); |
| |
| cpu_features(abcd); |
| if (EDX & kSSE) { features |= SkX64:: SSE1; } |
| if (EDX & kSSE2) { features |= SkX64:: SSE2; } |
| if (ECX & kSSE3) { features |= SkX64:: SSE3; } |
| if (ECX & kSSSE3) { features |= SkX64::SSSE3; } |
| if (ECX & kSSE41) { features |= SkX64::SSE41; } |
| if (ECX & kSSE42) { features |= SkX64::SSE42; } |
| |
| // From Intel® 64 and IA-32 Architectures Software Developer's Manual |
| // 14.3 DETECTION OF INTEL® AVX INSTRUCTIONS |
| // 1) Detect we have XGETBV, i.e. cpuid(1) returns ECX with bit 27 set |
| // (In Intel's docs, this is represented by CPUID.01H:ECX.OSXSAVE[27] = 1) |
| // For historical reasons, we check bit 26 (kXSAVE; see |
| // https://codereview.chromium.org/1428153003) which is the hardware support. Bit 27 |
| // (kOSXSAVE) means the OS *also* supports this, which the Intel docs say is important. |
| if ((ECX & (kXSAVE | kOSXSAVE)) == (kXSAVE | kOSXSAVE)) { |
| // 2) Call XGETBV to get eXtended Control Register info. |
| const uint64_t xcr = xgetbv(); |
| if ((xcr & kXCR0_XMM_YMM_STATE) == kXCR0_XMM_YMM_STATE) { |
| // 3) Check ECX bit 28 for AVX support |
| if (ECX & kAVX) { features |= SkX64::AVX; } |
| // Now that we have AVX, we can detect other features that list it as a prereq. |
| if (ECX & kF16C) { features |= SkX64::F16C; } // 14.4.1 Detection of F16C Instr... |
| if (ECX & kFMA) { features |= SkX64::FMA; } // 14.5.3 Detection of FMA |
| |
| // Fill the register values with values from leaf 7. |
| cpu_flags(abcd); |
| if (EBX & kAVX2) { features |= SkX64::AVX2; } // 14.7.1 Detection of Intel® AVX2 |
| |
| // These don't strictly require AVX support, but only exist on newer chips anyway. |
| if (EBX & kBMI1) { features |= SkX64::BMI1; } |
| if (EBX & kBMI2) { features |= SkX64::BMI2; } |
| if (EBX & kERMS) { features |= SkX64::ERMS; } |
| |
| // 15.2 DETECTION OF AVX-512 FOUNDATION INSTRUCTIONS |
| // (for Intel; AMD just needs flag checks) |
| // 1) Detect we have XGETBV (which we did above). |
| // 2) Execute XGETBV and verify that XCR0[7:5] = '111b' ... |
| // and that XCR0[2:1] = 11b' (which we did above) |
| if ((xcr & kXCR0_ZMM_STATE) == kXCR0_ZMM_STATE) { |
| // AVX-512 can be slow on early Intel processors (like Skylake or Cascade Lake) |
| // due to thermal throttling (see https://stackoverflow.com/a/63484551), |
| // but works well on Ice Lake and later, and all AMD processors. We detect |
| // AMD above, but to identify Intel with Gen 3+ AVX512 support, we look for |
| // VBMI2 support, which was added (along with other features) in Ice Lake. |
| // Table 16-2. Intel® AVX-512 CPUID Feature Flags Included in Intel® AVX10 |
| // has a nice table of AVX512 commands and when they were introduced. |
| const bool isNewerIntel = (ECX & kAVX512VBMI2); |
| if (isAMD || isNewerIntel) { |
| // 3) Check EBX bit 16 for AVX support |
| if (EBX & kAVX512F) { features |= SkX64::AVX512F; } |
| // ... and any other extensions. |
| if (EBX & kAVX512DQ) { features |= SkX64::AVX512DQ; } |
| if (EBX & kAVX512IFMA) { features |= SkX64::AVX512IFMA; } |
| if (EBX & kAVX512PF) { features |= SkX64::AVX512PF; } |
| if (EBX & kAVX512ER) { features |= SkX64::AVX512ER; } |
| if (EBX & kAVX512CD) { features |= SkX64::AVX512CD; } |
| if (EBX & kAVX512BW) { features |= SkX64::AVX512BW; } |
| if (EBX & kAVX512VL) { features |= SkX64::AVX512VL; } |
| } |
| } |
| } |
| } |
| return features; |
| } |
| #elif defined(SK_CPU_LOONGARCH) |
| uint32_t read_cpu_features() { |
| uint32_t features = 0; |
| uint64_t hwcap = getauxval(AT_HWCAP); |
| |
| if (hwcap & HWCAP_LOONGARCH_LSX) { features |= SkLoongArch::SX; } |
| if (hwcap & HWCAP_LOONGARCH_LASX) { features |= SkLoongArch::ASX; } |
| |
| return features; |
| } |
| #else |
| uint32_t read_cpu_features() { |
| return 0; |
| } |
| #endif |
| } // anonymous namespace |
| |
| uint32_t SkCpu::gCachedFeatures = 0; |
| |
| void SkCpu::CacheRuntimeFeatures() { |
| static SkOnce once; |
| once([] { gCachedFeatures = read_cpu_features(); }); |
| } |
