| /* | |
| * Copyright 2016 Google Inc. | |
| * | |
| * Use of this source code is governed by a BSD-style license that can be | |
| * found in the LICENSE file. | |
| * | |
| * This code detects what features the CPU we are currently running on has using cpuid | |
| * (a built-in x86 instruction). The canonical source for the magic numbers/bits is | |
| * Intel® Architecture Instruction Set Extensions Programming Reference (specifically | |
| * 1.4 DETECTION OF FUTURE INSTRUCTIONS AND FEATURES) and AMD64 Architecture Programmer's Manual | |
| * Volume 3: General Purpose and System Programming Instructions (D.2 CPUID Feature Flags Related | |
| * to Instruction Support) | |
| * | |
| * https://www.sandpile.org/x86/cpuid.htm also visualizes this and is easier to reference. | |
| * | |
| * Intel® 64 and IA-32 Architectures Software Developer's Manual gives more details | |
| * for some of the more intricate detection of features (e.g. AVX). | |
| * | |
| * See the Team Drive Skia > CPU Backend > Reference Manuals | |
| */ | |
| #include "src/core/SkCpu.h" | |
| #include "include/private/SkFeatures.h" | |
| #include "include/private/SkOnce.h" | |
| #if defined(SK_CPU_X86) | |
| #if defined(_MSC_VER) | |
| #include <intrin.h> | |
| #else | |
| #include <cpuid.h> | |
| #endif | |
| #elif defined(SK_CPU_LOONGARCH) | |
| #include <sys/auxv.h> | |
| #endif | |
| namespace { | |
| #if defined(SK_CPU_X86) | |
| // Vendor String https://www.sandpile.org/x86/cpuid.htm#leaf_0000_0000h | |
| constexpr uint32_t kVendorLeaf = 0; | |
| // Family/Model/Stepping/Feature Flags https://www.sandpile.org/x86/cpuid.htm#leaf_0000_0001h | |
| constexpr uint32_t kFMSFLeaf = 1; | |
| // Feature Flags https://www.sandpile.org/x86/cpuid.htm#leaf_0000_0007h | |
| constexpr uint32_t kFlagsLeaf = 7; | |
| constexpr uint32_t kFlagsSubleaf = 0; | |
| #if defined(_MSC_VER) | |
| // https://learn.microsoft.com/en-us/cpp/intrinsics/cpuid-cpuidex | |
| // The output will be written to the input arrays. | |
| void cpu_vendor(uint32_t abcd[4]) { __cpuid((int*)abcd, kVendorLeaf); } | |
| void cpu_features(uint32_t abcd[4]) { __cpuid((int*)abcd, kFMSFLeaf); } | |
| void cpu_flags(uint32_t abcd[4]) { __cpuidex((int*)abcd, kFlagsLeaf, kFlagsSubleaf); } | |
| uint64_t xgetbv() { | |
| // https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html | |
| constexpr uint32_t xcr = 0; | |
| return _xgetbv(xcr); | |
| } | |
| #else | |
| #if !defined(__cpuid_count) // Old Mac Clang doesn't have this defined. | |
| #define __cpuid_count(eax, ecx, a, b, c, d) \ | |
| __asm__("cpuid" : "=a"(a), "=b"(b), "=c"(c), "=d"(d) : "0"(eax), "2"(ecx)) | |
| #endif | |
| // https://www.felixcloutier.com/x86/cpuid | |
| // The output will be written to the input arrays. | |
| void cpu_vendor(uint32_t abcd[4]) { | |
| __cpuid(kVendorLeaf, abcd[0], abcd[1], abcd[2], abcd[3]); | |
| } | |
| void cpu_features(uint32_t abcd[4]) { | |
| __cpuid(kFMSFLeaf, abcd[0], abcd[1], abcd[2], abcd[3]); | |
| } | |
| void cpu_flags(uint32_t abcd[4]) { | |
| // __cpuid_count is just like __cpuid except it also takes a subleaf option (which | |
| // we set to 0, as that's where the info we care about is). | |
| __cpuid_count(kFlagsLeaf, kFlagsSubleaf, abcd[0], abcd[1], abcd[2], abcd[3]); | |
| } | |
| uint64_t xgetbv() { | |
| // https://www.felixcloutier.com/x86/xgetbv | |
| // "Execute XGETBV with ECX = 0 to discover the value of XCR0 ... | |
| constexpr uint32_t xcr = 0; | |
| uint32_t eax, edx; | |
| __asm__ __volatile__ ( "xgetbv" : "=a"(eax), "=d"(edx) : "c"(xcr)); | |
| return (uint64_t)(edx) << 32 | eax; | |
| } | |
| #endif | |
| // The comments on each of the below lines correspond to constants defined in <cpuid.h> | |
| // cpuid(1) -> EDX | |
| constexpr uint32_t kSSE = (1u << 25); // bit_SSE | |
| constexpr uint32_t kSSE2 = (1u << 26); // bit_SSE2 | |
| // cpuid(1) -> ECX | |
| constexpr uint32_t kSSE3 = (1u << 0); // bit_SSE3 | |
| constexpr uint32_t kSSSE3 = (1u << 9); // bit_SSSE3 | |
| constexpr uint32_t kSSE41 = (1u << 19); // bit_SSE4_1 | |
| constexpr uint32_t kSSE42 = (1u << 20); // bit_SSE4_2 | |
| constexpr uint32_t kFMA = (1u << 12); // bit_FMA | |
| constexpr uint32_t kAVX = (1u << 28); // bit_AVX | |
| constexpr uint32_t kF16C = (1u << 29); // bit_F16C | |
| constexpr uint32_t kXSAVE = (1u << 26); // bit_XSAVE | |
| constexpr uint32_t kOSXSAVE = (1u << 27); // bit_OSXSAVE | |
| // cpuid(7,0) -> EBX | |
| constexpr uint32_t kBMI1 = (1u << 3); // bit_BMI | |
| constexpr uint32_t kAVX2 = (1u << 5); // bit_AVX2 | |
| constexpr uint32_t kBMI2 = (1u << 8); // bit_BMI2 | |
| constexpr uint32_t kERMS = (1u << 9); // bit_ENH_MOVSB | |
| constexpr uint32_t kAVX512F = (1u << 16); // bit_AVX512F | |
| constexpr uint32_t kAVX512DQ = (1u << 17); // bit_AVX512DQ | |
| constexpr uint32_t kAVX512IFMA = (1u << 21); // bit_AVX512IFMA | |
| constexpr uint32_t kAVX512PF = (1u << 26); // bit_AVX512PF | |
| constexpr uint32_t kAVX512ER = (1u << 27); // bit_AVX512ER | |
| constexpr uint32_t kAVX512CD = (1u << 28); // bit_AVX512CD | |
| constexpr uint32_t kAVX512BW = (1u << 30); // bit_AVX512BW | |
| constexpr uint32_t kAVX512VL = (1u << 31); // bit_AVX512VL | |
| // cpuid(7,0) -> ECX | |
| constexpr uint32_t kAVX512VBMI2 = (1u << 6); // bit_AVX512VBMI2 | |
| // xgetbv(0) -> XCR (eXtended Control Register) | |
| constexpr uint64_t kXCR0_XMM_YMM_STATE = 0b00000110; // Bits 1 and 2 | |
| constexpr uint64_t kXCR0_ZMM_STATE = 0b11100000; // Bits 5, 6, and 7 | |
| // Combine 4 ASCII characters together in little-endian order. | |
| constexpr uint32_t ASCII_LE(const char str[4]) { | |
| auto a = str[0], b = str[1], c = str[2], d = str[3]; | |
| return (((uint32_t)d << 24) | ((uint32_t)c << 16) | ((uint32_t)b << 8) | (uint32_t)a); | |
| } | |
| uint32_t read_cpu_features() { | |
| uint32_t features = 0; | |
| uint32_t abcd[4] = {0,0,0,0}; | |
| #define EAX abcd[0] | |
| #define EBX abcd[1] | |
| #define ECX abcd[2] | |
| #define EDX abcd[3] | |
| cpu_vendor(abcd); | |
| // The vendor string in EBX, EDX, ECX (yes, in that order). | |
| // For AMD, this is "AuthenticAMD" encoded as ASCII (little-endian) | |
| // Intel happens to be "GenuineIntel" https://www.sandpile.org/x86/cpuid.htm#leaf_0000_0000h | |
| const bool isAMD = (EBX == ASCII_LE("Auth")) && | |
| (EDX == ASCII_LE("enti")) && | |
| (ECX == ASCII_LE("cAMD")); | |
| cpu_features(abcd); | |
| if (EDX & kSSE) { features |= SkX64:: SSE1; } | |
| if (EDX & kSSE2) { features |= SkX64:: SSE2; } | |
| if (ECX & kSSE3) { features |= SkX64:: SSE3; } | |
| if (ECX & kSSSE3) { features |= SkX64::SSSE3; } | |
| if (ECX & kSSE41) { features |= SkX64::SSE41; } | |
| if (ECX & kSSE42) { features |= SkX64::SSE42; } | |
| // From Intel® 64 and IA-32 Architectures Software Developer's Manual | |
| // 14.3 DETECTION OF INTEL® AVX INSTRUCTIONS | |
| // 1) Detect we have XGETBV, i.e. cpuid(1) returns ECX with bit 27 set | |
| // (In Intel's docs, this is represented by CPUID.01H:ECX.OSXSAVE[27] = 1) | |
| // For historical reasons, we check bit 26 (kXSAVE; see | |
| // https://codereview.chromium.org/1428153003) which is the hardware support. Bit 27 | |
| // (kOSXSAVE) means the OS *also* supports this, which the Intel docs say is important. | |
| if ((ECX & (kXSAVE | kOSXSAVE)) == (kXSAVE | kOSXSAVE)) { | |
| // 2) Call XGETBV to get eXtended Control Register info. | |
| const uint64_t xcr = xgetbv(); | |
| if ((xcr & kXCR0_XMM_YMM_STATE) == kXCR0_XMM_YMM_STATE) { | |
| // 3) Check ECX bit 28 for AVX support | |
| if (ECX & kAVX) { features |= SkX64::AVX; } | |
| // Now that we have AVX, we can detect other features that list it as a prereq. | |
| if (ECX & kF16C) { features |= SkX64::F16C; } // 14.4.1 Detection of F16C Instr... | |
| if (ECX & kFMA) { features |= SkX64::FMA; } // 14.5.3 Detection of FMA | |
| // Fill the register values with values from leaf 7. | |
| cpu_flags(abcd); | |
| if (EBX & kAVX2) { features |= SkX64::AVX2; } // 14.7.1 Detection of Intel® AVX2 | |
| // These don't strictly require AVX support, but only exist on newer chips anyway. | |
| if (EBX & kBMI1) { features |= SkX64::BMI1; } | |
| if (EBX & kBMI2) { features |= SkX64::BMI2; } | |
| if (EBX & kERMS) { features |= SkX64::ERMS; } | |
| // 15.2 DETECTION OF AVX-512 FOUNDATION INSTRUCTIONS | |
| // (for Intel; AMD just needs flag checks) | |
| // 1) Detect we have XGETBV (which we did above). | |
| // 2) Execute XGETBV and verify that XCR0[7:5] = '111b' ... | |
| // and that XCR0[2:1] = 11b' (which we did above) | |
| if ((xcr & kXCR0_ZMM_STATE) == kXCR0_ZMM_STATE) { | |
| // AVX-512 can be slow on early Intel processors (like Skylake or Cascade Lake) | |
| // due to thermal throttling (see https://stackoverflow.com/a/63484551), | |
| // but works well on Ice Lake and later, and all AMD processors. We detect | |
| // AMD above, but to identify Intel with Gen 3+ AVX512 support, we look for | |
| // VBMI2 support, which was added (along with other features) in Ice Lake. | |
| // Table 16-2. Intel® AVX-512 CPUID Feature Flags Included in Intel® AVX10 | |
| // has a nice table of AVX512 commands and when they were introduced. | |
| const bool isNewerIntel = (ECX & kAVX512VBMI2); | |
| if (isAMD || isNewerIntel) { | |
| // 3) Check EBX bit 16 for AVX support | |
| if (EBX & kAVX512F) { features |= SkX64::AVX512F; } | |
| // ... and any other extensions. | |
| if (EBX & kAVX512DQ) { features |= SkX64::AVX512DQ; } | |
| if (EBX & kAVX512IFMA) { features |= SkX64::AVX512IFMA; } | |
| if (EBX & kAVX512PF) { features |= SkX64::AVX512PF; } | |
| if (EBX & kAVX512ER) { features |= SkX64::AVX512ER; } | |
| if (EBX & kAVX512CD) { features |= SkX64::AVX512CD; } | |
| if (EBX & kAVX512BW) { features |= SkX64::AVX512BW; } | |
| if (EBX & kAVX512VL) { features |= SkX64::AVX512VL; } | |
| } | |
| } | |
| } | |
| } | |
| return features; | |
| } | |
| #elif defined(SK_CPU_LOONGARCH) | |
| uint32_t read_cpu_features() { | |
| uint32_t features = 0; | |
| uint64_t hwcap = getauxval(AT_HWCAP); | |
| if (hwcap & HWCAP_LOONGARCH_LSX) { features |= SkLoongArch::SX; } | |
| if (hwcap & HWCAP_LOONGARCH_LASX) { features |= SkLoongArch::ASX; } | |
| return features; | |
| } | |
| #else | |
| uint32_t read_cpu_features() { | |
| return 0; | |
| } | |
| #endif | |
| } // anonymous namespace | |
| uint32_t SkCpu::gCachedFeatures = 0; | |
| void SkCpu::CacheRuntimeFeatures() { | |
| static SkOnce once; | |
| once([] { gCachedFeatures = read_cpu_features(); }); | |
| } |