| 1 |
//! x86/x86-64 CPU feature detection support. |
| 2 |
//! |
| 3 |
//! Portable, `no_std`-friendly implementation that relies on the x86 `CPUID` |
| 4 |
//! instruction for feature detection. |
| 5 |
|
| 6 |
/// Evaluate the given `$body` expression any of the supplied target features |
| 7 |
/// are not enabled. Otherwise returns true. |
| 8 |
/// |
| 9 |
/// The `$body` expression is not evaluated on SGX targets, and returns false |
| 10 |
/// on these targets unless *all* supplied target features are enabled. |
| 11 |
#[macro_export] |
| 12 |
#[doc(hidden)] |
| 13 |
macro_rules! __unless_target_features { |
| 14 |
($($tf:tt),+ => $body:expr ) => {{ |
| 15 |
#[cfg(not(all($(target_feature=$tf,)*)))] |
| 16 |
{ |
| 17 |
#[cfg(not(any(target_env = "sgx", target_os = "", target_os = "uefi")))] |
| 18 |
$body |
| 19 |
|
| 20 |
// CPUID is not available on SGX. Freestanding and UEFI targets |
| 21 |
// do not support SIMD features with default compilation flags. |
| 22 |
#[cfg(any(target_env = "sgx", target_os = "", target_os = "uefi"))] |
| 23 |
false |
| 24 |
} |
| 25 |
|
| 26 |
#[cfg(all($(target_feature=$tf,)*))] |
| 27 |
true |
| 28 |
}}; |
| 29 |
} |
| 30 |
|
| 31 |
/// Use CPUID to detect the presence of all supplied target features. |
| 32 |
#[macro_export] |
| 33 |
#[doc(hidden)] |
| 34 |
macro_rules! __detect_target_features { |
| 35 |
($($tf:tt),+) => {{ |
| 36 |
#[cfg(target_arch = "x86")] |
| 37 |
use core::arch::x86::{__cpuid, __cpuid_count, CpuidResult}; |
| 38 |
#[cfg(target_arch = "x86_64")] |
| 39 |
use core::arch::x86_64::{__cpuid, __cpuid_count, CpuidResult}; |
| 40 |
|
| 41 |
// These wrappers are workarounds around |
| 42 |
// https://github.com/rust-lang/rust/issues/101346 |
| 43 |
// |
| 44 |
// DO NOT remove it until MSRV is bumped to a version |
| 45 |
// with the issue fix (at least 1.64). |
| 46 |
#[inline(never)] |
| 47 |
unsafe fn cpuid(leaf: u32) -> CpuidResult { |
| 48 |
__cpuid(leaf) |
| 49 |
} |
| 50 |
|
| 51 |
#[inline(never)] |
| 52 |
unsafe fn cpuid_count(leaf: u32, sub_leaf: u32) -> CpuidResult { |
| 53 |
__cpuid_count(leaf, sub_leaf) |
| 54 |
} |
| 55 |
|
| 56 |
let cr = unsafe { |
| 57 |
[cpuid(1), cpuid_count(7, 0)] |
| 58 |
}; |
| 59 |
|
| 60 |
$($crate::check!(cr, $tf) & )+ true |
| 61 |
}}; |
| 62 |
} |
| 63 |
|
| 64 |
/// Check that OS supports required SIMD registers |
| 65 |
#[macro_export] |
| 66 |
#[doc(hidden)] |
| 67 |
macro_rules! __xgetbv { |
| 68 |
($cr:expr, $mask:expr) => {{ |
| 69 |
#[cfg(target_arch = "x86")] |
| 70 |
use core::arch::x86 as arch; |
| 71 |
#[cfg(target_arch = "x86_64")] |
| 72 |
use core::arch::x86_64 as arch; |
| 73 |
|
| 74 |
// Check bits 26 and 27 |
| 75 |
let xmask = 0b11 << 26; |
| 76 |
let xsave = $cr[0].ecx & xmask == xmask; |
| 77 |
if xsave { |
| 78 |
let xcr0 = unsafe { arch::_xgetbv(arch::_XCR_XFEATURE_ENABLED_MASK) }; |
| 79 |
(xcr0 & $mask) == $mask |
| 80 |
} else { |
| 81 |
false |
| 82 |
} |
| 83 |
}}; |
| 84 |
} |
| 85 |
|
| 86 |
macro_rules! __expand_check_macro { |
| 87 |
($(($name:tt, $reg_cap:tt $(, $i:expr, $reg:ident, $offset:expr)*)),* $(,)?) => { |
| 88 |
#[macro_export] |
| 89 |
#[doc(hidden)] |
| 90 |
macro_rules! check { |
| 91 |
$( |
| 92 |
($cr:expr, $name) => {{ |
| 93 |
// Register bits are listed here: |
| 94 |
// https://wiki.osdev.org/CPU_Registers_x86#Extended_Control_Registers |
| 95 |
let reg_cap = match $reg_cap { |
| 96 |
// Bit 1 |
| 97 |
"xmm" => $crate::__xgetbv!($cr, 0b10), |
| 98 |
// Bits 1 and 2 |
| 99 |
"ymm" => $crate::__xgetbv!($cr, 0b110), |
| 100 |
// Bits 1, 2, 5, 6, and 7 |
| 101 |
"zmm" => $crate::__xgetbv!($cr, 0b1110_0110), |
| 102 |
_ => true, |
| 103 |
}; |
| 104 |
reg_cap |
| 105 |
$( |
| 106 |
& ($cr[$i].$reg & (1 << $offset) != 0) |
| 107 |
)* |
| 108 |
}}; |
| 109 |
)* |
| 110 |
} |
| 111 |
}; |
| 112 |
} |
| 113 |
|
| 114 |
__expand_check_macro! { |
| 115 |
("sse3", "xmm", 0, ecx, 0), |
| 116 |
("pclmulqdq", "xmm", 0, ecx, 1), |
| 117 |
("ssse3", "xmm", 0, ecx, 9), |
| 118 |
("fma", "xmm", 0, ecx, 12, 0, ecx, 28), |
| 119 |
("sse4.1", "xmm", 0, ecx, 19), |
| 120 |
("sse4.2", "xmm", 0, ecx, 20), |
| 121 |
("popcnt", "", 0, ecx, 23), |
| 122 |
("aes", "xmm", 0, ecx, 25), |
| 123 |
("avx", "xmm", 0, ecx, 28), |
| 124 |
("rdrand", "", 0, ecx, 30), |
| 125 |
|
| 126 |
("mmx", "", 0, edx, 23), |
| 127 |
("sse", "xmm", 0, edx, 25), |
| 128 |
("sse2", "xmm", 0, edx, 26), |
| 129 |
|
| 130 |
("sgx", "", 1, ebx, 2), |
| 131 |
("bmi1", "", 1, ebx, 3), |
| 132 |
("bmi2", "", 1, ebx, 8), |
| 133 |
("avx2", "ymm", 1, ebx, 5, 0, ecx, 28), |
| 134 |
("avx512f", "zmm", 1, ebx, 16), |
| 135 |
("avx512dq", "zmm", 1, ebx, 17), |
| 136 |
("rdseed", "", 1, ebx, 18), |
| 137 |
("adx", "", 1, ebx, 19), |
| 138 |
("avx512ifma", "zmm", 1, ebx, 21), |
| 139 |
("avx512pf", "zmm", 1, ebx, 26), |
| 140 |
("avx512er", "zmm", 1, ebx, 27), |
| 141 |
("avx512cd", "zmm", 1, ebx, 28), |
| 142 |
("sha", "xmm", 1, ebx, 29), |
| 143 |
("avx512bw", "zmm", 1, ebx, 30), |
| 144 |
("avx512vl", "zmm", 1, ebx, 31), |
| 145 |
("avx512vbmi", "zmm", 1, ecx, 1), |
| 146 |
("avx512vbmi2", "zmm", 1, ecx, 6), |
| 147 |
("gfni", "zmm", 1, ecx, 8), |
| 148 |
("vaes", "zmm", 1, ecx, 9), |
| 149 |
("vpclmulqdq", "zmm", 1, ecx, 10), |
| 150 |
("avx512bitalg", "zmm", 1, ecx, 12), |
| 151 |
("avx512vpopcntdq", "zmm", 1, ecx, 14), |
| 152 |
} |