diff --git a/library/stdarch/crates/std_detect/README.md b/library/stdarch/crates/std_detect/README.md index 5191b2f6654..0053d777aac 100644 --- a/library/stdarch/crates/std_detect/README.md +++ b/library/stdarch/crates/std_detect/README.md @@ -56,11 +56,14 @@ crate from working on applications in which `std` is not available. [`cupid`](https://crates.io/crates/cupid) crate. * Linux/Android: - * `arm{32, 64}`, `mips{32,64}{,el}`, `powerpc{32,64}{,le}`, `riscv{32,64}`, `loongarch64`, `s390x`: + * `arm{32, 64}`, `mips{32,64}{,el}`, `powerpc{32,64}{,le}`, `loongarch64`, `s390x`: `std_detect` supports these on Linux by querying ELF auxiliary vectors (using `getauxval` when available), and if that fails, by querying `/proc/cpuinfo`. * `arm64`: partial support for doing run-time feature detection by directly querying `mrs` is implemented for Linux >= 4.11, but not enabled by default. + * `riscv{32,64}`: + `std_detect` supports these on Linux by querying `riscv_hwprobe`, and + by querying ELF auxiliary vectors (using `getauxval` when available). * FreeBSD: * `arm32`, `powerpc64`: `std_detect` supports these on FreeBSD by querying ELF diff --git a/library/stdarch/crates/std_detect/src/detect/arch/riscv.rs b/library/stdarch/crates/std_detect/src/detect/arch/riscv.rs index 4b1cbdcda5a..e9a6c96af1e 100644 --- a/library/stdarch/crates/std_detect/src/detect/arch/riscv.rs +++ b/library/stdarch/crates/std_detect/src/detect/arch/riscv.rs @@ -37,22 +37,39 @@ features! { /// * Zbb: `"zbb"` /// * Zbs: `"zbs"` /// * C: `"c"` + /// * Zca: `"zca"` + /// * Zcd: `"zcd"` (if D is enabled) + /// * Zcf: `"zcf"` (if F is enabled on RV32) /// * D: `"d"` /// * F: `"f"` /// * M: `"m"` /// * Q: `"q"` /// * V: `"v"` + /// * Zve32x: `"zve32x"` + /// * Zve32f: `"zve32f"` + /// * Zve64x: `"zve64x"` + /// * Zve64f: `"zve64f"` + /// * Zve64d: `"zve64d"` + /// * Zicboz: `"zicboz"` /// * Zicntr: `"zicntr"` + /// * Zicond: `"zicond"` /// * Zicsr: `"zicsr"` /// * Zifencei: `"zifencei"` + /// * Zihintntl: `"zihintntl"` /// * Zihintpause: `"zihintpause"` /// * Zihpm: `"zihpm"` + /// * Zimop: `"zimop"` + /// * Zacas: `"zacas"` + /// * Zawrs: `"zawrs"` + /// * Zfa: `"zfa"` /// * Zfh: `"zfh"` /// * Zfhmin: `"zfhmin"` /// * Zfinx: `"zfinx"` /// * Zdinx: `"zdinx"` /// * Zhinx: `"zhinx"` /// * Zhinxmin: `"zhinxmin"` + /// * Zcb: `"zcb"` + /// * Zcmop: `"zcmop"` /// * Zbc: `"zbc"` /// * Zbkb: `"zbkb"` /// * Zbkc: `"zbkc"` @@ -67,6 +84,24 @@ features! { /// * Zksed: `"zksed"` /// * Zksh: `"zksh"` /// * Zkt: `"zkt"` + /// * Zvbb: `"zvbb"` + /// * Zvbc: `"zvbc"` + /// * Zvfh: `"zvfh"` + /// * Zvfhmin: `"zvfhmin"` + /// * Zvkb: `"zvkb"` + /// * Zvkg: `"zvkg"` + /// * Zvkn: `"zvkn"` + /// * Zvkned: `"zvkned"` + /// * Zvknha: `"zvknha"` + /// * Zvknhb: `"zvknhb"` + /// * Zvknc: `"zvknc"` + /// * Zvkng: `"zvkng"` + /// * Zvks: `"zvks"` + /// * Zvksed: `"zvksed"` + /// * Zvksh: `"zvksh"` + /// * Zvksc: `"zvksc"` + /// * Zvksg: `"zvksg"` + /// * Zvkt: `"zvkt"` /// * Ztso: `"ztso"` /// /// There's also bases and extensions marked as standard instruction set, @@ -87,6 +122,15 @@ features! { /// * Svnapot: `"svnapot"` /// * Svpbmt: `"svpbmt"` /// * Svinval: `"svinval"` + /// + /// # Performance Hints + /// + /// The two features below define performance hints for unaligned + /// scalar/vector memory accesses, respectively. If enabled, it denotes that + /// corresponding unaligned memory access is reasonably fast. + /// + /// * `"unaligned-scalar-mem"` + /// * `"unaligned-vector-mem"` #[stable(feature = "riscv_ratified", since = "1.78.0")] @FEATURE: #[unstable(feature = "stdarch_riscv_feature_detection", issue = "111192")] rv32i: "rv32i"; @@ -102,6 +146,11 @@ features! { without cfg check: true; /// RV128I Base Integer Instruction Set + @FEATURE: #[unstable(feature = "stdarch_riscv_feature_detection", issue = "111192")] unaligned_scalar_mem: "unaligned-scalar-mem"; + /// Has reasonably performant unaligned scalar + @FEATURE: #[unstable(feature = "stdarch_riscv_feature_detection", issue = "111192")] unaligned_vector_mem: "unaligned-vector-mem"; + /// Has reasonably performant unaligned vector + @FEATURE: #[unstable(feature = "stdarch_riscv_feature_detection", issue = "111192")] zicsr: "zicsr"; without cfg check: true; /// "Zicsr" Extension for Control and Status Register (CSR) Instructions @@ -115,9 +164,21 @@ features! { without cfg check: true; /// "Zifencei" Extension for Instruction-Fetch Fence + @FEATURE: #[unstable(feature = "stdarch_riscv_feature_detection", issue = "111192")] zihintntl: "zihintntl"; + without cfg check: true; + /// "Zihintntl" Extension for Non-Temporal Locality Hints @FEATURE: #[unstable(feature = "stdarch_riscv_feature_detection", issue = "111192")] zihintpause: "zihintpause"; without cfg check: true; /// "Zihintpause" Extension for Pause Hint + @FEATURE: #[unstable(feature = "stdarch_riscv_feature_detection", issue = "111192")] zimop: "zimop"; + without cfg check: true; + /// "Zimop" Extension for May-Be-Operations + @FEATURE: #[unstable(feature = "stdarch_riscv_feature_detection", issue = "111192")] zicboz: "zicboz"; + without cfg check: true; + /// "Zicboz" Extension for Cache-Block Zero Instruction + @FEATURE: #[unstable(feature = "stdarch_riscv_feature_detection", issue = "111192")] zicond: "zicond"; + without cfg check: true; + /// "Zicond" Extension for Integer Conditional Operations @FEATURE: #[stable(feature = "riscv_ratified", since = "1.78.0")] m: "m"; /// "M" Extension for Integer Multiplication and Division @@ -128,6 +189,10 @@ features! { /// "Zalrsc" Extension for Load-Reserved/Store-Conditional Instructions @FEATURE: #[unstable(feature = "stdarch_riscv_feature_detection", issue = "111192")] zaamo: "zaamo"; /// "Zaamo" Extension for Atomic Memory Operations + @FEATURE: #[unstable(feature = "stdarch_riscv_feature_detection", issue = "111192")] zawrs: "zawrs"; + /// "Zawrs" Extension for Wait-on-Reservation-Set Instructions + @FEATURE: #[unstable(feature = "stdarch_riscv_feature_detection", issue = "111192")] zacas: "zacas"; + /// "Zacas" Extension for Atomic Compare-and-Swap (CAS) Instructions @FEATURE: #[unstable(feature = "stdarch_riscv_feature_detection", issue = "111192")] zam: "zam"; without cfg check: true; /// "Zam" Extension for Misaligned Atomics @@ -146,6 +211,9 @@ features! { /// "Zfh" Extension for Half-Precision Floating-Point @FEATURE: #[unstable(feature = "stdarch_riscv_feature_detection", issue = "111192")] zfhmin: "zfhmin"; /// "Zfhmin" Extension for Minimal Half-Precision Floating-Point + @FEATURE: #[unstable(feature = "stdarch_riscv_feature_detection", issue = "111192")] zfa: "zfa"; + without cfg check: true; + /// "Zfa" Extension for Additional Floating-Point Instructions @FEATURE: #[unstable(feature = "stdarch_riscv_feature_detection", issue = "111192")] zfinx: "zfinx"; /// "Zfinx" Extension for Single-Precision Floating-Point in Integer Registers @@ -158,6 +226,21 @@ features! { @FEATURE: #[stable(feature = "riscv_ratified", since = "1.78.0")] c: "c"; /// "C" Extension for Compressed Instructions + @FEATURE: #[unstable(feature = "stdarch_riscv_feature_detection", issue = "111192")] zca: "zca"; + without cfg check: true; + /// "Zca" Compressed Instructions excluding Floating-Point Loads/Stores + @FEATURE: #[unstable(feature = "stdarch_riscv_feature_detection", issue = "111192")] zcf: "zcf"; + without cfg check: true; + /// "Zcf" Compressed Instructions for Single-Precision Floating-Point Loads/Stores on RV32 + @FEATURE: #[unstable(feature = "stdarch_riscv_feature_detection", issue = "111192")] zcd: "zcd"; + without cfg check: true; + /// "Zcd" Compressed Instructions for Double-Precision Floating-Point Loads/Stores + @FEATURE: #[unstable(feature = "stdarch_riscv_feature_detection", issue = "111192")] zcb: "zcb"; + without cfg check: true; + /// "Zcb" Simple Code-size Saving Compressed Instructions + @FEATURE: #[unstable(feature = "stdarch_riscv_feature_detection", issue = "111192")] zcmop: "zcmop"; + without cfg check: true; + /// "Zcmop" Extension for Compressed May-Be-Operations @FEATURE: #[unstable(feature = "stdarch_riscv_feature_detection", issue = "111192")] b: "b"; without cfg check: true; @@ -200,6 +283,53 @@ features! { @FEATURE: #[unstable(feature = "stdarch_riscv_feature_detection", issue = "111192")] v: "v"; /// "V" Extension for Vector Operations + @FEATURE: #[unstable(feature = "stdarch_riscv_feature_detection", issue = "111192")] zve32x: "zve32x"; + /// "Zve32x" Vector Extension for Embedded Processors (32-bit+; Integer) + @FEATURE: #[unstable(feature = "stdarch_riscv_feature_detection", issue = "111192")] zve32f: "zve32f"; + /// "Zve32f" Vector Extension for Embedded Processors (32-bit+; with Single-Precision Floating-Point) + @FEATURE: #[unstable(feature = "stdarch_riscv_feature_detection", issue = "111192")] zve64x: "zve64x"; + /// "Zve64x" Vector Extension for Embedded Processors (64-bit+; Integer) + @FEATURE: #[unstable(feature = "stdarch_riscv_feature_detection", issue = "111192")] zve64f: "zve64f"; + /// "Zve64f" Vector Extension for Embedded Processors (64-bit+; with Single-Precision Floating-Point) + @FEATURE: #[unstable(feature = "stdarch_riscv_feature_detection", issue = "111192")] zve64d: "zve64d"; + /// "Zve64d" Vector Extension for Embedded Processors (64-bit+; with Double-Precision Floating-Point) + @FEATURE: #[unstable(feature = "stdarch_riscv_feature_detection", issue = "111192")] zvfh: "zvfh"; + /// "Zvfh" Vector Extension for Half-Precision Floating-Point + @FEATURE: #[unstable(feature = "stdarch_riscv_feature_detection", issue = "111192")] zvfhmin: "zvfhmin"; + /// "Zvfhmin" Vector Extension for Minimal Half-Precision Floating-Point + + @FEATURE: #[unstable(feature = "stdarch_riscv_feature_detection", issue = "111192")] zvbb: "zvbb"; + /// "Zvbb" Extension for Vector Basic Bit-Manipulation + @FEATURE: #[unstable(feature = "stdarch_riscv_feature_detection", issue = "111192")] zvbc: "zvbc"; + /// "Zvbc" Extension for Vector Carryless Multiplication + @FEATURE: #[unstable(feature = "stdarch_riscv_feature_detection", issue = "111192")] zvkb: "zvkb"; + /// "Zvkb" Extension for Vector Cryptography Bit-Manipulation + @FEATURE: #[unstable(feature = "stdarch_riscv_feature_detection", issue = "111192")] zvkg: "zvkg"; + /// "Zvkg" Cryptography Extension for Vector GCM/GMAC + @FEATURE: #[unstable(feature = "stdarch_riscv_feature_detection", issue = "111192")] zvkned: "zvkned"; + /// "Zvkned" Cryptography Extension for NIST Suite: Vector AES Block Cipher + @FEATURE: #[unstable(feature = "stdarch_riscv_feature_detection", issue = "111192")] zvknha: "zvknha"; + /// "Zvknha" Cryptography Extension for Vector SHA-2 Secure Hash (SHA-256) + @FEATURE: #[unstable(feature = "stdarch_riscv_feature_detection", issue = "111192")] zvknhb: "zvknhb"; + /// "Zvknhb" Cryptography Extension for Vector SHA-2 Secure Hash (SHA-256/512) + @FEATURE: #[unstable(feature = "stdarch_riscv_feature_detection", issue = "111192")] zvksed: "zvksed"; + /// "Zvksed" Cryptography Extension for ShangMi Suite: Vector SM4 Block Cipher + @FEATURE: #[unstable(feature = "stdarch_riscv_feature_detection", issue = "111192")] zvksh: "zvksh"; + /// "Zvksh" Cryptography Extension for ShangMi Suite: Vector SM3 Secure Hash + @FEATURE: #[unstable(feature = "stdarch_riscv_feature_detection", issue = "111192")] zvkn: "zvkn"; + /// "Zvkn" Cryptography Extension for NIST Algorithm Suite + @FEATURE: #[unstable(feature = "stdarch_riscv_feature_detection", issue = "111192")] zvknc: "zvknc"; + /// "Zvknc" Cryptography Extension for NIST Algorithm Suite with Carryless Multiply + @FEATURE: #[unstable(feature = "stdarch_riscv_feature_detection", issue = "111192")] zvkng: "zvkng"; + /// "Zvkng" Cryptography Extension for NIST Algorithm Suite with GCM + @FEATURE: #[unstable(feature = "stdarch_riscv_feature_detection", issue = "111192")] zvks: "zvks"; + /// "Zvks" Cryptography Extension for ShangMi Algorithm Suite + @FEATURE: #[unstable(feature = "stdarch_riscv_feature_detection", issue = "111192")] zvksc: "zvksc"; + /// "Zvksc" Cryptography Extension for ShangMi Algorithm Suite with Carryless Multiply + @FEATURE: #[unstable(feature = "stdarch_riscv_feature_detection", issue = "111192")] zvksg: "zvksg"; + /// "Zvksg" Cryptography Extension for ShangMi Algorithm Suite with GCM + @FEATURE: #[unstable(feature = "stdarch_riscv_feature_detection", issue = "111192")] zvkt: "zvkt"; + /// "Zvkt" Extension for Vector Data-Independent Execution Latency @FEATURE: #[unstable(feature = "stdarch_riscv_feature_detection", issue = "111192")] svnapot: "svnapot"; without cfg check: true; diff --git a/library/stdarch/crates/std_detect/src/detect/os/linux/riscv.rs b/library/stdarch/crates/std_detect/src/detect/os/linux/riscv.rs index 6cdc72dff5a..5cf7b064e8e 100644 --- a/library/stdarch/crates/std_detect/src/detect/os/linux/riscv.rs +++ b/library/stdarch/crates/std_detect/src/detect/os/linux/riscv.rs @@ -1,10 +1,127 @@ //! Run-time feature detection for RISC-V on Linux. +//! +//! On RISC-V, detection using auxv only supports single-letter extensions. +//! So, we use riscv_hwprobe that supports multi-letter extensions if available. +//! + +use core::ptr; use super::super::riscv::imply_features; use super::auxvec; use crate::detect::{Feature, bit, cache}; -/// Read list of supported features from the auxiliary vector. +// See +// for runtime status query constants. +const PR_RISCV_V_GET_CONTROL: libc::c_int = 70; +const PR_RISCV_V_VSTATE_CTRL_ON: libc::c_int = 2; +const PR_RISCV_V_VSTATE_CTRL_CUR_MASK: libc::c_int = 3; + +// See +// for riscv_hwprobe struct and hardware probing constants. + +#[repr(C)] +struct riscv_hwprobe { + key: i64, + value: u64, +} + +#[allow(non_upper_case_globals)] +const __NR_riscv_hwprobe: libc::c_long = 258; + +const RISCV_HWPROBE_KEY_BASE_BEHAVIOR: i64 = 3; +const RISCV_HWPROBE_BASE_BEHAVIOR_IMA: u64 = 1 << 0; + +const RISCV_HWPROBE_KEY_IMA_EXT_0: i64 = 4; +const RISCV_HWPROBE_IMA_FD: u64 = 1 << 0; +const RISCV_HWPROBE_IMA_C: u64 = 1 << 1; +const RISCV_HWPROBE_IMA_V: u64 = 1 << 2; +const RISCV_HWPROBE_EXT_ZBA: u64 = 1 << 3; +const RISCV_HWPROBE_EXT_ZBB: u64 = 1 << 4; +const RISCV_HWPROBE_EXT_ZBS: u64 = 1 << 5; +const RISCV_HWPROBE_EXT_ZICBOZ: u64 = 1 << 6; +const RISCV_HWPROBE_EXT_ZBC: u64 = 1 << 7; +const RISCV_HWPROBE_EXT_ZBKB: u64 = 1 << 8; +const RISCV_HWPROBE_EXT_ZBKC: u64 = 1 << 9; +const RISCV_HWPROBE_EXT_ZBKX: u64 = 1 << 10; +const RISCV_HWPROBE_EXT_ZKND: u64 = 1 << 11; +const RISCV_HWPROBE_EXT_ZKNE: u64 = 1 << 12; +const RISCV_HWPROBE_EXT_ZKNH: u64 = 1 << 13; +const RISCV_HWPROBE_EXT_ZKSED: u64 = 1 << 14; +const RISCV_HWPROBE_EXT_ZKSH: u64 = 1 << 15; +const RISCV_HWPROBE_EXT_ZKT: u64 = 1 << 16; +const RISCV_HWPROBE_EXT_ZVBB: u64 = 1 << 17; +const RISCV_HWPROBE_EXT_ZVBC: u64 = 1 << 18; +const RISCV_HWPROBE_EXT_ZVKB: u64 = 1 << 19; +const RISCV_HWPROBE_EXT_ZVKG: u64 = 1 << 20; +const RISCV_HWPROBE_EXT_ZVKNED: u64 = 1 << 21; +const RISCV_HWPROBE_EXT_ZVKNHA: u64 = 1 << 22; +const RISCV_HWPROBE_EXT_ZVKNHB: u64 = 1 << 23; +const RISCV_HWPROBE_EXT_ZVKSED: u64 = 1 << 24; +const RISCV_HWPROBE_EXT_ZVKSH: u64 = 1 << 25; +const RISCV_HWPROBE_EXT_ZVKT: u64 = 1 << 26; +const RISCV_HWPROBE_EXT_ZFH: u64 = 1 << 27; +const RISCV_HWPROBE_EXT_ZFHMIN: u64 = 1 << 28; +const RISCV_HWPROBE_EXT_ZIHINTNTL: u64 = 1 << 29; +const RISCV_HWPROBE_EXT_ZVFH: u64 = 1 << 30; +const RISCV_HWPROBE_EXT_ZVFHMIN: u64 = 1 << 31; +const RISCV_HWPROBE_EXT_ZFA: u64 = 1 << 32; +const RISCV_HWPROBE_EXT_ZTSO: u64 = 1 << 33; +const RISCV_HWPROBE_EXT_ZACAS: u64 = 1 << 34; +const RISCV_HWPROBE_EXT_ZICOND: u64 = 1 << 35; +const RISCV_HWPROBE_EXT_ZIHINTPAUSE: u64 = 1 << 36; +const RISCV_HWPROBE_EXT_ZVE32X: u64 = 1 << 37; +const RISCV_HWPROBE_EXT_ZVE32F: u64 = 1 << 38; +const RISCV_HWPROBE_EXT_ZVE64X: u64 = 1 << 39; +const RISCV_HWPROBE_EXT_ZVE64F: u64 = 1 << 40; +const RISCV_HWPROBE_EXT_ZVE64D: u64 = 1 << 41; +const RISCV_HWPROBE_EXT_ZIMOP: u64 = 1 << 42; +const RISCV_HWPROBE_EXT_ZCA: u64 = 1 << 43; +const RISCV_HWPROBE_EXT_ZCB: u64 = 1 << 44; +const RISCV_HWPROBE_EXT_ZCD: u64 = 1 << 45; +const RISCV_HWPROBE_EXT_ZCF: u64 = 1 << 46; +const RISCV_HWPROBE_EXT_ZCMOP: u64 = 1 << 47; +const RISCV_HWPROBE_EXT_ZAWRS: u64 = 1 << 48; +// Excluded because it only reports the existence of `prctl`-based pointer masking control. +// const RISCV_HWPROBE_EXT_SUPM: u64 = 1 << 49; + +const RISCV_HWPROBE_KEY_CPUPERF_0: i64 = 5; +const RISCV_HWPROBE_MISALIGNED_FAST: u64 = 3; +const RISCV_HWPROBE_MISALIGNED_MASK: u64 = 7; + +const RISCV_HWPROBE_KEY_MISALIGNED_SCALAR_PERF: i64 = 9; +const RISCV_HWPROBE_MISALIGNED_SCALAR_FAST: u64 = 3; + +const RISCV_HWPROBE_KEY_MISALIGNED_VECTOR_PERF: i64 = 10; +const RISCV_HWPROBE_MISALIGNED_VECTOR_FAST: u64 = 3; + +// syscall returns an unsupported error if riscv_hwprobe is not supported, +// so we can safely use this function on older versions of Linux. +fn _riscv_hwprobe(out: &mut [riscv_hwprobe]) -> bool { + unsafe fn __riscv_hwprobe( + pairs: *mut riscv_hwprobe, + pair_count: libc::size_t, + cpu_set_size: libc::size_t, + cpus: *mut libc::c_ulong, + flags: libc::c_uint, + ) -> libc::c_long { + unsafe { + libc::syscall( + __NR_riscv_hwprobe, + pairs, + pair_count, + cpu_set_size, + cpus, + flags, + ) + } + } + + let len = out.len(); + unsafe { __riscv_hwprobe(out.as_mut_ptr(), len, 0, ptr::null_mut(), 0) == 0 } +} + +/// Read list of supported features from (1) the auxiliary vector +/// and (2) the results of `riscv_hwprobe` and `prctl` system calls. pub(crate) fn detect_features() -> cache::Initializer { let mut value = cache::Initializer::default(); let mut enable_feature = |feature, enable| { @@ -18,6 +135,7 @@ pub(crate) fn detect_features() -> cache::Initializer { // // [hwcap]: https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git/tree/arch/riscv/include/uapi/asm/hwcap.h?h=v6.14 let auxv = auxvec::auxv().expect("read auxvec"); // should not fail on RISC-V platform + let mut has_i = bit::test(auxv.hwcap, (b'i' - b'a').into()); #[allow(clippy::eq_op)] enable_feature(Feature::a, bit::test(auxv.hwcap, (b'a' - b'a').into())); enable_feature(Feature::c, bit::test(auxv.hwcap, (b'c' - b'a').into())); @@ -25,9 +143,167 @@ pub(crate) fn detect_features() -> cache::Initializer { enable_feature(Feature::f, bit::test(auxv.hwcap, (b'f' - b'a').into())); enable_feature(Feature::h, bit::test(auxv.hwcap, (b'h' - b'a').into())); enable_feature(Feature::m, bit::test(auxv.hwcap, (b'm' - b'a').into())); + let has_v = bit::test(auxv.hwcap, (b'v' - b'a').into()); + let mut is_v_set = false; + + // Use riscv_hwprobe syscall to query more extensions and + // performance-related capabilities. + 'hwprobe: { + let mut out = [ + riscv_hwprobe { + key: RISCV_HWPROBE_KEY_BASE_BEHAVIOR, + value: 0, + }, + riscv_hwprobe { + key: RISCV_HWPROBE_KEY_IMA_EXT_0, + value: 0, + }, + riscv_hwprobe { + key: RISCV_HWPROBE_KEY_MISALIGNED_SCALAR_PERF, + value: 0, + }, + riscv_hwprobe { + key: RISCV_HWPROBE_KEY_MISALIGNED_VECTOR_PERF, + value: 0, + }, + riscv_hwprobe { + key: RISCV_HWPROBE_KEY_CPUPERF_0, + value: 0, + }, + ]; + if !_riscv_hwprobe(&mut out) { + break 'hwprobe; + } + + // Query scalar/vector misaligned behavior. + if out[2].key != -1 { + enable_feature( + Feature::unaligned_scalar_mem, + out[2].value == RISCV_HWPROBE_MISALIGNED_SCALAR_FAST, + ); + } else if out[4].key != -1 { + // Deprecated method for fallback + enable_feature( + Feature::unaligned_scalar_mem, + out[4].value & RISCV_HWPROBE_MISALIGNED_MASK == RISCV_HWPROBE_MISALIGNED_FAST, + ); + } + if out[3].key != -1 { + enable_feature( + Feature::unaligned_vector_mem, + out[3].value == RISCV_HWPROBE_MISALIGNED_VECTOR_FAST, + ); + } + + // Query whether "I" base and extensions "M" and "A" (as in the ISA + // manual version 2.2) are enabled. "I" base at that time corresponds + // to "I", "Zicsr", "Zicntr" and "Zifencei" (as in the ISA manual version + // 20240411) and we chose to imply "Zicsr" and "Zifencei" (not "Zicntr") + // because there will be a separate RISCV_HWPROBE_EXT_ZICNTR constant to + // determine existence of the "Zicntr" extension in Linux 6.15 (as of rc1). + // "fence.i" ("Zifencei") is conditionally valid on the Linux userland + // (when CMODX is enabled). + // This is a requirement of `RISCV_HWPROBE_KEY_IMA_EXT_0`-based tests. + let has_ima = (out[0].key != -1) && (out[0].value & RISCV_HWPROBE_BASE_BEHAVIOR_IMA != 0); + if !has_ima { + break 'hwprobe; + } + has_i |= has_ima; + enable_feature(Feature::zicsr, has_ima); + enable_feature(Feature::zifencei, has_ima); + enable_feature(Feature::m, has_ima); + enable_feature(Feature::a, has_ima); + + // Enable features based on `RISCV_HWPROBE_KEY_IMA_EXT_0`. + if out[1].key == -1 { + break 'hwprobe; + } + let ima_ext_0 = out[1].value; + let test = |mask| (ima_ext_0 & mask) != 0; + + enable_feature(Feature::d, test(RISCV_HWPROBE_IMA_FD)); // F is implied. + enable_feature(Feature::c, test(RISCV_HWPROBE_IMA_C)); + + enable_feature(Feature::zihintntl, test(RISCV_HWPROBE_EXT_ZIHINTNTL)); + enable_feature(Feature::zihintpause, test(RISCV_HWPROBE_EXT_ZIHINTPAUSE)); + enable_feature(Feature::zimop, test(RISCV_HWPROBE_EXT_ZIMOP)); + enable_feature(Feature::zicboz, test(RISCV_HWPROBE_EXT_ZICBOZ)); + enable_feature(Feature::zicond, test(RISCV_HWPROBE_EXT_ZICOND)); + + enable_feature(Feature::zawrs, test(RISCV_HWPROBE_EXT_ZAWRS)); + enable_feature(Feature::zacas, test(RISCV_HWPROBE_EXT_ZACAS)); + enable_feature(Feature::ztso, test(RISCV_HWPROBE_EXT_ZTSO)); + + enable_feature(Feature::zba, test(RISCV_HWPROBE_EXT_ZBA)); + enable_feature(Feature::zbb, test(RISCV_HWPROBE_EXT_ZBB)); + enable_feature(Feature::zbs, test(RISCV_HWPROBE_EXT_ZBS)); + enable_feature(Feature::zbc, test(RISCV_HWPROBE_EXT_ZBC)); + + enable_feature(Feature::zbkb, test(RISCV_HWPROBE_EXT_ZBKB)); + enable_feature(Feature::zbkc, test(RISCV_HWPROBE_EXT_ZBKC)); + enable_feature(Feature::zbkx, test(RISCV_HWPROBE_EXT_ZBKX)); + enable_feature(Feature::zknd, test(RISCV_HWPROBE_EXT_ZKND)); + enable_feature(Feature::zkne, test(RISCV_HWPROBE_EXT_ZKNE)); + enable_feature(Feature::zknh, test(RISCV_HWPROBE_EXT_ZKNH)); + enable_feature(Feature::zksed, test(RISCV_HWPROBE_EXT_ZKSED)); + enable_feature(Feature::zksh, test(RISCV_HWPROBE_EXT_ZKSH)); + enable_feature(Feature::zkt, test(RISCV_HWPROBE_EXT_ZKT)); + + enable_feature(Feature::zcmop, test(RISCV_HWPROBE_EXT_ZCMOP)); + enable_feature(Feature::zca, test(RISCV_HWPROBE_EXT_ZCA)); + enable_feature(Feature::zcf, test(RISCV_HWPROBE_EXT_ZCF)); + enable_feature(Feature::zcd, test(RISCV_HWPROBE_EXT_ZCD)); + enable_feature(Feature::zcb, test(RISCV_HWPROBE_EXT_ZCB)); + + enable_feature(Feature::zfh, test(RISCV_HWPROBE_EXT_ZFH)); + enable_feature(Feature::zfhmin, test(RISCV_HWPROBE_EXT_ZFHMIN)); + enable_feature(Feature::zfa, test(RISCV_HWPROBE_EXT_ZFA)); + + // Use prctl (if any) to determine whether the vector extension + // is enabled on the current thread (assuming the entire process + // share the same status). If prctl fails (e.g. QEMU userland emulator + // as of version 9.2.3), use auxiliary vector to retrieve the default + // vector status on the process startup. + let has_vectors = { + let v_status = unsafe { libc::prctl(PR_RISCV_V_GET_CONTROL) }; + if v_status >= 0 { + (v_status & PR_RISCV_V_VSTATE_CTRL_CUR_MASK) == PR_RISCV_V_VSTATE_CTRL_ON + } else { + has_v + } + }; + if has_vectors { + enable_feature(Feature::v, test(RISCV_HWPROBE_IMA_V)); + enable_feature(Feature::zve32x, test(RISCV_HWPROBE_EXT_ZVE32X)); + enable_feature(Feature::zve32f, test(RISCV_HWPROBE_EXT_ZVE32F)); + enable_feature(Feature::zve64x, test(RISCV_HWPROBE_EXT_ZVE64X)); + enable_feature(Feature::zve64f, test(RISCV_HWPROBE_EXT_ZVE64F)); + enable_feature(Feature::zve64d, test(RISCV_HWPROBE_EXT_ZVE64D)); + + enable_feature(Feature::zvbb, test(RISCV_HWPROBE_EXT_ZVBB)); + enable_feature(Feature::zvbc, test(RISCV_HWPROBE_EXT_ZVBC)); + enable_feature(Feature::zvkb, test(RISCV_HWPROBE_EXT_ZVKB)); + enable_feature(Feature::zvkg, test(RISCV_HWPROBE_EXT_ZVKG)); + enable_feature(Feature::zvkned, test(RISCV_HWPROBE_EXT_ZVKNED)); + enable_feature(Feature::zvknha, test(RISCV_HWPROBE_EXT_ZVKNHA)); + enable_feature(Feature::zvknhb, test(RISCV_HWPROBE_EXT_ZVKNHB)); + enable_feature(Feature::zvksed, test(RISCV_HWPROBE_EXT_ZVKSED)); + enable_feature(Feature::zvksh, test(RISCV_HWPROBE_EXT_ZVKSH)); + enable_feature(Feature::zvkt, test(RISCV_HWPROBE_EXT_ZVKT)); + + enable_feature(Feature::zvfh, test(RISCV_HWPROBE_EXT_ZVFH)); + enable_feature(Feature::zvfhmin, test(RISCV_HWPROBE_EXT_ZVFHMIN)); + } + is_v_set = true; + }; + + // Set V purely depending on the auxiliary vector + // only if no fine-grained vector extension detection is available. + if !is_v_set { + enable_feature(Feature::v, has_v); + } // Handle base ISA. - let has_i = bit::test(auxv.hwcap, (b'i' - b'a').into()); // If future RV128I is supported, implement with `enable_feature` here. // Note that we should use `target_arch` instead of `target_pointer_width` // to avoid misdetection caused by experimental ABIs such as RV64ILP32. diff --git a/library/stdarch/crates/std_detect/src/detect/os/riscv.rs b/library/stdarch/crates/std_detect/src/detect/os/riscv.rs index b268eff8a14..fc218d51ab5 100644 --- a/library/stdarch/crates/std_detect/src/detect/os/riscv.rs +++ b/library/stdarch/crates/std_detect/src/detect/os/riscv.rs @@ -62,25 +62,67 @@ pub(crate) fn imply_features(mut value: cache::Initializer) -> cache::Initialize "defined as subset": The latter extension is described as a subset of the former (but the evidence is weak). + "functional": + The former extension is functionally a superset of the latter + (no direct references though). */ + imply!(zvbb => zvkb); + + // Certain set of vector cryptography extensions form a group. + group!(zvkn == zvkned & zvknhb & zvkb & zvkt); + group!(zvknc == zvkn & zvbc); + group!(zvkng == zvkn & zvkg); + group!(zvks == zvksed & zvksh & zvkb & zvkt); + group!(zvksc == zvks & zvbc); + group!(zvksg == zvks & zvkg); + + imply!(zvknhb => zvknha); // functional + + // For vector cryptography, Zvknhb and Zvbc require integer arithmetic + // with EEW=64 (Zve64x) while others not depending on them + // require EEW=32 (Zve32x). + imply!(zvknhb | zvbc => zve64x); + imply!(zvbb | zvkb | zvkg | zvkned | zvknha | zvksed | zvksh => zve32x); + imply!(zbc => zbkc); // defined as subset group!(zkn == zbkb & zbkc & zbkx & zkne & zknd & zknh); group!(zks == zbkb & zbkc & zbkx & zksed & zksh); group!(zk == zkn & zkr & zkt); + imply!(zacas => zaamo); group!(a == zalrsc & zaamo); group!(b == zba & zbb & zbs); + imply!(zcf => zca & f); + imply!(zcd => zca & d); + imply!(zcmop | zcb => zca); + imply!(zhinx => zhinxmin); imply!(zdinx | zhinxmin => zfinx); + imply!(zvfh => zvfhmin); // functional + imply!(zvfh => zve32f & zfhmin); + imply!(zvfhmin => zve32f); + + imply!(v => zve64d); + imply!(zve64d => zve64f & d); + imply!(zve64f => zve64x & zve32f); + imply!(zve64x => zve32x); + imply!(zve32f => zve32x & f); + imply!(zfh => zfhmin); imply!(q => d); - imply!(d | zfhmin => f); + imply!(d | zfhmin | zfa => f); - imply!(zicntr | zihpm | f | zfinx => zicsr); + // Relatively complex implication rules from the "C" extension. + imply!(c => zca); + imply!(c & d => zcd); + #[cfg(target_arch = "riscv32")] + imply!(c & f => zcf); + + imply!(zicntr | zihpm | f | zfinx | zve32x => zicsr); imply!(s | h => zicsr); // Loop until the feature flags converge. @@ -110,6 +152,16 @@ mod tests { assert!(imply_features(value).test(Feature::zicsr as u32)); } + #[test] + fn complex_zcd() { + let mut value = cache::Initializer::default(); + // C & D -> Zcd + value.set(Feature::c as u32); + assert!(!imply_features(value).test(Feature::zcd as u32)); + value.set(Feature::d as u32); + assert!(imply_features(value).test(Feature::zcd as u32)); + } + #[test] fn group_simple_forward() { let mut value = cache::Initializer::default(); @@ -132,17 +184,18 @@ mod tests { #[test] fn group_complex_convergence() { let mut value = cache::Initializer::default(); - // Needs 2 iterations to converge - // (and 3rd iteration for convergence checking): - // 1. [Zk] -> Zkn & Zkr & Zkt - // 2. Zkn -> {Zbkb} & {Zbkc} & {Zbkx} & {Zkne} & {Zknd} & {Zknh} - value.set(Feature::zk as u32); + // Needs 3 iterations to converge + // (and 4th iteration for convergence checking): + // 1. [Zvksc] -> Zvks & Zvbc + // 2. Zvks -> Zvksed & Zvksh & Zvkb & Zvkt + // 3a. [Zvkned] & [Zvknhb] & [Zvkb] & Zvkt -> {Zvkn} + // 3b. Zvkn & Zvbc -> {Zvknc} + value.set(Feature::zvksc as u32); + value.set(Feature::zvkned as u32); + value.set(Feature::zvknhb as u32); + value.set(Feature::zvkb as u32); let value = imply_features(value); - assert!(value.test(Feature::zbkb as u32)); - assert!(value.test(Feature::zbkc as u32)); - assert!(value.test(Feature::zbkx as u32)); - assert!(value.test(Feature::zkne as u32)); - assert!(value.test(Feature::zknd as u32)); - assert!(value.test(Feature::zknh as u32)); + assert!(value.test(Feature::zvkn as u32)); + assert!(value.test(Feature::zvknc as u32)); } } diff --git a/library/stdarch/crates/std_detect/tests/cpu-detection.rs b/library/stdarch/crates/std_detect/tests/cpu-detection.rs index 8a40997f75d..c0a76898d41 100644 --- a/library/stdarch/crates/std_detect/tests/cpu-detection.rs +++ b/library/stdarch/crates/std_detect/tests/cpu-detection.rs @@ -236,15 +236,29 @@ fn riscv_linux() { println!("rv32e: {}", is_riscv_feature_detected!("rv32e")); println!("rv64i: {}", is_riscv_feature_detected!("rv64i")); println!("rv128i: {}", is_riscv_feature_detected!("rv128i")); + println!( + "unaligned-scalar-mem: {}", + is_riscv_feature_detected!("unaligned-scalar-mem") + ); + println!( + "unaligned-vector-mem: {}", + is_riscv_feature_detected!("unaligned-vector-mem") + ); println!("zicsr: {}", is_riscv_feature_detected!("zicsr")); println!("zicntr: {}", is_riscv_feature_detected!("zicntr")); println!("zihpm: {}", is_riscv_feature_detected!("zihpm")); println!("zifencei: {}", is_riscv_feature_detected!("zifencei")); + println!("zihintntl: {}", is_riscv_feature_detected!("zihintntl")); println!("zihintpause: {}", is_riscv_feature_detected!("zihintpause")); + println!("zimop: {}", is_riscv_feature_detected!("zimop")); + println!("zicboz: {}", is_riscv_feature_detected!("zicboz")); + println!("zicond: {}", is_riscv_feature_detected!("zicond")); println!("m: {}", is_riscv_feature_detected!("m")); println!("a: {}", is_riscv_feature_detected!("a")); println!("zalrsc: {}", is_riscv_feature_detected!("zalrsc")); println!("zaamo: {}", is_riscv_feature_detected!("zaamo")); + println!("zawrs: {}", is_riscv_feature_detected!("zawrs")); + println!("zacas: {}", is_riscv_feature_detected!("zacas")); println!("zam: {}", is_riscv_feature_detected!("zam")); println!("ztso: {}", is_riscv_feature_detected!("ztso")); println!("f: {}", is_riscv_feature_detected!("f")); @@ -252,11 +266,17 @@ fn riscv_linux() { println!("q: {}", is_riscv_feature_detected!("q")); println!("zfh: {}", is_riscv_feature_detected!("zfh")); println!("zfhmin: {}", is_riscv_feature_detected!("zfhmin")); + println!("zfa: {}", is_riscv_feature_detected!("zfa")); println!("zfinx: {}", is_riscv_feature_detected!("zfinx")); println!("zdinx: {}", is_riscv_feature_detected!("zdinx")); println!("zhinx: {}", is_riscv_feature_detected!("zhinx")); println!("zhinxmin: {}", is_riscv_feature_detected!("zhinxmin")); println!("c: {}", is_riscv_feature_detected!("c")); + println!("zca: {}", is_riscv_feature_detected!("zca")); + println!("zcf: {}", is_riscv_feature_detected!("zcf")); + println!("zcd: {}", is_riscv_feature_detected!("zcd")); + println!("zcb: {}", is_riscv_feature_detected!("zcb")); + println!("zcmop: {}", is_riscv_feature_detected!("zcmop")); println!("b: {}", is_riscv_feature_detected!("b")); println!("zba: {}", is_riscv_feature_detected!("zba")); println!("zbb: {}", is_riscv_feature_detected!("zbb")); @@ -276,6 +296,29 @@ fn riscv_linux() { println!("zk: {}", is_riscv_feature_detected!("zk")); println!("zkt: {}", is_riscv_feature_detected!("zkt")); println!("v: {}", is_riscv_feature_detected!("v")); + println!("zve32x: {}", is_riscv_feature_detected!("zve32x")); + println!("zve32f: {}", is_riscv_feature_detected!("zve32f")); + println!("zve64x: {}", is_riscv_feature_detected!("zve64x")); + println!("zve64f: {}", is_riscv_feature_detected!("zve64f")); + println!("zve64d: {}", is_riscv_feature_detected!("zve64d")); + println!("zvfh: {}", is_riscv_feature_detected!("zvfh")); + println!("zvfhmin: {}", is_riscv_feature_detected!("zvfhmin")); + println!("zvbb: {}", is_riscv_feature_detected!("zvbb")); + println!("zvbc: {}", is_riscv_feature_detected!("zvbc")); + println!("zvkb: {}", is_riscv_feature_detected!("zvkb")); + println!("zvkg: {}", is_riscv_feature_detected!("zvkg")); + println!("zvkned: {}", is_riscv_feature_detected!("zvkned")); + println!("zvknha: {}", is_riscv_feature_detected!("zvknha")); + println!("zvknhb: {}", is_riscv_feature_detected!("zvknhb")); + println!("zvksed: {}", is_riscv_feature_detected!("zvksed")); + println!("zvksh: {}", is_riscv_feature_detected!("zvksh")); + println!("zvkn: {}", is_riscv_feature_detected!("zvkn")); + println!("zvknc: {}", is_riscv_feature_detected!("zvknc")); + println!("zvkng: {}", is_riscv_feature_detected!("zvkng")); + println!("zvks: {}", is_riscv_feature_detected!("zvks")); + println!("zvksc: {}", is_riscv_feature_detected!("zvksc")); + println!("zvksg: {}", is_riscv_feature_detected!("zvksg")); + println!("zvkt: {}", is_riscv_feature_detected!("zvkt")); println!("svnapot: {}", is_riscv_feature_detected!("svnapot")); println!("svpbmt: {}", is_riscv_feature_detected!("svpbmt")); println!("svinval: {}", is_riscv_feature_detected!("svinval"));