RISC-V: riscv_hwprobe-based feature detection on Linux / Android

This commit implements `riscv_hwprobe`-based feature detection as available
on newer versions of the Linux kernel.  It also queries whether the vector
extensions are enabled using `prctl` but this is not supported on QEMU's
userland emulator (as of version 9.2.3) and use the auxiliary vector
as a fallback.

Currently, all extensions discoverable from the Linux kernel version 6.14
and related extension groups (except "Supm", which reports the existence of
`prctl`-based pointer masking control and too OS-dependent) are implemented.

Co-Authored-By: Taiki Endo <te316e89@gmail.com>
This commit is contained in:
Tsukasa OI 2025-04-13 04:35:58 +00:00 committed by Amanieu d'Antras
parent db188b33b3
commit d5baf4da91
5 changed files with 521 additions and 16 deletions

View File

@ -56,11 +56,14 @@ crate from working on applications in which `std` is not available.
[`cupid`](https://crates.io/crates/cupid) crate.
* Linux/Android:
* `arm{32, 64}`, `mips{32,64}{,el}`, `powerpc{32,64}{,le}`, `riscv{32,64}`, `loongarch64`, `s390x`:
* `arm{32, 64}`, `mips{32,64}{,el}`, `powerpc{32,64}{,le}`, `loongarch64`, `s390x`:
`std_detect` supports these on Linux by querying ELF auxiliary vectors (using `getauxval`
when available), and if that fails, by querying `/proc/cpuinfo`.
* `arm64`: partial support for doing run-time feature detection by directly
querying `mrs` is implemented for Linux >= 4.11, but not enabled by default.
* `riscv{32,64}`:
`std_detect` supports these on Linux by querying `riscv_hwprobe`, and
by querying ELF auxiliary vectors (using `getauxval` when available).
* FreeBSD:
* `arm32`, `powerpc64`: `std_detect` supports these on FreeBSD by querying ELF

View File

@ -37,22 +37,39 @@ features! {
/// * Zbb: `"zbb"`
/// * Zbs: `"zbs"`
/// * C: `"c"`
/// * Zca: `"zca"`
/// * Zcd: `"zcd"` (if D is enabled)
/// * Zcf: `"zcf"` (if F is enabled on RV32)
/// * D: `"d"`
/// * F: `"f"`
/// * M: `"m"`
/// * Q: `"q"`
/// * V: `"v"`
/// * Zve32x: `"zve32x"`
/// * Zve32f: `"zve32f"`
/// * Zve64x: `"zve64x"`
/// * Zve64f: `"zve64f"`
/// * Zve64d: `"zve64d"`
/// * Zicboz: `"zicboz"`
/// * Zicntr: `"zicntr"`
/// * Zicond: `"zicond"`
/// * Zicsr: `"zicsr"`
/// * Zifencei: `"zifencei"`
/// * Zihintntl: `"zihintntl"`
/// * Zihintpause: `"zihintpause"`
/// * Zihpm: `"zihpm"`
/// * Zimop: `"zimop"`
/// * Zacas: `"zacas"`
/// * Zawrs: `"zawrs"`
/// * Zfa: `"zfa"`
/// * Zfh: `"zfh"`
/// * Zfhmin: `"zfhmin"`
/// * Zfinx: `"zfinx"`
/// * Zdinx: `"zdinx"`
/// * Zhinx: `"zhinx"`
/// * Zhinxmin: `"zhinxmin"`
/// * Zcb: `"zcb"`
/// * Zcmop: `"zcmop"`
/// * Zbc: `"zbc"`
/// * Zbkb: `"zbkb"`
/// * Zbkc: `"zbkc"`
@ -67,6 +84,24 @@ features! {
/// * Zksed: `"zksed"`
/// * Zksh: `"zksh"`
/// * Zkt: `"zkt"`
/// * Zvbb: `"zvbb"`
/// * Zvbc: `"zvbc"`
/// * Zvfh: `"zvfh"`
/// * Zvfhmin: `"zvfhmin"`
/// * Zvkb: `"zvkb"`
/// * Zvkg: `"zvkg"`
/// * Zvkn: `"zvkn"`
/// * Zvkned: `"zvkned"`
/// * Zvknha: `"zvknha"`
/// * Zvknhb: `"zvknhb"`
/// * Zvknc: `"zvknc"`
/// * Zvkng: `"zvkng"`
/// * Zvks: `"zvks"`
/// * Zvksed: `"zvksed"`
/// * Zvksh: `"zvksh"`
/// * Zvksc: `"zvksc"`
/// * Zvksg: `"zvksg"`
/// * Zvkt: `"zvkt"`
/// * Ztso: `"ztso"`
///
/// There's also bases and extensions marked as standard instruction set,
@ -87,6 +122,15 @@ features! {
/// * Svnapot: `"svnapot"`
/// * Svpbmt: `"svpbmt"`
/// * Svinval: `"svinval"`
///
/// # Performance Hints
///
/// The two features below define performance hints for unaligned
/// scalar/vector memory accesses, respectively. If enabled, it denotes that
/// corresponding unaligned memory access is reasonably fast.
///
/// * `"unaligned-scalar-mem"`
/// * `"unaligned-vector-mem"`
#[stable(feature = "riscv_ratified", since = "1.78.0")]
@FEATURE: #[unstable(feature = "stdarch_riscv_feature_detection", issue = "111192")] rv32i: "rv32i";
@ -102,6 +146,11 @@ features! {
without cfg check: true;
/// RV128I Base Integer Instruction Set
@FEATURE: #[unstable(feature = "stdarch_riscv_feature_detection", issue = "111192")] unaligned_scalar_mem: "unaligned-scalar-mem";
/// Has reasonably performant unaligned scalar
@FEATURE: #[unstable(feature = "stdarch_riscv_feature_detection", issue = "111192")] unaligned_vector_mem: "unaligned-vector-mem";
/// Has reasonably performant unaligned vector
@FEATURE: #[unstable(feature = "stdarch_riscv_feature_detection", issue = "111192")] zicsr: "zicsr";
without cfg check: true;
/// "Zicsr" Extension for Control and Status Register (CSR) Instructions
@ -115,9 +164,21 @@ features! {
without cfg check: true;
/// "Zifencei" Extension for Instruction-Fetch Fence
@FEATURE: #[unstable(feature = "stdarch_riscv_feature_detection", issue = "111192")] zihintntl: "zihintntl";
without cfg check: true;
/// "Zihintntl" Extension for Non-Temporal Locality Hints
@FEATURE: #[unstable(feature = "stdarch_riscv_feature_detection", issue = "111192")] zihintpause: "zihintpause";
without cfg check: true;
/// "Zihintpause" Extension for Pause Hint
@FEATURE: #[unstable(feature = "stdarch_riscv_feature_detection", issue = "111192")] zimop: "zimop";
without cfg check: true;
/// "Zimop" Extension for May-Be-Operations
@FEATURE: #[unstable(feature = "stdarch_riscv_feature_detection", issue = "111192")] zicboz: "zicboz";
without cfg check: true;
/// "Zicboz" Extension for Cache-Block Zero Instruction
@FEATURE: #[unstable(feature = "stdarch_riscv_feature_detection", issue = "111192")] zicond: "zicond";
without cfg check: true;
/// "Zicond" Extension for Integer Conditional Operations
@FEATURE: #[stable(feature = "riscv_ratified", since = "1.78.0")] m: "m";
/// "M" Extension for Integer Multiplication and Division
@ -128,6 +189,10 @@ features! {
/// "Zalrsc" Extension for Load-Reserved/Store-Conditional Instructions
@FEATURE: #[unstable(feature = "stdarch_riscv_feature_detection", issue = "111192")] zaamo: "zaamo";
/// "Zaamo" Extension for Atomic Memory Operations
@FEATURE: #[unstable(feature = "stdarch_riscv_feature_detection", issue = "111192")] zawrs: "zawrs";
/// "Zawrs" Extension for Wait-on-Reservation-Set Instructions
@FEATURE: #[unstable(feature = "stdarch_riscv_feature_detection", issue = "111192")] zacas: "zacas";
/// "Zacas" Extension for Atomic Compare-and-Swap (CAS) Instructions
@FEATURE: #[unstable(feature = "stdarch_riscv_feature_detection", issue = "111192")] zam: "zam";
without cfg check: true;
/// "Zam" Extension for Misaligned Atomics
@ -146,6 +211,9 @@ features! {
/// "Zfh" Extension for Half-Precision Floating-Point
@FEATURE: #[unstable(feature = "stdarch_riscv_feature_detection", issue = "111192")] zfhmin: "zfhmin";
/// "Zfhmin" Extension for Minimal Half-Precision Floating-Point
@FEATURE: #[unstable(feature = "stdarch_riscv_feature_detection", issue = "111192")] zfa: "zfa";
without cfg check: true;
/// "Zfa" Extension for Additional Floating-Point Instructions
@FEATURE: #[unstable(feature = "stdarch_riscv_feature_detection", issue = "111192")] zfinx: "zfinx";
/// "Zfinx" Extension for Single-Precision Floating-Point in Integer Registers
@ -158,6 +226,21 @@ features! {
@FEATURE: #[stable(feature = "riscv_ratified", since = "1.78.0")] c: "c";
/// "C" Extension for Compressed Instructions
@FEATURE: #[unstable(feature = "stdarch_riscv_feature_detection", issue = "111192")] zca: "zca";
without cfg check: true;
/// "Zca" Compressed Instructions excluding Floating-Point Loads/Stores
@FEATURE: #[unstable(feature = "stdarch_riscv_feature_detection", issue = "111192")] zcf: "zcf";
without cfg check: true;
/// "Zcf" Compressed Instructions for Single-Precision Floating-Point Loads/Stores on RV32
@FEATURE: #[unstable(feature = "stdarch_riscv_feature_detection", issue = "111192")] zcd: "zcd";
without cfg check: true;
/// "Zcd" Compressed Instructions for Double-Precision Floating-Point Loads/Stores
@FEATURE: #[unstable(feature = "stdarch_riscv_feature_detection", issue = "111192")] zcb: "zcb";
without cfg check: true;
/// "Zcb" Simple Code-size Saving Compressed Instructions
@FEATURE: #[unstable(feature = "stdarch_riscv_feature_detection", issue = "111192")] zcmop: "zcmop";
without cfg check: true;
/// "Zcmop" Extension for Compressed May-Be-Operations
@FEATURE: #[unstable(feature = "stdarch_riscv_feature_detection", issue = "111192")] b: "b";
without cfg check: true;
@ -200,6 +283,53 @@ features! {
@FEATURE: #[unstable(feature = "stdarch_riscv_feature_detection", issue = "111192")] v: "v";
/// "V" Extension for Vector Operations
@FEATURE: #[unstable(feature = "stdarch_riscv_feature_detection", issue = "111192")] zve32x: "zve32x";
/// "Zve32x" Vector Extension for Embedded Processors (32-bit+; Integer)
@FEATURE: #[unstable(feature = "stdarch_riscv_feature_detection", issue = "111192")] zve32f: "zve32f";
/// "Zve32f" Vector Extension for Embedded Processors (32-bit+; with Single-Precision Floating-Point)
@FEATURE: #[unstable(feature = "stdarch_riscv_feature_detection", issue = "111192")] zve64x: "zve64x";
/// "Zve64x" Vector Extension for Embedded Processors (64-bit+; Integer)
@FEATURE: #[unstable(feature = "stdarch_riscv_feature_detection", issue = "111192")] zve64f: "zve64f";
/// "Zve64f" Vector Extension for Embedded Processors (64-bit+; with Single-Precision Floating-Point)
@FEATURE: #[unstable(feature = "stdarch_riscv_feature_detection", issue = "111192")] zve64d: "zve64d";
/// "Zve64d" Vector Extension for Embedded Processors (64-bit+; with Double-Precision Floating-Point)
@FEATURE: #[unstable(feature = "stdarch_riscv_feature_detection", issue = "111192")] zvfh: "zvfh";
/// "Zvfh" Vector Extension for Half-Precision Floating-Point
@FEATURE: #[unstable(feature = "stdarch_riscv_feature_detection", issue = "111192")] zvfhmin: "zvfhmin";
/// "Zvfhmin" Vector Extension for Minimal Half-Precision Floating-Point
@FEATURE: #[unstable(feature = "stdarch_riscv_feature_detection", issue = "111192")] zvbb: "zvbb";
/// "Zvbb" Extension for Vector Basic Bit-Manipulation
@FEATURE: #[unstable(feature = "stdarch_riscv_feature_detection", issue = "111192")] zvbc: "zvbc";
/// "Zvbc" Extension for Vector Carryless Multiplication
@FEATURE: #[unstable(feature = "stdarch_riscv_feature_detection", issue = "111192")] zvkb: "zvkb";
/// "Zvkb" Extension for Vector Cryptography Bit-Manipulation
@FEATURE: #[unstable(feature = "stdarch_riscv_feature_detection", issue = "111192")] zvkg: "zvkg";
/// "Zvkg" Cryptography Extension for Vector GCM/GMAC
@FEATURE: #[unstable(feature = "stdarch_riscv_feature_detection", issue = "111192")] zvkned: "zvkned";
/// "Zvkned" Cryptography Extension for NIST Suite: Vector AES Block Cipher
@FEATURE: #[unstable(feature = "stdarch_riscv_feature_detection", issue = "111192")] zvknha: "zvknha";
/// "Zvknha" Cryptography Extension for Vector SHA-2 Secure Hash (SHA-256)
@FEATURE: #[unstable(feature = "stdarch_riscv_feature_detection", issue = "111192")] zvknhb: "zvknhb";
/// "Zvknhb" Cryptography Extension for Vector SHA-2 Secure Hash (SHA-256/512)
@FEATURE: #[unstable(feature = "stdarch_riscv_feature_detection", issue = "111192")] zvksed: "zvksed";
/// "Zvksed" Cryptography Extension for ShangMi Suite: Vector SM4 Block Cipher
@FEATURE: #[unstable(feature = "stdarch_riscv_feature_detection", issue = "111192")] zvksh: "zvksh";
/// "Zvksh" Cryptography Extension for ShangMi Suite: Vector SM3 Secure Hash
@FEATURE: #[unstable(feature = "stdarch_riscv_feature_detection", issue = "111192")] zvkn: "zvkn";
/// "Zvkn" Cryptography Extension for NIST Algorithm Suite
@FEATURE: #[unstable(feature = "stdarch_riscv_feature_detection", issue = "111192")] zvknc: "zvknc";
/// "Zvknc" Cryptography Extension for NIST Algorithm Suite with Carryless Multiply
@FEATURE: #[unstable(feature = "stdarch_riscv_feature_detection", issue = "111192")] zvkng: "zvkng";
/// "Zvkng" Cryptography Extension for NIST Algorithm Suite with GCM
@FEATURE: #[unstable(feature = "stdarch_riscv_feature_detection", issue = "111192")] zvks: "zvks";
/// "Zvks" Cryptography Extension for ShangMi Algorithm Suite
@FEATURE: #[unstable(feature = "stdarch_riscv_feature_detection", issue = "111192")] zvksc: "zvksc";
/// "Zvksc" Cryptography Extension for ShangMi Algorithm Suite with Carryless Multiply
@FEATURE: #[unstable(feature = "stdarch_riscv_feature_detection", issue = "111192")] zvksg: "zvksg";
/// "Zvksg" Cryptography Extension for ShangMi Algorithm Suite with GCM
@FEATURE: #[unstable(feature = "stdarch_riscv_feature_detection", issue = "111192")] zvkt: "zvkt";
/// "Zvkt" Extension for Vector Data-Independent Execution Latency
@FEATURE: #[unstable(feature = "stdarch_riscv_feature_detection", issue = "111192")] svnapot: "svnapot";
without cfg check: true;

View File

@ -1,10 +1,127 @@
//! Run-time feature detection for RISC-V on Linux.
//!
//! On RISC-V, detection using auxv only supports single-letter extensions.
//! So, we use riscv_hwprobe that supports multi-letter extensions if available.
//! <https://www.kernel.org/doc/html/latest/arch/riscv/hwprobe.html>
use core::ptr;
use super::super::riscv::imply_features;
use super::auxvec;
use crate::detect::{Feature, bit, cache};
/// Read list of supported features from the auxiliary vector.
// See <https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git/tree/include/uapi/linux/prctl.h?h=v6.14>
// for runtime status query constants.
const PR_RISCV_V_GET_CONTROL: libc::c_int = 70;
const PR_RISCV_V_VSTATE_CTRL_ON: libc::c_int = 2;
const PR_RISCV_V_VSTATE_CTRL_CUR_MASK: libc::c_int = 3;
// See <https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git/tree/arch/riscv/include/uapi/asm/hwprobe.h?h=v6.14>
// for riscv_hwprobe struct and hardware probing constants.
#[repr(C)]
struct riscv_hwprobe {
key: i64,
value: u64,
}
#[allow(non_upper_case_globals)]
const __NR_riscv_hwprobe: libc::c_long = 258;
const RISCV_HWPROBE_KEY_BASE_BEHAVIOR: i64 = 3;
const RISCV_HWPROBE_BASE_BEHAVIOR_IMA: u64 = 1 << 0;
const RISCV_HWPROBE_KEY_IMA_EXT_0: i64 = 4;
const RISCV_HWPROBE_IMA_FD: u64 = 1 << 0;
const RISCV_HWPROBE_IMA_C: u64 = 1 << 1;
const RISCV_HWPROBE_IMA_V: u64 = 1 << 2;
const RISCV_HWPROBE_EXT_ZBA: u64 = 1 << 3;
const RISCV_HWPROBE_EXT_ZBB: u64 = 1 << 4;
const RISCV_HWPROBE_EXT_ZBS: u64 = 1 << 5;
const RISCV_HWPROBE_EXT_ZICBOZ: u64 = 1 << 6;
const RISCV_HWPROBE_EXT_ZBC: u64 = 1 << 7;
const RISCV_HWPROBE_EXT_ZBKB: u64 = 1 << 8;
const RISCV_HWPROBE_EXT_ZBKC: u64 = 1 << 9;
const RISCV_HWPROBE_EXT_ZBKX: u64 = 1 << 10;
const RISCV_HWPROBE_EXT_ZKND: u64 = 1 << 11;
const RISCV_HWPROBE_EXT_ZKNE: u64 = 1 << 12;
const RISCV_HWPROBE_EXT_ZKNH: u64 = 1 << 13;
const RISCV_HWPROBE_EXT_ZKSED: u64 = 1 << 14;
const RISCV_HWPROBE_EXT_ZKSH: u64 = 1 << 15;
const RISCV_HWPROBE_EXT_ZKT: u64 = 1 << 16;
const RISCV_HWPROBE_EXT_ZVBB: u64 = 1 << 17;
const RISCV_HWPROBE_EXT_ZVBC: u64 = 1 << 18;
const RISCV_HWPROBE_EXT_ZVKB: u64 = 1 << 19;
const RISCV_HWPROBE_EXT_ZVKG: u64 = 1 << 20;
const RISCV_HWPROBE_EXT_ZVKNED: u64 = 1 << 21;
const RISCV_HWPROBE_EXT_ZVKNHA: u64 = 1 << 22;
const RISCV_HWPROBE_EXT_ZVKNHB: u64 = 1 << 23;
const RISCV_HWPROBE_EXT_ZVKSED: u64 = 1 << 24;
const RISCV_HWPROBE_EXT_ZVKSH: u64 = 1 << 25;
const RISCV_HWPROBE_EXT_ZVKT: u64 = 1 << 26;
const RISCV_HWPROBE_EXT_ZFH: u64 = 1 << 27;
const RISCV_HWPROBE_EXT_ZFHMIN: u64 = 1 << 28;
const RISCV_HWPROBE_EXT_ZIHINTNTL: u64 = 1 << 29;
const RISCV_HWPROBE_EXT_ZVFH: u64 = 1 << 30;
const RISCV_HWPROBE_EXT_ZVFHMIN: u64 = 1 << 31;
const RISCV_HWPROBE_EXT_ZFA: u64 = 1 << 32;
const RISCV_HWPROBE_EXT_ZTSO: u64 = 1 << 33;
const RISCV_HWPROBE_EXT_ZACAS: u64 = 1 << 34;
const RISCV_HWPROBE_EXT_ZICOND: u64 = 1 << 35;
const RISCV_HWPROBE_EXT_ZIHINTPAUSE: u64 = 1 << 36;
const RISCV_HWPROBE_EXT_ZVE32X: u64 = 1 << 37;
const RISCV_HWPROBE_EXT_ZVE32F: u64 = 1 << 38;
const RISCV_HWPROBE_EXT_ZVE64X: u64 = 1 << 39;
const RISCV_HWPROBE_EXT_ZVE64F: u64 = 1 << 40;
const RISCV_HWPROBE_EXT_ZVE64D: u64 = 1 << 41;
const RISCV_HWPROBE_EXT_ZIMOP: u64 = 1 << 42;
const RISCV_HWPROBE_EXT_ZCA: u64 = 1 << 43;
const RISCV_HWPROBE_EXT_ZCB: u64 = 1 << 44;
const RISCV_HWPROBE_EXT_ZCD: u64 = 1 << 45;
const RISCV_HWPROBE_EXT_ZCF: u64 = 1 << 46;
const RISCV_HWPROBE_EXT_ZCMOP: u64 = 1 << 47;
const RISCV_HWPROBE_EXT_ZAWRS: u64 = 1 << 48;
// Excluded because it only reports the existence of `prctl`-based pointer masking control.
// const RISCV_HWPROBE_EXT_SUPM: u64 = 1 << 49;
const RISCV_HWPROBE_KEY_CPUPERF_0: i64 = 5;
const RISCV_HWPROBE_MISALIGNED_FAST: u64 = 3;
const RISCV_HWPROBE_MISALIGNED_MASK: u64 = 7;
const RISCV_HWPROBE_KEY_MISALIGNED_SCALAR_PERF: i64 = 9;
const RISCV_HWPROBE_MISALIGNED_SCALAR_FAST: u64 = 3;
const RISCV_HWPROBE_KEY_MISALIGNED_VECTOR_PERF: i64 = 10;
const RISCV_HWPROBE_MISALIGNED_VECTOR_FAST: u64 = 3;
// syscall returns an unsupported error if riscv_hwprobe is not supported,
// so we can safely use this function on older versions of Linux.
fn _riscv_hwprobe(out: &mut [riscv_hwprobe]) -> bool {
unsafe fn __riscv_hwprobe(
pairs: *mut riscv_hwprobe,
pair_count: libc::size_t,
cpu_set_size: libc::size_t,
cpus: *mut libc::c_ulong,
flags: libc::c_uint,
) -> libc::c_long {
unsafe {
libc::syscall(
__NR_riscv_hwprobe,
pairs,
pair_count,
cpu_set_size,
cpus,
flags,
)
}
}
let len = out.len();
unsafe { __riscv_hwprobe(out.as_mut_ptr(), len, 0, ptr::null_mut(), 0) == 0 }
}
/// Read list of supported features from (1) the auxiliary vector
/// and (2) the results of `riscv_hwprobe` and `prctl` system calls.
pub(crate) fn detect_features() -> cache::Initializer {
let mut value = cache::Initializer::default();
let mut enable_feature = |feature, enable| {
@ -18,6 +135,7 @@ pub(crate) fn detect_features() -> cache::Initializer {
//
// [hwcap]: https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git/tree/arch/riscv/include/uapi/asm/hwcap.h?h=v6.14
let auxv = auxvec::auxv().expect("read auxvec"); // should not fail on RISC-V platform
let mut has_i = bit::test(auxv.hwcap, (b'i' - b'a').into());
#[allow(clippy::eq_op)]
enable_feature(Feature::a, bit::test(auxv.hwcap, (b'a' - b'a').into()));
enable_feature(Feature::c, bit::test(auxv.hwcap, (b'c' - b'a').into()));
@ -25,9 +143,167 @@ pub(crate) fn detect_features() -> cache::Initializer {
enable_feature(Feature::f, bit::test(auxv.hwcap, (b'f' - b'a').into()));
enable_feature(Feature::h, bit::test(auxv.hwcap, (b'h' - b'a').into()));
enable_feature(Feature::m, bit::test(auxv.hwcap, (b'm' - b'a').into()));
let has_v = bit::test(auxv.hwcap, (b'v' - b'a').into());
let mut is_v_set = false;
// Use riscv_hwprobe syscall to query more extensions and
// performance-related capabilities.
'hwprobe: {
let mut out = [
riscv_hwprobe {
key: RISCV_HWPROBE_KEY_BASE_BEHAVIOR,
value: 0,
},
riscv_hwprobe {
key: RISCV_HWPROBE_KEY_IMA_EXT_0,
value: 0,
},
riscv_hwprobe {
key: RISCV_HWPROBE_KEY_MISALIGNED_SCALAR_PERF,
value: 0,
},
riscv_hwprobe {
key: RISCV_HWPROBE_KEY_MISALIGNED_VECTOR_PERF,
value: 0,
},
riscv_hwprobe {
key: RISCV_HWPROBE_KEY_CPUPERF_0,
value: 0,
},
];
if !_riscv_hwprobe(&mut out) {
break 'hwprobe;
}
// Query scalar/vector misaligned behavior.
if out[2].key != -1 {
enable_feature(
Feature::unaligned_scalar_mem,
out[2].value == RISCV_HWPROBE_MISALIGNED_SCALAR_FAST,
);
} else if out[4].key != -1 {
// Deprecated method for fallback
enable_feature(
Feature::unaligned_scalar_mem,
out[4].value & RISCV_HWPROBE_MISALIGNED_MASK == RISCV_HWPROBE_MISALIGNED_FAST,
);
}
if out[3].key != -1 {
enable_feature(
Feature::unaligned_vector_mem,
out[3].value == RISCV_HWPROBE_MISALIGNED_VECTOR_FAST,
);
}
// Query whether "I" base and extensions "M" and "A" (as in the ISA
// manual version 2.2) are enabled. "I" base at that time corresponds
// to "I", "Zicsr", "Zicntr" and "Zifencei" (as in the ISA manual version
// 20240411) and we chose to imply "Zicsr" and "Zifencei" (not "Zicntr")
// because there will be a separate RISCV_HWPROBE_EXT_ZICNTR constant to
// determine existence of the "Zicntr" extension in Linux 6.15 (as of rc1).
// "fence.i" ("Zifencei") is conditionally valid on the Linux userland
// (when CMODX is enabled).
// This is a requirement of `RISCV_HWPROBE_KEY_IMA_EXT_0`-based tests.
let has_ima = (out[0].key != -1) && (out[0].value & RISCV_HWPROBE_BASE_BEHAVIOR_IMA != 0);
if !has_ima {
break 'hwprobe;
}
has_i |= has_ima;
enable_feature(Feature::zicsr, has_ima);
enable_feature(Feature::zifencei, has_ima);
enable_feature(Feature::m, has_ima);
enable_feature(Feature::a, has_ima);
// Enable features based on `RISCV_HWPROBE_KEY_IMA_EXT_0`.
if out[1].key == -1 {
break 'hwprobe;
}
let ima_ext_0 = out[1].value;
let test = |mask| (ima_ext_0 & mask) != 0;
enable_feature(Feature::d, test(RISCV_HWPROBE_IMA_FD)); // F is implied.
enable_feature(Feature::c, test(RISCV_HWPROBE_IMA_C));
enable_feature(Feature::zihintntl, test(RISCV_HWPROBE_EXT_ZIHINTNTL));
enable_feature(Feature::zihintpause, test(RISCV_HWPROBE_EXT_ZIHINTPAUSE));
enable_feature(Feature::zimop, test(RISCV_HWPROBE_EXT_ZIMOP));
enable_feature(Feature::zicboz, test(RISCV_HWPROBE_EXT_ZICBOZ));
enable_feature(Feature::zicond, test(RISCV_HWPROBE_EXT_ZICOND));
enable_feature(Feature::zawrs, test(RISCV_HWPROBE_EXT_ZAWRS));
enable_feature(Feature::zacas, test(RISCV_HWPROBE_EXT_ZACAS));
enable_feature(Feature::ztso, test(RISCV_HWPROBE_EXT_ZTSO));
enable_feature(Feature::zba, test(RISCV_HWPROBE_EXT_ZBA));
enable_feature(Feature::zbb, test(RISCV_HWPROBE_EXT_ZBB));
enable_feature(Feature::zbs, test(RISCV_HWPROBE_EXT_ZBS));
enable_feature(Feature::zbc, test(RISCV_HWPROBE_EXT_ZBC));
enable_feature(Feature::zbkb, test(RISCV_HWPROBE_EXT_ZBKB));
enable_feature(Feature::zbkc, test(RISCV_HWPROBE_EXT_ZBKC));
enable_feature(Feature::zbkx, test(RISCV_HWPROBE_EXT_ZBKX));
enable_feature(Feature::zknd, test(RISCV_HWPROBE_EXT_ZKND));
enable_feature(Feature::zkne, test(RISCV_HWPROBE_EXT_ZKNE));
enable_feature(Feature::zknh, test(RISCV_HWPROBE_EXT_ZKNH));
enable_feature(Feature::zksed, test(RISCV_HWPROBE_EXT_ZKSED));
enable_feature(Feature::zksh, test(RISCV_HWPROBE_EXT_ZKSH));
enable_feature(Feature::zkt, test(RISCV_HWPROBE_EXT_ZKT));
enable_feature(Feature::zcmop, test(RISCV_HWPROBE_EXT_ZCMOP));
enable_feature(Feature::zca, test(RISCV_HWPROBE_EXT_ZCA));
enable_feature(Feature::zcf, test(RISCV_HWPROBE_EXT_ZCF));
enable_feature(Feature::zcd, test(RISCV_HWPROBE_EXT_ZCD));
enable_feature(Feature::zcb, test(RISCV_HWPROBE_EXT_ZCB));
enable_feature(Feature::zfh, test(RISCV_HWPROBE_EXT_ZFH));
enable_feature(Feature::zfhmin, test(RISCV_HWPROBE_EXT_ZFHMIN));
enable_feature(Feature::zfa, test(RISCV_HWPROBE_EXT_ZFA));
// Use prctl (if any) to determine whether the vector extension
// is enabled on the current thread (assuming the entire process
// share the same status). If prctl fails (e.g. QEMU userland emulator
// as of version 9.2.3), use auxiliary vector to retrieve the default
// vector status on the process startup.
let has_vectors = {
let v_status = unsafe { libc::prctl(PR_RISCV_V_GET_CONTROL) };
if v_status >= 0 {
(v_status & PR_RISCV_V_VSTATE_CTRL_CUR_MASK) == PR_RISCV_V_VSTATE_CTRL_ON
} else {
has_v
}
};
if has_vectors {
enable_feature(Feature::v, test(RISCV_HWPROBE_IMA_V));
enable_feature(Feature::zve32x, test(RISCV_HWPROBE_EXT_ZVE32X));
enable_feature(Feature::zve32f, test(RISCV_HWPROBE_EXT_ZVE32F));
enable_feature(Feature::zve64x, test(RISCV_HWPROBE_EXT_ZVE64X));
enable_feature(Feature::zve64f, test(RISCV_HWPROBE_EXT_ZVE64F));
enable_feature(Feature::zve64d, test(RISCV_HWPROBE_EXT_ZVE64D));
enable_feature(Feature::zvbb, test(RISCV_HWPROBE_EXT_ZVBB));
enable_feature(Feature::zvbc, test(RISCV_HWPROBE_EXT_ZVBC));
enable_feature(Feature::zvkb, test(RISCV_HWPROBE_EXT_ZVKB));
enable_feature(Feature::zvkg, test(RISCV_HWPROBE_EXT_ZVKG));
enable_feature(Feature::zvkned, test(RISCV_HWPROBE_EXT_ZVKNED));
enable_feature(Feature::zvknha, test(RISCV_HWPROBE_EXT_ZVKNHA));
enable_feature(Feature::zvknhb, test(RISCV_HWPROBE_EXT_ZVKNHB));
enable_feature(Feature::zvksed, test(RISCV_HWPROBE_EXT_ZVKSED));
enable_feature(Feature::zvksh, test(RISCV_HWPROBE_EXT_ZVKSH));
enable_feature(Feature::zvkt, test(RISCV_HWPROBE_EXT_ZVKT));
enable_feature(Feature::zvfh, test(RISCV_HWPROBE_EXT_ZVFH));
enable_feature(Feature::zvfhmin, test(RISCV_HWPROBE_EXT_ZVFHMIN));
}
is_v_set = true;
};
// Set V purely depending on the auxiliary vector
// only if no fine-grained vector extension detection is available.
if !is_v_set {
enable_feature(Feature::v, has_v);
}
// Handle base ISA.
let has_i = bit::test(auxv.hwcap, (b'i' - b'a').into());
// If future RV128I is supported, implement with `enable_feature` here.
// Note that we should use `target_arch` instead of `target_pointer_width`
// to avoid misdetection caused by experimental ABIs such as RV64ILP32.

View File

@ -62,25 +62,67 @@ pub(crate) fn imply_features(mut value: cache::Initializer) -> cache::Initialize
"defined as subset":
The latter extension is described as a subset of the former
(but the evidence is weak).
"functional":
The former extension is functionally a superset of the latter
(no direct references though).
*/
imply!(zvbb => zvkb);
// Certain set of vector cryptography extensions form a group.
group!(zvkn == zvkned & zvknhb & zvkb & zvkt);
group!(zvknc == zvkn & zvbc);
group!(zvkng == zvkn & zvkg);
group!(zvks == zvksed & zvksh & zvkb & zvkt);
group!(zvksc == zvks & zvbc);
group!(zvksg == zvks & zvkg);
imply!(zvknhb => zvknha); // functional
// For vector cryptography, Zvknhb and Zvbc require integer arithmetic
// with EEW=64 (Zve64x) while others not depending on them
// require EEW=32 (Zve32x).
imply!(zvknhb | zvbc => zve64x);
imply!(zvbb | zvkb | zvkg | zvkned | zvknha | zvksed | zvksh => zve32x);
imply!(zbc => zbkc); // defined as subset
group!(zkn == zbkb & zbkc & zbkx & zkne & zknd & zknh);
group!(zks == zbkb & zbkc & zbkx & zksed & zksh);
group!(zk == zkn & zkr & zkt);
imply!(zacas => zaamo);
group!(a == zalrsc & zaamo);
group!(b == zba & zbb & zbs);
imply!(zcf => zca & f);
imply!(zcd => zca & d);
imply!(zcmop | zcb => zca);
imply!(zhinx => zhinxmin);
imply!(zdinx | zhinxmin => zfinx);
imply!(zvfh => zvfhmin); // functional
imply!(zvfh => zve32f & zfhmin);
imply!(zvfhmin => zve32f);
imply!(v => zve64d);
imply!(zve64d => zve64f & d);
imply!(zve64f => zve64x & zve32f);
imply!(zve64x => zve32x);
imply!(zve32f => zve32x & f);
imply!(zfh => zfhmin);
imply!(q => d);
imply!(d | zfhmin => f);
imply!(d | zfhmin | zfa => f);
imply!(zicntr | zihpm | f | zfinx => zicsr);
// Relatively complex implication rules from the "C" extension.
imply!(c => zca);
imply!(c & d => zcd);
#[cfg(target_arch = "riscv32")]
imply!(c & f => zcf);
imply!(zicntr | zihpm | f | zfinx | zve32x => zicsr);
imply!(s | h => zicsr);
// Loop until the feature flags converge.
@ -110,6 +152,16 @@ mod tests {
assert!(imply_features(value).test(Feature::zicsr as u32));
}
#[test]
fn complex_zcd() {
let mut value = cache::Initializer::default();
// C & D -> Zcd
value.set(Feature::c as u32);
assert!(!imply_features(value).test(Feature::zcd as u32));
value.set(Feature::d as u32);
assert!(imply_features(value).test(Feature::zcd as u32));
}
#[test]
fn group_simple_forward() {
let mut value = cache::Initializer::default();
@ -132,17 +184,18 @@ mod tests {
#[test]
fn group_complex_convergence() {
let mut value = cache::Initializer::default();
// Needs 2 iterations to converge
// (and 3rd iteration for convergence checking):
// 1. [Zk] -> Zkn & Zkr & Zkt
// 2. Zkn -> {Zbkb} & {Zbkc} & {Zbkx} & {Zkne} & {Zknd} & {Zknh}
value.set(Feature::zk as u32);
// Needs 3 iterations to converge
// (and 4th iteration for convergence checking):
// 1. [Zvksc] -> Zvks & Zvbc
// 2. Zvks -> Zvksed & Zvksh & Zvkb & Zvkt
// 3a. [Zvkned] & [Zvknhb] & [Zvkb] & Zvkt -> {Zvkn}
// 3b. Zvkn & Zvbc -> {Zvknc}
value.set(Feature::zvksc as u32);
value.set(Feature::zvkned as u32);
value.set(Feature::zvknhb as u32);
value.set(Feature::zvkb as u32);
let value = imply_features(value);
assert!(value.test(Feature::zbkb as u32));
assert!(value.test(Feature::zbkc as u32));
assert!(value.test(Feature::zbkx as u32));
assert!(value.test(Feature::zkne as u32));
assert!(value.test(Feature::zknd as u32));
assert!(value.test(Feature::zknh as u32));
assert!(value.test(Feature::zvkn as u32));
assert!(value.test(Feature::zvknc as u32));
}
}

View File

@ -236,15 +236,29 @@ fn riscv_linux() {
println!("rv32e: {}", is_riscv_feature_detected!("rv32e"));
println!("rv64i: {}", is_riscv_feature_detected!("rv64i"));
println!("rv128i: {}", is_riscv_feature_detected!("rv128i"));
println!(
"unaligned-scalar-mem: {}",
is_riscv_feature_detected!("unaligned-scalar-mem")
);
println!(
"unaligned-vector-mem: {}",
is_riscv_feature_detected!("unaligned-vector-mem")
);
println!("zicsr: {}", is_riscv_feature_detected!("zicsr"));
println!("zicntr: {}", is_riscv_feature_detected!("zicntr"));
println!("zihpm: {}", is_riscv_feature_detected!("zihpm"));
println!("zifencei: {}", is_riscv_feature_detected!("zifencei"));
println!("zihintntl: {}", is_riscv_feature_detected!("zihintntl"));
println!("zihintpause: {}", is_riscv_feature_detected!("zihintpause"));
println!("zimop: {}", is_riscv_feature_detected!("zimop"));
println!("zicboz: {}", is_riscv_feature_detected!("zicboz"));
println!("zicond: {}", is_riscv_feature_detected!("zicond"));
println!("m: {}", is_riscv_feature_detected!("m"));
println!("a: {}", is_riscv_feature_detected!("a"));
println!("zalrsc: {}", is_riscv_feature_detected!("zalrsc"));
println!("zaamo: {}", is_riscv_feature_detected!("zaamo"));
println!("zawrs: {}", is_riscv_feature_detected!("zawrs"));
println!("zacas: {}", is_riscv_feature_detected!("zacas"));
println!("zam: {}", is_riscv_feature_detected!("zam"));
println!("ztso: {}", is_riscv_feature_detected!("ztso"));
println!("f: {}", is_riscv_feature_detected!("f"));
@ -252,11 +266,17 @@ fn riscv_linux() {
println!("q: {}", is_riscv_feature_detected!("q"));
println!("zfh: {}", is_riscv_feature_detected!("zfh"));
println!("zfhmin: {}", is_riscv_feature_detected!("zfhmin"));
println!("zfa: {}", is_riscv_feature_detected!("zfa"));
println!("zfinx: {}", is_riscv_feature_detected!("zfinx"));
println!("zdinx: {}", is_riscv_feature_detected!("zdinx"));
println!("zhinx: {}", is_riscv_feature_detected!("zhinx"));
println!("zhinxmin: {}", is_riscv_feature_detected!("zhinxmin"));
println!("c: {}", is_riscv_feature_detected!("c"));
println!("zca: {}", is_riscv_feature_detected!("zca"));
println!("zcf: {}", is_riscv_feature_detected!("zcf"));
println!("zcd: {}", is_riscv_feature_detected!("zcd"));
println!("zcb: {}", is_riscv_feature_detected!("zcb"));
println!("zcmop: {}", is_riscv_feature_detected!("zcmop"));
println!("b: {}", is_riscv_feature_detected!("b"));
println!("zba: {}", is_riscv_feature_detected!("zba"));
println!("zbb: {}", is_riscv_feature_detected!("zbb"));
@ -276,6 +296,29 @@ fn riscv_linux() {
println!("zk: {}", is_riscv_feature_detected!("zk"));
println!("zkt: {}", is_riscv_feature_detected!("zkt"));
println!("v: {}", is_riscv_feature_detected!("v"));
println!("zve32x: {}", is_riscv_feature_detected!("zve32x"));
println!("zve32f: {}", is_riscv_feature_detected!("zve32f"));
println!("zve64x: {}", is_riscv_feature_detected!("zve64x"));
println!("zve64f: {}", is_riscv_feature_detected!("zve64f"));
println!("zve64d: {}", is_riscv_feature_detected!("zve64d"));
println!("zvfh: {}", is_riscv_feature_detected!("zvfh"));
println!("zvfhmin: {}", is_riscv_feature_detected!("zvfhmin"));
println!("zvbb: {}", is_riscv_feature_detected!("zvbb"));
println!("zvbc: {}", is_riscv_feature_detected!("zvbc"));
println!("zvkb: {}", is_riscv_feature_detected!("zvkb"));
println!("zvkg: {}", is_riscv_feature_detected!("zvkg"));
println!("zvkned: {}", is_riscv_feature_detected!("zvkned"));
println!("zvknha: {}", is_riscv_feature_detected!("zvknha"));
println!("zvknhb: {}", is_riscv_feature_detected!("zvknhb"));
println!("zvksed: {}", is_riscv_feature_detected!("zvksed"));
println!("zvksh: {}", is_riscv_feature_detected!("zvksh"));
println!("zvkn: {}", is_riscv_feature_detected!("zvkn"));
println!("zvknc: {}", is_riscv_feature_detected!("zvknc"));
println!("zvkng: {}", is_riscv_feature_detected!("zvkng"));
println!("zvks: {}", is_riscv_feature_detected!("zvks"));
println!("zvksc: {}", is_riscv_feature_detected!("zvksc"));
println!("zvksg: {}", is_riscv_feature_detected!("zvksg"));
println!("zvkt: {}", is_riscv_feature_detected!("zvkt"));
println!("svnapot: {}", is_riscv_feature_detected!("svnapot"));
println!("svpbmt: {}", is_riscv_feature_detected!("svpbmt"));
println!("svinval: {}", is_riscv_feature_detected!("svinval"));