From 51df7aabbe3140ccca1f63617d7cfdb883509239 Mon Sep 17 00:00:00 2001 From: Folkert de Vries Date: Tue, 19 Aug 2025 21:12:53 +0200 Subject: [PATCH 1/2] add a fallback implementation for the `prefetch_*` intrinsics The fallback is to just ignore the arguments. That is a valid implementation because this intrinsic is just a hint. I also added `miri::intrinsic_fallback_is_spec` annotation, so that miri now supports these operations. A prefetch intrinsic call is valid on any pointer. --- library/core/src/intrinsics/mod.rs | 39 +++++++++++++++++++++------ src/tools/miri/tests/pass/prefetch.rs | 23 ++++++++++++++++ 2 files changed, 54 insertions(+), 8 deletions(-) create mode 100644 src/tools/miri/tests/pass/prefetch.rs diff --git a/library/core/src/intrinsics/mod.rs b/library/core/src/intrinsics/mod.rs index dd838d494bc..346c7825b74 100644 --- a/library/core/src/intrinsics/mod.rs +++ b/library/core/src/intrinsics/mod.rs @@ -261,7 +261,7 @@ pub unsafe fn atomic_fence(); pub unsafe fn atomic_singlethreadfence(); /// The `prefetch` intrinsic is a hint to the code generator to insert a prefetch instruction -/// if supported; otherwise, it is a no-op. +/// for the given address if supported; otherwise, it is a no-op. /// Prefetches have no effect on the behavior of the program but can change its performance /// characteristics. /// @@ -271,9 +271,15 @@ pub unsafe fn atomic_singlethreadfence(); /// This intrinsic does not have a stable counterpart. #[rustc_intrinsic] #[rustc_nounwind] -pub unsafe fn prefetch_read_data(data: *const T, locality: i32); +#[miri::intrinsic_fallback_is_spec] +pub unsafe fn prefetch_read_data(data: *const T, locality: i32) { + // This operation is a no-op, unless it is overridden by the backend. + let _ = data; + let _ = locality; +} + /// The `prefetch` intrinsic is a hint to the code generator to insert a prefetch instruction -/// if supported; otherwise, it is a no-op. +/// for the given address if supported; otherwise, it is a no-op. /// Prefetches have no effect on the behavior of the program but can change its performance /// characteristics. /// @@ -283,9 +289,15 @@ pub unsafe fn prefetch_read_data(data: *const T, locality: i32); /// This intrinsic does not have a stable counterpart. #[rustc_intrinsic] #[rustc_nounwind] -pub unsafe fn prefetch_write_data(data: *const T, locality: i32); +#[miri::intrinsic_fallback_is_spec] +pub unsafe fn prefetch_write_data(data: *const T, locality: i32) { + // This operation is a no-op, unless it is overridden by the backend. + let _ = data; + let _ = locality; +} + /// The `prefetch` intrinsic is a hint to the code generator to insert a prefetch instruction -/// if supported; otherwise, it is a no-op. +/// for the given address if supported; otherwise, it is a no-op. /// Prefetches have no effect on the behavior of the program but can change its performance /// characteristics. /// @@ -295,9 +307,15 @@ pub unsafe fn prefetch_write_data(data: *const T, locality: i32); /// This intrinsic does not have a stable counterpart. #[rustc_intrinsic] #[rustc_nounwind] -pub unsafe fn prefetch_read_instruction(data: *const T, locality: i32); +#[miri::intrinsic_fallback_is_spec] +pub unsafe fn prefetch_read_instruction(data: *const T, locality: i32) { + // This operation is a no-op, unless it is overridden by the backend. + let _ = data; + let _ = locality; +} + /// The `prefetch` intrinsic is a hint to the code generator to insert a prefetch instruction -/// if supported; otherwise, it is a no-op. +/// for the given address if supported; otherwise, it is a no-op. /// Prefetches have no effect on the behavior of the program but can change its performance /// characteristics. /// @@ -307,7 +325,12 @@ pub unsafe fn prefetch_read_instruction(data: *const T, locality: i32); /// This intrinsic does not have a stable counterpart. #[rustc_intrinsic] #[rustc_nounwind] -pub unsafe fn prefetch_write_instruction(data: *const T, locality: i32); +#[miri::intrinsic_fallback_is_spec] +pub unsafe fn prefetch_write_instruction(data: *const T, locality: i32) { + // This operation is a no-op, unless it is overridden by the backend. + let _ = data; + let _ = locality; +} /// Executes a breakpoint trap, for inspection by a debugger. /// diff --git a/src/tools/miri/tests/pass/prefetch.rs b/src/tools/miri/tests/pass/prefetch.rs new file mode 100644 index 00000000000..09352695989 --- /dev/null +++ b/src/tools/miri/tests/pass/prefetch.rs @@ -0,0 +1,23 @@ +#![feature(core_intrinsics)] + +fn main() { + static X: [u8; 8] = [0; 8]; + + unsafe { + ::std::intrinsics::prefetch_read_data(::std::ptr::null::(), 1); + ::std::intrinsics::prefetch_read_data(::std::ptr::dangling::(), 2); + ::std::intrinsics::prefetch_read_data(X.as_ptr(), 3); + + ::std::intrinsics::prefetch_write_data(::std::ptr::null::(), 1); + ::std::intrinsics::prefetch_write_data(::std::ptr::dangling::(), 2); + ::std::intrinsics::prefetch_write_data(X.as_ptr(), 3); + + ::std::intrinsics::prefetch_read_instruction(::std::ptr::null::(), 1); + ::std::intrinsics::prefetch_read_instruction(::std::ptr::dangling::(), 2); + ::std::intrinsics::prefetch_read_instruction(X.as_ptr(), 3); + + ::std::intrinsics::prefetch_write_instruction(::std::ptr::null::(), 1); + ::std::intrinsics::prefetch_write_instruction(::std::ptr::dangling::(), 2); + ::std::intrinsics::prefetch_write_instruction(X.as_ptr(), 3); + } +} From d25910eaeb3eab25d3bbf9b9815e6d215c202d1f Mon Sep 17 00:00:00 2001 From: Folkert de Vries Date: Tue, 19 Aug 2025 22:16:58 +0200 Subject: [PATCH 2/2] make `prefetch` intrinsics safe --- compiler/rustc_codegen_llvm/src/intrinsic.rs | 8 ++- .../rustc_hir_analysis/src/check/intrinsic.rs | 6 +- library/core/src/intrinsics/mod.rs | 28 ++++---- src/tools/miri/tests/pass/prefetch.rs | 28 ++++---- tests/codegen-llvm/intrinsics/prefetch.rs | 72 +++++++++---------- 5 files changed, 70 insertions(+), 72 deletions(-) diff --git a/compiler/rustc_codegen_llvm/src/intrinsic.rs b/compiler/rustc_codegen_llvm/src/intrinsic.rs index 4935f8d7dff..06c3d8ed6bc 100644 --- a/compiler/rustc_codegen_llvm/src/intrinsic.rs +++ b/compiler/rustc_codegen_llvm/src/intrinsic.rs @@ -330,10 +330,16 @@ impl<'ll, 'tcx> IntrinsicCallBuilderMethods<'tcx> for Builder<'_, 'll, 'tcx> { _ => bug!(), }; let ptr = args[0].immediate(); + let locality = fn_args.const_at(1).to_value().valtree.unwrap_leaf().to_u32() as i32; self.call_intrinsic( "llvm.prefetch", &[self.val_ty(ptr)], - &[ptr, self.const_i32(rw), args[1].immediate(), self.const_i32(cache_type)], + &[ + ptr, + self.const_i32(rw), + self.const_i32(locality), + self.const_i32(cache_type), + ], ) } sym::carrying_mul_add => { diff --git a/compiler/rustc_hir_analysis/src/check/intrinsic.rs b/compiler/rustc_hir_analysis/src/check/intrinsic.rs index f50aed0b3c2..cfc6bc2f3a0 100644 --- a/compiler/rustc_hir_analysis/src/check/intrinsic.rs +++ b/compiler/rustc_hir_analysis/src/check/intrinsic.rs @@ -136,6 +136,10 @@ fn intrinsic_operation_unsafety(tcx: TyCtxt<'_>, intrinsic_id: LocalDefId) -> hi | sym::round_ties_even_f64 | sym::round_ties_even_f128 | sym::autodiff + | sym::prefetch_read_data + | sym::prefetch_write_data + | sym::prefetch_read_instruction + | sym::prefetch_write_instruction | sym::const_eval_select => hir::Safety::Safe, _ => hir::Safety::Unsafe, }; @@ -218,7 +222,7 @@ pub(crate) fn check_intrinsic_type( | sym::prefetch_write_data | sym::prefetch_read_instruction | sym::prefetch_write_instruction => { - (1, 0, vec![Ty::new_imm_ptr(tcx, param(0)), tcx.types.i32], tcx.types.unit) + (1, 1, vec![Ty::new_imm_ptr(tcx, param(0))], tcx.types.unit) } sym::needs_drop => (1, 0, vec![], tcx.types.bool), diff --git a/library/core/src/intrinsics/mod.rs b/library/core/src/intrinsics/mod.rs index 346c7825b74..904aa52c784 100644 --- a/library/core/src/intrinsics/mod.rs +++ b/library/core/src/intrinsics/mod.rs @@ -265,17 +265,16 @@ pub unsafe fn atomic_singlethreadfence(); /// Prefetches have no effect on the behavior of the program but can change its performance /// characteristics. /// -/// The `locality` argument must be a constant integer and is a temporal locality specifier -/// ranging from (0) - no locality, to (3) - extremely local keep in cache. +/// The `LOCALITY` argument is a temporal locality specifier ranging from (0) - no locality, +/// to (3) - extremely local keep in cache. /// /// This intrinsic does not have a stable counterpart. #[rustc_intrinsic] #[rustc_nounwind] #[miri::intrinsic_fallback_is_spec] -pub unsafe fn prefetch_read_data(data: *const T, locality: i32) { +pub const fn prefetch_read_data(data: *const T) { // This operation is a no-op, unless it is overridden by the backend. let _ = data; - let _ = locality; } /// The `prefetch` intrinsic is a hint to the code generator to insert a prefetch instruction @@ -283,17 +282,16 @@ pub unsafe fn prefetch_read_data(data: *const T, locality: i32) { /// Prefetches have no effect on the behavior of the program but can change its performance /// characteristics. /// -/// The `locality` argument must be a constant integer and is a temporal locality specifier -/// ranging from (0) - no locality, to (3) - extremely local keep in cache. +/// The `LOCALITY` argument is a temporal locality specifier ranging from (0) - no locality, +/// to (3) - extremely local keep in cache. /// /// This intrinsic does not have a stable counterpart. #[rustc_intrinsic] #[rustc_nounwind] #[miri::intrinsic_fallback_is_spec] -pub unsafe fn prefetch_write_data(data: *const T, locality: i32) { +pub const fn prefetch_write_data(data: *const T) { // This operation is a no-op, unless it is overridden by the backend. let _ = data; - let _ = locality; } /// The `prefetch` intrinsic is a hint to the code generator to insert a prefetch instruction @@ -301,17 +299,16 @@ pub unsafe fn prefetch_write_data(data: *const T, locality: i32) { /// Prefetches have no effect on the behavior of the program but can change its performance /// characteristics. /// -/// The `locality` argument must be a constant integer and is a temporal locality specifier -/// ranging from (0) - no locality, to (3) - extremely local keep in cache. +/// The `LOCALITY` argument is a temporal locality specifier ranging from (0) - no locality, +/// to (3) - extremely local keep in cache. /// /// This intrinsic does not have a stable counterpart. #[rustc_intrinsic] #[rustc_nounwind] #[miri::intrinsic_fallback_is_spec] -pub unsafe fn prefetch_read_instruction(data: *const T, locality: i32) { +pub const fn prefetch_read_instruction(data: *const T) { // This operation is a no-op, unless it is overridden by the backend. let _ = data; - let _ = locality; } /// The `prefetch` intrinsic is a hint to the code generator to insert a prefetch instruction @@ -319,17 +316,16 @@ pub unsafe fn prefetch_read_instruction(data: *const T, locality: i32) { /// Prefetches have no effect on the behavior of the program but can change its performance /// characteristics. /// -/// The `locality` argument must be a constant integer and is a temporal locality specifier -/// ranging from (0) - no locality, to (3) - extremely local keep in cache. +/// The `LOCALITY` argument is a temporal locality specifier ranging from (0) - no locality, +/// to (3) - extremely local keep in cache. /// /// This intrinsic does not have a stable counterpart. #[rustc_intrinsic] #[rustc_nounwind] #[miri::intrinsic_fallback_is_spec] -pub unsafe fn prefetch_write_instruction(data: *const T, locality: i32) { +pub const fn prefetch_write_instruction(data: *const T) { // This operation is a no-op, unless it is overridden by the backend. let _ = data; - let _ = locality; } /// Executes a breakpoint trap, for inspection by a debugger. diff --git a/src/tools/miri/tests/pass/prefetch.rs b/src/tools/miri/tests/pass/prefetch.rs index 09352695989..99c75c38bde 100644 --- a/src/tools/miri/tests/pass/prefetch.rs +++ b/src/tools/miri/tests/pass/prefetch.rs @@ -1,23 +1,23 @@ #![feature(core_intrinsics)] +// Test that these intrinsics work. Their behavior should be a no-op. + fn main() { static X: [u8; 8] = [0; 8]; - unsafe { - ::std::intrinsics::prefetch_read_data(::std::ptr::null::(), 1); - ::std::intrinsics::prefetch_read_data(::std::ptr::dangling::(), 2); - ::std::intrinsics::prefetch_read_data(X.as_ptr(), 3); + ::std::intrinsics::prefetch_read_data::<_, 1>(::std::ptr::null::()); + ::std::intrinsics::prefetch_read_data::<_, 2>(::std::ptr::dangling::()); + ::std::intrinsics::prefetch_read_data::<_, 3>(X.as_ptr()); - ::std::intrinsics::prefetch_write_data(::std::ptr::null::(), 1); - ::std::intrinsics::prefetch_write_data(::std::ptr::dangling::(), 2); - ::std::intrinsics::prefetch_write_data(X.as_ptr(), 3); + ::std::intrinsics::prefetch_write_data::<_, 1>(::std::ptr::null::()); + ::std::intrinsics::prefetch_write_data::<_, 2>(::std::ptr::dangling::()); + ::std::intrinsics::prefetch_write_data::<_, 3>(X.as_ptr()); - ::std::intrinsics::prefetch_read_instruction(::std::ptr::null::(), 1); - ::std::intrinsics::prefetch_read_instruction(::std::ptr::dangling::(), 2); - ::std::intrinsics::prefetch_read_instruction(X.as_ptr(), 3); + ::std::intrinsics::prefetch_read_instruction::<_, 1>(::std::ptr::null::()); + ::std::intrinsics::prefetch_read_instruction::<_, 2>(::std::ptr::dangling::()); + ::std::intrinsics::prefetch_read_instruction::<_, 3>(X.as_ptr()); - ::std::intrinsics::prefetch_write_instruction(::std::ptr::null::(), 1); - ::std::intrinsics::prefetch_write_instruction(::std::ptr::dangling::(), 2); - ::std::intrinsics::prefetch_write_instruction(X.as_ptr(), 3); - } + ::std::intrinsics::prefetch_write_instruction::<_, 1>(::std::ptr::null::()); + ::std::intrinsics::prefetch_write_instruction::<_, 2>(::std::ptr::dangling::()); + ::std::intrinsics::prefetch_write_instruction::<_, 3>(X.as_ptr()); } diff --git a/tests/codegen-llvm/intrinsics/prefetch.rs b/tests/codegen-llvm/intrinsics/prefetch.rs index 3f9f21c85cb..41877872019 100644 --- a/tests/codegen-llvm/intrinsics/prefetch.rs +++ b/tests/codegen-llvm/intrinsics/prefetch.rs @@ -9,56 +9,48 @@ use std::intrinsics::{ #[no_mangle] pub fn check_prefetch_read_data(data: &[i8]) { - unsafe { - // CHECK: call void @llvm.prefetch{{.*}}({{.*}}, i32 0, i32 0, i32 1) - prefetch_read_data(data.as_ptr(), 0); - // CHECK: call void @llvm.prefetch{{.*}}({{.*}}, i32 0, i32 1, i32 1) - prefetch_read_data(data.as_ptr(), 1); - // CHECK: call void @llvm.prefetch{{.*}}({{.*}}, i32 0, i32 2, i32 1) - prefetch_read_data(data.as_ptr(), 2); - // CHECK: call void @llvm.prefetch{{.*}}({{.*}}, i32 0, i32 3, i32 1) - prefetch_read_data(data.as_ptr(), 3); - } + // CHECK: call void @llvm.prefetch{{.*}}({{.*}}, i32 0, i32 0, i32 1) + prefetch_read_data::<_, 0>(data.as_ptr()); + // CHECK: call void @llvm.prefetch{{.*}}({{.*}}, i32 0, i32 1, i32 1) + prefetch_read_data::<_, 1>(data.as_ptr()); + // CHECK: call void @llvm.prefetch{{.*}}({{.*}}, i32 0, i32 2, i32 1) + prefetch_read_data::<_, 2>(data.as_ptr()); + // CHECK: call void @llvm.prefetch{{.*}}({{.*}}, i32 0, i32 3, i32 1) + prefetch_read_data::<_, 3>(data.as_ptr()); } #[no_mangle] pub fn check_prefetch_write_data(data: &[i8]) { - unsafe { - // CHECK: call void @llvm.prefetch{{.*}}({{.*}}, i32 1, i32 0, i32 1) - prefetch_write_data(data.as_ptr(), 0); - // CHECK: call void @llvm.prefetch{{.*}}({{.*}}, i32 1, i32 1, i32 1) - prefetch_write_data(data.as_ptr(), 1); - // CHECK: call void @llvm.prefetch{{.*}}({{.*}}, i32 1, i32 2, i32 1) - prefetch_write_data(data.as_ptr(), 2); - // CHECK: call void @llvm.prefetch{{.*}}({{.*}}, i32 1, i32 3, i32 1) - prefetch_write_data(data.as_ptr(), 3); - } + // CHECK: call void @llvm.prefetch{{.*}}({{.*}}, i32 1, i32 0, i32 1) + prefetch_write_data::<_, 0>(data.as_ptr()); + // CHECK: call void @llvm.prefetch{{.*}}({{.*}}, i32 1, i32 1, i32 1) + prefetch_write_data::<_, 1>(data.as_ptr()); + // CHECK: call void @llvm.prefetch{{.*}}({{.*}}, i32 1, i32 2, i32 1) + prefetch_write_data::<_, 2>(data.as_ptr()); + // CHECK: call void @llvm.prefetch{{.*}}({{.*}}, i32 1, i32 3, i32 1) + prefetch_write_data::<_, 3>(data.as_ptr()); } #[no_mangle] pub fn check_prefetch_read_instruction(data: &[i8]) { - unsafe { - // CHECK: call void @llvm.prefetch{{.*}}({{.*}}, i32 0, i32 0, i32 0) - prefetch_read_instruction(data.as_ptr(), 0); - // CHECK: call void @llvm.prefetch{{.*}}({{.*}}, i32 0, i32 1, i32 0) - prefetch_read_instruction(data.as_ptr(), 1); - // CHECK: call void @llvm.prefetch{{.*}}({{.*}}, i32 0, i32 2, i32 0) - prefetch_read_instruction(data.as_ptr(), 2); - // CHECK: call void @llvm.prefetch{{.*}}({{.*}}, i32 0, i32 3, i32 0) - prefetch_read_instruction(data.as_ptr(), 3); - } + // CHECK: call void @llvm.prefetch{{.*}}({{.*}}, i32 0, i32 0, i32 0) + prefetch_read_instruction::<_, 0>(data.as_ptr()); + // CHECK: call void @llvm.prefetch{{.*}}({{.*}}, i32 0, i32 1, i32 0) + prefetch_read_instruction::<_, 1>(data.as_ptr()); + // CHECK: call void @llvm.prefetch{{.*}}({{.*}}, i32 0, i32 2, i32 0) + prefetch_read_instruction::<_, 2>(data.as_ptr()); + // CHECK: call void @llvm.prefetch{{.*}}({{.*}}, i32 0, i32 3, i32 0) + prefetch_read_instruction::<_, 3>(data.as_ptr()); } #[no_mangle] pub fn check_prefetch_write_instruction(data: &[i8]) { - unsafe { - // CHECK: call void @llvm.prefetch{{.*}}({{.*}}, i32 1, i32 0, i32 0) - prefetch_write_instruction(data.as_ptr(), 0); - // CHECK: call void @llvm.prefetch{{.*}}({{.*}}, i32 1, i32 1, i32 0) - prefetch_write_instruction(data.as_ptr(), 1); - // CHECK: call void @llvm.prefetch{{.*}}({{.*}}, i32 1, i32 2, i32 0) - prefetch_write_instruction(data.as_ptr(), 2); - // CHECK: call void @llvm.prefetch{{.*}}({{.*}}, i32 1, i32 3, i32 0) - prefetch_write_instruction(data.as_ptr(), 3); - } + // CHECK: call void @llvm.prefetch{{.*}}({{.*}}, i32 1, i32 0, i32 0) + prefetch_write_instruction::<_, 0>(data.as_ptr()); + // CHECK: call void @llvm.prefetch{{.*}}({{.*}}, i32 1, i32 1, i32 0) + prefetch_write_instruction::<_, 1>(data.as_ptr()); + // CHECK: call void @llvm.prefetch{{.*}}({{.*}}, i32 1, i32 2, i32 0) + prefetch_write_instruction::<_, 2>(data.as_ptr()); + // CHECK: call void @llvm.prefetch{{.*}}({{.*}}, i32 1, i32 3, i32 0) + prefetch_write_instruction::<_, 3>(data.as_ptr()); }