Rollup merge of #145626 - folkertdev:prefetch-fallback, r=Amanieu

add a fallback implementation for the `prefetch_*` intrinsics

related ACP: https://github.com/rust-lang/libs-team/issues/638

The fallback is to just ignore the arguments. That is a valid implementation because this intrinsic is just a hint.

I also added the `miri::intrinsic_fallback_is_spec` annotation, so that miri now supports these operations. A prefetch intrinsic call is valid on any pointer. (specifically LLVM guarantees this https://llvm.org/docs/LangRef.html#llvm-prefetch-intrinsic)

Next, I made the `LOCALITY` argument a const generic. That argument must be const (otherwise LLVM crashes), but that was not reflected in the type.

Finally, with these changes, the intrinsic can be safe and `const` (a prefetch at const evaluation time is just a no-op).

cc `@Amanieu`
r? `@RalfJung`

compiler/rustc_codegen_llvm/src/intrinsic.rs

@@ -330,10 +330,16 @@ impl<'ll, 'tcx> IntrinsicCallBuilderMethods<'tcx> for Builder<'_, 'll, 'tcx> {
                     _ => bug!(),
                 };
                 let ptr = args[0].immediate();
+                let locality = fn_args.const_at(1).to_value().valtree.unwrap_leaf().to_u32() as i32;
                 self.call_intrinsic(
                     "llvm.prefetch",
                     &[self.val_ty(ptr)],
-                    &[ptr, self.const_i32(rw), args[1].immediate(), self.const_i32(cache_type)],
+                    &[
+                        ptr,
+                        self.const_i32(rw),
+                        self.const_i32(locality),
+                        self.const_i32(cache_type),
+                    ],
                 )
             }
             sym::carrying_mul_add => {

compiler/rustc_hir_analysis/src/check/intrinsic.rs

@@ -136,6 +136,10 @@ fn intrinsic_operation_unsafety(tcx: TyCtxt<'_>, intrinsic_id: LocalDefId) -> hi
         | sym::round_ties_even_f64
         | sym::round_ties_even_f128
         | sym::autodiff
+        | sym::prefetch_read_data
+        | sym::prefetch_write_data
+        | sym::prefetch_read_instruction
+        | sym::prefetch_write_instruction
         | sym::const_eval_select => hir::Safety::Safe,
         _ => hir::Safety::Unsafe,
     };
@@ -218,7 +222,7 @@ pub(crate) fn check_intrinsic_type(
         | sym::prefetch_write_data
         | sym::prefetch_read_instruction
         | sym::prefetch_write_instruction => {
-            (1, 0, vec![Ty::new_imm_ptr(tcx, param(0)), tcx.types.i32], tcx.types.unit)
+            (1, 1, vec![Ty::new_imm_ptr(tcx, param(0))], tcx.types.unit)
         }
         sym::needs_drop => (1, 0, vec![], tcx.types.bool),
 

library/core/src/intrinsics/mod.rs

@@ -261,53 +261,72 @@ pub unsafe fn atomic_fence<const ORD: AtomicOrdering>();
 pub unsafe fn atomic_singlethreadfence<const ORD: AtomicOrdering>();
 
 /// The `prefetch` intrinsic is a hint to the code generator to insert a prefetch instruction
-/// if supported; otherwise, it is a no-op.
+/// for the given address if supported; otherwise, it is a no-op.
 /// Prefetches have no effect on the behavior of the program but can change its performance
 /// characteristics.
 ///
-/// The `locality` argument must be a constant integer and is a temporal locality specifier
-/// ranging from (0) - no locality, to (3) - extremely local keep in cache.
+/// The `LOCALITY` argument is a temporal locality specifier ranging from (0) - no locality,
+/// to (3) - extremely local keep in cache.
 ///
 /// This intrinsic does not have a stable counterpart.
 #[rustc_intrinsic]
 #[rustc_nounwind]
-pub unsafe fn prefetch_read_data<T>(data: *const T, locality: i32);
+#[miri::intrinsic_fallback_is_spec]
+pub const fn prefetch_read_data<T, const LOCALITY: i32>(data: *const T) {
+    // This operation is a no-op, unless it is overridden by the backend.
+    let _ = data;
+}
+
 /// The `prefetch` intrinsic is a hint to the code generator to insert a prefetch instruction
-/// if supported; otherwise, it is a no-op.
+/// for the given address if supported; otherwise, it is a no-op.
 /// Prefetches have no effect on the behavior of the program but can change its performance
 /// characteristics.
 ///
-/// The `locality` argument must be a constant integer and is a temporal locality specifier
-/// ranging from (0) - no locality, to (3) - extremely local keep in cache.
+/// The `LOCALITY` argument is a temporal locality specifier ranging from (0) - no locality,
+/// to (3) - extremely local keep in cache.
 ///
 /// This intrinsic does not have a stable counterpart.
 #[rustc_intrinsic]
 #[rustc_nounwind]
-pub unsafe fn prefetch_write_data<T>(data: *const T, locality: i32);
+#[miri::intrinsic_fallback_is_spec]
+pub const fn prefetch_write_data<T, const LOCALITY: i32>(data: *const T) {
+    // This operation is a no-op, unless it is overridden by the backend.
+    let _ = data;
+}
+
 /// The `prefetch` intrinsic is a hint to the code generator to insert a prefetch instruction
-/// if supported; otherwise, it is a no-op.
+/// for the given address if supported; otherwise, it is a no-op.
 /// Prefetches have no effect on the behavior of the program but can change its performance
 /// characteristics.
 ///
-/// The `locality` argument must be a constant integer and is a temporal locality specifier
-/// ranging from (0) - no locality, to (3) - extremely local keep in cache.
+/// The `LOCALITY` argument is a temporal locality specifier ranging from (0) - no locality,
+/// to (3) - extremely local keep in cache.
 ///
 /// This intrinsic does not have a stable counterpart.
 #[rustc_intrinsic]
 #[rustc_nounwind]
-pub unsafe fn prefetch_read_instruction<T>(data: *const T, locality: i32);
+#[miri::intrinsic_fallback_is_spec]
+pub const fn prefetch_read_instruction<T, const LOCALITY: i32>(data: *const T) {
+    // This operation is a no-op, unless it is overridden by the backend.
+    let _ = data;
+}
+
 /// The `prefetch` intrinsic is a hint to the code generator to insert a prefetch instruction
-/// if supported; otherwise, it is a no-op.
+/// for the given address if supported; otherwise, it is a no-op.
 /// Prefetches have no effect on the behavior of the program but can change its performance
 /// characteristics.
 ///
-/// The `locality` argument must be a constant integer and is a temporal locality specifier
-/// ranging from (0) - no locality, to (3) - extremely local keep in cache.
+/// The `LOCALITY` argument is a temporal locality specifier ranging from (0) - no locality,
+/// to (3) - extremely local keep in cache.
 ///
 /// This intrinsic does not have a stable counterpart.
 #[rustc_intrinsic]
 #[rustc_nounwind]
-pub unsafe fn prefetch_write_instruction<T>(data: *const T, locality: i32);
+#[miri::intrinsic_fallback_is_spec]
+pub const fn prefetch_write_instruction<T, const LOCALITY: i32>(data: *const T) {
+    // This operation is a no-op, unless it is overridden by the backend.
+    let _ = data;
+}
 
 /// Executes a breakpoint trap, for inspection by a debugger.
 ///

src/tools/miri/tests/pass/prefetch.rs

@@ -0,0 +1,23 @@
+#![feature(core_intrinsics)]
+
+// Test that these intrinsics work. Their behavior should be a no-op.
+
+fn main() {
+    static X: [u8; 8] = [0; 8];
+
+    ::std::intrinsics::prefetch_read_data::<_, 1>(::std::ptr::null::<u8>());
+    ::std::intrinsics::prefetch_read_data::<_, 2>(::std::ptr::dangling::<u8>());
+    ::std::intrinsics::prefetch_read_data::<_, 3>(X.as_ptr());
+
+    ::std::intrinsics::prefetch_write_data::<_, 1>(::std::ptr::null::<u8>());
+    ::std::intrinsics::prefetch_write_data::<_, 2>(::std::ptr::dangling::<u8>());
+    ::std::intrinsics::prefetch_write_data::<_, 3>(X.as_ptr());
+
+    ::std::intrinsics::prefetch_read_instruction::<_, 1>(::std::ptr::null::<u8>());
+    ::std::intrinsics::prefetch_read_instruction::<_, 2>(::std::ptr::dangling::<u8>());
+    ::std::intrinsics::prefetch_read_instruction::<_, 3>(X.as_ptr());
+
+    ::std::intrinsics::prefetch_write_instruction::<_, 1>(::std::ptr::null::<u8>());
+    ::std::intrinsics::prefetch_write_instruction::<_, 2>(::std::ptr::dangling::<u8>());
+    ::std::intrinsics::prefetch_write_instruction::<_, 3>(X.as_ptr());
+}

tests/codegen-llvm/intrinsics/prefetch.rs

@@ -9,56 +9,48 @@ use std::intrinsics::{
 
 #[no_mangle]
 pub fn check_prefetch_read_data(data: &[i8]) {
-    unsafe {
-        // CHECK: call void @llvm.prefetch{{.*}}({{.*}}, i32 0, i32 0, i32 1)
-        prefetch_read_data(data.as_ptr(), 0);
-        // CHECK: call void @llvm.prefetch{{.*}}({{.*}}, i32 0, i32 1, i32 1)
-        prefetch_read_data(data.as_ptr(), 1);
-        // CHECK: call void @llvm.prefetch{{.*}}({{.*}}, i32 0, i32 2, i32 1)
-        prefetch_read_data(data.as_ptr(), 2);
-        // CHECK: call void @llvm.prefetch{{.*}}({{.*}}, i32 0, i32 3, i32 1)
-        prefetch_read_data(data.as_ptr(), 3);
-    }
+    // CHECK: call void @llvm.prefetch{{.*}}({{.*}}, i32 0, i32 0, i32 1)
+    prefetch_read_data::<_, 0>(data.as_ptr());
+    // CHECK: call void @llvm.prefetch{{.*}}({{.*}}, i32 0, i32 1, i32 1)
+    prefetch_read_data::<_, 1>(data.as_ptr());
+    // CHECK: call void @llvm.prefetch{{.*}}({{.*}}, i32 0, i32 2, i32 1)
+    prefetch_read_data::<_, 2>(data.as_ptr());
+    // CHECK: call void @llvm.prefetch{{.*}}({{.*}}, i32 0, i32 3, i32 1)
+    prefetch_read_data::<_, 3>(data.as_ptr());
 }
 
 #[no_mangle]
 pub fn check_prefetch_write_data(data: &[i8]) {
-    unsafe {
-        // CHECK: call void @llvm.prefetch{{.*}}({{.*}}, i32 1, i32 0, i32 1)
-        prefetch_write_data(data.as_ptr(), 0);
-        // CHECK: call void @llvm.prefetch{{.*}}({{.*}}, i32 1, i32 1, i32 1)
-        prefetch_write_data(data.as_ptr(), 1);
-        // CHECK: call void @llvm.prefetch{{.*}}({{.*}}, i32 1, i32 2, i32 1)
-        prefetch_write_data(data.as_ptr(), 2);
-        // CHECK: call void @llvm.prefetch{{.*}}({{.*}}, i32 1, i32 3, i32 1)
-        prefetch_write_data(data.as_ptr(), 3);
-    }
+    // CHECK: call void @llvm.prefetch{{.*}}({{.*}}, i32 1, i32 0, i32 1)
+    prefetch_write_data::<_, 0>(data.as_ptr());
+    // CHECK: call void @llvm.prefetch{{.*}}({{.*}}, i32 1, i32 1, i32 1)
+    prefetch_write_data::<_, 1>(data.as_ptr());
+    // CHECK: call void @llvm.prefetch{{.*}}({{.*}}, i32 1, i32 2, i32 1)
+    prefetch_write_data::<_, 2>(data.as_ptr());
+    // CHECK: call void @llvm.prefetch{{.*}}({{.*}}, i32 1, i32 3, i32 1)
+    prefetch_write_data::<_, 3>(data.as_ptr());
 }
 
 #[no_mangle]
 pub fn check_prefetch_read_instruction(data: &[i8]) {
-    unsafe {
-        // CHECK: call void @llvm.prefetch{{.*}}({{.*}}, i32 0, i32 0, i32 0)
-        prefetch_read_instruction(data.as_ptr(), 0);
-        // CHECK: call void @llvm.prefetch{{.*}}({{.*}}, i32 0, i32 1, i32 0)
-        prefetch_read_instruction(data.as_ptr(), 1);
-        // CHECK: call void @llvm.prefetch{{.*}}({{.*}}, i32 0, i32 2, i32 0)
-        prefetch_read_instruction(data.as_ptr(), 2);
-        // CHECK: call void @llvm.prefetch{{.*}}({{.*}}, i32 0, i32 3, i32 0)
-        prefetch_read_instruction(data.as_ptr(), 3);
-    }
+    // CHECK: call void @llvm.prefetch{{.*}}({{.*}}, i32 0, i32 0, i32 0)
+    prefetch_read_instruction::<_, 0>(data.as_ptr());
+    // CHECK: call void @llvm.prefetch{{.*}}({{.*}}, i32 0, i32 1, i32 0)
+    prefetch_read_instruction::<_, 1>(data.as_ptr());
+    // CHECK: call void @llvm.prefetch{{.*}}({{.*}}, i32 0, i32 2, i32 0)
+    prefetch_read_instruction::<_, 2>(data.as_ptr());
+    // CHECK: call void @llvm.prefetch{{.*}}({{.*}}, i32 0, i32 3, i32 0)
+    prefetch_read_instruction::<_, 3>(data.as_ptr());
 }
 
 #[no_mangle]
 pub fn check_prefetch_write_instruction(data: &[i8]) {
-    unsafe {
-        // CHECK: call void @llvm.prefetch{{.*}}({{.*}}, i32 1, i32 0, i32 0)
-        prefetch_write_instruction(data.as_ptr(), 0);
-        // CHECK: call void @llvm.prefetch{{.*}}({{.*}}, i32 1, i32 1, i32 0)
-        prefetch_write_instruction(data.as_ptr(), 1);
-        // CHECK: call void @llvm.prefetch{{.*}}({{.*}}, i32 1, i32 2, i32 0)
-        prefetch_write_instruction(data.as_ptr(), 2);
-        // CHECK: call void @llvm.prefetch{{.*}}({{.*}}, i32 1, i32 3, i32 0)
-        prefetch_write_instruction(data.as_ptr(), 3);
-    }
+    // CHECK: call void @llvm.prefetch{{.*}}({{.*}}, i32 1, i32 0, i32 0)
+    prefetch_write_instruction::<_, 0>(data.as_ptr());
+    // CHECK: call void @llvm.prefetch{{.*}}({{.*}}, i32 1, i32 1, i32 0)
+    prefetch_write_instruction::<_, 1>(data.as_ptr());
+    // CHECK: call void @llvm.prefetch{{.*}}({{.*}}, i32 1, i32 2, i32 0)
+    prefetch_write_instruction::<_, 2>(data.as_ptr());
+    // CHECK: call void @llvm.prefetch{{.*}}({{.*}}, i32 1, i32 3, i32 0)
+    prefetch_write_instruction::<_, 3>(data.as_ptr());
 }