itsscb/rust
1
0
Fork 0
mirror of https://github.com/rust-lang/rust.git synced 2025-10-02 18:27:37 +00:00
Code Issues Packages Projects Releases Wiki Activity

intrinsics: use const generic to set atomic ordering

Browse Source
This commit is contained in:
Ralf Jung 2025-05-28 18:37:58 +02:00
parent 5e0bdaa9dd
commit 8808c9d34b
23 changed files with 653 additions and 1474 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

28
compiler/rustc_codegen_cranelift/src/intrinsics/mod.rs
View File

@ -875,7 +875,6 @@ fn codegen_regular_intrinsic_call<'tcx>(
let ptr = ptr.load_scalar(fx);
let ty = generic_args.type_at(0);
let _ord = generic_args.const_at(1).to_value(); // FIXME: forward this to cranelift once they support that
match ty.kind() {
ty::Uint(UintTy::U128) | ty::Int(IntTy::I128) => {
// FIXME implement 128bit atomics
@ -906,7 +905,7 @@ fn codegen_regular_intrinsic_call<'tcx>(
let val = CValue::by_val(val, fx.layout_of(ty));
ret.write_cvalue(fx, val);
}
_ if intrinsic.as_str().starts_with("atomic_store") => {
sym::atomic_store => {
intrinsic_args!(fx, args => (ptr, val); intrinsic);
let ptr = ptr.load_scalar(fx);
@ -939,7 +938,7 @@ fn codegen_regular_intrinsic_call<'tcx>(
fx.bcx.ins().atomic_store(MemFlags::trusted(), val, ptr);
}
_ if intrinsic.as_str().starts_with("atomic_xchg") => {
sym::atomic_xchg => {
intrinsic_args!(fx, args => (ptr, new); intrinsic);
let ptr = ptr.load_scalar(fx);
@ -960,8 +959,7 @@ fn codegen_regular_intrinsic_call<'tcx>(
let old = CValue::by_val(old, layout);
ret.write_cvalue(fx, old);
}
_ if intrinsic.as_str().starts_with("atomic_cxchg") => {
// both atomic_cxchg_* and atomic_cxchgweak_*
sym::atomic_cxchg | sym::atomic_cxchgweak => {
intrinsic_args!(fx, args => (ptr, test_old, new); intrinsic);
let ptr = ptr.load_scalar(fx);
@ -984,7 +982,7 @@ fn codegen_regular_intrinsic_call<'tcx>(
ret.write_cvalue(fx, ret_val)
}
_ if intrinsic.as_str().starts_with("atomic_xadd") => {
sym::atomic_xadd => {
intrinsic_args!(fx, args => (ptr, amount); intrinsic);
let ptr = ptr.load_scalar(fx);
@ -1006,7 +1004,7 @@ fn codegen_regular_intrinsic_call<'tcx>(
let old = CValue::by_val(old, layout);
ret.write_cvalue(fx, old);
}
_ if intrinsic.as_str().starts_with("atomic_xsub") => {
sym::atomic_xsub => {
intrinsic_args!(fx, args => (ptr, amount); intrinsic);
let ptr = ptr.load_scalar(fx);
@ -1028,7 +1026,7 @@ fn codegen_regular_intrinsic_call<'tcx>(
let old = CValue::by_val(old, layout);
ret.write_cvalue(fx, old);
}
_ if intrinsic.as_str().starts_with("atomic_and") => {
sym::atomic_and => {
intrinsic_args!(fx, args => (ptr, src); intrinsic);
let ptr = ptr.load_scalar(fx);
@ -1049,7 +1047,7 @@ fn codegen_regular_intrinsic_call<'tcx>(
let old = CValue::by_val(old, layout);
ret.write_cvalue(fx, old);
}
_ if intrinsic.as_str().starts_with("atomic_or") => {
sym::atomic_or => {
intrinsic_args!(fx, args => (ptr, src); intrinsic);
let ptr = ptr.load_scalar(fx);
@ -1070,7 +1068,7 @@ fn codegen_regular_intrinsic_call<'tcx>(
let old = CValue::by_val(old, layout);
ret.write_cvalue(fx, old);
}
_ if intrinsic.as_str().starts_with("atomic_xor") => {
sym::atomic_xor => {
intrinsic_args!(fx, args => (ptr, src); intrinsic);
let ptr = ptr.load_scalar(fx);
@ -1091,7 +1089,7 @@ fn codegen_regular_intrinsic_call<'tcx>(
let old = CValue::by_val(old, layout);
ret.write_cvalue(fx, old);
}
_ if intrinsic.as_str().starts_with("atomic_nand") => {
sym::atomic_nand => {
intrinsic_args!(fx, args => (ptr, src); intrinsic);
let ptr = ptr.load_scalar(fx);
@ -1112,7 +1110,7 @@ fn codegen_regular_intrinsic_call<'tcx>(
let old = CValue::by_val(old, layout);
ret.write_cvalue(fx, old);
}
_ if intrinsic.as_str().starts_with("atomic_max") => {
sym::atomic_max => {
intrinsic_args!(fx, args => (ptr, src); intrinsic);
let ptr = ptr.load_scalar(fx);
@ -1133,7 +1131,7 @@ fn codegen_regular_intrinsic_call<'tcx>(
let old = CValue::by_val(old, layout);
ret.write_cvalue(fx, old);
}
_ if intrinsic.as_str().starts_with("atomic_umax") => {
sym::atomic_umax => {
intrinsic_args!(fx, args => (ptr, src); intrinsic);
let ptr = ptr.load_scalar(fx);
@ -1154,7 +1152,7 @@ fn codegen_regular_intrinsic_call<'tcx>(
let old = CValue::by_val(old, layout);
ret.write_cvalue(fx, old);
}
_ if intrinsic.as_str().starts_with("atomic_min") => {
sym::atomic_min => {
intrinsic_args!(fx, args => (ptr, src); intrinsic);
let ptr = ptr.load_scalar(fx);
@ -1175,7 +1173,7 @@ fn codegen_regular_intrinsic_call<'tcx>(
let old = CValue::by_val(old, layout);
ret.write_cvalue(fx, old);
}
_ if intrinsic.as_str().starts_with("atomic_umin") => {
sym::atomic_umin => {
intrinsic_args!(fx, args => (ptr, src); intrinsic);
let ptr = ptr.load_scalar(fx);

8
compiler/rustc_codegen_ssa/messages.ftl
View File

@ -8,8 +8,6 @@ codegen_ssa_aix_strip_not_used = using host's `strip` binary to cross-compile to
codegen_ssa_archive_build_failure = failed to build archive at `{$path}`: {$error}
codegen_ssa_atomic_compare_exchange = Atomic compare-exchange intrinsic missing failure memory ordering
codegen_ssa_autodiff_without_lto = using the autodiff feature requires using fat-lto
codegen_ssa_bare_instruction_set = `#[instruction_set]` requires an argument
@ -206,8 +204,6 @@ codegen_ssa_missing_cpp_build_tool_component = or a necessary component may be m
codegen_ssa_missing_features = add the missing features in a `target_feature` attribute
codegen_ssa_missing_memory_ordering = Atomic intrinsic missing memory ordering
codegen_ssa_missing_query_depgraph =
found CGU-reuse attribute but `-Zquery-dep-graph` was not specified
@ -374,10 +370,6 @@ codegen_ssa_unexpected_parameter_name = unexpected parameter name
codegen_ssa_unknown_archive_kind =
Don't know how to build archive of type: {$kind}
codegen_ssa_unknown_atomic_operation = unknown atomic operation
codegen_ssa_unknown_atomic_ordering = unknown ordering in atomic intrinsic
codegen_ssa_unknown_reuse_kind = unknown cgu-reuse-kind `{$kind}` specified
codegen_ssa_unsupported_instruction_set = target does not support `#[instruction_set]`

16
compiler/rustc_codegen_ssa/src/errors.rs
View File

@ -796,22 +796,6 @@ pub(crate) struct ShuffleIndicesEvaluation {
pub span: Span,
}
#[derive(Diagnostic)]
#[diag(codegen_ssa_missing_memory_ordering)]
pub(crate) struct MissingMemoryOrdering;
#[derive(Diagnostic)]
#[diag(codegen_ssa_unknown_atomic_ordering)]
pub(crate) struct UnknownAtomicOrdering;
#[derive(Diagnostic)]
#[diag(codegen_ssa_atomic_compare_exchange)]
pub(crate) struct AtomicCompareExchange;
#[derive(Diagnostic)]
#[diag(codegen_ssa_unknown_atomic_operation)]
pub(crate) struct UnknownAtomicOperation;
#[derive(Diagnostic)]
pub enum InvalidMonomorphization<'tcx> {
#[diag(codegen_ssa_invalid_monomorphization_basic_integer_type, code = E0511)]

338
compiler/rustc_codegen_ssa/src/mir/intrinsic.rs
View File

@ -8,9 +8,10 @@ use rustc_span::sym;
use super::FunctionCx;
use super::operand::OperandRef;
use super::place::PlaceRef;
use crate::common::{AtomicRmwBinOp, SynchronizationScope};
use crate::errors::InvalidMonomorphization;
use crate::traits::*;
use crate::{MemFlags, errors, meth, size_of_val};
use crate::{MemFlags, meth, size_of_val};
fn copy_intrinsic<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>>(
bx: &mut Bx,
@ -62,7 +63,6 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
let span = source_info.span;
let name = bx.tcx().item_name(instance.def_id());
let name_str = name.as_str();
let fn_args = instance.args;
// If we're swapping something that's *not* an `OperandValue::Ref`,
@ -89,14 +89,13 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
}
}
let ret_llval = |bx: &mut Bx, llval| {
if result.layout.ty.is_bool() {
let val = bx.from_immediate(llval);
bx.store_to_place(val, result.val);
} else if !result.layout.ty.is_unit() {
bx.store_to_place(llval, result.val);
}
Ok(())
let invalid_monomorphization_int_type = |ty| {
bx.tcx().dcx().emit_err(InvalidMonomorphization::BasicIntegerType { span, name, ty });
};
let parse_atomic_ordering = |ord: ty::Value<'tcx>| {
let discr = ord.valtree.unwrap_branch()[0].unwrap_leaf();
discr.to_atomic_ordering()
};
let llval = match name {
@ -336,183 +335,144 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
}
}
// This requires that atomic intrinsics follow a specific naming pattern:
// "atomic_<operation>[_<ordering>]"
name if let Some(atomic) = name_str.strip_prefix("atomic_") => {
use rustc_middle::ty::AtomicOrdering::*;
use crate::common::{AtomicRmwBinOp, SynchronizationScope};
let invalid_monomorphization = |ty| {
bx.tcx().dcx().emit_err(InvalidMonomorphization::BasicIntegerType {
span,
name,
ty,
});
};
let parse_const_generic_ordering = |ord: ty::Value<'tcx>| {
let discr = ord.valtree.unwrap_branch()[0].unwrap_leaf();
discr.to_atomic_ordering()
};
// Some intrinsics have the ordering already converted to a const generic parameter, we handle those first.
match name {
sym::atomic_load => {
let ty = fn_args.type_at(0);
let ordering = fn_args.const_at(1).to_value();
if !(int_type_width_signed(ty, bx.tcx()).is_some() || ty.is_raw_ptr()) {
invalid_monomorphization(ty);
return Ok(());
}
let layout = bx.layout_of(ty);
let source = args[0].immediate();
let llval = bx.atomic_load(
bx.backend_type(layout),
source,
parse_const_generic_ordering(ordering),
layout.size,
);
return ret_llval(bx, llval);
}
// The rest falls back to below.
_ => {}
sym::atomic_load => {
let ty = fn_args.type_at(0);
if !(int_type_width_signed(ty, bx.tcx()).is_some() || ty.is_raw_ptr()) {
invalid_monomorphization_int_type(ty);
return Ok(());
}
let Some((instruction, ordering)) = atomic.split_once('_') else {
bx.sess().dcx().emit_fatal(errors::MissingMemoryOrdering);
};
let parse_ordering = |bx: &Bx, s| match s {
"relaxed" => Relaxed,
"acquire" => Acquire,
"release" => Release,
"acqrel" => AcqRel,
"seqcst" => SeqCst,
_ => bx.sess().dcx().emit_fatal(errors::UnknownAtomicOrdering),
};
match instruction {
"cxchg" | "cxchgweak" => {
let Some((success, failure)) = ordering.split_once('_') else {
bx.sess().dcx().emit_fatal(errors::AtomicCompareExchange);
};
let ty = fn_args.type_at(0);
if int_type_width_signed(ty, bx.tcx()).is_some() || ty.is_raw_ptr() {
let weak = instruction == "cxchgweak";
let dst = args[0].immediate();
let cmp = args[1].immediate();
let src = args[2].immediate();
let (val, success) = bx.atomic_cmpxchg(
dst,
cmp,
src,
parse_ordering(bx, success),
parse_ordering(bx, failure),
weak,
);
let val = bx.from_immediate(val);
let success = bx.from_immediate(success);
let dest = result.project_field(bx, 0);
bx.store_to_place(val, dest.val);
let dest = result.project_field(bx, 1);
bx.store_to_place(success, dest.val);
} else {
invalid_monomorphization(ty);
}
return Ok(());
}
"store" => {
let ty = fn_args.type_at(0);
if int_type_width_signed(ty, bx.tcx()).is_some() || ty.is_raw_ptr() {
let size = bx.layout_of(ty).size;
let val = args[1].immediate();
let ptr = args[0].immediate();
bx.atomic_store(val, ptr, parse_ordering(bx, ordering), size);
} else {
invalid_monomorphization(ty);
}
return Ok(());
}
"fence" => {
bx.atomic_fence(
parse_ordering(bx, ordering),
SynchronizationScope::CrossThread,
);
return Ok(());
}
"singlethreadfence" => {
bx.atomic_fence(
parse_ordering(bx, ordering),
SynchronizationScope::SingleThread,
);
return Ok(());
}
// These are all AtomicRMW ops
"max" | "min" => {
let atom_op = if instruction == "max" {
AtomicRmwBinOp::AtomicMax
} else {
AtomicRmwBinOp::AtomicMin
};
let ty = fn_args.type_at(0);
if matches!(ty.kind(), ty::Int(_)) {
let ptr = args[0].immediate();
let val = args[1].immediate();
bx.atomic_rmw(atom_op, ptr, val, parse_ordering(bx, ordering))
} else {
invalid_monomorphization(ty);
return Ok(());
}
}
"umax" | "umin" => {
let atom_op = if instruction == "umax" {
AtomicRmwBinOp::AtomicUMax
} else {
AtomicRmwBinOp::AtomicUMin
};
let ty = fn_args.type_at(0);
if matches!(ty.kind(), ty::Uint(_)) {
let ptr = args[0].immediate();
let val = args[1].immediate();
bx.atomic_rmw(atom_op, ptr, val, parse_ordering(bx, ordering))
} else {
invalid_monomorphization(ty);
return Ok(());
}
}
op => {
let atom_op = match op {
"xchg" => AtomicRmwBinOp::AtomicXchg,
"xadd" => AtomicRmwBinOp::AtomicAdd,
"xsub" => AtomicRmwBinOp::AtomicSub,
"and" => AtomicRmwBinOp::AtomicAnd,
"nand" => AtomicRmwBinOp::AtomicNand,
"or" => AtomicRmwBinOp::AtomicOr,
"xor" => AtomicRmwBinOp::AtomicXor,
_ => bx.sess().dcx().emit_fatal(errors::UnknownAtomicOperation),
};
let ty = fn_args.type_at(0);
if int_type_width_signed(ty, bx.tcx()).is_some() || ty.is_raw_ptr() {
let ptr = args[0].immediate();
let val = args[1].immediate();
bx.atomic_rmw(atom_op, ptr, val, parse_ordering(bx, ordering))
} else {
invalid_monomorphization(ty);
return Ok(());
}
}
let ordering = fn_args.const_at(1).to_value();
let layout = bx.layout_of(ty);
let source = args[0].immediate();
bx.atomic_load(
bx.backend_type(layout),
source,
parse_atomic_ordering(ordering),
layout.size,
)
}
sym::atomic_store => {
let ty = fn_args.type_at(0);
if !(int_type_width_signed(ty, bx.tcx()).is_some() || ty.is_raw_ptr()) {
invalid_monomorphization_int_type(ty);
return Ok(());
}
let ordering = fn_args.const_at(1).to_value();
let size = bx.layout_of(ty).size;
let val = args[1].immediate();
let ptr = args[0].immediate();
bx.atomic_store(val, ptr, parse_atomic_ordering(ordering), size);
return Ok(());
}
sym::atomic_cxchg | sym::atomic_cxchgweak => {
let ty = fn_args.type_at(0);
if !(int_type_width_signed(ty, bx.tcx()).is_some() || ty.is_raw_ptr()) {
invalid_monomorphization_int_type(ty);
return Ok(());
}
let succ_ordering = fn_args.const_at(1).to_value();
let fail_ordering = fn_args.const_at(2).to_value();
let weak = name == sym::atomic_cxchgweak;
let dst = args[0].immediate();
let cmp = args[1].immediate();
let src = args[2].immediate();
let (val, success) = bx.atomic_cmpxchg(
dst,
cmp,
src,
parse_atomic_ordering(succ_ordering),
parse_atomic_ordering(fail_ordering),
weak,
);
let val = bx.from_immediate(val);
let success = bx.from_immediate(success);
let dest = result.project_field(bx, 0);
bx.store_to_place(val, dest.val);
let dest = result.project_field(bx, 1);
bx.store_to_place(success, dest.val);
return Ok(());
}
// These are all AtomicRMW ops
sym::atomic_max | sym::atomic_min => {
let atom_op = if name == sym::atomic_max {
AtomicRmwBinOp::AtomicMax
} else {
AtomicRmwBinOp::AtomicMin
};
let ty = fn_args.type_at(0);
if matches!(ty.kind(), ty::Int(_)) {
let ordering = fn_args.const_at(1).to_value();
let ptr = args[0].immediate();
let val = args[1].immediate();
bx.atomic_rmw(atom_op, ptr, val, parse_atomic_ordering(ordering))
} else {
invalid_monomorphization_int_type(ty);
return Ok(());
}
}
sym::atomic_umax | sym::atomic_umin => {
let atom_op = if name == sym::atomic_umax {
AtomicRmwBinOp::AtomicUMax
} else {
AtomicRmwBinOp::AtomicUMin
};
let ty = fn_args.type_at(0);
if matches!(ty.kind(), ty::Uint(_)) {
let ordering = fn_args.const_at(1).to_value();
let ptr = args[0].immediate();
let val = args[1].immediate();
bx.atomic_rmw(atom_op, ptr, val, parse_atomic_ordering(ordering))
} else {
invalid_monomorphization_int_type(ty);
return Ok(());
}
}
sym::atomic_xchg
| sym::atomic_xadd
| sym::atomic_xsub
| sym::atomic_and
| sym::atomic_nand
| sym::atomic_or
| sym::atomic_xor => {
let atom_op = match name {
sym::atomic_xchg => AtomicRmwBinOp::AtomicXchg,
sym::atomic_xadd => AtomicRmwBinOp::AtomicAdd,
sym::atomic_xsub => AtomicRmwBinOp::AtomicSub,
sym::atomic_and => AtomicRmwBinOp::AtomicAnd,
sym::atomic_nand => AtomicRmwBinOp::AtomicNand,
sym::atomic_or => AtomicRmwBinOp::AtomicOr,
sym::atomic_xor => AtomicRmwBinOp::AtomicXor,
_ => unreachable!(),
};
let ty = fn_args.type_at(0);
if int_type_width_signed(ty, bx.tcx()).is_some() || ty.is_raw_ptr() {
let ordering = fn_args.const_at(1).to_value();
let ptr = args[0].immediate();
let val = args[1].immediate();
bx.atomic_rmw(atom_op, ptr, val, parse_atomic_ordering(ordering))
} else {
invalid_monomorphization_int_type(ty);
return Ok(());
}
}
sym::atomic_fence => {
let ordering = fn_args.const_at(0).to_value();
bx.atomic_fence(parse_atomic_ordering(ordering), SynchronizationScope::CrossThread);
return Ok(());
}
sym::atomic_singlethreadfence => {
let ordering = fn_args.const_at(0).to_value();
bx.atomic_fence(
parse_atomic_ordering(ordering),
SynchronizationScope::SingleThread,
);
return Ok(());
}
sym::nontemporal_store => {
@ -556,7 +516,13 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
}
};
ret_llval(bx, llval)
if result.layout.ty.is_bool() {
let val = bx.from_immediate(llval);
bx.store_to_place(val, result.val);
} else if !result.layout.ty.is_unit() {
bx.store_to_place(llval, result.val);
}
Ok(())
}
}

15
compiler/rustc_error_codes/src/error_codes/E0092.md
View File

@ -1,8 +1,10 @@
#### Note: this error code is no longer emitted by the compiler.
An undefined atomic operation function was declared.
Erroneous code example:
```compile_fail,E0092
```ignore (no longer emitted)
#![feature(intrinsics)]
#![allow(internal_features)]
@ -12,13 +14,4 @@ unsafe fn atomic_foo(); // error: unrecognized atomic operation
```
Please check you didn't make a mistake in the function's name. All intrinsic
functions are defined in `compiler/rustc_codegen_llvm/src/intrinsic.rs` and in
`library/core/src/intrinsics.rs` in the Rust source code. Example:
```
#![feature(intrinsics)]
#![allow(internal_features)]
#[rustc_intrinsic]
unsafe fn atomic_fence_seqcst(); // ok!
```
functions are defined in `library/core/src/intrinsics` in the Rust source code.

17
compiler/rustc_error_codes/src/error_codes/E0093.md
View File

@ -17,19 +17,4 @@ fn main() {
```
Please check you didn't make a mistake in the function's name. All intrinsic
functions are defined in `compiler/rustc_codegen_llvm/src/intrinsic.rs` and in
`library/core/src/intrinsics.rs` in the Rust source code. Example:
```
#![feature(intrinsics)]
#![allow(internal_features)]
#[rustc_intrinsic]
unsafe fn atomic_fence_seqcst(); // ok!
fn main() {
unsafe {
atomic_fence_seqcst();
}
}
```
functions are defined in `library/core/src/intrinsics` in the Rust source code.

4
compiler/rustc_error_codes/src/error_codes/E0622.md
View File

@ -4,7 +4,7 @@ An intrinsic was declared without being a function.
Erroneous code example:
```no_run
```ignore (no longer emitted)
#![feature(intrinsics)]
#![allow(internal_features)]
@ -21,7 +21,7 @@ An intrinsic is a function available for use in a given programming language
whose implementation is handled specially by the compiler. In order to fix this
error, just declare a function. Example:
```no_run
```ignore (no longer emitted)
#![feature(intrinsics)]
#![allow(internal_features)]

4
compiler/rustc_hir_analysis/messages.ftl
View File

@ -565,10 +565,6 @@ hir_analysis_unconstrained_generic_parameter = the {$param_def_kind} `{$param_na
hir_analysis_unconstrained_opaque_type = unconstrained opaque type
.note = `{$name}` must be used in combination with a concrete type within the same {$what}
hir_analysis_unrecognized_atomic_operation =
unrecognized atomic operation function: `{$op}`
.label = unrecognized atomic operation
hir_analysis_unrecognized_intrinsic_function =
unrecognized intrinsic function: `{$name}`
.label = unrecognized intrinsic

59
compiler/rustc_hir_analysis/src/check/intrinsic.rs
View File

@ -9,10 +9,7 @@ use rustc_span::def_id::LocalDefId;
use rustc_span::{Span, Symbol, sym};
use crate::check::check_function_signature;
use crate::errors::{
UnrecognizedAtomicOperation, UnrecognizedIntrinsicFunction,
WrongNumberOfGenericArgumentsToIntrinsic,
};
use crate::errors::{UnrecognizedIntrinsicFunction, WrongNumberOfGenericArgumentsToIntrinsic};
fn equate_intrinsic_type<'tcx>(
tcx: TyCtxt<'tcx>,
@ -172,7 +169,6 @@ pub(crate) fn check_intrinsic_type(
Ty::new_error_with_message(tcx, span, "expected param")
}
};
let name_str = intrinsic_name.as_str();
let bound_vars = tcx.mk_bound_variable_kinds(&[
ty::BoundVariableKind::Region(ty::BoundRegionKind::Anon),
@ -198,32 +194,9 @@ pub(crate) fn check_intrinsic_type(
(Ty::new_ref(tcx, env_region, va_list_ty, mutbl), va_list_ty)
};
let (n_tps, n_lts, n_cts, inputs, output, safety) = if name_str.starts_with("atomic_") {
let split: Vec<&str> = name_str.split('_').collect();
assert!(split.len() >= 2, "Atomic intrinsic in an incorrect format");
// Each atomic op has variants with different suffixes (`_seq_cst`, `_acquire`, etc.). Use
// string ops to strip the suffixes, because the variants all get the same treatment here.
let (n_tps, n_cts, inputs, output) = match split[1] {
"cxchg" | "cxchgweak" => (
1,
0,
vec![Ty::new_mut_ptr(tcx, param(0)), param(0), param(0)],
Ty::new_tup(tcx, &[param(0), tcx.types.bool]),
),
"load" => (1, 1, vec![Ty::new_imm_ptr(tcx, param(0))], param(0)),
"store" => (1, 0, vec![Ty::new_mut_ptr(tcx, param(0)), param(0)], tcx.types.unit),
"xchg" | "xadd" | "xsub" | "and" | "nand" | "or" | "xor" | "max" | "min" | "umax"
| "umin" => (1, 0, vec![Ty::new_mut_ptr(tcx, param(0)), param(0)], param(0)),
"fence" | "singlethreadfence" => (0, 0, Vec::new(), tcx.types.unit),
op => {
tcx.dcx().emit_err(UnrecognizedAtomicOperation { span, op });
return;
}
};
(n_tps, 0, n_cts, inputs, output, hir::Safety::Unsafe)
} else if intrinsic_name == sym::contract_check_ensures {
let (n_tps, n_lts, n_cts, inputs, output, safety) = if intrinsic_name
== sym::contract_check_ensures
{
// contract_check_ensures::<Ret, C>(Ret, C) -> Ret
// where C: for<'a> Fn(&'a Ret) -> bool,
//
@ -694,6 +667,30 @@ pub(crate) fn check_intrinsic_type(
sym::simd_shuffle => (3, 0, vec![param(0), param(0), param(1)], param(2)),
sym::simd_shuffle_const_generic => (2, 1, vec![param(0), param(0)], param(1)),
sym::atomic_cxchg | sym::atomic_cxchgweak => (
1,
2,
vec![Ty::new_mut_ptr(tcx, param(0)), param(0), param(0)],
Ty::new_tup(tcx, &[param(0), tcx.types.bool]),
),
sym::atomic_load => (1, 1, vec![Ty::new_imm_ptr(tcx, param(0))], param(0)),
sym::atomic_store => {
(1, 1, vec![Ty::new_mut_ptr(tcx, param(0)), param(0)], tcx.types.unit)
}
sym::atomic_xchg
| sym::atomic_xadd
| sym::atomic_xsub
| sym::atomic_and
| sym::atomic_nand
| sym::atomic_or
| sym::atomic_xor
| sym::atomic_max
| sym::atomic_min
| sym::atomic_umax
| sym::atomic_umin => (1, 1, vec![Ty::new_mut_ptr(tcx, param(0)), param(0)], param(0)),
sym::atomic_fence | sym::atomic_singlethreadfence => (0, 1, Vec::new(), tcx.types.unit),
other => {
tcx.dcx().emit_err(UnrecognizedIntrinsicFunction { span, name: other });
return;

9
compiler/rustc_hir_analysis/src/errors.rs
View File

@ -161,15 +161,6 @@ pub(crate) enum AssocItemNotFoundSugg<'a> {
},
}
#[derive(Diagnostic)]
#[diag(hir_analysis_unrecognized_atomic_operation, code = E0092)]
pub(crate) struct UnrecognizedAtomicOperation<'a> {
#[primary_span]
#[label]
pub span: Span,
pub op: &'a str,
}
#[derive(Diagnostic)]
#[diag(hir_analysis_wrong_number_of_generic_arguments_to_intrinsic, code = E0094)]
pub(crate) struct WrongNumberOfGenericArgumentsToIntrinsic<'a> {

16
compiler/rustc_span/src/symbol.rs
View File

@ -515,8 +515,24 @@ symbols! {
async_iterator_poll_next,
async_trait_bounds,
atomic,
atomic_and,
atomic_cxchg,
atomic_cxchgweak,
atomic_fence,
atomic_load,
atomic_max,
atomic_min,
atomic_mod,
atomic_nand,
atomic_or,
atomic_singlethreadfence,
atomic_store,
atomic_umax,
atomic_umin,
atomic_xadd,
atomic_xchg,
atomic_xor,
atomic_xsub,
atomics,
att_syntax,
attr,

926
library/core/src/intrinsics/mod.rs
View File

File diff suppressed because it is too large Load Diff

270
library/core/src/sync/atomic.rs
View File

@ -245,6 +245,7 @@
use self::Ordering::*;
use crate::cell::UnsafeCell;
use crate::hint::spin_loop;
use crate::intrinsics::AtomicOrdering as AO;
use crate::{fmt, intrinsics};
trait Sealed {}
@ -3811,9 +3812,9 @@ unsafe fn atomic_store<T: Copy>(dst: *mut T, val: T, order: Ordering) {
// SAFETY: the caller must uphold the safety contract for `atomic_store`.
unsafe {
match order {
Relaxed => intrinsics::atomic_store_relaxed(dst, val),
Release => intrinsics::atomic_store_release(dst, val),
SeqCst => intrinsics::atomic_store_seqcst(dst, val),
Relaxed => intrinsics::atomic_store::<T, { AO::Relaxed }>(dst, val),
Release => intrinsics::atomic_store::<T, { AO::Release }>(dst, val),
SeqCst => intrinsics::atomic_store::<T, { AO::SeqCst }>(dst, val),
Acquire => panic!("there is no such thing as an acquire store"),
AcqRel => panic!("there is no such thing as an acquire-release store"),
}
@ -3823,13 +3824,12 @@ unsafe fn atomic_store<T: Copy>(dst: *mut T, val: T, order: Ordering) {
#[inline]
#[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces
unsafe fn atomic_load<T: Copy>(dst: *const T, order: Ordering) -> T {
use intrinsics::AtomicOrdering;
// SAFETY: the caller must uphold the safety contract for `atomic_load`.
unsafe {
match order {
Relaxed => intrinsics::atomic_load::<T, { AtomicOrdering::Relaxed }>(dst),
Acquire => intrinsics::atomic_load::<T, { AtomicOrdering::Acquire }>(dst),
SeqCst => intrinsics::atomic_load::<T, { AtomicOrdering::SeqCst }>(dst),
Relaxed => intrinsics::atomic_load::<T, { AO::Relaxed }>(dst),
Acquire => intrinsics::atomic_load::<T, { AO::Acquire }>(dst),
SeqCst => intrinsics::atomic_load::<T, { AO::SeqCst }>(dst),
Release => panic!("there is no such thing as a release load"),
AcqRel => panic!("there is no such thing as an acquire-release load"),
}
@ -3843,11 +3843,11 @@ unsafe fn atomic_swap<T: Copy>(dst: *mut T, val: T, order: Ordering) -> T {
// SAFETY: the caller must uphold the safety contract for `atomic_swap`.
unsafe {
match order {
Relaxed => intrinsics::atomic_xchg_relaxed(dst, val),
Acquire => intrinsics::atomic_xchg_acquire(dst, val),
Release => intrinsics::atomic_xchg_release(dst, val),
AcqRel => intrinsics::atomic_xchg_acqrel(dst, val),
SeqCst => intrinsics::atomic_xchg_seqcst(dst, val),
Relaxed => intrinsics::atomic_xchg::<T, { AO::Relaxed }>(dst, val),
Acquire => intrinsics::atomic_xchg::<T, { AO::Acquire }>(dst, val),
Release => intrinsics::atomic_xchg::<T, { AO::Release }>(dst, val),
AcqRel => intrinsics::atomic_xchg::<T, { AO::AcqRel }>(dst, val),
SeqCst => intrinsics::atomic_xchg::<T, { AO::SeqCst }>(dst, val),
}
}
}
@ -3860,11 +3860,11 @@ unsafe fn atomic_add<T: Copy>(dst: *mut T, val: T, order: Ordering) -> T {
// SAFETY: the caller must uphold the safety contract for `atomic_add`.
unsafe {
match order {
Relaxed => intrinsics::atomic_xadd_relaxed(dst, val),
Acquire => intrinsics::atomic_xadd_acquire(dst, val),
Release => intrinsics::atomic_xadd_release(dst, val),
AcqRel => intrinsics::atomic_xadd_acqrel(dst, val),
SeqCst => intrinsics::atomic_xadd_seqcst(dst, val),
Relaxed => intrinsics::atomic_xadd::<T, { AO::Relaxed }>(dst, val),
Acquire => intrinsics::atomic_xadd::<T, { AO::Acquire }>(dst, val),
Release => intrinsics::atomic_xadd::<T, { AO::Release }>(dst, val),
AcqRel => intrinsics::atomic_xadd::<T, { AO::AcqRel }>(dst, val),
SeqCst => intrinsics::atomic_xadd::<T, { AO::SeqCst }>(dst, val),
}
}
}
@ -3877,11 +3877,11 @@ unsafe fn atomic_sub<T: Copy>(dst: *mut T, val: T, order: Ordering) -> T {
// SAFETY: the caller must uphold the safety contract for `atomic_sub`.
unsafe {
match order {
Relaxed => intrinsics::atomic_xsub_relaxed(dst, val),
Acquire => intrinsics::atomic_xsub_acquire(dst, val),
Release => intrinsics::atomic_xsub_release(dst, val),
AcqRel => intrinsics::atomic_xsub_acqrel(dst, val),
SeqCst => intrinsics::atomic_xsub_seqcst(dst, val),
Relaxed => intrinsics::atomic_xsub::<T, { AO::Relaxed }>(dst, val),
Acquire => intrinsics::atomic_xsub::<T, { AO::Acquire }>(dst, val),
Release => intrinsics::atomic_xsub::<T, { AO::Release }>(dst, val),
AcqRel => intrinsics::atomic_xsub::<T, { AO::AcqRel }>(dst, val),
SeqCst => intrinsics::atomic_xsub::<T, { AO::SeqCst }>(dst, val),
}
}
}
@ -3902,21 +3902,51 @@ pub unsafe fn atomic_compare_exchange<T: Copy>(
// SAFETY: the caller must uphold the safety contract for `atomic_compare_exchange`.
let (val, ok) = unsafe {
match (success, failure) {
(Relaxed, Relaxed) => intrinsics::atomic_cxchg_relaxed_relaxed(dst, old, new),
(Relaxed, Acquire) => intrinsics::atomic_cxchg_relaxed_acquire(dst, old, new),
(Relaxed, SeqCst) => intrinsics::atomic_cxchg_relaxed_seqcst(dst, old, new),
(Acquire, Relaxed) => intrinsics::atomic_cxchg_acquire_relaxed(dst, old, new),
(Acquire, Acquire) => intrinsics::atomic_cxchg_acquire_acquire(dst, old, new),
(Acquire, SeqCst) => intrinsics::atomic_cxchg_acquire_seqcst(dst, old, new),
(Release, Relaxed) => intrinsics::atomic_cxchg_release_relaxed(dst, old, new),
(Release, Acquire) => intrinsics::atomic_cxchg_release_acquire(dst, old, new),
(Release, SeqCst) => intrinsics::atomic_cxchg_release_seqcst(dst, old, new),
(AcqRel, Relaxed) => intrinsics::atomic_cxchg_acqrel_relaxed(dst, old, new),
(AcqRel, Acquire) => intrinsics::atomic_cxchg_acqrel_acquire(dst, old, new),
(AcqRel, SeqCst) => intrinsics::atomic_cxchg_acqrel_seqcst(dst, old, new),
(SeqCst, Relaxed) => intrinsics::atomic_cxchg_seqcst_relaxed(dst, old, new),
(SeqCst, Acquire) => intrinsics::atomic_cxchg_seqcst_acquire(dst, old, new),
(SeqCst, SeqCst) => intrinsics::atomic_cxchg_seqcst_seqcst(dst, old, new),
(Relaxed, Relaxed) => {
intrinsics::atomic_cxchg::<T, { AO::Relaxed }, { AO::Relaxed }>(dst, old, new)
}
(Relaxed, Acquire) => {
intrinsics::atomic_cxchg::<T, { AO::Relaxed }, { AO::Acquire }>(dst, old, new)
}
(Relaxed, SeqCst) => {
intrinsics::atomic_cxchg::<T, { AO::Relaxed }, { AO::SeqCst }>(dst, old, new)
}
(Acquire, Relaxed) => {
intrinsics::atomic_cxchg::<T, { AO::Acquire }, { AO::Relaxed }>(dst, old, new)
}
(Acquire, Acquire) => {
intrinsics::atomic_cxchg::<T, { AO::Acquire }, { AO::Acquire }>(dst, old, new)
}
(Acquire, SeqCst) => {
intrinsics::atomic_cxchg::<T, { AO::Acquire }, { AO::SeqCst }>(dst, old, new)
}
(Release, Relaxed) => {
intrinsics::atomic_cxchg::<T, { AO::Release }, { AO::Relaxed }>(dst, old, new)
}
(Release, Acquire) => {
intrinsics::atomic_cxchg::<T, { AO::Release }, { AO::Acquire }>(dst, old, new)
}
(Release, SeqCst) => {
intrinsics::atomic_cxchg::<T, { AO::Release }, { AO::SeqCst }>(dst, old, new)
}
(AcqRel, Relaxed) => {
intrinsics::atomic_cxchg::<T, { AO::AcqRel }, { AO::Relaxed }>(dst, old, new)
}
(AcqRel, Acquire) => {
intrinsics::atomic_cxchg::<T, { AO::AcqRel }, { AO::Acquire }>(dst, old, new)
}
(AcqRel, SeqCst) => {
intrinsics::atomic_cxchg::<T, { AO::AcqRel }, { AO::SeqCst }>(dst, old, new)
}
(SeqCst, Relaxed) => {
intrinsics::atomic_cxchg::<T, { AO::SeqCst }, { AO::Relaxed }>(dst, old, new)
}
(SeqCst, Acquire) => {
intrinsics::atomic_cxchg::<T, { AO::SeqCst }, { AO::Acquire }>(dst, old, new)
}
(SeqCst, SeqCst) => {
intrinsics::atomic_cxchg::<T, { AO::SeqCst }, { AO::SeqCst }>(dst, old, new)
}
(_, AcqRel) => panic!("there is no such thing as an acquire-release failure ordering"),
(_, Release) => panic!("there is no such thing as a release failure ordering"),
}
@ -3937,21 +3967,51 @@ unsafe fn atomic_compare_exchange_weak<T: Copy>(
// SAFETY: the caller must uphold the safety contract for `atomic_compare_exchange_weak`.
let (val, ok) = unsafe {
match (success, failure) {
(Relaxed, Relaxed) => intrinsics::atomic_cxchgweak_relaxed_relaxed(dst, old, new),
(Relaxed, Acquire) => intrinsics::atomic_cxchgweak_relaxed_acquire(dst, old, new),
(Relaxed, SeqCst) => intrinsics::atomic_cxchgweak_relaxed_seqcst(dst, old, new),
(Acquire, Relaxed) => intrinsics::atomic_cxchgweak_acquire_relaxed(dst, old, new),
(Acquire, Acquire) => intrinsics::atomic_cxchgweak_acquire_acquire(dst, old, new),
(Acquire, SeqCst) => intrinsics::atomic_cxchgweak_acquire_seqcst(dst, old, new),
(Release, Relaxed) => intrinsics::atomic_cxchgweak_release_relaxed(dst, old, new),
(Release, Acquire) => intrinsics::atomic_cxchgweak_release_acquire(dst, old, new),
(Release, SeqCst) => intrinsics::atomic_cxchgweak_release_seqcst(dst, old, new),
(AcqRel, Relaxed) => intrinsics::atomic_cxchgweak_acqrel_relaxed(dst, old, new),
(AcqRel, Acquire) => intrinsics::atomic_cxchgweak_acqrel_acquire(dst, old, new),
(AcqRel, SeqCst) => intrinsics::atomic_cxchgweak_acqrel_seqcst(dst, old, new),
(SeqCst, Relaxed) => intrinsics::atomic_cxchgweak_seqcst_relaxed(dst, old, new),
(SeqCst, Acquire) => intrinsics::atomic_cxchgweak_seqcst_acquire(dst, old, new),
(SeqCst, SeqCst) => intrinsics::atomic_cxchgweak_seqcst_seqcst(dst, old, new),
(Relaxed, Relaxed) => {
intrinsics::atomic_cxchgweak::<T, { AO::Relaxed }, { AO::Relaxed }>(dst, old, new)
}
(Relaxed, Acquire) => {
intrinsics::atomic_cxchgweak::<T, { AO::Relaxed }, { AO::Acquire }>(dst, old, new)
}
(Relaxed, SeqCst) => {
intrinsics::atomic_cxchgweak::<T, { AO::Relaxed }, { AO::SeqCst }>(dst, old, new)
}
(Acquire, Relaxed) => {
intrinsics::atomic_cxchgweak::<T, { AO::Acquire }, { AO::Relaxed }>(dst, old, new)
}
(Acquire, Acquire) => {
intrinsics::atomic_cxchgweak::<T, { AO::Acquire }, { AO::Acquire }>(dst, old, new)
}
(Acquire, SeqCst) => {
intrinsics::atomic_cxchgweak::<T, { AO::Acquire }, { AO::SeqCst }>(dst, old, new)
}
(Release, Relaxed) => {
intrinsics::atomic_cxchgweak::<T, { AO::Release }, { AO::Relaxed }>(dst, old, new)
}
(Release, Acquire) => {
intrinsics::atomic_cxchgweak::<T, { AO::Release }, { AO::Acquire }>(dst, old, new)
}
(Release, SeqCst) => {
intrinsics::atomic_cxchgweak::<T, { AO::Release }, { AO::SeqCst }>(dst, old, new)
}
(AcqRel, Relaxed) => {
intrinsics::atomic_cxchgweak::<T, { AO::AcqRel }, { AO::Relaxed }>(dst, old, new)
}
(AcqRel, Acquire) => {
intrinsics::atomic_cxchgweak::<T, { AO::AcqRel }, { AO::Acquire }>(dst, old, new)
}
(AcqRel, SeqCst) => {
intrinsics::atomic_cxchgweak::<T, { AO::AcqRel }, { AO::SeqCst }>(dst, old, new)
}
(SeqCst, Relaxed) => {
intrinsics::atomic_cxchgweak::<T, { AO::SeqCst }, { AO::Relaxed }>(dst, old, new)
}
(SeqCst, Acquire) => {
intrinsics::atomic_cxchgweak::<T, { AO::SeqCst }, { AO::Acquire }>(dst, old, new)
}
(SeqCst, SeqCst) => {
intrinsics::atomic_cxchgweak::<T, { AO::SeqCst }, { AO::SeqCst }>(dst, old, new)
}
(_, AcqRel) => panic!("there is no such thing as an acquire-release failure ordering"),
(_, Release) => panic!("there is no such thing as a release failure ordering"),
}
@ -3966,11 +4026,11 @@ unsafe fn atomic_and<T: Copy>(dst: *mut T, val: T, order: Ordering) -> T {
// SAFETY: the caller must uphold the safety contract for `atomic_and`
unsafe {
match order {
Relaxed => intrinsics::atomic_and_relaxed(dst, val),
Acquire => intrinsics::atomic_and_acquire(dst, val),
Release => intrinsics::atomic_and_release(dst, val),
AcqRel => intrinsics::atomic_and_acqrel(dst, val),
SeqCst => intrinsics::atomic_and_seqcst(dst, val),
Relaxed => intrinsics::atomic_and::<T, { AO::Relaxed }>(dst, val),
Acquire => intrinsics::atomic_and::<T, { AO::Acquire }>(dst, val),
Release => intrinsics::atomic_and::<T, { AO::Release }>(dst, val),
AcqRel => intrinsics::atomic_and::<T, { AO::AcqRel }>(dst, val),
SeqCst => intrinsics::atomic_and::<T, { AO::SeqCst }>(dst, val),
}
}
}
@ -3982,11 +4042,11 @@ unsafe fn atomic_nand<T: Copy>(dst: *mut T, val: T, order: Ordering) -> T {
// SAFETY: the caller must uphold the safety contract for `atomic_nand`
unsafe {
match order {
Relaxed => intrinsics::atomic_nand_relaxed(dst, val),
Acquire => intrinsics::atomic_nand_acquire(dst, val),
Release => intrinsics::atomic_nand_release(dst, val),
AcqRel => intrinsics::atomic_nand_acqrel(dst, val),
SeqCst => intrinsics::atomic_nand_seqcst(dst, val),
Relaxed => intrinsics::atomic_nand::<T, { AO::Relaxed }>(dst, val),
Acquire => intrinsics::atomic_nand::<T, { AO::Acquire }>(dst, val),
Release => intrinsics::atomic_nand::<T, { AO::Release }>(dst, val),
AcqRel => intrinsics::atomic_nand::<T, { AO::AcqRel }>(dst, val),
SeqCst => intrinsics::atomic_nand::<T, { AO::SeqCst }>(dst, val),
}
}
}
@ -3998,11 +4058,11 @@ unsafe fn atomic_or<T: Copy>(dst: *mut T, val: T, order: Ordering) -> T {
// SAFETY: the caller must uphold the safety contract for `atomic_or`
unsafe {
match order {
SeqCst => intrinsics::atomic_or_seqcst(dst, val),
Acquire => intrinsics::atomic_or_acquire(dst, val),
Release => intrinsics::atomic_or_release(dst, val),
AcqRel => intrinsics::atomic_or_acqrel(dst, val),
Relaxed => intrinsics::atomic_or_relaxed(dst, val),
SeqCst => intrinsics::atomic_or::<T, { AO::SeqCst }>(dst, val),
Acquire => intrinsics::atomic_or::<T, { AO::Acquire }>(dst, val),
Release => intrinsics::atomic_or::<T, { AO::Release }>(dst, val),
AcqRel => intrinsics::atomic_or::<T, { AO::AcqRel }>(dst, val),
Relaxed => intrinsics::atomic_or::<T, { AO::Relaxed }>(dst, val),
}
}
}
@ -4014,16 +4074,16 @@ unsafe fn atomic_xor<T: Copy>(dst: *mut T, val: T, order: Ordering) -> T {
// SAFETY: the caller must uphold the safety contract for `atomic_xor`
unsafe {
match order {
SeqCst => intrinsics::atomic_xor_seqcst(dst, val),
Acquire => intrinsics::atomic_xor_acquire(dst, val),
Release => intrinsics::atomic_xor_release(dst, val),
AcqRel => intrinsics::atomic_xor_acqrel(dst, val),
Relaxed => intrinsics::atomic_xor_relaxed(dst, val),
SeqCst => intrinsics::atomic_xor::<T, { AO::SeqCst }>(dst, val),
Acquire => intrinsics::atomic_xor::<T, { AO::Acquire }>(dst, val),
Release => intrinsics::atomic_xor::<T, { AO::Release }>(dst, val),
AcqRel => intrinsics::atomic_xor::<T, { AO::AcqRel }>(dst, val),
Relaxed => intrinsics::atomic_xor::<T, { AO::Relaxed }>(dst, val),
}
}
}
/// returns the max value (signed comparison)
/// Updates `*dst` to the max value of `val` and the old value (signed comparison)
#[inline]
#[cfg(target_has_atomic)]
#[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces
@ -4031,16 +4091,16 @@ unsafe fn atomic_max<T: Copy>(dst: *mut T, val: T, order: Ordering) -> T {
// SAFETY: the caller must uphold the safety contract for `atomic_max`
unsafe {
match order {
Relaxed => intrinsics::atomic_max_relaxed(dst, val),
Acquire => intrinsics::atomic_max_acquire(dst, val),
Release => intrinsics::atomic_max_release(dst, val),
AcqRel => intrinsics::atomic_max_acqrel(dst, val),
SeqCst => intrinsics::atomic_max_seqcst(dst, val),
Relaxed => intrinsics::atomic_max::<T, { AO::Relaxed }>(dst, val),
Acquire => intrinsics::atomic_max::<T, { AO::Acquire }>(dst, val),
Release => intrinsics::atomic_max::<T, { AO::Release }>(dst, val),
AcqRel => intrinsics::atomic_max::<T, { AO::AcqRel }>(dst, val),
SeqCst => intrinsics::atomic_max::<T, { AO::SeqCst }>(dst, val),
}
}
}
/// returns the min value (signed comparison)
/// Updates `*dst` to the min value of `val` and the old value (signed comparison)
#[inline]
#[cfg(target_has_atomic)]
#[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces
@ -4048,16 +4108,16 @@ unsafe fn atomic_min<T: Copy>(dst: *mut T, val: T, order: Ordering) -> T {
// SAFETY: the caller must uphold the safety contract for `atomic_min`
unsafe {
match order {
Relaxed => intrinsics::atomic_min_relaxed(dst, val),
Acquire => intrinsics::atomic_min_acquire(dst, val),
Release => intrinsics::atomic_min_release(dst, val),
AcqRel => intrinsics::atomic_min_acqrel(dst, val),
SeqCst => intrinsics::atomic_min_seqcst(dst, val),
Relaxed => intrinsics::atomic_min::<T, { AO::Relaxed }>(dst, val),
Acquire => intrinsics::atomic_min::<T, { AO::Acquire }>(dst, val),
Release => intrinsics::atomic_min::<T, { AO::Release }>(dst, val),
AcqRel => intrinsics::atomic_min::<T, { AO::AcqRel }>(dst, val),
SeqCst => intrinsics::atomic_min::<T, { AO::SeqCst }>(dst, val),
}
}
}
/// returns the max value (unsigned comparison)
/// Updates `*dst` to the max value of `val` and the old value (unsigned comparison)
#[inline]
#[cfg(target_has_atomic)]
#[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces
@ -4065,16 +4125,16 @@ unsafe fn atomic_umax<T: Copy>(dst: *mut T, val: T, order: Ordering) -> T {
// SAFETY: the caller must uphold the safety contract for `atomic_umax`
unsafe {
match order {
Relaxed => intrinsics::atomic_umax_relaxed(dst, val),
Acquire => intrinsics::atomic_umax_acquire(dst, val),
Release => intrinsics::atomic_umax_release(dst, val),
AcqRel => intrinsics::atomic_umax_acqrel(dst, val),
SeqCst => intrinsics::atomic_umax_seqcst(dst, val),
Relaxed => intrinsics::atomic_umax::<T, { AO::Relaxed }>(dst, val),
Acquire => intrinsics::atomic_umax::<T, { AO::Acquire }>(dst, val),
Release => intrinsics::atomic_umax::<T, { AO::Release }>(dst, val),
AcqRel => intrinsics::atomic_umax::<T, { AO::AcqRel }>(dst, val),
SeqCst => intrinsics::atomic_umax::<T, { AO::SeqCst }>(dst, val),
}
}
}
/// returns the min value (unsigned comparison)
/// Updates `*dst` to the min value of `val` and the old value (unsigned comparison)
#[inline]
#[cfg(target_has_atomic)]
#[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces
@ -4082,11 +4142,11 @@ unsafe fn atomic_umin<T: Copy>(dst: *mut T, val: T, order: Ordering) -> T {
// SAFETY: the caller must uphold the safety contract for `atomic_umin`
unsafe {
match order {
Relaxed => intrinsics::atomic_umin_relaxed(dst, val),
Acquire => intrinsics::atomic_umin_acquire(dst, val),
Release => intrinsics::atomic_umin_release(dst, val),
AcqRel => intrinsics::atomic_umin_acqrel(dst, val),
SeqCst => intrinsics::atomic_umin_seqcst(dst, val),
Relaxed => intrinsics::atomic_umin::<T, { AO::Relaxed }>(dst, val),
Acquire => intrinsics::atomic_umin::<T, { AO::Acquire }>(dst, val),
Release => intrinsics::atomic_umin::<T, { AO::Release }>(dst, val),
AcqRel => intrinsics::atomic_umin::<T, { AO::AcqRel }>(dst, val),
SeqCst => intrinsics::atomic_umin::<T, { AO::SeqCst }>(dst, val),
}
}
}
@ -4178,10 +4238,10 @@ pub fn fence(order: Ordering) {
// SAFETY: using an atomic fence is safe.
unsafe {
match order {
Acquire => intrinsics::atomic_fence_acquire(),
Release => intrinsics::atomic_fence_release(),
AcqRel => intrinsics::atomic_fence_acqrel(),
SeqCst => intrinsics::atomic_fence_seqcst(),
Acquire => intrinsics::atomic_fence::<{ AO::Acquire }>(),
Release => intrinsics::atomic_fence::<{ AO::Release }>(),
AcqRel => intrinsics::atomic_fence::<{ AO::AcqRel }>(),
SeqCst => intrinsics::atomic_fence::<{ AO::SeqCst }>(),
Relaxed => panic!("there is no such thing as a relaxed fence"),
}
}
@ -4256,11 +4316,11 @@ pub fn compiler_fence(order: Ordering) {
// SAFETY: using an atomic fence is safe.
unsafe {
match order {
Acquire => intrinsics::atomic_singlethreadfence_acquire(),
Release => intrinsics::atomic_singlethreadfence_release(),
AcqRel => intrinsics::atomic_singlethreadfence_acqrel(),
SeqCst => intrinsics::atomic_singlethreadfence_seqcst(),
Relaxed => panic!("there is no such thing as a relaxed compiler fence"),
Acquire => intrinsics::atomic_singlethreadfence::<{ AO::Acquire }>(),
Release => intrinsics::atomic_singlethreadfence::<{ AO::Release }>(),
AcqRel => intrinsics::atomic_singlethreadfence::<{ AO::AcqRel }>(),
SeqCst => intrinsics::atomic_singlethreadfence::<{ AO::SeqCst }>(),
Relaxed => panic!("there is no such thing as a relaxed fence"),
}
}
}

12
library/panic_unwind/src/seh.rs
View File

@ -291,7 +291,7 @@ cfg_if::cfg_if! {
}
pub(crate) unsafe fn panic(data: Box<dyn Any + Send>) -> u32 {
use core::intrinsics::atomic_store_seqcst;
use core::intrinsics::{AtomicOrdering, atomic_store};
// _CxxThrowException executes entirely on this stack frame, so there's no
// need to otherwise transfer `data` to the heap. We just pass a stack
@ -325,23 +325,23 @@ pub(crate) unsafe fn panic(data: Box<dyn Any + Send>) -> u32 {
// In any case, we basically need to do something like this until we can
// express more operations in statics (and we may never be able to).
unsafe {
atomic_store_seqcst(
atomic_store::<_, { AtomicOrdering::SeqCst }>(
(&raw mut THROW_INFO.pmfnUnwind).cast(),
ptr_t::new(exception_cleanup as *mut u8).raw(),
);
atomic_store_seqcst(
atomic_store::<_, { AtomicOrdering::SeqCst }>(
(&raw mut THROW_INFO.pCatchableTypeArray).cast(),
ptr_t::new((&raw mut CATCHABLE_TYPE_ARRAY).cast()).raw(),
);
atomic_store_seqcst(
atomic_store::<_, { AtomicOrdering::SeqCst }>(
(&raw mut CATCHABLE_TYPE_ARRAY.arrayOfCatchableTypes[0]).cast(),
ptr_t::new((&raw mut CATCHABLE_TYPE).cast()).raw(),
);
atomic_store_seqcst(
atomic_store::<_, { AtomicOrdering::SeqCst }>(
(&raw mut CATCHABLE_TYPE.pType).cast(),
ptr_t::new((&raw mut TYPE_DESCRIPTOR).cast()).raw(),
);
atomic_store_seqcst(
atomic_store::<_, { AtomicOrdering::SeqCst }>(
(&raw mut CATCHABLE_TYPE.copyFunction).cast(),
ptr_t::new(exception_copy as *mut u8).raw(),
);

147
src/tools/miri/src/intrinsics/atomic.rs
View File

@ -26,108 +26,131 @@ pub trait EvalContextExt<'tcx>: crate::MiriInterpCxExt<'tcx> {
) -> InterpResult<'tcx, EmulateItemResult> {
let this = self.eval_context_mut();
let intrinsic_structure: Vec<_> = intrinsic_name.split('_').collect();
let get_ord_at = |i: usize| {
let ordering = generic_args.const_at(i).to_value();
ordering.valtree.unwrap_branch()[0].unwrap_leaf().to_atomic_ordering()
};
fn read_ord(ord: &str) -> AtomicReadOrd {
fn read_ord(ord: AtomicOrdering) -> AtomicReadOrd {
match ord {
"seqcst" => AtomicReadOrd::SeqCst,
"acquire" => AtomicReadOrd::Acquire,
"relaxed" => AtomicReadOrd::Relaxed,
_ => panic!("invalid read ordering `{ord}`"),
}
}
fn read_ord_const_generic(o: AtomicOrdering) -> AtomicReadOrd {
match o {
AtomicOrdering::SeqCst => AtomicReadOrd::SeqCst,
AtomicOrdering::Acquire => AtomicReadOrd::Acquire,
AtomicOrdering::Relaxed => AtomicReadOrd::Relaxed,
_ => panic!("invalid read ordering `{o:?}`"),
_ => panic!("invalid read ordering `{ord:?}`"),
}
}
fn write_ord(ord: &str) -> AtomicWriteOrd {
fn write_ord(ord: AtomicOrdering) -> AtomicWriteOrd {
match ord {
"seqcst" => AtomicWriteOrd::SeqCst,
"release" => AtomicWriteOrd::Release,
"relaxed" => AtomicWriteOrd::Relaxed,
_ => panic!("invalid write ordering `{ord}`"),
AtomicOrdering::SeqCst => AtomicWriteOrd::SeqCst,
AtomicOrdering::Release => AtomicWriteOrd::Release,
AtomicOrdering::Relaxed => AtomicWriteOrd::Relaxed,
_ => panic!("invalid write ordering `{ord:?}`"),
}
}
fn rw_ord(ord: &str) -> AtomicRwOrd {
fn rw_ord(ord: AtomicOrdering) -> AtomicRwOrd {
match ord {
"seqcst" => AtomicRwOrd::SeqCst,
"acqrel" => AtomicRwOrd::AcqRel,
"acquire" => AtomicRwOrd::Acquire,
"release" => AtomicRwOrd::Release,
"relaxed" => AtomicRwOrd::Relaxed,
_ => panic!("invalid read-write ordering `{ord}`"),
AtomicOrdering::SeqCst => AtomicRwOrd::SeqCst,
AtomicOrdering::AcqRel => AtomicRwOrd::AcqRel,
AtomicOrdering::Acquire => AtomicRwOrd::Acquire,
AtomicOrdering::Release => AtomicRwOrd::Release,
AtomicOrdering::Relaxed => AtomicRwOrd::Relaxed,
}
}
fn fence_ord(ord: &str) -> AtomicFenceOrd {
fn fence_ord(ord: AtomicOrdering) -> AtomicFenceOrd {
match ord {
"seqcst" => AtomicFenceOrd::SeqCst,
"acqrel" => AtomicFenceOrd::AcqRel,
"acquire" => AtomicFenceOrd::Acquire,
"release" => AtomicFenceOrd::Release,
_ => panic!("invalid fence ordering `{ord}`"),
AtomicOrdering::SeqCst => AtomicFenceOrd::SeqCst,
AtomicOrdering::AcqRel => AtomicFenceOrd::AcqRel,
AtomicOrdering::Acquire => AtomicFenceOrd::Acquire,
AtomicOrdering::Release => AtomicFenceOrd::Release,
_ => panic!("invalid fence ordering `{ord:?}`"),
}
}
match &*intrinsic_structure {
// New-style intrinsics that use const generics
["load"] => {
let ordering = generic_args.const_at(1).to_value();
let ordering =
ordering.valtree.unwrap_branch()[0].unwrap_leaf().to_atomic_ordering();
this.atomic_load(args, dest, read_ord_const_generic(ordering))?;
match intrinsic_name {
"load" => {
let ord = get_ord_at(1);
this.atomic_load(args, dest, read_ord(ord))?;
}
// Old-style intrinsics that have the ordering in the intrinsic name
["store", ord] => this.atomic_store(args, write_ord(ord))?,
"store" => {
let ord = get_ord_at(1);
this.atomic_store(args, write_ord(ord))?
}
["fence", ord] => this.atomic_fence_intrinsic(args, fence_ord(ord))?,
["singlethreadfence", ord] => this.compiler_fence_intrinsic(args, fence_ord(ord))?,
"fence" => {
let ord = get_ord_at(0);
this.atomic_fence_intrinsic(args, fence_ord(ord))?
}
"singlethreadfence" => {
let ord = get_ord_at(0);
this.compiler_fence_intrinsic(args, fence_ord(ord))?;
}
["xchg", ord] => this.atomic_exchange(args, dest, rw_ord(ord))?,
["cxchg", ord1, ord2] =>
this.atomic_compare_exchange(args, dest, rw_ord(ord1), read_ord(ord2))?,
["cxchgweak", ord1, ord2] =>
this.atomic_compare_exchange_weak(args, dest, rw_ord(ord1), read_ord(ord2))?,
"xchg" => {
let ord = get_ord_at(1);
this.atomic_exchange(args, dest, rw_ord(ord))?;
}
"cxchg" => {
let ord1 = get_ord_at(1);
let ord2 = get_ord_at(2);
this.atomic_compare_exchange(args, dest, rw_ord(ord1), read_ord(ord2))?;
}
"cxchgweak" => {
let ord1 = get_ord_at(1);
let ord2 = get_ord_at(2);
this.atomic_compare_exchange_weak(args, dest, rw_ord(ord1), read_ord(ord2))?;
}
["or", ord] =>
this.atomic_rmw_op(args, dest, AtomicOp::MirOp(BinOp::BitOr, false), rw_ord(ord))?,
["xor", ord] =>
this.atomic_rmw_op(args, dest, AtomicOp::MirOp(BinOp::BitXor, false), rw_ord(ord))?,
["and", ord] =>
this.atomic_rmw_op(args, dest, AtomicOp::MirOp(BinOp::BitAnd, false), rw_ord(ord))?,
["nand", ord] =>
this.atomic_rmw_op(args, dest, AtomicOp::MirOp(BinOp::BitAnd, true), rw_ord(ord))?,
["xadd", ord] =>
this.atomic_rmw_op(args, dest, AtomicOp::MirOp(BinOp::Add, false), rw_ord(ord))?,
["xsub", ord] =>
this.atomic_rmw_op(args, dest, AtomicOp::MirOp(BinOp::Sub, false), rw_ord(ord))?,
["min", ord] => {
"or" => {
let ord = get_ord_at(1);
this.atomic_rmw_op(args, dest, AtomicOp::MirOp(BinOp::BitOr, false), rw_ord(ord))?;
}
"xor" => {
let ord = get_ord_at(1);
this.atomic_rmw_op(args, dest, AtomicOp::MirOp(BinOp::BitXor, false), rw_ord(ord))?;
}
"and" => {
let ord = get_ord_at(1);
this.atomic_rmw_op(args, dest, AtomicOp::MirOp(BinOp::BitAnd, false), rw_ord(ord))?;
}
"nand" => {
let ord = get_ord_at(1);
this.atomic_rmw_op(args, dest, AtomicOp::MirOp(BinOp::BitAnd, true), rw_ord(ord))?;
}
"xadd" => {
let ord = get_ord_at(1);
this.atomic_rmw_op(args, dest, AtomicOp::MirOp(BinOp::Add, false), rw_ord(ord))?;
}
"xsub" => {
let ord = get_ord_at(1);
this.atomic_rmw_op(args, dest, AtomicOp::MirOp(BinOp::Sub, false), rw_ord(ord))?;
}
"min" => {
let ord = get_ord_at(1);
// Later we will use the type to indicate signed vs unsigned,
// so make sure it matches the intrinsic name.
assert!(matches!(args[1].layout.ty.kind(), ty::Int(_)));
this.atomic_rmw_op(args, dest, AtomicOp::Min, rw_ord(ord))?;
}
["umin", ord] => {
"umin" => {
let ord = get_ord_at(1);
// Later we will use the type to indicate signed vs unsigned,
// so make sure it matches the intrinsic name.
assert!(matches!(args[1].layout.ty.kind(), ty::Uint(_)));
this.atomic_rmw_op(args, dest, AtomicOp::Min, rw_ord(ord))?;
}
["max", ord] => {
"max" => {
let ord = get_ord_at(1);
// Later we will use the type to indicate signed vs unsigned,
// so make sure it matches the intrinsic name.
assert!(matches!(args[1].layout.ty.kind(), ty::Int(_)));
this.atomic_rmw_op(args, dest, AtomicOp::Max, rw_ord(ord))?;
}
["umax", ord] => {
"umax" => {
let ord = get_ord_at(1);
// Later we will use the type to indicate signed vs unsigned,
// so make sure it matches the intrinsic name.
assert!(matches!(args[1].layout.ty.kind(), ty::Uint(_)));

42
tests/run-make/atomic-lock-free/atomic_lock_free.rs
View File

@ -1,9 +1,20 @@
#![feature(no_core, intrinsics, lang_items)]
#![feature(adt_const_params)]
#![crate_type = "rlib"]
#![no_core]
pub enum AtomicOrdering {
// These values must match the compiler's `AtomicOrdering` defined in
// `rustc_middle/src/ty/consts/int.rs`!
Relaxed = 0,
Release = 1,
Acquire = 2,
AcqRel = 3,
SeqCst = 4,
}
#[rustc_intrinsic]
unsafe fn atomic_xadd_seqcst<T>(dst: *mut T, src: T) -> T;
unsafe fn atomic_xadd<T, const ORD: AtomicOrdering>(dst: *mut T, src: T) -> T;
#[lang = "sized"]
trait Sized {}
@ -11,55 +22,58 @@ trait Sized {}
trait Copy {}
#[lang = "freeze"]
trait Freeze {}
#[lang = "const_param_ty"]
pub trait ConstParamTy {}
impl<T: ?Sized> Copy for *mut T {}
impl ConstParamTy for AtomicOrdering {}
#[cfg(target_has_atomic = "8")]
pub unsafe fn atomic_u8(x: *mut u8) {
atomic_xadd_seqcst(x, 1);
atomic_xadd_seqcst(x, 1);
atomic_xadd::<_, { AtomicOrdering::SeqCst }>(x, 1);
atomic_xadd::<_, { AtomicOrdering::SeqCst }>(x, 1);
}
#[cfg(target_has_atomic = "8")]
pub unsafe fn atomic_i8(x: *mut i8) {
atomic_xadd_seqcst(x, 1);
atomic_xadd::<_, { AtomicOrdering::SeqCst }>(x, 1);
}
#[cfg(target_has_atomic = "16")]
pub unsafe fn atomic_u16(x: *mut u16) {
atomic_xadd_seqcst(x, 1);
atomic_xadd::<_, { AtomicOrdering::SeqCst }>(x, 1);
}
#[cfg(target_has_atomic = "16")]
pub unsafe fn atomic_i16(x: *mut i16) {
atomic_xadd_seqcst(x, 1);
atomic_xadd::<_, { AtomicOrdering::SeqCst }>(x, 1);
}
#[cfg(target_has_atomic = "32")]
pub unsafe fn atomic_u32(x: *mut u32) {
atomic_xadd_seqcst(x, 1);
atomic_xadd::<_, { AtomicOrdering::SeqCst }>(x, 1);
}
#[cfg(target_has_atomic = "32")]
pub unsafe fn atomic_i32(x: *mut i32) {
atomic_xadd_seqcst(x, 1);
atomic_xadd::<_, { AtomicOrdering::SeqCst }>(x, 1);
}
#[cfg(target_has_atomic = "64")]
pub unsafe fn atomic_u64(x: *mut u64) {
atomic_xadd_seqcst(x, 1);
atomic_xadd::<_, { AtomicOrdering::SeqCst }>(x, 1);
}
#[cfg(target_has_atomic = "64")]
pub unsafe fn atomic_i64(x: *mut i64) {
atomic_xadd_seqcst(x, 1);
atomic_xadd::<_, { AtomicOrdering::SeqCst }>(x, 1);
}
#[cfg(target_has_atomic = "128")]
pub unsafe fn atomic_u128(x: *mut u128) {
atomic_xadd_seqcst(x, 1);
atomic_xadd::<_, { AtomicOrdering::SeqCst }>(x, 1);
}
#[cfg(target_has_atomic = "128")]
pub unsafe fn atomic_i128(x: *mut i128) {
atomic_xadd_seqcst(x, 1);
atomic_xadd::<_, { AtomicOrdering::SeqCst }>(x, 1);
}
#[cfg(target_has_atomic = "ptr")]
pub unsafe fn atomic_usize(x: *mut usize) {
atomic_xadd_seqcst(x, 1);
atomic_xadd::<_, { AtomicOrdering::SeqCst }>(x, 1);
}
#[cfg(target_has_atomic = "ptr")]
pub unsafe fn atomic_isize(x: *mut isize) {
atomic_xadd_seqcst(x, 1);
atomic_xadd::<_, { AtomicOrdering::SeqCst }>(x, 1);
}

6
tests/ui/error-codes/E0092.rs
View File

@ -1,6 +0,0 @@
#![feature(intrinsics)]
#[rustc_intrinsic]
unsafe fn atomic_foo(); //~ ERROR E0092
fn main() {}

9
tests/ui/error-codes/E0092.stderr
View File

@ -1,9 +0,0 @@
error[E0092]: unrecognized atomic operation function: `foo`
--> $DIR/E0092.rs:4:11
|
LL | unsafe fn atomic_foo();
| ^^^^^^^^^^ unrecognized atomic operation
error: aborting due to 1 previous error
For more information about this error, try `rustc --explain E0092`.

2
tests/ui/feature-gates/feature-gated-feature-in-macro-arg.stderr
View File

@ -7,7 +7,7 @@ LL | #[rustc_intrinsic]
= help: add `#![feature(intrinsics)]` to the crate attributes to enable
= note: this compiler was built on YYYY-MM-DD; consider upgrading it if it is out of date
error[E0133]: call to unsafe function `atomic_fence` is unsafe and requires unsafe function or block
error[E0133]: call to unsafe function `main::atomic_fence` is unsafe and requires unsafe function or block
--> $DIR/feature-gated-feature-in-macro-arg.rs:11:9
|
LL | atomic_fence();

19
tests/ui/intrinsics/auxiliary/cci_intrinsic.rs
View File

@ -1,11 +1,24 @@
#![feature(intrinsics)]
#![feature(intrinsics, adt_const_params)]
pub mod rusti {
use std::marker::ConstParamTy;
#[derive(Debug, ConstParamTy, PartialEq, Eq)]
pub enum AtomicOrdering {
// These values must match the compiler's `AtomicOrdering` defined in
// `rustc_middle/src/ty/consts/int.rs`!
Relaxed = 0,
Release = 1,
Acquire = 2,
AcqRel = 3,
SeqCst = 4,
}
#[rustc_intrinsic]
pub unsafe fn atomic_xchg_seqcst<T>(dst: *mut T, src: T) -> T;
pub unsafe fn atomic_xchg<T, const ORD: AtomicOrdering>(dst: *mut T, src: T) -> T;
}
#[inline(always)]
pub fn atomic_xchg_seqcst(dst: *mut isize, src: isize) -> isize {
unsafe { rusti::atomic_xchg_seqcst(dst, src) }
unsafe { rusti::atomic_xchg::<_, { rusti::AtomicOrdering::SeqCst }>(dst, src) }
}

40
tests/ui/intrinsics/intrinsic-atomics.rs
View File

@ -1,50 +1,50 @@
//@ run-pass
#![feature(core_intrinsics)]
use std::intrinsics::{self as rusti, AtomicOrdering};
use std::intrinsics::{self as rusti, AtomicOrdering::*};
pub fn main() {
unsafe {
let mut x: Box<_> = Box::new(1);
assert_eq!(rusti::atomic_load::<_, { AtomicOrdering::SeqCst }>(&*x), 1);
assert_eq!(rusti::atomic_load::<_, { SeqCst }>(&*x), 1);
*x = 5;
assert_eq!(rusti::atomic_load::<_, { AtomicOrdering::Acquire }>(&*x), 5);
assert_eq!(rusti::atomic_load::<_, { Acquire }>(&*x), 5);
rusti::atomic_store_seqcst(&mut *x, 3);
rusti::atomic_store::<_, { SeqCst }>(&mut *x, 3);
assert_eq!(*x, 3);
rusti::atomic_store_release(&mut *x, 1);
rusti::atomic_store::<_, { Release }>(&mut *x, 1);
assert_eq!(*x, 1);
assert_eq!(rusti::atomic_cxchg_seqcst_seqcst(&mut *x, 1, 2), (1, true));
assert_eq!(rusti::atomic_cxchg::<_, { SeqCst }, { SeqCst }>(&mut *x, 1, 2), (1, true));
assert_eq!(*x, 2);
assert_eq!(rusti::atomic_cxchg_acquire_acquire(&mut *x, 1, 3), (2, false));
assert_eq!(rusti::atomic_cxchg::<_, { Acquire }, { Acquire }>(&mut *x, 1, 3), (2, false));
assert_eq!(*x, 2);
assert_eq!(rusti::atomic_cxchg_release_relaxed(&mut *x, 2, 1), (2, true));
assert_eq!(rusti::atomic_cxchg::<_, { Release }, { Relaxed }>(&mut *x, 2, 1), (2, true));
assert_eq!(*x, 1);
assert_eq!(rusti::atomic_xchg_seqcst(&mut *x, 0), 1);
assert_eq!(rusti::atomic_xchg::<_, { SeqCst }>(&mut *x, 0), 1);
assert_eq!(*x, 0);
assert_eq!(rusti::atomic_xchg_acquire(&mut *x, 1), 0);
assert_eq!(rusti::atomic_xchg::<_, { Acquire }>(&mut *x, 1), 0);
assert_eq!(*x, 1);
assert_eq!(rusti::atomic_xchg_release(&mut *x, 0), 1);
assert_eq!(rusti::atomic_xchg::<_, { Release }>(&mut *x, 0), 1);
assert_eq!(*x, 0);
assert_eq!(rusti::atomic_xadd_seqcst(&mut *x, 1), 0);
assert_eq!(rusti::atomic_xadd_acquire(&mut *x, 1), 1);
assert_eq!(rusti::atomic_xadd_release(&mut *x, 1), 2);
assert_eq!(rusti::atomic_xadd::<_, { SeqCst }>(&mut *x, 1), 0);
assert_eq!(rusti::atomic_xadd::<_, { Acquire }>(&mut *x, 1), 1);
assert_eq!(rusti::atomic_xadd::<_, { Release }>(&mut *x, 1), 2);
assert_eq!(*x, 3);
assert_eq!(rusti::atomic_xsub_seqcst(&mut *x, 1), 3);
assert_eq!(rusti::atomic_xsub_acquire(&mut *x, 1), 2);
assert_eq!(rusti::atomic_xsub_release(&mut *x, 1), 1);
assert_eq!(rusti::atomic_xsub::<_, { SeqCst }>(&mut *x, 1), 3);
assert_eq!(rusti::atomic_xsub::<_, { Acquire }>(&mut *x, 1), 2);
assert_eq!(rusti::atomic_xsub::<_, { Release }>(&mut *x, 1), 1);
assert_eq!(*x, 0);
loop {
let res = rusti::atomic_cxchgweak_seqcst_seqcst(&mut *x, 0, 1);
let res = rusti::atomic_cxchgweak::<_, { SeqCst }, { SeqCst }>(&mut *x, 0, 1);
assert_eq!(res.0, 0);
if res.1 {
break;
@ -53,7 +53,7 @@ pub fn main() {
assert_eq!(*x, 1);
loop {
let res = rusti::atomic_cxchgweak_acquire_acquire(&mut *x, 1, 2);
let res = rusti::atomic_cxchgweak::<_, { Acquire }, { Acquire }>(&mut *x, 1, 2);
assert_eq!(res.0, 1);
if res.1 {
break;
@ -62,7 +62,7 @@ pub fn main() {
assert_eq!(*x, 2);
loop {
let res = rusti::atomic_cxchgweak_release_relaxed(&mut *x, 2, 3);
let res = rusti::atomic_cxchgweak::<_, { Release }, { Relaxed }>(&mut *x, 2, 3);
assert_eq!(res.0, 2);
if res.1 {
break;

52
tests/ui/intrinsics/non-integer-atomic.rs
View File

@ -4,7 +4,7 @@
#![allow(warnings)]
#![crate_type = "rlib"]
use std::intrinsics::{self, AtomicOrdering};
use std::intrinsics::{self, AtomicOrdering::*};
#[derive(Copy, Clone)]
pub struct Foo(i64);
@ -12,81 +12,81 @@ pub type Bar = &'static dyn Fn();
pub type Quux = [u8; 100];
pub unsafe fn test_bool_load(p: &mut bool, v: bool) {
intrinsics::atomic_load::<_, { AtomicOrdering::SeqCst }>(p);
intrinsics::atomic_load::<_, { SeqCst }>(p);
//~^ ERROR `atomic_load` intrinsic: expected basic integer type, found `bool`
}
pub unsafe fn test_bool_store(p: &mut bool, v: bool) {
intrinsics::atomic_store_seqcst(p, v);
//~^ ERROR `atomic_store_seqcst` intrinsic: expected basic integer type, found `bool`
intrinsics::atomic_store::<_, { SeqCst }>(p, v);
//~^ ERROR `atomic_store` intrinsic: expected basic integer type, found `bool`
}
pub unsafe fn test_bool_xchg(p: &mut bool, v: bool) {
intrinsics::atomic_xchg_seqcst(p, v);
//~^ ERROR `atomic_xchg_seqcst` intrinsic: expected basic integer type, found `bool`
intrinsics::atomic_xchg::<_, { SeqCst }>(p, v);
//~^ ERROR `atomic_xchg` intrinsic: expected basic integer type, found `bool`
}
pub unsafe fn test_bool_cxchg(p: &mut bool, v: bool) {
intrinsics::atomic_cxchg_seqcst_seqcst(p, v, v);
//~^ ERROR `atomic_cxchg_seqcst_seqcst` intrinsic: expected basic integer type, found `bool`
intrinsics::atomic_cxchg::<_, { SeqCst }, { SeqCst }>(p, v, v);
//~^ ERROR `atomic_cxchg` intrinsic: expected basic integer type, found `bool`
}
pub unsafe fn test_Foo_load(p: &mut Foo, v: Foo) {
intrinsics::atomic_load::<_, { AtomicOrdering::SeqCst }>(p);
intrinsics::atomic_load::<_, { SeqCst }>(p);
//~^ ERROR `atomic_load` intrinsic: expected basic integer type, found `Foo`
}
pub unsafe fn test_Foo_store(p: &mut Foo, v: Foo) {
intrinsics::atomic_store_seqcst(p, v);
//~^ ERROR `atomic_store_seqcst` intrinsic: expected basic integer type, found `Foo`
intrinsics::atomic_store::<_, { SeqCst }>(p, v);
//~^ ERROR `atomic_store` intrinsic: expected basic integer type, found `Foo`
}
pub unsafe fn test_Foo_xchg(p: &mut Foo, v: Foo) {
intrinsics::atomic_xchg_seqcst(p, v);
//~^ ERROR `atomic_xchg_seqcst` intrinsic: expected basic integer type, found `Foo`
intrinsics::atomic_xchg::<_, { SeqCst }>(p, v);
//~^ ERROR `atomic_xchg` intrinsic: expected basic integer type, found `Foo`
}
pub unsafe fn test_Foo_cxchg(p: &mut Foo, v: Foo) {
intrinsics::atomic_cxchg_seqcst_seqcst(p, v, v);
//~^ ERROR `atomic_cxchg_seqcst_seqcst` intrinsic: expected basic integer type, found `Foo`
intrinsics::atomic_cxchg::<_, { SeqCst }, { SeqCst }>(p, v, v);
//~^ ERROR `atomic_cxchg` intrinsic: expected basic integer type, found `Foo`
}
pub unsafe fn test_Bar_load(p: &mut Bar, v: Bar) {
intrinsics::atomic_load::<_, { AtomicOrdering::SeqCst }>(p);
intrinsics::atomic_load::<_, { SeqCst }>(p);
//~^ ERROR expected basic integer type, found `&dyn Fn()`
}
pub unsafe fn test_Bar_store(p: &mut Bar, v: Bar) {
intrinsics::atomic_store_seqcst(p, v);
intrinsics::atomic_store::<_, { SeqCst }>(p, v);
//~^ ERROR expected basic integer type, found `&dyn Fn()`
}
pub unsafe fn test_Bar_xchg(p: &mut Bar, v: Bar) {
intrinsics::atomic_xchg_seqcst(p, v);
intrinsics::atomic_xchg::<_, { SeqCst }>(p, v);
//~^ ERROR expected basic integer type, found `&dyn Fn()`
}
pub unsafe fn test_Bar_cxchg(p: &mut Bar, v: Bar) {
intrinsics::atomic_cxchg_seqcst_seqcst(p, v, v);
intrinsics::atomic_cxchg::<_, { SeqCst }, { SeqCst }>(p, v, v);
//~^ ERROR expected basic integer type, found `&dyn Fn()`
}
pub unsafe fn test_Quux_load(p: &mut Quux, v: Quux) {
intrinsics::atomic_load::<_, { AtomicOrdering::SeqCst }>(p);
intrinsics::atomic_load::<_, { SeqCst }>(p);
//~^ ERROR `atomic_load` intrinsic: expected basic integer type, found `[u8; 100]`
}
pub unsafe fn test_Quux_store(p: &mut Quux, v: Quux) {
intrinsics::atomic_store_seqcst(p, v);
//~^ ERROR `atomic_store_seqcst` intrinsic: expected basic integer type, found `[u8; 100]`
intrinsics::atomic_store::<_, { SeqCst }>(p, v);
//~^ ERROR `atomic_store` intrinsic: expected basic integer type, found `[u8; 100]`
}
pub unsafe fn test_Quux_xchg(p: &mut Quux, v: Quux) {
intrinsics::atomic_xchg_seqcst(p, v);
//~^ ERROR `atomic_xchg_seqcst` intrinsic: expected basic integer type, found `[u8; 100]`
intrinsics::atomic_xchg::<_, { SeqCst }>(p, v);
//~^ ERROR `atomic_xchg` intrinsic: expected basic integer type, found `[u8; 100]`
}
pub unsafe fn test_Quux_cxchg(p: &mut Quux, v: Quux) {
intrinsics::atomic_cxchg_seqcst_seqcst(p, v, v);
//~^ ERROR `atomic_cxchg_seqcst_seqcst` intrinsic: expected basic integer type, found `[u8; 100]`
intrinsics::atomic_cxchg::<_, { SeqCst }, { SeqCst }>(p, v, v);
//~^ ERROR `atomic_cxchg` intrinsic: expected basic integer type, found `[u8; 100]`
}

88
tests/ui/intrinsics/non-integer-atomic.stderr
View File

@ -1,98 +1,98 @@
error[E0511]: invalid monomorphization of `atomic_load` intrinsic: expected basic integer type, found `bool`
--> $DIR/non-integer-atomic.rs:15:5
|
LL | intrinsics::atomic_load::<_, { AtomicOrdering::SeqCst }>(p);
| ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
LL | intrinsics::atomic_load::<_, { SeqCst }>(p);
| ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
error[E0511]: invalid monomorphization of `atomic_store_seqcst` intrinsic: expected basic integer type, found `bool`
error[E0511]: invalid monomorphization of `atomic_store` intrinsic: expected basic integer type, found `bool`
--> $DIR/non-integer-atomic.rs:20:5
|
LL | intrinsics::atomic_store_seqcst(p, v);
| ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
LL | intrinsics::atomic_store::<_, { SeqCst }>(p, v);
| ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
error[E0511]: invalid monomorphization of `atomic_xchg_seqcst` intrinsic: expected basic integer type, found `bool`
error[E0511]: invalid monomorphization of `atomic_xchg` intrinsic: expected basic integer type, found `bool`
--> $DIR/non-integer-atomic.rs:25:5
|
LL | intrinsics::atomic_xchg_seqcst(p, v);
| ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
LL | intrinsics::atomic_xchg::<_, { SeqCst }>(p, v);
| ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
error[E0511]: invalid monomorphization of `atomic_cxchg_seqcst_seqcst` intrinsic: expected basic integer type, found `bool`
error[E0511]: invalid monomorphization of `atomic_cxchg` intrinsic: expected basic integer type, found `bool`
--> $DIR/non-integer-atomic.rs:30:5
|
LL | intrinsics::atomic_cxchg_seqcst_seqcst(p, v, v);
| ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
LL | intrinsics::atomic_cxchg::<_, { SeqCst }, { SeqCst }>(p, v, v);
| ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
error[E0511]: invalid monomorphization of `atomic_load` intrinsic: expected basic integer type, found `Foo`
--> $DIR/non-integer-atomic.rs:35:5
|
LL | intrinsics::atomic_load::<_, { AtomicOrdering::SeqCst }>(p);
| ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
LL | intrinsics::atomic_load::<_, { SeqCst }>(p);
| ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
error[E0511]: invalid monomorphization of `atomic_store_seqcst` intrinsic: expected basic integer type, found `Foo`
error[E0511]: invalid monomorphization of `atomic_store` intrinsic: expected basic integer type, found `Foo`
--> $DIR/non-integer-atomic.rs:40:5
|
LL | intrinsics::atomic_store_seqcst(p, v);
| ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
LL | intrinsics::atomic_store::<_, { SeqCst }>(p, v);
| ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
error[E0511]: invalid monomorphization of `atomic_xchg_seqcst` intrinsic: expected basic integer type, found `Foo`
error[E0511]: invalid monomorphization of `atomic_xchg` intrinsic: expected basic integer type, found `Foo`
--> $DIR/non-integer-atomic.rs:45:5
|
LL | intrinsics::atomic_xchg_seqcst(p, v);
| ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
LL | intrinsics::atomic_xchg::<_, { SeqCst }>(p, v);
| ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
error[E0511]: invalid monomorphization of `atomic_cxchg_seqcst_seqcst` intrinsic: expected basic integer type, found `Foo`
error[E0511]: invalid monomorphization of `atomic_cxchg` intrinsic: expected basic integer type, found `Foo`
--> $DIR/non-integer-atomic.rs:50:5
|
LL | intrinsics::atomic_cxchg_seqcst_seqcst(p, v, v);
| ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
LL | intrinsics::atomic_cxchg::<_, { SeqCst }, { SeqCst }>(p, v, v);
| ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
error[E0511]: invalid monomorphization of `atomic_load` intrinsic: expected basic integer type, found `&dyn Fn()`
--> $DIR/non-integer-atomic.rs:55:5
|
LL | intrinsics::atomic_load::<_, { AtomicOrdering::SeqCst }>(p);
| ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
LL | intrinsics::atomic_load::<_, { SeqCst }>(p);
| ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
error[E0511]: invalid monomorphization of `atomic_store_seqcst` intrinsic: expected basic integer type, found `&dyn Fn()`
error[E0511]: invalid monomorphization of `atomic_store` intrinsic: expected basic integer type, found `&dyn Fn()`
--> $DIR/non-integer-atomic.rs:60:5
|
LL | intrinsics::atomic_store_seqcst(p, v);
| ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
LL | intrinsics::atomic_store::<_, { SeqCst }>(p, v);
| ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
error[E0511]: invalid monomorphization of `atomic_xchg_seqcst` intrinsic: expected basic integer type, found `&dyn Fn()`
error[E0511]: invalid monomorphization of `atomic_xchg` intrinsic: expected basic integer type, found `&dyn Fn()`
--> $DIR/non-integer-atomic.rs:65:5
|
LL | intrinsics::atomic_xchg_seqcst(p, v);
| ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
LL | intrinsics::atomic_xchg::<_, { SeqCst }>(p, v);
| ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
error[E0511]: invalid monomorphization of `atomic_cxchg_seqcst_seqcst` intrinsic: expected basic integer type, found `&dyn Fn()`
error[E0511]: invalid monomorphization of `atomic_cxchg` intrinsic: expected basic integer type, found `&dyn Fn()`
--> $DIR/non-integer-atomic.rs:70:5
|
LL | intrinsics::atomic_cxchg_seqcst_seqcst(p, v, v);
| ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
LL | intrinsics::atomic_cxchg::<_, { SeqCst }, { SeqCst }>(p, v, v);
| ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
error[E0511]: invalid monomorphization of `atomic_load` intrinsic: expected basic integer type, found `[u8; 100]`
--> $DIR/non-integer-atomic.rs:75:5
|
LL | intrinsics::atomic_load::<_, { AtomicOrdering::SeqCst }>(p);
| ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
LL | intrinsics::atomic_load::<_, { SeqCst }>(p);
| ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
error[E0511]: invalid monomorphization of `atomic_store_seqcst` intrinsic: expected basic integer type, found `[u8; 100]`
error[E0511]: invalid monomorphization of `atomic_store` intrinsic: expected basic integer type, found `[u8; 100]`
--> $DIR/non-integer-atomic.rs:80:5
|
LL | intrinsics::atomic_store_seqcst(p, v);
| ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
LL | intrinsics::atomic_store::<_, { SeqCst }>(p, v);
| ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
error[E0511]: invalid monomorphization of `atomic_xchg_seqcst` intrinsic: expected basic integer type, found `[u8; 100]`
error[E0511]: invalid monomorphization of `atomic_xchg` intrinsic: expected basic integer type, found `[u8; 100]`
--> $DIR/non-integer-atomic.rs:85:5
|
LL | intrinsics::atomic_xchg_seqcst(p, v);
| ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
LL | intrinsics::atomic_xchg::<_, { SeqCst }>(p, v);
| ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
error[E0511]: invalid monomorphization of `atomic_cxchg_seqcst_seqcst` intrinsic: expected basic integer type, found `[u8; 100]`
error[E0511]: invalid monomorphization of `atomic_cxchg` intrinsic: expected basic integer type, found `[u8; 100]`
--> $DIR/non-integer-atomic.rs:90:5
|
LL | intrinsics::atomic_cxchg_seqcst_seqcst(p, v, v);
| ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
LL | intrinsics::atomic_cxchg::<_, { SeqCst }, { SeqCst }>(p, v, v);
| ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
error: aborting due to 16 previous errors