From 64ddcb33f431be554449116706eb0d6af2666a11 Mon Sep 17 00:00:00 2001 From: Amanieu d'Antras Date: Sat, 16 Jan 2016 23:40:11 +0000 Subject: [PATCH] Add intrinsics for compare_exchange and compare_exchange_weak --- src/libcore/intrinsics.rs | 27 +++++++++++ src/librustc_llvm/lib.rs | 3 +- src/librustc_trans/trans/build.rs | 5 +- src/librustc_trans/trans/builder.rs | 5 +- src/librustc_trans/trans/intrinsic.rs | 65 ++++++++++++++------------ src/librustc_typeck/check/intrinsic.rs | 4 ++ src/rustllvm/RustWrapper.cpp | 14 ++++-- src/test/run-pass/intrinsic-atomics.rs | 31 ++++++++++++ 8 files changed, 114 insertions(+), 40 deletions(-) diff --git a/src/libcore/intrinsics.rs b/src/libcore/intrinsics.rs index 2e2292d63f4c..47f5d68f3115 100644 --- a/src/libcore/intrinsics.rs +++ b/src/libcore/intrinsics.rs @@ -58,6 +58,33 @@ extern "rust-intrinsic" { pub fn atomic_cxchg_rel(dst: *mut T, old: T, src: T) -> T; pub fn atomic_cxchg_acqrel(dst: *mut T, old: T, src: T) -> T; pub fn atomic_cxchg_relaxed(dst: *mut T, old: T, src: T) -> T; + #[cfg(not(stage0))] + pub fn atomic_cxchg_failrelaxed(dst: *mut T, old: T, src: T) -> T; + #[cfg(not(stage0))] + pub fn atomic_cxchg_failacq(dst: *mut T, old: T, src: T) -> T; + #[cfg(not(stage0))] + pub fn atomic_cxchg_acq_failrelaxed(dst: *mut T, old: T, src: T) -> T; + #[cfg(not(stage0))] + pub fn atomic_cxchg_acqrel_failrelaxed(dst: *mut T, old: T, src: T) -> T; + + #[cfg(not(stage0))] + pub fn atomic_cxchgweak(dst: *mut T, old: T, src: T) -> (T, bool); + #[cfg(not(stage0))] + pub fn atomic_cxchgweak_acq(dst: *mut T, old: T, src: T) -> (T, bool); + #[cfg(not(stage0))] + pub fn atomic_cxchgweak_rel(dst: *mut T, old: T, src: T) -> (T, bool); + #[cfg(not(stage0))] + pub fn atomic_cxchgweak_acqrel(dst: *mut T, old: T, src: T) -> (T, bool); + #[cfg(not(stage0))] + pub fn atomic_cxchgweak_relaxed(dst: *mut T, old: T, src: T) -> (T, bool); + #[cfg(not(stage0))] + pub fn atomic_cxchgweak_failrelaxed(dst: *mut T, old: T, src: T) -> (T, bool); + #[cfg(not(stage0))] + pub fn atomic_cxchgweak_failacq(dst: *mut T, old: T, src: T) -> (T, bool); + #[cfg(not(stage0))] + pub fn atomic_cxchgweak_acq_failrelaxed(dst: *mut T, old: T, src: T) -> (T, bool); + #[cfg(not(stage0))] + pub fn atomic_cxchgweak_acqrel_failrelaxed(dst: *mut T, old: T, src: T) -> (T, bool); pub fn atomic_load(src: *const T) -> T; pub fn atomic_load_acq(src: *const T) -> T; diff --git a/src/librustc_llvm/lib.rs b/src/librustc_llvm/lib.rs index 1933c926e301..d55a801875be 100644 --- a/src/librustc_llvm/lib.rs +++ b/src/librustc_llvm/lib.rs @@ -1584,7 +1584,8 @@ extern { CMP: ValueRef, RHS: ValueRef, Order: AtomicOrdering, - FailureOrder: AtomicOrdering) + FailureOrder: AtomicOrdering, + Weak: Bool) -> ValueRef; pub fn LLVMBuildAtomicRMW(B: BuilderRef, Op: AtomicBinOp, diff --git a/src/librustc_trans/trans/build.rs b/src/librustc_trans/trans/build.rs index ce541c8d411b..2b4fcf436cbc 100644 --- a/src/librustc_trans/trans/build.rs +++ b/src/librustc_trans/trans/build.rs @@ -1067,8 +1067,9 @@ pub fn Resume(cx: Block, exn: ValueRef) -> ValueRef { pub fn AtomicCmpXchg(cx: Block, dst: ValueRef, cmp: ValueRef, src: ValueRef, order: AtomicOrdering, - failure_order: AtomicOrdering) -> ValueRef { - B(cx).atomic_cmpxchg(dst, cmp, src, order, failure_order) + failure_order: AtomicOrdering, + weak: llvm::Bool) -> ValueRef { + B(cx).atomic_cmpxchg(dst, cmp, src, order, failure_order, weak) } pub fn AtomicRMW(cx: Block, op: AtomicBinOp, dst: ValueRef, src: ValueRef, diff --git a/src/librustc_trans/trans/builder.rs b/src/librustc_trans/trans/builder.rs index 878d01f46b63..434fca416885 100644 --- a/src/librustc_trans/trans/builder.rs +++ b/src/librustc_trans/trans/builder.rs @@ -1077,10 +1077,11 @@ impl<'a, 'tcx> Builder<'a, 'tcx> { pub fn atomic_cmpxchg(&self, dst: ValueRef, cmp: ValueRef, src: ValueRef, order: AtomicOrdering, - failure_order: AtomicOrdering) -> ValueRef { + failure_order: AtomicOrdering, + weak: llvm::Bool) -> ValueRef { unsafe { llvm::LLVMBuildAtomicCmpXchg(self.llbuilder, dst, cmp, src, - order, failure_order) + order, failure_order, weak) } } pub fn atomic_rmw(&self, op: AtomicBinOp, diff --git a/src/librustc_trans/trans/intrinsic.rs b/src/librustc_trans/trans/intrinsic.rs index 188fb7de9dd8..b7b520f6c825 100644 --- a/src/librustc_trans/trans/intrinsic.rs +++ b/src/librustc_trans/trans/intrinsic.rs @@ -678,49 +678,54 @@ pub fn trans_intrinsic_call<'a, 'blk, 'tcx>(mut bcx: Block<'blk, 'tcx>, // "atomic_[_]", and no ordering means SeqCst (_, name) if name.starts_with("atomic_") => { let split: Vec<&str> = name.split('_').collect(); - assert!(split.len() >= 2, "Atomic intrinsic not correct format"); - let order = if split.len() == 2 { - llvm::SequentiallyConsistent - } else { - match split[2] { - "unordered" => llvm::Unordered, - "relaxed" => llvm::Monotonic, - "acq" => llvm::Acquire, - "rel" => llvm::Release, - "acqrel" => llvm::AcquireRelease, + let (order, failorder) = match split.len() { + 2 => (llvm::SequentiallyConsistent, llvm::SequentiallyConsistent), + 3 => match split[2] { + "unordered" => (llvm::Unordered, llvm::Unordered), + "relaxed" => (llvm::Monotonic, llvm::Monotonic), + "acq" => (llvm::Acquire, llvm::Acquire), + "rel" => (llvm::Release, llvm::Monotonic), + "acqrel" => (llvm::AcquireRelease, llvm::Acquire), + "failrelaxed" if split[1] == "cxchg" || split[1] == "cxchgweak" => + (llvm::SequentiallyConsistent, llvm::Monotonic), + "failacq" if split[1] == "cxchg" || split[1] == "cxchgweak" => + (llvm::SequentiallyConsistent, llvm::Acquire), _ => ccx.sess().fatal("unknown ordering in atomic intrinsic") - } + }, + 4 => match (split[2], split[3]) { + ("acq", "failrelaxed") if split[1] == "cxchg" || split[1] == "cxchgweak" => + (llvm::Acquire, llvm::Monotonic), + ("acqrel", "failrelaxed") if split[1] == "cxchg" || split[1] == "cxchgweak" => + (llvm::AcquireRelease, llvm::Monotonic), + _ => ccx.sess().fatal("unknown ordering in atomic intrinsic") + }, + _ => ccx.sess().fatal("Atomic intrinsic not in correct format"), }; match split[1] { "cxchg" => { - // See include/llvm/IR/Instructions.h for their implementation - // of this, I assume that it's good enough for us to use for - // now. - let strongest_failure_ordering = match order { - llvm::NotAtomic | llvm::Unordered => - ccx.sess().fatal("cmpxchg must be atomic"), - - llvm::Monotonic | llvm::Release => - llvm::Monotonic, - - llvm::Acquire | llvm::AcquireRelease => - llvm::Acquire, - - llvm::SequentiallyConsistent => - llvm::SequentiallyConsistent - }; - let tp_ty = *substs.types.get(FnSpace, 0); let ptr = to_arg_ty_ptr(bcx, llargs[0], tp_ty); let cmp = from_arg_ty(bcx, llargs[1], tp_ty); let src = from_arg_ty(bcx, llargs[2], tp_ty); - let res = AtomicCmpXchg(bcx, ptr, cmp, src, order, - strongest_failure_ordering); + let res = AtomicCmpXchg(bcx, ptr, cmp, src, order, failorder, llvm::False); ExtractValue(bcx, res, 0) } + "cxchgweak" => { + let tp_ty = *substs.types.get(FnSpace, 0); + let ptr = to_arg_ty_ptr(bcx, llargs[0], tp_ty); + let cmp = from_arg_ty(bcx, llargs[1], tp_ty); + let src = from_arg_ty(bcx, llargs[2], tp_ty); + let val = AtomicCmpXchg(bcx, ptr, cmp, src, order, failorder, llvm::True); + let result = ExtractValue(bcx, val, 0); + let success = ZExt(bcx, ExtractValue(bcx, val, 1), Type::bool(bcx.ccx())); + Store(bcx, result, StructGEP(bcx, llresult, 0)); + Store(bcx, success, StructGEP(bcx, llresult, 1)); + C_nil(ccx) + } + "load" => { let tp_ty = *substs.types.get(FnSpace, 0); let ptr = to_arg_ty_ptr(bcx, llargs[0], tp_ty); diff --git a/src/librustc_typeck/check/intrinsic.rs b/src/librustc_typeck/check/intrinsic.rs index ba6fa9aed3d9..5e1dc35870bb 100644 --- a/src/librustc_typeck/check/intrinsic.rs +++ b/src/librustc_typeck/check/intrinsic.rs @@ -83,6 +83,10 @@ pub fn check_intrinsic_type(ccx: &CrateCtxt, it: &hir::ForeignItem) { param(ccx, 0), param(ccx, 0)), param(ccx, 0)), + "cxchgweak" => (1, vec!(tcx.mk_mut_ptr(param(ccx, 0)), + param(ccx, 0), + param(ccx, 0)), + tcx.mk_tup(vec!(param(ccx, 0), tcx.types.bool))), "load" => (1, vec!(tcx.mk_imm_ptr(param(ccx, 0))), param(ccx, 0)), "store" => (1, vec!(tcx.mk_mut_ptr(param(ccx, 0)), param(ccx, 0)), diff --git a/src/rustllvm/RustWrapper.cpp b/src/rustllvm/RustWrapper.cpp index ecf9146a1d01..4ebe49512d74 100644 --- a/src/rustllvm/RustWrapper.cpp +++ b/src/rustllvm/RustWrapper.cpp @@ -191,11 +191,15 @@ extern "C" LLVMValueRef LLVMBuildAtomicCmpXchg(LLVMBuilderRef B, LLVMValueRef old, LLVMValueRef source, AtomicOrdering order, - AtomicOrdering failure_order) { - return wrap(unwrap(B)->CreateAtomicCmpXchg(unwrap(target), unwrap(old), - unwrap(source), order, - failure_order - )); + AtomicOrdering failure_order, + LLVMBool weak) { + AtomicCmpXchgInst* acxi = unwrap(B)->CreateAtomicCmpXchg(unwrap(target), + unwrap(old), + unwrap(source), + order, + failure_order); + acxi->setWeak(weak); + return wrap(acxi); } extern "C" LLVMValueRef LLVMBuildAtomicFence(LLVMBuilderRef B, AtomicOrdering order, diff --git a/src/test/run-pass/intrinsic-atomics.rs b/src/test/run-pass/intrinsic-atomics.rs index 4ccab55e9434..3cc125e95133 100644 --- a/src/test/run-pass/intrinsic-atomics.rs +++ b/src/test/run-pass/intrinsic-atomics.rs @@ -19,6 +19,10 @@ mod rusti { pub fn atomic_cxchg_acq(dst: *mut T, old: T, src: T) -> T; pub fn atomic_cxchg_rel(dst: *mut T, old: T, src: T) -> T; + pub fn atomic_cxchgweak(dst: *mut T, old: T, src: T) -> (T, bool); + pub fn atomic_cxchgweak_acq(dst: *mut T, old: T, src: T) -> (T, bool); + pub fn atomic_cxchgweak_rel(dst: *mut T, old: T, src: T) -> (T, bool); + pub fn atomic_load(src: *const T) -> T; pub fn atomic_load_acq(src: *const T) -> T; @@ -79,5 +83,32 @@ pub fn main() { assert_eq!(rusti::atomic_xsub_acq(&mut *x, 1), 2); assert_eq!(rusti::atomic_xsub_rel(&mut *x, 1), 1); assert_eq!(*x, 0); + + loop { + let res = rusti::atomic_cxchgweak(&mut *x, 0, 1); + assert_eq!(res.0, 0); + if res.1 { + break; + } + } + assert_eq!(*x, 1); + + loop { + let res = rusti::atomic_cxchgweak_acq(&mut *x, 1, 2); + assert_eq!(res.0, 1); + if res.1 { + break; + } + } + assert_eq!(*x, 2); + + loop { + let res = rusti::atomic_cxchgweak_rel(&mut *x, 2, 3); + assert_eq!(res.0, 2); + if res.1 { + break; + } + } + assert_eq!(*x, 3); } }