| 1 |
#[cfg(not(crossbeam_no_atomic))] |
| 2 |
use core::sync::atomic::Ordering; |
| 3 |
|
| 4 |
/// Trait which allows reading from primitive atomic types with "consume" ordering. |
| 5 |
pub trait AtomicConsume { |
| 6 |
/// Type returned by `load_consume`. |
| 7 |
type Val; |
| 8 |
|
| 9 |
/// Loads a value from the atomic using a "consume" memory ordering. |
| 10 |
/// |
| 11 |
/// This is similar to the "acquire" ordering, except that an ordering is |
| 12 |
/// only guaranteed with operations that "depend on" the result of the load. |
| 13 |
/// However consume loads are usually much faster than acquire loads on |
| 14 |
/// architectures with a weak memory model since they don't require memory |
| 15 |
/// fence instructions. |
| 16 |
/// |
| 17 |
/// The exact definition of "depend on" is a bit vague, but it works as you |
| 18 |
/// would expect in practice since a lot of software, especially the Linux |
| 19 |
/// kernel, rely on this behavior. |
| 20 |
/// |
| 21 |
/// This is currently only implemented on ARM and AArch64, where a fence |
| 22 |
/// can be avoided. On other architectures this will fall back to a simple |
| 23 |
/// `load(Ordering::Acquire)`. |
| 24 |
fn load_consume(&self) -> Self::Val; |
| 25 |
} |
| 26 |
|
| 27 |
#[cfg(not(crossbeam_no_atomic))] |
| 28 |
// Miri and Loom don't support "consume" ordering and ThreadSanitizer doesn't treat |
| 29 |
// load(Relaxed) + compiler_fence(Acquire) as "consume" load. |
| 30 |
// LLVM generates machine code equivalent to fence(Acquire) in compiler_fence(Acquire) |
| 31 |
// on PowerPC, MIPS, etc. (https://godbolt.org/z/hffvjvW7h), so for now the fence |
| 32 |
// can be actually avoided here only on ARM and AArch64. See also |
| 33 |
// https://github.com/rust-lang/rust/issues/62256. |
| 34 |
#[cfg(all( |
| 35 |
any(target_arch = "arm", target_arch = "aarch64"), |
| 36 |
not(any(miri, crossbeam_loom, crossbeam_sanitize_thread)), |
| 37 |
))] |
| 38 |
macro_rules! impl_consume { |
| 39 |
() => { |
| 40 |
#[inline] |
| 41 |
fn load_consume(&self) -> Self::Val { |
| 42 |
use crate::primitive::sync::atomic::compiler_fence; |
| 43 |
let result = self.load(Ordering::Relaxed); |
| 44 |
compiler_fence(Ordering::Acquire); |
| 45 |
result |
| 46 |
} |
| 47 |
}; |
| 48 |
} |
| 49 |
|
| 50 |
#[cfg(not(crossbeam_no_atomic))] |
| 51 |
#[cfg(not(all( |
| 52 |
any(target_arch = "arm", target_arch = "aarch64"), |
| 53 |
not(any(miri, crossbeam_loom, crossbeam_sanitize_thread)), |
| 54 |
)))] |
| 55 |
macro_rules! impl_consume { |
| 56 |
() => { |
| 57 |
#[inline] |
| 58 |
fn load_consume(&self) -> Self::Val { |
| 59 |
self.load(Ordering::Acquire) |
| 60 |
} |
| 61 |
}; |
| 62 |
} |
| 63 |
|
| 64 |
macro_rules! impl_atomic { |
| 65 |
($atomic:ident, $val:ty) => { |
| 66 |
#[cfg(not(crossbeam_no_atomic))] |
| 67 |
impl AtomicConsume for core::sync::atomic::$atomic { |
| 68 |
type Val = $val; |
| 69 |
impl_consume!(); |
| 70 |
} |
| 71 |
#[cfg(crossbeam_loom)] |
| 72 |
impl AtomicConsume for loom::sync::atomic::$atomic { |
| 73 |
type Val = $val; |
| 74 |
impl_consume!(); |
| 75 |
} |
| 76 |
}; |
| 77 |
} |
| 78 |
|
| 79 |
impl_atomic!(AtomicBool, bool); |
| 80 |
impl_atomic!(AtomicUsize, usize); |
| 81 |
impl_atomic!(AtomicIsize, isize); |
| 82 |
impl_atomic!(AtomicU8, u8); |
| 83 |
impl_atomic!(AtomicI8, i8); |
| 84 |
impl_atomic!(AtomicU16, u16); |
| 85 |
impl_atomic!(AtomicI16, i16); |
| 86 |
#[cfg(any(target_has_atomic = "32", not(target_pointer_width = "16")))] |
| 87 |
impl_atomic!(AtomicU32, u32); |
| 88 |
#[cfg(any(target_has_atomic = "32", not(target_pointer_width = "16")))] |
| 89 |
impl_atomic!(AtomicI32, i32); |
| 90 |
#[cfg(any( |
| 91 |
target_has_atomic = "64", |
| 92 |
not(any(target_pointer_width = "16", target_pointer_width = "32")), |
| 93 |
))] |
| 94 |
impl_atomic!(AtomicU64, u64); |
| 95 |
#[cfg(any( |
| 96 |
target_has_atomic = "64", |
| 97 |
not(any(target_pointer_width = "16", target_pointer_width = "32")), |
| 98 |
))] |
| 99 |
impl_atomic!(AtomicI64, i64); |
| 100 |
|
| 101 |
#[cfg(not(crossbeam_no_atomic))] |
| 102 |
impl<T> AtomicConsume for core::sync::atomic::AtomicPtr<T> { |
| 103 |
type Val = *mut T; |
| 104 |
impl_consume!(); |
| 105 |
} |
| 106 |
|
| 107 |
#[cfg(crossbeam_loom)] |
| 108 |
impl<T> AtomicConsume for loom::sync::atomic::AtomicPtr<T> { |
| 109 |
type Val = *mut T; |
| 110 |
impl_consume!(); |
| 111 |
} |