| 1 |
// Start of atomics8.h |
| 2 |
|
| 3 |
SCALAR_FUN_ATTR int8_t atomic_cmpxchg_i8_global(volatile __global int8_t *p, |
| 4 |
int8_t cmp, int8_t val); |
| 5 |
SCALAR_FUN_ATTR int8_t atomic_cmpxchg_i8_shared(volatile __local int8_t *p, |
| 6 |
int8_t cmp, int8_t val); |
| 7 |
SCALAR_FUN_ATTR int8_t atomic_add_i8_global(volatile __global int8_t *p, int8_t x); |
| 8 |
SCALAR_FUN_ATTR int8_t atomic_add_i8_shared(volatile __local int8_t *p, int8_t x); |
| 9 |
SCALAR_FUN_ATTR int8_t atomic_smax_i8_global(volatile __global int8_t *p, int8_t x); |
| 10 |
SCALAR_FUN_ATTR int8_t atomic_smax_i8_shared(volatile __local int8_t *p, int8_t x); |
| 11 |
SCALAR_FUN_ATTR int8_t atomic_smin_i8_global(volatile __global int8_t *p, int8_t x); |
| 12 |
SCALAR_FUN_ATTR int8_t atomic_smin_i8_shared(volatile __local int8_t *p, int8_t x); |
| 13 |
SCALAR_FUN_ATTR uint8_t atomic_umax_i8_global(volatile __global uint8_t *p, uint8_t x); |
| 14 |
SCALAR_FUN_ATTR uint8_t atomic_umax_i8_shared(volatile __local uint8_t *p, uint8_t x); |
| 15 |
SCALAR_FUN_ATTR uint8_t atomic_umin_i8_global(volatile __global uint8_t *p, uint8_t x); |
| 16 |
SCALAR_FUN_ATTR uint8_t atomic_umin_i8_shared(volatile __local uint8_t *p, uint8_t x); |
| 17 |
SCALAR_FUN_ATTR int8_t atomic_and_i8_global(volatile __global int8_t *p, int8_t x); |
| 18 |
SCALAR_FUN_ATTR int8_t atomic_and_i8_shared(volatile __local int8_t *p, int8_t x); |
| 19 |
SCALAR_FUN_ATTR int8_t atomic_or_i8_global(volatile __global int8_t *p, int8_t x); |
| 20 |
SCALAR_FUN_ATTR int8_t atomic_or_i8_shared(volatile __local int8_t *p, int8_t x); |
| 21 |
SCALAR_FUN_ATTR int8_t atomic_xor_i8_global(volatile __global int8_t *p, int8_t x); |
| 22 |
SCALAR_FUN_ATTR int8_t atomic_xor_i8_shared(volatile __local int8_t *p, int8_t x); |
| 23 |
|
| 24 |
SCALAR_FUN_ATTR int8_t atomic_cmpxchg_i8_global(volatile __global int8_t *p, |
| 25 |
int8_t cmp, int8_t val) { |
| 26 |
int offset = ((uintptr_t)p & 3); |
| 27 |
volatile __global int32_t *p32 = (volatile __global int32_t*)((uintptr_t)p & ~0x3); |
| 28 |
|
| 29 |
int shift = offset * 8; |
| 30 |
int32_t mask = 0xff << shift; |
| 31 |
int32_t shifted_val = val << shift; |
| 32 |
int32_t shifted_cmp = cmp << shift; |
| 33 |
|
| 34 |
uint32_t old = shifted_cmp; |
| 35 |
uint32_t upd = shifted_val; |
| 36 |
uint32_t got; |
| 37 |
|
| 38 |
while ((got=atomic_cmpxchg_i32_global(p32, old, upd)) != old) { |
| 39 |
old = got; |
| 40 |
upd = (old & ~mask) | shifted_val; |
| 41 |
} |
| 42 |
|
| 43 |
return old >> shift; |
| 44 |
} |
| 45 |
|
| 46 |
SCALAR_FUN_ATTR int8_t atomic_cmpxchg_i8_shared(volatile __local int8_t *p, |
| 47 |
int8_t cmp, int8_t val) { |
| 48 |
int offset = ((uintptr_t)p >> 1 & 3); |
| 49 |
volatile __local int32_t *p32 = (volatile __local int32_t*)((uintptr_t)p & ~0x3); |
| 50 |
|
| 51 |
int shift = offset * 8; |
| 52 |
int32_t mask = 0xff << shift; |
| 53 |
int32_t shifted_val = val << shift; |
| 54 |
int32_t shifted_cmp = cmp << shift; |
| 55 |
|
| 56 |
uint32_t old = shifted_cmp; |
| 57 |
uint32_t upd = shifted_val; |
| 58 |
uint32_t got; |
| 59 |
|
| 60 |
while ((got=atomic_cmpxchg_i32_shared(p32, old, upd)) != old) { |
| 61 |
old = got; |
| 62 |
upd = (old & ~mask) | shifted_val; |
| 63 |
} |
| 64 |
|
| 65 |
return old >> shift; |
| 66 |
} |
| 67 |
|
| 68 |
// Convenience macro for arithmetic. |
| 69 |
#define DEFINE_8BIT_ATOMIC(name, T, op) \ |
| 70 |
SCALAR_FUN_ATTR T \ |
| 71 |
atomic_##name##_i8_global(volatile __global T *p, T val) { \ |
| 72 |
int offset = ((uintptr_t)p & 3); \ |
| 73 |
volatile __global int32_t *p32 = (volatile __global int32_t*)((uintptr_t)p & ~0x3); \ |
| 74 |
int shift = offset * 8; \ |
| 75 |
int32_t mask = 0xff << shift; \ |
| 76 |
int32_t old = 0; \ |
| 77 |
int32_t upd = mask & (op(old >> shift, val) << shift); \ |
| 78 |
int32_t saw; \ |
| 79 |
while ((saw=atomic_cmpxchg_i32_global(p32, old, upd)) != old) { \ |
| 80 |
old = saw; \ |
| 81 |
upd = (old & ~mask) | ((op(old >> shift, val)) << shift); \ |
| 82 |
} \ |
| 83 |
return old >> shift; \ |
| 84 |
} \ |
| 85 |
SCALAR_FUN_ATTR T \ |
| 86 |
atomic_##name##_i8_shared(volatile __local T *p, T val) { \ |
| 87 |
int offset = ((uintptr_t)p & 3); \ |
| 88 |
volatile __local int32_t *p32 = (volatile __local int32_t*)((uintptr_t)p & ~0x3); \ |
| 89 |
int shift = offset * 8; \ |
| 90 |
int32_t mask = 0xff << shift; \ |
| 91 |
int32_t old = 0; \ |
| 92 |
int32_t upd = mask & ((op(old >> shift, val)) << shift); \ |
| 93 |
int32_t saw; \ |
| 94 |
while ((saw=atomic_cmpxchg_i32_shared(p32, old, upd)) != old) { \ |
| 95 |
old = saw; \ |
| 96 |
upd = (old & ~mask) | ((op(old >> shift, val)) << shift); \ |
| 97 |
} \ |
| 98 |
return old >> shift; \ |
| 99 |
} |
| 100 |
|
| 101 |
DEFINE_8BIT_ATOMIC(add, int8_t, add8); |
| 102 |
DEFINE_8BIT_ATOMIC(smax, int8_t, smax8); |
| 103 |
DEFINE_8BIT_ATOMIC(smin, int8_t, smin8); |
| 104 |
DEFINE_8BIT_ATOMIC(umax, uint8_t, umax8); |
| 105 |
DEFINE_8BIT_ATOMIC(umin, uint8_t, umin8); |
| 106 |
|
| 107 |
SCALAR_FUN_ATTR int8_t atomic_and_i8_global(volatile __global int8_t *p, int8_t val) { |
| 108 |
volatile __global int32_t *p32 = (volatile __global int32_t*)((uintptr_t)p & ~0x3); |
| 109 |
int shift = ((uintptr_t)p & 3) * 8; |
| 110 |
int32_t mask = 0xff << shift; |
| 111 |
return atomic_and_i32_global(p32, ~mask | (val<<shift)) >> shift; |
| 112 |
} |
| 113 |
|
| 114 |
SCALAR_FUN_ATTR int8_t atomic_and_i8_shared(volatile __local int8_t *p, int8_t val) { |
| 115 |
volatile __local int32_t *p32 = (volatile __local int32_t*)((uintptr_t)p & ~0x3); |
| 116 |
int shift = ((uintptr_t)p & 3) * 8; |
| 117 |
int32_t mask = 0xff << shift; |
| 118 |
return atomic_and_i32_shared(p32, ~mask | (val<<shift)) >> shift; |
| 119 |
} |
| 120 |
|
| 121 |
SCALAR_FUN_ATTR int8_t atomic_or_i8_global(volatile __global int8_t *p, int8_t val) { |
| 122 |
volatile __global int32_t *p32 = (volatile __global int32_t*)((uintptr_t)p & ~0x3); |
| 123 |
int shift = ((uintptr_t)p & 3) * 8; |
| 124 |
return atomic_or_i32_global(p32, (uint8_t)val<<shift) >> shift; |
| 125 |
} |
| 126 |
|
| 127 |
SCALAR_FUN_ATTR int8_t atomic_or_i8_shared(volatile __local int8_t *p, int8_t val) { |
| 128 |
volatile __local int32_t *p32 = (volatile __local int32_t*)((uintptr_t)p & ~0x3); |
| 129 |
int shift = ((uintptr_t)p & 3) * 8; |
| 130 |
return atomic_or_i32_shared(p32, (uint8_t)val<<shift) >> shift; |
| 131 |
} |
| 132 |
|
| 133 |
SCALAR_FUN_ATTR int8_t atomic_xor_i8_global(volatile __global int8_t *p, int8_t val) { |
| 134 |
volatile __global int32_t *p32 = (volatile __global int32_t*)((uintptr_t)p & ~0x3); |
| 135 |
int shift = ((uintptr_t)p & 3) * 8; |
| 136 |
return atomic_xor_i32_global(p32, (uint8_t)val<<shift) >> shift; |
| 137 |
} |
| 138 |
|
| 139 |
SCALAR_FUN_ATTR int8_t atomic_xor_i8_shared(volatile __local int8_t *p, int8_t val) { |
| 140 |
volatile __local int32_t *p32 = (volatile __local int32_t*)((uintptr_t)p & ~0x3); |
| 141 |
int shift = ((uintptr_t)p & 3) * 8; |
| 142 |
return atomic_xor_i32_shared(p32, (uint8_t)val<<shift) >> shift; |
| 143 |
} |
| 144 |
|
| 145 |
// End of atomics8.h |