| 1 |
#ifndef _ATOMIC_H |
| 2 |
#define _ATOMIC_H |
| 3 |
|
| 4 |
#include <stdint.h> |
| 5 |
|
| 6 |
#include "atomic_arch.h" |
| 7 |
|
| 8 |
#ifdef a_ll |
| 9 |
|
| 10 |
#ifndef a_pre_llsc |
| 11 |
#define a_pre_llsc() |
| 12 |
#endif |
| 13 |
|
| 14 |
#ifndef a_post_llsc |
| 15 |
#define a_post_llsc() |
| 16 |
#endif |
| 17 |
|
| 18 |
#ifndef a_cas |
| 19 |
#define a_cas a_cas |
| 20 |
static inline int a_cas(volatile int *p, int t, int s) |
| 21 |
{ |
| 22 |
int old; |
| 23 |
a_pre_llsc(); |
| 24 |
do old = a_ll(p); |
| 25 |
while (old==t && !a_sc(p, s)); |
| 26 |
a_post_llsc(); |
| 27 |
return old; |
| 28 |
} |
| 29 |
#endif |
| 30 |
|
| 31 |
#ifndef a_swap |
| 32 |
#define a_swap a_swap |
| 33 |
static inline int a_swap(volatile int *p, int v) |
| 34 |
{ |
| 35 |
int old; |
| 36 |
a_pre_llsc(); |
| 37 |
do old = a_ll(p); |
| 38 |
while (!a_sc(p, v)); |
| 39 |
a_post_llsc(); |
| 40 |
return old; |
| 41 |
} |
| 42 |
#endif |
| 43 |
|
| 44 |
#ifndef a_fetch_add |
| 45 |
#define a_fetch_add a_fetch_add |
| 46 |
static inline int a_fetch_add(volatile int *p, int v) |
| 47 |
{ |
| 48 |
int old; |
| 49 |
a_pre_llsc(); |
| 50 |
do old = a_ll(p); |
| 51 |
while (!a_sc(p, (unsigned)old + v)); |
| 52 |
a_post_llsc(); |
| 53 |
return old; |
| 54 |
} |
| 55 |
#endif |
| 56 |
|
| 57 |
#ifndef a_fetch_and |
| 58 |
#define a_fetch_and a_fetch_and |
| 59 |
static inline int a_fetch_and(volatile int *p, int v) |
| 60 |
{ |
| 61 |
int old; |
| 62 |
a_pre_llsc(); |
| 63 |
do old = a_ll(p); |
| 64 |
while (!a_sc(p, old & v)); |
| 65 |
a_post_llsc(); |
| 66 |
return old; |
| 67 |
} |
| 68 |
#endif |
| 69 |
|
| 70 |
#ifndef a_fetch_or |
| 71 |
#define a_fetch_or a_fetch_or |
| 72 |
static inline int a_fetch_or(volatile int *p, int v) |
| 73 |
{ |
| 74 |
int old; |
| 75 |
a_pre_llsc(); |
| 76 |
do old = a_ll(p); |
| 77 |
while (!a_sc(p, old | v)); |
| 78 |
a_post_llsc(); |
| 79 |
return old; |
| 80 |
} |
| 81 |
#endif |
| 82 |
|
| 83 |
#endif |
| 84 |
|
| 85 |
#ifdef a_ll_p |
| 86 |
|
| 87 |
#ifndef a_cas_p |
| 88 |
#define a_cas_p a_cas_p |
| 89 |
static inline void *a_cas_p(volatile void *p, void *t, void *s) |
| 90 |
{ |
| 91 |
void *old; |
| 92 |
a_pre_llsc(); |
| 93 |
do old = a_ll_p(p); |
| 94 |
while (old==t && !a_sc_p(p, s)); |
| 95 |
a_post_llsc(); |
| 96 |
return old; |
| 97 |
} |
| 98 |
#endif |
| 99 |
|
| 100 |
#endif |
| 101 |
|
| 102 |
#ifndef a_cas |
| 103 |
#error missing definition of a_cas |
| 104 |
#endif |
| 105 |
|
| 106 |
#ifndef a_swap |
| 107 |
#define a_swap a_swap |
| 108 |
static inline int a_swap(volatile int *p, int v) |
| 109 |
{ |
| 110 |
int old; |
| 111 |
do old = *p; |
| 112 |
while (a_cas(p, old, v) != old); |
| 113 |
return old; |
| 114 |
} |
| 115 |
#endif |
| 116 |
|
| 117 |
#ifndef a_fetch_add |
| 118 |
#define a_fetch_add a_fetch_add |
| 119 |
static inline int a_fetch_add(volatile int *p, int v) |
| 120 |
{ |
| 121 |
int old; |
| 122 |
do old = *p; |
| 123 |
while (a_cas(p, old, (unsigned)old+v) != old); |
| 124 |
return old; |
| 125 |
} |
| 126 |
#endif |
| 127 |
|
| 128 |
#ifndef a_fetch_and |
| 129 |
#define a_fetch_and a_fetch_and |
| 130 |
static inline int a_fetch_and(volatile int *p, int v) |
| 131 |
{ |
| 132 |
int old; |
| 133 |
do old = *p; |
| 134 |
while (a_cas(p, old, old&v) != old); |
| 135 |
return old; |
| 136 |
} |
| 137 |
#endif |
| 138 |
#ifndef a_fetch_or |
| 139 |
#define a_fetch_or a_fetch_or |
| 140 |
static inline int a_fetch_or(volatile int *p, int v) |
| 141 |
{ |
| 142 |
int old; |
| 143 |
do old = *p; |
| 144 |
while (a_cas(p, old, old|v) != old); |
| 145 |
return old; |
| 146 |
} |
| 147 |
#endif |
| 148 |
|
| 149 |
#ifndef a_and |
| 150 |
#define a_and a_and |
| 151 |
static inline void a_and(volatile int *p, int v) |
| 152 |
{ |
| 153 |
a_fetch_and(p, v); |
| 154 |
} |
| 155 |
#endif |
| 156 |
|
| 157 |
#ifndef a_or |
| 158 |
#define a_or a_or |
| 159 |
static inline void a_or(volatile int *p, int v) |
| 160 |
{ |
| 161 |
a_fetch_or(p, v); |
| 162 |
} |
| 163 |
#endif |
| 164 |
|
| 165 |
#ifndef a_inc |
| 166 |
#define a_inc a_inc |
| 167 |
static inline void a_inc(volatile int *p) |
| 168 |
{ |
| 169 |
a_fetch_add(p, 1); |
| 170 |
} |
| 171 |
#endif |
| 172 |
|
| 173 |
#ifndef a_dec |
| 174 |
#define a_dec a_dec |
| 175 |
static inline void a_dec(volatile int *p) |
| 176 |
{ |
| 177 |
a_fetch_add(p, -1); |
| 178 |
} |
| 179 |
#endif |
| 180 |
|
| 181 |
#ifndef a_store |
| 182 |
#define a_store a_store |
| 183 |
static inline void a_store(volatile int *p, int v) |
| 184 |
{ |
| 185 |
#ifdef a_barrier |
| 186 |
a_barrier(); |
| 187 |
*p = v; |
| 188 |
a_barrier(); |
| 189 |
#else |
| 190 |
a_swap(p, v); |
| 191 |
#endif |
| 192 |
} |
| 193 |
#endif |
| 194 |
|
| 195 |
#ifndef a_barrier |
| 196 |
#define a_barrier a_barrier |
| 197 |
static void a_barrier() |
| 198 |
{ |
| 199 |
volatile int tmp = 0; |
| 200 |
a_cas(&tmp, 0, 0); |
| 201 |
} |
| 202 |
#endif |
| 203 |
|
| 204 |
#ifndef a_spin |
| 205 |
#define a_spin a_barrier |
| 206 |
#endif |
| 207 |
|
| 208 |
#ifndef a_and_64 |
| 209 |
#define a_and_64 a_and_64 |
| 210 |
static inline void a_and_64(volatile uint64_t *p, uint64_t v) |
| 211 |
{ |
| 212 |
union { uint64_t v; uint32_t r[2]; } u = { v }; |
| 213 |
if (u.r[0]+1) a_and((int *)p, u.r[0]); |
| 214 |
if (u.r[1]+1) a_and((int *)p+1, u.r[1]); |
| 215 |
} |
| 216 |
#endif |
| 217 |
|
| 218 |
#ifndef a_or_64 |
| 219 |
#define a_or_64 a_or_64 |
| 220 |
static inline void a_or_64(volatile uint64_t *p, uint64_t v) |
| 221 |
{ |
| 222 |
union { uint64_t v; uint32_t r[2]; } u = { v }; |
| 223 |
if (u.r[0]) a_or((int *)p, u.r[0]); |
| 224 |
if (u.r[1]) a_or((int *)p+1, u.r[1]); |
| 225 |
} |
| 226 |
#endif |
| 227 |
|
| 228 |
#ifndef a_cas_p |
| 229 |
typedef char a_cas_p_undefined_but_pointer_not_32bit[-sizeof(char) == 0xffffffff ? 1 : -1]; |
| 230 |
#define a_cas_p a_cas_p |
| 231 |
static inline void *a_cas_p(volatile void *p, void *t, void *s) |
| 232 |
{ |
| 233 |
return (void *)a_cas((volatile int *)p, (int)t, (int)s); |
| 234 |
} |
| 235 |
#endif |
| 236 |
|
| 237 |
#ifndef a_or_l |
| 238 |
#define a_or_l a_or_l |
| 239 |
static inline void a_or_l(volatile void *p, long v) |
| 240 |
{ |
| 241 |
if (sizeof(long) == sizeof(int)) a_or(p, v); |
| 242 |
else a_or_64(p, v); |
| 243 |
} |
| 244 |
#endif |
| 245 |
|
| 246 |
#ifndef a_crash |
| 247 |
#define a_crash a_crash |
| 248 |
static inline void a_crash() |
| 249 |
{ |
| 250 |
*(volatile char *)0=0; |
| 251 |
} |
| 252 |
#endif |
| 253 |
|
| 254 |
#ifndef a_ctz_32 |
| 255 |
#define a_ctz_32 a_ctz_32 |
| 256 |
static inline int a_ctz_32(uint32_t x) |
| 257 |
{ |
| 258 |
#ifdef a_clz_32 |
| 259 |
return 31-a_clz_32(x&-x); |
| 260 |
#else |
| 261 |
static const char debruijn32[32] = { |
| 262 |
0, 1, 23, 2, 29, 24, 19, 3, 30, 27, 25, 11, 20, 8, 4, 13, |
| 263 |
31, 22, 28, 18, 26, 10, 7, 12, 21, 17, 9, 6, 16, 5, 15, 14 |
| 264 |
}; |
| 265 |
return debruijn32[(x&-x)*0x076be629 >> 27]; |
| 266 |
#endif |
| 267 |
} |
| 268 |
#endif |
| 269 |
|
| 270 |
#ifndef a_ctz_64 |
| 271 |
#define a_ctz_64 a_ctz_64 |
| 272 |
static inline int a_ctz_64(uint64_t x) |
| 273 |
{ |
| 274 |
static const char debruijn64[64] = { |
| 275 |
0, 1, 2, 53, 3, 7, 54, 27, 4, 38, 41, 8, 34, 55, 48, 28, |
| 276 |
62, 5, 39, 46, 44, 42, 22, 9, 24, 35, 59, 56, 49, 18, 29, 11, |
| 277 |
63, 52, 6, 26, 37, 40, 33, 47, 61, 45, 43, 21, 23, 58, 17, 10, |
| 278 |
51, 25, 36, 32, 60, 20, 57, 16, 50, 31, 19, 15, 30, 14, 13, 12 |
| 279 |
}; |
| 280 |
if (sizeof(long) < 8) { |
| 281 |
uint32_t y = x; |
| 282 |
if (!y) { |
| 283 |
y = x>>32; |
| 284 |
return 32 + a_ctz_32(y); |
| 285 |
} |
| 286 |
return a_ctz_32(y); |
| 287 |
} |
| 288 |
return debruijn64[(x&-x)*0x022fdd63cc95386dull >> 58]; |
| 289 |
} |
| 290 |
#endif |
| 291 |
|
| 292 |
static inline int a_ctz_l(unsigned long x) |
| 293 |
{ |
| 294 |
return (sizeof(long) < 8) ? a_ctz_32(x) : a_ctz_64(x); |
| 295 |
} |
| 296 |
|
| 297 |
#ifndef a_clz_64 |
| 298 |
#define a_clz_64 a_clz_64 |
| 299 |
static inline int a_clz_64(uint64_t x) |
| 300 |
{ |
| 301 |
#ifdef a_clz_32 |
| 302 |
if (x>>32) |
| 303 |
return a_clz_32(x>>32); |
| 304 |
return a_clz_32(x) + 32; |
| 305 |
#else |
| 306 |
uint32_t y; |
| 307 |
int r; |
| 308 |
if (x>>32) y=x>>32, r=0; else y=x, r=32; |
| 309 |
if (y>>16) y>>=16; else r |= 16; |
| 310 |
if (y>>8) y>>=8; else r |= 8; |
| 311 |
if (y>>4) y>>=4; else r |= 4; |
| 312 |
if (y>>2) y>>=2; else r |= 2; |
| 313 |
return r | !(y>>1); |
| 314 |
#endif |
| 315 |
} |
| 316 |
#endif |
| 317 |
|
| 318 |
#ifndef a_clz_32 |
| 319 |
#define a_clz_32 a_clz_32 |
| 320 |
static inline int a_clz_32(uint32_t x) |
| 321 |
{ |
| 322 |
x >>= 1; |
| 323 |
x |= x >> 1; |
| 324 |
x |= x >> 2; |
| 325 |
x |= x >> 4; |
| 326 |
x |= x >> 8; |
| 327 |
x |= x >> 16; |
| 328 |
x++; |
| 329 |
return 31-a_ctz_32(x); |
| 330 |
} |
| 331 |
#endif |
| 332 |
|
| 333 |
#endif |