| 1 |
/* Linux-specific atomic operations for ARM EABI. |
| 2 |
Copyright (C) 2008-2025 Free Software Foundation, Inc. |
| 3 |
Contributed by CodeSourcery. |
| 4 |
|
| 5 |
This file is part of GCC. |
| 6 |
|
| 7 |
GCC is free software; you can redistribute it and/or modify it under |
| 8 |
the terms of the GNU General Public License as published by the Free |
| 9 |
Software Foundation; either version 3, or (at your option) any later |
| 10 |
version. |
| 11 |
|
| 12 |
GCC is distributed in the hope that it will be useful, but WITHOUT ANY |
| 13 |
WARRANTY; without even the implied warranty of MERCHANTABILITY or |
| 14 |
FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License |
| 15 |
for more details. |
| 16 |
|
| 17 |
Under Section 7 of GPL version 3, you are granted additional |
| 18 |
permissions described in the GCC Runtime Library Exception, version |
| 19 |
3.1, as published by the Free Software Foundation. |
| 20 |
|
| 21 |
You should have received a copy of the GNU General Public License and |
| 22 |
a copy of the GCC Runtime Library Exception along with this program; |
| 23 |
see the files COPYING3 and COPYING.RUNTIME respectively. If not, see |
| 24 |
<http://www.gnu.org/licenses/>. */ |
| 25 |
|
| 26 |
/* Kernel helper for compare-and-exchange. */ |
| 27 |
typedef int (__kernel_cmpxchg_t) (int oldval, int newval, int *ptr); |
| 28 |
|
| 29 |
#define STR(X) #X |
| 30 |
#define XSTR(X) STR(X) |
| 31 |
|
| 32 |
#define KERNEL_CMPXCHG 0xffff0fc0 |
| 33 |
|
| 34 |
#if __FDPIC__ |
| 35 |
/* Non-FDPIC ABIs call __kernel_cmpxchg directly by dereferencing its |
| 36 |
address, but under FDPIC we would generate a broken call |
| 37 |
sequence. That's why we have to implement __kernel_cmpxchg and |
| 38 |
__kernel_dmb here: this way, the FDPIC call sequence works. */ |
| 39 |
#define __kernel_cmpxchg __fdpic_cmpxchg |
| 40 |
#else |
| 41 |
#define __kernel_cmpxchg (*(__kernel_cmpxchg_t *) KERNEL_CMPXCHG) |
| 42 |
#endif |
| 43 |
|
| 44 |
/* Kernel helper for memory barrier. */ |
| 45 |
typedef void (__kernel_dmb_t) (void); |
| 46 |
|
| 47 |
#define KERNEL_DMB 0xffff0fa0 |
| 48 |
|
| 49 |
#if __FDPIC__ |
| 50 |
#define __kernel_dmb __fdpic_dmb |
| 51 |
#else |
| 52 |
#define __kernel_dmb (*(__kernel_dmb_t *) KERNEL_DMB) |
| 53 |
#endif |
| 54 |
|
| 55 |
#if __FDPIC__ |
| 56 |
static int __fdpic_cmpxchg (int oldval, int newval, int *ptr) |
| 57 |
{ |
| 58 |
int result; |
| 59 |
|
| 60 |
asm volatile ( |
| 61 |
"ldr ip, 1f\n\t" |
| 62 |
"bx ip\n\t" |
| 63 |
"1:\n\t" |
| 64 |
".word " XSTR(KERNEL_CMPXCHG) "\n\t" |
| 65 |
: "=r" (result) |
| 66 |
: "r" (oldval) , "r" (newval), "r" (ptr) |
| 67 |
: "r3", "memory"); |
| 68 |
/* The result is actually returned by the kernel helper, we need |
| 69 |
this to avoid a warning. */ |
| 70 |
return result; |
| 71 |
} |
| 72 |
|
| 73 |
static void __fdpic_dmb (void) |
| 74 |
{ |
| 75 |
asm volatile ( |
| 76 |
"ldr ip, 1f\n\t" |
| 77 |
"bx ip\n\t" |
| 78 |
"1:\n\t" |
| 79 |
".word " XSTR(KERNEL_DMB) "\n\t" |
| 80 |
); |
| 81 |
} |
| 82 |
|
| 83 |
#endif |
| 84 |
|
| 85 |
/* Note: we implement byte, short and int versions of atomic operations using |
| 86 |
the above kernel helpers; see linux-atomic-64bit.c for "long long" (64-bit) |
| 87 |
operations. */ |
| 88 |
|
| 89 |
#define HIDDEN __attribute__ ((visibility ("hidden"))) |
| 90 |
|
| 91 |
#ifdef __ARMEL__ |
| 92 |
#define INVERT_MASK_1 0 |
| 93 |
#define INVERT_MASK_2 0 |
| 94 |
#else |
| 95 |
#define INVERT_MASK_1 24 |
| 96 |
#define INVERT_MASK_2 16 |
| 97 |
#endif |
| 98 |
|
| 99 |
#define MASK_1 0xffu |
| 100 |
#define MASK_2 0xffffu |
| 101 |
|
| 102 |
#define FETCH_AND_OP_WORD(OP, PFX_OP, INF_OP) \ |
| 103 |
int HIDDEN \ |
| 104 |
__sync_fetch_and_##OP##_4 (int *ptr, int val) \ |
| 105 |
{ \ |
| 106 |
int failure, tmp; \ |
| 107 |
\ |
| 108 |
do { \ |
| 109 |
tmp = *ptr; \ |
| 110 |
failure = __kernel_cmpxchg (tmp, PFX_OP (tmp INF_OP val), ptr); \ |
| 111 |
} while (failure != 0); \ |
| 112 |
\ |
| 113 |
return tmp; \ |
| 114 |
} |
| 115 |
|
| 116 |
FETCH_AND_OP_WORD (add, , +) |
| 117 |
FETCH_AND_OP_WORD (sub, , -) |
| 118 |
FETCH_AND_OP_WORD (or, , |) |
| 119 |
FETCH_AND_OP_WORD (and, , &) |
| 120 |
FETCH_AND_OP_WORD (xor, , ^) |
| 121 |
FETCH_AND_OP_WORD (nand, ~, &) |
| 122 |
|
| 123 |
#define NAME_oldval(OP, WIDTH) __sync_fetch_and_##OP##_##WIDTH |
| 124 |
#define NAME_newval(OP, WIDTH) __sync_##OP##_and_fetch_##WIDTH |
| 125 |
|
| 126 |
/* Implement both __sync_<op>_and_fetch and __sync_fetch_and_<op> for |
| 127 |
subword-sized quantities. */ |
| 128 |
|
| 129 |
#define SUBWORD_SYNC_OP(OP, PFX_OP, INF_OP, TYPE, WIDTH, RETURN) \ |
| 130 |
TYPE HIDDEN \ |
| 131 |
NAME##_##RETURN (OP, WIDTH) (TYPE *ptr, TYPE val) \ |
| 132 |
{ \ |
| 133 |
int *wordptr = (int *) ((unsigned int) ptr & ~3); \ |
| 134 |
unsigned int mask, shift, oldval, newval; \ |
| 135 |
int failure; \ |
| 136 |
\ |
| 137 |
shift = (((unsigned int) ptr & 3) << 3) ^ INVERT_MASK_##WIDTH; \ |
| 138 |
mask = MASK_##WIDTH << shift; \ |
| 139 |
\ |
| 140 |
do { \ |
| 141 |
oldval = *wordptr; \ |
| 142 |
newval = ((PFX_OP (((oldval & mask) >> shift) \ |
| 143 |
INF_OP (unsigned int) val)) << shift) & mask; \ |
| 144 |
newval |= oldval & ~mask; \ |
| 145 |
failure = __kernel_cmpxchg (oldval, newval, wordptr); \ |
| 146 |
} while (failure != 0); \ |
| 147 |
\ |
| 148 |
return (RETURN & mask) >> shift; \ |
| 149 |
} |
| 150 |
|
| 151 |
SUBWORD_SYNC_OP (add, , +, short, 2, oldval) |
| 152 |
SUBWORD_SYNC_OP (sub, , -, short, 2, oldval) |
| 153 |
SUBWORD_SYNC_OP (or, , |, short, 2, oldval) |
| 154 |
SUBWORD_SYNC_OP (and, , &, short, 2, oldval) |
| 155 |
SUBWORD_SYNC_OP (xor, , ^, short, 2, oldval) |
| 156 |
SUBWORD_SYNC_OP (nand, ~, &, short, 2, oldval) |
| 157 |
|
| 158 |
SUBWORD_SYNC_OP (add, , +, signed char, 1, oldval) |
| 159 |
SUBWORD_SYNC_OP (sub, , -, signed char, 1, oldval) |
| 160 |
SUBWORD_SYNC_OP (or, , |, signed char, 1, oldval) |
| 161 |
SUBWORD_SYNC_OP (and, , &, signed char, 1, oldval) |
| 162 |
SUBWORD_SYNC_OP (xor, , ^, signed char, 1, oldval) |
| 163 |
SUBWORD_SYNC_OP (nand, ~, &, signed char, 1, oldval) |
| 164 |
|
| 165 |
#define OP_AND_FETCH_WORD(OP, PFX_OP, INF_OP) \ |
| 166 |
int HIDDEN \ |
| 167 |
__sync_##OP##_and_fetch_4 (int *ptr, int val) \ |
| 168 |
{ \ |
| 169 |
int tmp, failure; \ |
| 170 |
\ |
| 171 |
do { \ |
| 172 |
tmp = *ptr; \ |
| 173 |
failure = __kernel_cmpxchg (tmp, PFX_OP (tmp INF_OP val), ptr); \ |
| 174 |
} while (failure != 0); \ |
| 175 |
\ |
| 176 |
return PFX_OP (tmp INF_OP val); \ |
| 177 |
} |
| 178 |
|
| 179 |
OP_AND_FETCH_WORD (add, , +) |
| 180 |
OP_AND_FETCH_WORD (sub, , -) |
| 181 |
OP_AND_FETCH_WORD (or, , |) |
| 182 |
OP_AND_FETCH_WORD (and, , &) |
| 183 |
OP_AND_FETCH_WORD (xor, , ^) |
| 184 |
OP_AND_FETCH_WORD (nand, ~, &) |
| 185 |
|
| 186 |
SUBWORD_SYNC_OP (add, , +, short, 2, newval) |
| 187 |
SUBWORD_SYNC_OP (sub, , -, short, 2, newval) |
| 188 |
SUBWORD_SYNC_OP (or, , |, short, 2, newval) |
| 189 |
SUBWORD_SYNC_OP (and, , &, short, 2, newval) |
| 190 |
SUBWORD_SYNC_OP (xor, , ^, short, 2, newval) |
| 191 |
SUBWORD_SYNC_OP (nand, ~, &, short, 2, newval) |
| 192 |
|
| 193 |
SUBWORD_SYNC_OP (add, , +, signed char, 1, newval) |
| 194 |
SUBWORD_SYNC_OP (sub, , -, signed char, 1, newval) |
| 195 |
SUBWORD_SYNC_OP (or, , |, signed char, 1, newval) |
| 196 |
SUBWORD_SYNC_OP (and, , &, signed char, 1, newval) |
| 197 |
SUBWORD_SYNC_OP (xor, , ^, signed char, 1, newval) |
| 198 |
SUBWORD_SYNC_OP (nand, ~, &, signed char, 1, newval) |
| 199 |
|
| 200 |
int HIDDEN |
| 201 |
__sync_val_compare_and_swap_4 (int *ptr, int oldval, int newval) |
| 202 |
{ |
| 203 |
int actual_oldval, fail; |
| 204 |
|
| 205 |
while (1) |
| 206 |
{ |
| 207 |
actual_oldval = *ptr; |
| 208 |
|
| 209 |
if (__builtin_expect (oldval != actual_oldval, 0)) |
| 210 |
return actual_oldval; |
| 211 |
|
| 212 |
fail = __kernel_cmpxchg (actual_oldval, newval, ptr); |
| 213 |
|
| 214 |
if (__builtin_expect (!fail, 1)) |
| 215 |
return oldval; |
| 216 |
} |
| 217 |
} |
| 218 |
|
| 219 |
#define SUBWORD_VAL_CAS(TYPE, WIDTH) \ |
| 220 |
TYPE HIDDEN \ |
| 221 |
__sync_val_compare_and_swap_##WIDTH (TYPE *ptr, TYPE oldval, \ |
| 222 |
TYPE newval) \ |
| 223 |
{ \ |
| 224 |
int *wordptr = (int *)((unsigned int) ptr & ~3), fail; \ |
| 225 |
unsigned int mask, shift, actual_oldval, actual_newval; \ |
| 226 |
\ |
| 227 |
shift = (((unsigned int) ptr & 3) << 3) ^ INVERT_MASK_##WIDTH; \ |
| 228 |
mask = MASK_##WIDTH << shift; \ |
| 229 |
\ |
| 230 |
while (1) \ |
| 231 |
{ \ |
| 232 |
actual_oldval = *wordptr; \ |
| 233 |
\ |
| 234 |
if (__builtin_expect (((actual_oldval & mask) >> shift) != \ |
| 235 |
((unsigned int) oldval & MASK_##WIDTH), 0)) \ |
| 236 |
return (actual_oldval & mask) >> shift; \ |
| 237 |
\ |
| 238 |
actual_newval = (actual_oldval & ~mask) \ |
| 239 |
| (((unsigned int) newval << shift) & mask); \ |
| 240 |
\ |
| 241 |
fail = __kernel_cmpxchg (actual_oldval, actual_newval, \ |
| 242 |
wordptr); \ |
| 243 |
\ |
| 244 |
if (__builtin_expect (!fail, 1)) \ |
| 245 |
return oldval; \ |
| 246 |
} \ |
| 247 |
} |
| 248 |
|
| 249 |
SUBWORD_VAL_CAS (short, 2) |
| 250 |
SUBWORD_VAL_CAS (signed char, 1) |
| 251 |
|
| 252 |
bool HIDDEN |
| 253 |
__sync_bool_compare_and_swap_4 (int *ptr, int oldval, int newval) |
| 254 |
{ |
| 255 |
int failure = __kernel_cmpxchg (oldval, newval, ptr); |
| 256 |
return (failure == 0); |
| 257 |
} |
| 258 |
|
| 259 |
#define SUBWORD_BOOL_CAS(TYPE, WIDTH) \ |
| 260 |
bool HIDDEN \ |
| 261 |
__sync_bool_compare_and_swap_##WIDTH (TYPE *ptr, TYPE oldval, \ |
| 262 |
TYPE newval) \ |
| 263 |
{ \ |
| 264 |
TYPE actual_oldval \ |
| 265 |
= __sync_val_compare_and_swap_##WIDTH (ptr, oldval, newval); \ |
| 266 |
return (oldval == actual_oldval); \ |
| 267 |
} |
| 268 |
|
| 269 |
SUBWORD_BOOL_CAS (short, 2) |
| 270 |
SUBWORD_BOOL_CAS (signed char, 1) |
| 271 |
|
| 272 |
void HIDDEN |
| 273 |
__sync_synchronize (void) |
| 274 |
{ |
| 275 |
__kernel_dmb (); |
| 276 |
} |
| 277 |
|
| 278 |
int HIDDEN |
| 279 |
__sync_lock_test_and_set_4 (int *ptr, int val) |
| 280 |
{ |
| 281 |
int failure, oldval; |
| 282 |
|
| 283 |
do { |
| 284 |
oldval = *ptr; |
| 285 |
failure = __kernel_cmpxchg (oldval, val, ptr); |
| 286 |
} while (failure != 0); |
| 287 |
|
| 288 |
return oldval; |
| 289 |
} |
| 290 |
|
| 291 |
#define SUBWORD_TEST_AND_SET(TYPE, WIDTH) \ |
| 292 |
TYPE HIDDEN \ |
| 293 |
__sync_lock_test_and_set_##WIDTH (TYPE *ptr, TYPE val) \ |
| 294 |
{ \ |
| 295 |
int failure; \ |
| 296 |
unsigned int oldval, newval, shift, mask; \ |
| 297 |
int *wordptr = (int *) ((unsigned int) ptr & ~3); \ |
| 298 |
\ |
| 299 |
shift = (((unsigned int) ptr & 3) << 3) ^ INVERT_MASK_##WIDTH; \ |
| 300 |
mask = MASK_##WIDTH << shift; \ |
| 301 |
\ |
| 302 |
do { \ |
| 303 |
oldval = *wordptr; \ |
| 304 |
newval = (oldval & ~mask) \ |
| 305 |
| (((unsigned int) val << shift) & mask); \ |
| 306 |
failure = __kernel_cmpxchg (oldval, newval, wordptr); \ |
| 307 |
} while (failure != 0); \ |
| 308 |
\ |
| 309 |
return (oldval & mask) >> shift; \ |
| 310 |
} |
| 311 |
|
| 312 |
SUBWORD_TEST_AND_SET (short, 2) |
| 313 |
SUBWORD_TEST_AND_SET (signed char, 1) |
| 314 |
|
| 315 |
#define SYNC_LOCK_RELEASE(TYPE, WIDTH) \ |
| 316 |
void HIDDEN \ |
| 317 |
__sync_lock_release_##WIDTH (TYPE *ptr) \ |
| 318 |
{ \ |
| 319 |
/* All writes before this point must be seen before we release \ |
| 320 |
the lock itself. */ \ |
| 321 |
__kernel_dmb (); \ |
| 322 |
*ptr = 0; \ |
| 323 |
} |
| 324 |
|
| 325 |
SYNC_LOCK_RELEASE (long long, 8) |
| 326 |
SYNC_LOCK_RELEASE (int, 4) |
| 327 |
SYNC_LOCK_RELEASE (short, 2) |
| 328 |
SYNC_LOCK_RELEASE (char, 1) |