| 1 |
/* Out-of-line LSE atomics for AArch64 architecture. |
| 2 |
Copyright (C) 2019-2025 Free Software Foundation, Inc. |
| 3 |
Contributed by Linaro Ltd. |
| 4 |
|
| 5 |
This file is part of GCC. |
| 6 |
|
| 7 |
GCC is free software; you can redistribute it and/or modify it under |
| 8 |
the terms of the GNU General Public License as published by the Free |
| 9 |
Software Foundation; either version 3, or (at your option) any later |
| 10 |
version. |
| 11 |
|
| 12 |
GCC is distributed in the hope that it will be useful, but WITHOUT ANY |
| 13 |
WARRANTY; without even the implied warranty of MERCHANTABILITY or |
| 14 |
FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License |
| 15 |
for more details. |
| 16 |
|
| 17 |
Under Section 7 of GPL version 3, you are granted additional |
| 18 |
permissions described in the GCC Runtime Library Exception, version |
| 19 |
3.1, as published by the Free Software Foundation. |
| 20 |
|
| 21 |
You should have received a copy of the GNU General Public License and |
| 22 |
a copy of the GCC Runtime Library Exception along with this program; |
| 23 |
see the files COPYING3 and COPYING.RUNTIME respectively. If not, see |
| 24 |
<http://www.gnu.org/licenses/>. */ |
| 25 |
|
| 26 |
/* |
| 27 |
* The problem that we are trying to solve is operating system deployment |
| 28 |
* of ARMv8.1-Atomics, also known as Large System Exensions (LSE). |
| 29 |
* |
| 30 |
* There are a number of potential solutions for this problem which have |
| 31 |
* been proposed and rejected for various reasons. To recap: |
| 32 |
* |
| 33 |
* (1) Multiple builds. The dynamic linker will examine /lib64/atomics/ |
| 34 |
* if HWCAP_ATOMICS is set, allowing entire libraries to be overwritten. |
| 35 |
* However, not all Linux distributions are happy with multiple builds, |
| 36 |
* and anyway it has no effect on main applications. |
| 37 |
* |
| 38 |
* (2) IFUNC. We could put these functions into libgcc_s.so, and have |
| 39 |
* a single copy of each function for all DSOs. However, ARM is concerned |
| 40 |
* that the branch-to-indirect-branch that is implied by using a PLT, |
| 41 |
* as required by IFUNC, is too much overhead for smaller cpus. |
| 42 |
* |
| 43 |
* (3) Statically predicted direct branches. This is the approach that |
| 44 |
* is taken here. These functions are linked into every DSO that uses them. |
| 45 |
* All of the symbols are hidden, so that the functions are called via a |
| 46 |
* direct branch. The choice of LSE vs non-LSE is done via one byte load |
| 47 |
* followed by a well-predicted direct branch. The functions are compiled |
| 48 |
* separately to minimize code size. |
| 49 |
* |
| 50 |
* Since these functions have hidden visibility and are never called |
| 51 |
* indirectly, they do not need to start with a BTI instruction. |
| 52 |
*/ |
| 53 |
|
| 54 |
#include "aarch64-asm.h" |
| 55 |
#include "auto-target.h" |
| 56 |
|
| 57 |
/* L is defined in aarch64-asm.h for a different purpose than why we |
| 58 |
use it here. */ |
| 59 |
#undef L |
| 60 |
|
| 61 |
/* Tell the assembler to accept LSE instructions. */ |
| 62 |
#ifdef HAVE_AS_LSE |
| 63 |
.arch armv8-a+lse |
| 64 |
#else |
| 65 |
.arch armv8-a |
| 66 |
#endif |
| 67 |
|
| 68 |
/* Declare the symbol gating the LSE implementations. */ |
| 69 |
HIDDEN(__aarch64_have_lse_atomics) |
| 70 |
|
| 71 |
/* Turn size and memory model defines into mnemonic fragments. */ |
| 72 |
#if SIZE == 1 |
| 73 |
# define S b |
| 74 |
# define UXT uxtb |
| 75 |
# define B 0x00000000 |
| 76 |
#elif SIZE == 2 |
| 77 |
# define S h |
| 78 |
# define UXT uxth |
| 79 |
# define B 0x40000000 |
| 80 |
#elif SIZE == 4 || SIZE == 8 || SIZE == 16 |
| 81 |
# define S |
| 82 |
# define UXT mov |
| 83 |
# if SIZE == 4 |
| 84 |
# define B 0x80000000 |
| 85 |
# elif SIZE == 8 |
| 86 |
# define B 0xc0000000 |
| 87 |
# endif |
| 88 |
#else |
| 89 |
# error |
| 90 |
#endif |
| 91 |
|
| 92 |
#if MODEL == 1 |
| 93 |
# define SUFF _relax |
| 94 |
# define A |
| 95 |
# define L |
| 96 |
# define M 0x000000 |
| 97 |
# define N 0x000000 |
| 98 |
# define BARRIER |
| 99 |
#elif MODEL == 2 |
| 100 |
# define SUFF _acq |
| 101 |
# define A a |
| 102 |
# define L |
| 103 |
# define M 0x400000 |
| 104 |
# define N 0x800000 |
| 105 |
# define BARRIER |
| 106 |
#elif MODEL == 3 |
| 107 |
# define SUFF _rel |
| 108 |
# define A |
| 109 |
# define L l |
| 110 |
# define M 0x008000 |
| 111 |
# define N 0x400000 |
| 112 |
# define BARRIER |
| 113 |
#elif MODEL == 4 |
| 114 |
# define SUFF _acq_rel |
| 115 |
# define A a |
| 116 |
# define L l |
| 117 |
# define M 0x408000 |
| 118 |
# define N 0xc00000 |
| 119 |
# define BARRIER |
| 120 |
#elif MODEL == 5 |
| 121 |
# define SUFF _sync |
| 122 |
#ifdef L_swp |
| 123 |
/* swp has _acq semantics. */ |
| 124 |
# define A a |
| 125 |
# define L |
| 126 |
# define M 0x400000 |
| 127 |
# define N 0x800000 |
| 128 |
#else |
| 129 |
/* All other _sync functions have _seq semantics. */ |
| 130 |
# define A a |
| 131 |
# define L l |
| 132 |
# define M 0x408000 |
| 133 |
# define N 0xc00000 |
| 134 |
#endif |
| 135 |
# define BARRIER dmb ish |
| 136 |
#else |
| 137 |
# error |
| 138 |
#endif |
| 139 |
|
| 140 |
/* Concatenate symbols. */ |
| 141 |
#define glue2_(A, B) A ## B |
| 142 |
#define glue2(A, B) glue2_(A, B) |
| 143 |
#define glue3_(A, B, C) A ## B ## C |
| 144 |
#define glue3(A, B, C) glue3_(A, B, C) |
| 145 |
#define glue4_(A, B, C, D) A ## B ## C ## D |
| 146 |
#define glue4(A, B, C, D) glue4_(A, B, C, D) |
| 147 |
|
| 148 |
/* Select the size of a register, given a regno. */ |
| 149 |
#define x(N) glue2(x, N) |
| 150 |
#define w(N) glue2(w, N) |
| 151 |
#if SIZE < 8 |
| 152 |
# define s(N) w(N) |
| 153 |
#else |
| 154 |
# define s(N) x(N) |
| 155 |
#endif |
| 156 |
|
| 157 |
#define NAME(BASE) glue4(__aarch64_, BASE, SIZE, SUFF) |
| 158 |
#if MODEL == 5 |
| 159 |
/* Drop A for _sync functions. */ |
| 160 |
# define LDXR glue3(ld, xr, S) |
| 161 |
#else |
| 162 |
# define LDXR glue4(ld, A, xr, S) |
| 163 |
#endif |
| 164 |
#define STXR glue4(st, L, xr, S) |
| 165 |
|
| 166 |
/* Temporary registers used. Other than these, only the return value |
| 167 |
register (x0) and the flags are modified. */ |
| 168 |
#define tmp0 16 |
| 169 |
#define tmp1 17 |
| 170 |
#define tmp2 15 |
| 171 |
#define tmp3 14 |
| 172 |
#define tmp4 13 |
| 173 |
|
| 174 |
/* Start and end a function. */ |
| 175 |
.macro STARTFN name |
| 176 |
.text |
| 177 |
.balign 16 |
| 178 |
.globl \name |
| 179 |
HIDDEN(\name) |
| 180 |
SYMBOL_TYPE(\name, %function) |
| 181 |
.cfi_startproc |
| 182 |
\name: |
| 183 |
.endm |
| 184 |
|
| 185 |
.macro ENDFN name |
| 186 |
.cfi_endproc |
| 187 |
SYMBOL_SIZE(\name) |
| 188 |
.endm |
| 189 |
|
| 190 |
/* Branch to LABEL if LSE is disabled. */ |
| 191 |
.macro JUMP_IF_NOT_LSE label |
| 192 |
adrp x(tmp0), __aarch64_have_lse_atomics |
| 193 |
ldrb w(tmp0), [x(tmp0), :lo12:__aarch64_have_lse_atomics] |
| 194 |
cbz w(tmp0), \label |
| 195 |
.endm |
| 196 |
|
| 197 |
#ifdef L_cas |
| 198 |
|
| 199 |
STARTFN NAME(cas) |
| 200 |
JUMP_IF_NOT_LSE 8f |
| 201 |
|
| 202 |
#if SIZE < 16 |
| 203 |
#ifdef HAVE_AS_LSE |
| 204 |
# define CAS glue4(cas, A, L, S) s(0), s(1), [x2] |
| 205 |
#else |
| 206 |
# define CAS .inst 0x08a07c41 + B + M |
| 207 |
#endif |
| 208 |
|
| 209 |
CAS /* s(0), s(1), [x2] */ |
| 210 |
ret |
| 211 |
|
| 212 |
8: UXT s(tmp0), s(0) |
| 213 |
0: LDXR s(0), [x2] |
| 214 |
cmp s(0), s(tmp0) |
| 215 |
bne 1f |
| 216 |
STXR w(tmp1), s(1), [x2] |
| 217 |
cbnz w(tmp1), 0b |
| 218 |
1: BARRIER |
| 219 |
ret |
| 220 |
|
| 221 |
#else |
| 222 |
#if MODEL == 5 |
| 223 |
/* Drop A for _sync functions. */ |
| 224 |
# define LDXP glue2(ld, xp) |
| 225 |
#else |
| 226 |
# define LDXP glue3(ld, A, xp) |
| 227 |
#endif |
| 228 |
#define STXP glue3(st, L, xp) |
| 229 |
#ifdef HAVE_AS_LSE |
| 230 |
# define CASP glue3(casp, A, L) x0, x1, x2, x3, [x4] |
| 231 |
#else |
| 232 |
# define CASP .inst 0x48207c82 + M |
| 233 |
#endif |
| 234 |
|
| 235 |
CASP /* x0, x1, x2, x3, [x4] */ |
| 236 |
ret |
| 237 |
|
| 238 |
8: mov x(tmp0), x0 |
| 239 |
mov x(tmp1), x1 |
| 240 |
0: LDXP x0, x1, [x4] |
| 241 |
cmp x0, x(tmp0) |
| 242 |
ccmp x1, x(tmp1), #0, eq |
| 243 |
csel x(tmp2), x2, x0, eq |
| 244 |
csel x(tmp3), x3, x1, eq |
| 245 |
STXP w(tmp4), x(tmp2), x(tmp3), [x4] |
| 246 |
cbnz w(tmp4), 0b |
| 247 |
BARRIER |
| 248 |
ret |
| 249 |
|
| 250 |
#endif |
| 251 |
|
| 252 |
ENDFN NAME(cas) |
| 253 |
#endif |
| 254 |
|
| 255 |
#ifdef L_swp |
| 256 |
#ifdef HAVE_AS_LSE |
| 257 |
# define SWP glue4(swp, A, L, S) s(0), s(0), [x1] |
| 258 |
#else |
| 259 |
# define SWP .inst 0x38208020 + B + N |
| 260 |
#endif |
| 261 |
|
| 262 |
STARTFN NAME(swp) |
| 263 |
JUMP_IF_NOT_LSE 8f |
| 264 |
|
| 265 |
SWP /* s(0), s(0), [x1] */ |
| 266 |
ret |
| 267 |
|
| 268 |
8: mov s(tmp0), s(0) |
| 269 |
0: LDXR s(0), [x1] |
| 270 |
STXR w(tmp1), s(tmp0), [x1] |
| 271 |
cbnz w(tmp1), 0b |
| 272 |
BARRIER |
| 273 |
ret |
| 274 |
|
| 275 |
ENDFN NAME(swp) |
| 276 |
#endif |
| 277 |
|
| 278 |
#if defined(L_ldadd) || defined(L_ldclr) \ |
| 279 |
|| defined(L_ldeor) || defined(L_ldset) |
| 280 |
|
| 281 |
#ifdef L_ldadd |
| 282 |
#define LDNM ldadd |
| 283 |
#define OP add |
| 284 |
#define OPN 0x0000 |
| 285 |
#elif defined(L_ldclr) |
| 286 |
#define LDNM ldclr |
| 287 |
#define OP bic |
| 288 |
#define OPN 0x1000 |
| 289 |
#elif defined(L_ldeor) |
| 290 |
#define LDNM ldeor |
| 291 |
#define OP eor |
| 292 |
#define OPN 0x2000 |
| 293 |
#elif defined(L_ldset) |
| 294 |
#define LDNM ldset |
| 295 |
#define OP orr |
| 296 |
#define OPN 0x3000 |
| 297 |
#else |
| 298 |
#error |
| 299 |
#endif |
| 300 |
#ifdef HAVE_AS_LSE |
| 301 |
# define LDOP glue4(LDNM, A, L, S) s(0), s(0), [x1] |
| 302 |
#else |
| 303 |
# define LDOP .inst 0x38200020 + OPN + B + N |
| 304 |
#endif |
| 305 |
|
| 306 |
STARTFN NAME(LDNM) |
| 307 |
JUMP_IF_NOT_LSE 8f |
| 308 |
|
| 309 |
LDOP /* s(0), s(0), [x1] */ |
| 310 |
ret |
| 311 |
|
| 312 |
8: mov s(tmp0), s(0) |
| 313 |
0: LDXR s(0), [x1] |
| 314 |
OP s(tmp1), s(0), s(tmp0) |
| 315 |
STXR w(tmp2), s(tmp1), [x1] |
| 316 |
cbnz w(tmp2), 0b |
| 317 |
BARRIER |
| 318 |
ret |
| 319 |
|
| 320 |
ENDFN NAME(LDNM) |
| 321 |
#endif |