Debian Codegraph @

libgcc/config/aarch64/lse.S

raw
1 /* Out-of-line LSE atomics for AArch64 architecture.
2 Copyright (C) 2019-2025 Free Software Foundation, Inc.
3 Contributed by Linaro Ltd.
4
5 This file is part of GCC.
6
7 GCC is free software; you can redistribute it and/or modify it under
8 the terms of the GNU General Public License as published by the Free
9 Software Foundation; either version 3, or (at your option) any later
10 version.
11
12 GCC is distributed in the hope that it will be useful, but WITHOUT ANY
13 WARRANTY; without even the implied warranty of MERCHANTABILITY or
14 FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
15 for more details.
16
17 Under Section 7 of GPL version 3, you are granted additional
18 permissions described in the GCC Runtime Library Exception, version
19 3.1, as published by the Free Software Foundation.
20
21 You should have received a copy of the GNU General Public License and
22 a copy of the GCC Runtime Library Exception along with this program;
23 see the files COPYING3 and COPYING.RUNTIME respectively. If not, see
24 <http://www.gnu.org/licenses/>. */
25
26 /*
27 * The problem that we are trying to solve is operating system deployment
28 * of ARMv8.1-Atomics, also known as Large System Exensions (LSE).
29 *
30 * There are a number of potential solutions for this problem which have
31 * been proposed and rejected for various reasons. To recap:
32 *
33 * (1) Multiple builds. The dynamic linker will examine /lib64/atomics/
34 * if HWCAP_ATOMICS is set, allowing entire libraries to be overwritten.
35 * However, not all Linux distributions are happy with multiple builds,
36 * and anyway it has no effect on main applications.
37 *
38 * (2) IFUNC. We could put these functions into libgcc_s.so, and have
39 * a single copy of each function for all DSOs. However, ARM is concerned
40 * that the branch-to-indirect-branch that is implied by using a PLT,
41 * as required by IFUNC, is too much overhead for smaller cpus.
42 *
43 * (3) Statically predicted direct branches. This is the approach that
44 * is taken here. These functions are linked into every DSO that uses them.
45 * All of the symbols are hidden, so that the functions are called via a
46 * direct branch. The choice of LSE vs non-LSE is done via one byte load
47 * followed by a well-predicted direct branch. The functions are compiled
48 * separately to minimize code size.
49 *
50 * Since these functions have hidden visibility and are never called
51 * indirectly, they do not need to start with a BTI instruction.
52 */
53
54 #include "aarch64-asm.h"
55 #include "auto-target.h"
56
57 /* L is defined in aarch64-asm.h for a different purpose than why we
58 use it here. */
59 #undef L
60
61 /* Tell the assembler to accept LSE instructions. */
62 #ifdef HAVE_AS_LSE
63 .arch armv8-a+lse
64 #else
65 .arch armv8-a
66 #endif
67
68 /* Declare the symbol gating the LSE implementations. */
69 HIDDEN(__aarch64_have_lse_atomics)
70
71 /* Turn size and memory model defines into mnemonic fragments. */
72 #if SIZE == 1
73 # define S b
74 # define UXT uxtb
75 # define B 0x00000000
76 #elif SIZE == 2
77 # define S h
78 # define UXT uxth
79 # define B 0x40000000
80 #elif SIZE == 4 || SIZE == 8 || SIZE == 16
81 # define S
82 # define UXT mov
83 # if SIZE == 4
84 # define B 0x80000000
85 # elif SIZE == 8
86 # define B 0xc0000000
87 # endif
88 #else
89 # error
90 #endif
91
92 #if MODEL == 1
93 # define SUFF _relax
94 # define A
95 # define L
96 # define M 0x000000
97 # define N 0x000000
98 # define BARRIER
99 #elif MODEL == 2
100 # define SUFF _acq
101 # define A a
102 # define L
103 # define M 0x400000
104 # define N 0x800000
105 # define BARRIER
106 #elif MODEL == 3
107 # define SUFF _rel
108 # define A
109 # define L l
110 # define M 0x008000
111 # define N 0x400000
112 # define BARRIER
113 #elif MODEL == 4
114 # define SUFF _acq_rel
115 # define A a
116 # define L l
117 # define M 0x408000
118 # define N 0xc00000
119 # define BARRIER
120 #elif MODEL == 5
121 # define SUFF _sync
122 #ifdef L_swp
123 /* swp has _acq semantics. */
124 # define A a
125 # define L
126 # define M 0x400000
127 # define N 0x800000
128 #else
129 /* All other _sync functions have _seq semantics. */
130 # define A a
131 # define L l
132 # define M 0x408000
133 # define N 0xc00000
134 #endif
135 # define BARRIER dmb ish
136 #else
137 # error
138 #endif
139
140 /* Concatenate symbols. */
141 #define glue2_(A, B) A ## B
142 #define glue2(A, B) glue2_(A, B)
143 #define glue3_(A, B, C) A ## B ## C
144 #define glue3(A, B, C) glue3_(A, B, C)
145 #define glue4_(A, B, C, D) A ## B ## C ## D
146 #define glue4(A, B, C, D) glue4_(A, B, C, D)
147
148 /* Select the size of a register, given a regno. */
149 #define x(N) glue2(x, N)
150 #define w(N) glue2(w, N)
151 #if SIZE < 8
152 # define s(N) w(N)
153 #else
154 # define s(N) x(N)
155 #endif
156
157 #define NAME(BASE) glue4(__aarch64_, BASE, SIZE, SUFF)
158 #if MODEL == 5
159 /* Drop A for _sync functions. */
160 # define LDXR glue3(ld, xr, S)
161 #else
162 # define LDXR glue4(ld, A, xr, S)
163 #endif
164 #define STXR glue4(st, L, xr, S)
165
166 /* Temporary registers used. Other than these, only the return value
167 register (x0) and the flags are modified. */
168 #define tmp0 16
169 #define tmp1 17
170 #define tmp2 15
171 #define tmp3 14
172 #define tmp4 13
173
174 /* Start and end a function. */
175 .macro STARTFN name
176 .text
177 .balign 16
178 .globl \name
179 HIDDEN(\name)
180 SYMBOL_TYPE(\name, %function)
181 .cfi_startproc
182 \name:
183 .endm
184
185 .macro ENDFN name
186 .cfi_endproc
187 SYMBOL_SIZE(\name)
188 .endm
189
190 /* Branch to LABEL if LSE is disabled. */
191 .macro JUMP_IF_NOT_LSE label
192 adrp x(tmp0), __aarch64_have_lse_atomics
193 ldrb w(tmp0), [x(tmp0), :lo12:__aarch64_have_lse_atomics]
194 cbz w(tmp0), \label
195 .endm
196
197 #ifdef L_cas
198
199 STARTFN NAME(cas)
200 JUMP_IF_NOT_LSE 8f
201
202 #if SIZE < 16
203 #ifdef HAVE_AS_LSE
204 # define CAS glue4(cas, A, L, S) s(0), s(1), [x2]
205 #else
206 # define CAS .inst 0x08a07c41 + B + M
207 #endif
208
209 CAS /* s(0), s(1), [x2] */
210 ret
211
212 8: UXT s(tmp0), s(0)
213 0: LDXR s(0), [x2]
214 cmp s(0), s(tmp0)
215 bne 1f
216 STXR w(tmp1), s(1), [x2]
217 cbnz w(tmp1), 0b
218 1: BARRIER
219 ret
220
221 #else
222 #if MODEL == 5
223 /* Drop A for _sync functions. */
224 # define LDXP glue2(ld, xp)
225 #else
226 # define LDXP glue3(ld, A, xp)
227 #endif
228 #define STXP glue3(st, L, xp)
229 #ifdef HAVE_AS_LSE
230 # define CASP glue3(casp, A, L) x0, x1, x2, x3, [x4]
231 #else
232 # define CASP .inst 0x48207c82 + M
233 #endif
234
235 CASP /* x0, x1, x2, x3, [x4] */
236 ret
237
238 8: mov x(tmp0), x0
239 mov x(tmp1), x1
240 0: LDXP x0, x1, [x4]
241 cmp x0, x(tmp0)
242 ccmp x1, x(tmp1), #0, eq
243 csel x(tmp2), x2, x0, eq
244 csel x(tmp3), x3, x1, eq
245 STXP w(tmp4), x(tmp2), x(tmp3), [x4]
246 cbnz w(tmp4), 0b
247 BARRIER
248 ret
249
250 #endif
251
252 ENDFN NAME(cas)
253 #endif
254
255 #ifdef L_swp
256 #ifdef HAVE_AS_LSE
257 # define SWP glue4(swp, A, L, S) s(0), s(0), [x1]
258 #else
259 # define SWP .inst 0x38208020 + B + N
260 #endif
261
262 STARTFN NAME(swp)
263 JUMP_IF_NOT_LSE 8f
264
265 SWP /* s(0), s(0), [x1] */
266 ret
267
268 8: mov s(tmp0), s(0)
269 0: LDXR s(0), [x1]
270 STXR w(tmp1), s(tmp0), [x1]
271 cbnz w(tmp1), 0b
272 BARRIER
273 ret
274
275 ENDFN NAME(swp)
276 #endif
277
278 #if defined(L_ldadd) || defined(L_ldclr) \
279 || defined(L_ldeor) || defined(L_ldset)
280
281 #ifdef L_ldadd
282 #define LDNM ldadd
283 #define OP add
284 #define OPN 0x0000
285 #elif defined(L_ldclr)
286 #define LDNM ldclr
287 #define OP bic
288 #define OPN 0x1000
289 #elif defined(L_ldeor)
290 #define LDNM ldeor
291 #define OP eor
292 #define OPN 0x2000
293 #elif defined(L_ldset)
294 #define LDNM ldset
295 #define OP orr
296 #define OPN 0x3000
297 #else
298 #error
299 #endif
300 #ifdef HAVE_AS_LSE
301 # define LDOP glue4(LDNM, A, L, S) s(0), s(0), [x1]
302 #else
303 # define LDOP .inst 0x38200020 + OPN + B + N
304 #endif
305
306 STARTFN NAME(LDNM)
307 JUMP_IF_NOT_LSE 8f
308
309 LDOP /* s(0), s(0), [x1] */
310 ret
311
312 8: mov s(tmp0), s(0)
313 0: LDXR s(0), [x1]
314 OP s(tmp1), s(0), s(tmp0)
315 STXR w(tmp2), s(tmp1), [x1]
316 cbnz w(tmp2), 0b
317 BARRIER
318 ret
319
320 ENDFN NAME(LDNM)
321 #endif