Debian Codegraph @

libgcc/config/aarch64/cpuinfo.c

raw
1 /* CPU feature detection for AArch64 architecture.
2 Copyright (C) 2023-2025 Free Software Foundation, Inc.
3
4 This file is part of GCC.
5
6 This file is free software; you can redistribute it and/or modify it
7 under the terms of the GNU General Public License as published by the
8 Free Software Foundation; either version 3, or (at your option) any
9 later version.
10
11 This file is distributed in the hope that it will be useful, but
12 WITHOUT ANY WARRANTY; without even the implied warranty of
13 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14 General Public License for more details.
15
16 Under Section 7 of GPL version 3, you are granted additional
17 permissions described in the GCC Runtime Library Exception, version
18 3.1, as published by the Free Software Foundation.
19
20 You should have received a copy of the GNU General Public License and
21 a copy of the GCC Runtime Library Exception along with this program;
22 see the files COPYING3 and COPYING.RUNTIME respectively. If not, see
23 <http://www.gnu.org/licenses/>. */
24
25 #include "common/config/aarch64/cpuinfo.h"
26
27 #if __has_include(<sys/auxv.h>)
28 #include <sys/auxv.h>
29
30 #if __has_include(<sys/ifunc.h>)
31 #include <sys/ifunc.h>
32 #else
33 typedef struct __ifunc_arg_t {
34 unsigned long _size;
35 unsigned long _hwcap;
36 unsigned long _hwcap2;
37 } __ifunc_arg_t;
38 #endif
39
40 #if __has_include(<asm/hwcap.h>)
41 #include <asm/hwcap.h>
42
43 /* Architecture features used in Function Multi Versioning. */
44 struct {
45 unsigned long long features;
46 /* As features grows new fields could be added. */
47 } __aarch64_cpu_features __attribute__((visibility("hidden"), nocommon));
48
49 #ifndef _IFUNC_ARG_HWCAP
50 #define _IFUNC_ARG_HWCAP (1ULL << 62)
51 #endif
52 #ifndef AT_HWCAP
53 #define AT_HWCAP 16
54 #endif
55 #ifndef HWCAP_FP
56 #define HWCAP_FP (1 << 0)
57 #endif
58 #ifndef HWCAP_ASIMD
59 #define HWCAP_ASIMD (1 << 1)
60 #endif
61 #ifndef HWCAP_EVTSTRM
62 #define HWCAP_EVTSTRM (1 << 2)
63 #endif
64 #ifndef HWCAP_AES
65 #define HWCAP_AES (1 << 3)
66 #endif
67 #ifndef HWCAP_PMULL
68 #define HWCAP_PMULL (1 << 4)
69 #endif
70 #ifndef HWCAP_SHA1
71 #define HWCAP_SHA1 (1 << 5)
72 #endif
73 #ifndef HWCAP_SHA2
74 #define HWCAP_SHA2 (1 << 6)
75 #endif
76 #ifndef HWCAP_CRC32
77 #define HWCAP_CRC32 (1 << 7)
78 #endif
79 #ifndef HWCAP_ATOMICS
80 #define HWCAP_ATOMICS (1 << 8)
81 #endif
82 #ifndef HWCAP_FPHP
83 #define HWCAP_FPHP (1 << 9)
84 #endif
85 #ifndef HWCAP_ASIMDHP
86 #define HWCAP_ASIMDHP (1 << 10)
87 #endif
88 #ifndef HWCAP_CPUID
89 #define HWCAP_CPUID (1 << 11)
90 #endif
91 #ifndef HWCAP_ASIMDRDM
92 #define HWCAP_ASIMDRDM (1 << 12)
93 #endif
94 #ifndef HWCAP_JSCVT
95 #define HWCAP_JSCVT (1 << 13)
96 #endif
97 #ifndef HWCAP_FCMA
98 #define HWCAP_FCMA (1 << 14)
99 #endif
100 #ifndef HWCAP_LRCPC
101 #define HWCAP_LRCPC (1 << 15)
102 #endif
103 #ifndef HWCAP_DCPOP
104 #define HWCAP_DCPOP (1 << 16)
105 #endif
106 #ifndef HWCAP_SHA3
107 #define HWCAP_SHA3 (1 << 17)
108 #endif
109 #ifndef HWCAP_SM3
110 #define HWCAP_SM3 (1 << 18)
111 #endif
112 #ifndef HWCAP_SM4
113 #define HWCAP_SM4 (1 << 19)
114 #endif
115 #ifndef HWCAP_ASIMDDP
116 #define HWCAP_ASIMDDP (1 << 20)
117 #endif
118 #ifndef HWCAP_SHA512
119 #define HWCAP_SHA512 (1 << 21)
120 #endif
121 #ifndef HWCAP_SVE
122 #define HWCAP_SVE (1 << 22)
123 #endif
124 #ifndef HWCAP_ASIMDFHM
125 #define HWCAP_ASIMDFHM (1 << 23)
126 #endif
127 #ifndef HWCAP_DIT
128 #define HWCAP_DIT (1 << 24)
129 #endif
130 #ifndef HWCAP_ILRCPC
131 #define HWCAP_ILRCPC (1 << 26)
132 #endif
133 #ifndef HWCAP_FLAGM
134 #define HWCAP_FLAGM (1 << 27)
135 #endif
136 #ifndef HWCAP_SSBS
137 #define HWCAP_SSBS (1 << 28)
138 #endif
139 #ifndef HWCAP_SB
140 #define HWCAP_SB (1 << 29)
141 #endif
142 #ifndef HWCAP_PACA
143 #define HWCAP_PACA (1 << 30)
144 #endif
145 #ifndef HWCAP_PACG
146 #define HWCAP_PACG (1UL << 31)
147 #endif
148
149 #ifndef AT_HWCAP2
150 #define AT_HWCAP2 26
151 #endif
152 #ifndef HWCAP2_DCPODP
153 #define HWCAP2_DCPODP (1 << 0)
154 #endif
155 #ifndef HWCAP2_SVE2
156 #define HWCAP2_SVE2 (1 << 1)
157 #endif
158 #ifndef HWCAP2_SVEAES
159 #define HWCAP2_SVEAES (1 << 2)
160 #endif
161 #ifndef HWCAP2_SVEPMULL
162 #define HWCAP2_SVEPMULL (1 << 3)
163 #endif
164 #ifndef HWCAP2_SVEBITPERM
165 #define HWCAP2_SVEBITPERM (1 << 4)
166 #endif
167 #ifndef HWCAP2_SVESHA3
168 #define HWCAP2_SVESHA3 (1 << 5)
169 #endif
170 #ifndef HWCAP2_SVESM4
171 #define HWCAP2_SVESM4 (1 << 6)
172 #endif
173 #ifndef HWCAP2_FLAGM2
174 #define HWCAP2_FLAGM2 (1 << 7)
175 #endif
176 #ifndef HWCAP2_FRINT
177 #define HWCAP2_FRINT (1 << 8)
178 #endif
179 #ifndef HWCAP2_SVEI8MM
180 #define HWCAP2_SVEI8MM (1 << 9)
181 #endif
182 #ifndef HWCAP2_SVEF32MM
183 #define HWCAP2_SVEF32MM (1 << 10)
184 #endif
185 #ifndef HWCAP2_SVEF64MM
186 #define HWCAP2_SVEF64MM (1 << 11)
187 #endif
188 #ifndef HWCAP2_SVEBF16
189 #define HWCAP2_SVEBF16 (1 << 12)
190 #endif
191 #ifndef HWCAP2_I8MM
192 #define HWCAP2_I8MM (1 << 13)
193 #endif
194 #ifndef HWCAP2_BF16
195 #define HWCAP2_BF16 (1 << 14)
196 #endif
197 #ifndef HWCAP2_DGH
198 #define HWCAP2_DGH (1 << 15)
199 #endif
200 #ifndef HWCAP2_RNG
201 #define HWCAP2_RNG (1 << 16)
202 #endif
203 #ifndef HWCAP2_BTI
204 #define HWCAP2_BTI (1 << 17)
205 #endif
206 #ifndef HWCAP2_MTE
207 #define HWCAP2_MTE (1 << 18)
208 #endif
209 #ifndef HWCAP2_RPRES
210 #define HWCAP2_RPRES (1 << 21)
211 #endif
212 #ifndef HWCAP2_MTE3
213 #define HWCAP2_MTE3 (1 << 22)
214 #endif
215 #ifndef HWCAP2_SME
216 #define HWCAP2_SME (1 << 23)
217 #endif
218 #ifndef HWCAP2_SME_I16I64
219 #define HWCAP2_SME_I16I64 (1 << 24)
220 #endif
221 #ifndef HWCAP2_SME_F64F64
222 #define HWCAP2_SME_F64F64 (1 << 25)
223 #endif
224 #ifndef HWCAP2_WFXT
225 #define HWCAP2_WFXT (1UL << 31)
226 #endif
227 #ifndef HWCAP2_EBF16
228 #define HWCAP2_EBF16 (1UL << 32)
229 #endif
230 #ifndef HWCAP2_SVE_EBF16
231 #define HWCAP2_SVE_EBF16 (1UL << 33)
232 #endif
233 #ifndef HWCAP2_SME2
234 #define HWCAP2_SME2 (1UL << 37)
235 #endif
236 #ifndef HWCAP2_LRCPC3
237 #define HWCAP2_LRCPC3 (1UL << 46)
238 #endif
239
240 static void
241 __init_cpu_features_constructor (unsigned long hwcap,
242 const __ifunc_arg_t *arg)
243 {
244 unsigned long feat = 0;
245 #define setCPUFeature(F) feat |= 1UL << F
246 #define getCPUFeature(id, ftr) __asm__("mrs %0, " #id : "=r"(ftr))
247 #define extractBits(val, start, number) \
248 (val & ((1UL << number) - 1UL) << start) >> start
249 unsigned long hwcap2 = 0;
250 if (hwcap & _IFUNC_ARG_HWCAP)
251 hwcap2 = arg->_hwcap2;
252 if (hwcap & HWCAP_CRC32)
253 setCPUFeature(FEAT_CRC);
254 if (hwcap & HWCAP_PMULL)
255 setCPUFeature(FEAT_PMULL);
256 if (hwcap & HWCAP_FLAGM)
257 setCPUFeature(FEAT_FLAGM);
258 if (hwcap2 & HWCAP2_FLAGM2)
259 setCPUFeature(FEAT_FLAGM2);
260 if (hwcap & HWCAP_SM4)
261 setCPUFeature(FEAT_SM4);
262 if (hwcap & HWCAP_ASIMDDP)
263 setCPUFeature(FEAT_DOTPROD);
264 if (hwcap & HWCAP_ASIMDFHM)
265 setCPUFeature(FEAT_FP16FML);
266 if (hwcap & HWCAP_FPHP)
267 setCPUFeature(FEAT_FP16);
268 if (hwcap & HWCAP_DIT)
269 setCPUFeature(FEAT_DIT);
270 if (hwcap & HWCAP_ASIMDRDM)
271 setCPUFeature(FEAT_RDM);
272 if (hwcap & HWCAP_AES)
273 setCPUFeature(FEAT_AES);
274 if (hwcap & HWCAP_SHA1)
275 setCPUFeature(FEAT_SHA1);
276 if (hwcap & HWCAP_SHA2)
277 setCPUFeature(FEAT_SHA2);
278 if (hwcap & HWCAP_JSCVT)
279 setCPUFeature(FEAT_JSCVT);
280 if (hwcap & HWCAP_FCMA)
281 setCPUFeature(FEAT_FCMA);
282 if (hwcap & HWCAP_SB)
283 setCPUFeature(FEAT_SB);
284 if (hwcap & HWCAP_SSBS)
285 {
286 setCPUFeature(FEAT_SSBS);
287 setCPUFeature(FEAT_SSBS2);
288 }
289 if (hwcap2 & HWCAP2_MTE)
290 {
291 setCPUFeature(FEAT_MEMTAG);
292 setCPUFeature(FEAT_MEMTAG2);
293 }
294 if (hwcap2 & HWCAP2_MTE3)
295 setCPUFeature(FEAT_MEMTAG3);
296 if (hwcap2 & HWCAP2_SVEAES)
297 setCPUFeature(FEAT_SVE_AES);
298 if (hwcap2 & HWCAP2_SVEPMULL)
299 setCPUFeature(FEAT_SVE_PMULL128);
300 if (hwcap2 & HWCAP2_SVEBITPERM)
301 setCPUFeature(FEAT_SVE_BITPERM);
302 if (hwcap2 & HWCAP2_SVESHA3)
303 setCPUFeature(FEAT_SVE_SHA3);
304 if (hwcap2 & HWCAP2_SVESM4)
305 setCPUFeature(FEAT_SVE_SM4);
306 if (hwcap2 & HWCAP2_DCPODP)
307 setCPUFeature(FEAT_DPB2);
308 if (hwcap & HWCAP_ATOMICS)
309 setCPUFeature(FEAT_LSE);
310 if (hwcap2 & HWCAP2_RNG)
311 setCPUFeature(FEAT_RNG);
312 if (hwcap2 & HWCAP2_I8MM)
313 setCPUFeature(FEAT_I8MM);
314 if (hwcap2 & HWCAP2_EBF16)
315 setCPUFeature(FEAT_EBF16);
316 if (hwcap2 & HWCAP2_SVE_EBF16)
317 setCPUFeature(FEAT_SVE_EBF16);
318 if (hwcap2 & HWCAP2_DGH)
319 setCPUFeature(FEAT_DGH);
320 if (hwcap2 & HWCAP2_FRINT)
321 setCPUFeature(FEAT_FRINTTS);
322 if (hwcap2 & HWCAP2_SVEI8MM)
323 setCPUFeature(FEAT_SVE_I8MM);
324 if (hwcap2 & HWCAP2_SVEF32MM)
325 setCPUFeature(FEAT_SVE_F32MM);
326 if (hwcap2 & HWCAP2_SVEF64MM)
327 setCPUFeature(FEAT_SVE_F64MM);
328 if (hwcap2 & HWCAP2_BTI)
329 setCPUFeature(FEAT_BTI);
330 if (hwcap2 & HWCAP2_RPRES)
331 setCPUFeature(FEAT_RPRES);
332 if (hwcap2 & HWCAP2_WFXT)
333 setCPUFeature(FEAT_WFXT);
334 if (hwcap2 & HWCAP2_SME)
335 setCPUFeature(FEAT_SME);
336 if (hwcap2 & HWCAP2_SME2)
337 setCPUFeature(FEAT_SME2);
338 if (hwcap2 & HWCAP2_SME_I16I64)
339 setCPUFeature(FEAT_SME_I64);
340 if (hwcap2 & HWCAP2_SME_F64F64)
341 setCPUFeature(FEAT_SME_F64);
342 if (hwcap & HWCAP_FP)
343 {
344 setCPUFeature(FEAT_FP);
345 /* FP and AdvSIMD fields have the same value. */
346 setCPUFeature(FEAT_SIMD);
347 }
348 if (hwcap & HWCAP_DCPOP)
349 setCPUFeature(FEAT_DPB);
350 if (hwcap & HWCAP_LRCPC)
351 setCPUFeature(FEAT_RCPC);
352 if (hwcap & HWCAP_ILRCPC)
353 setCPUFeature(FEAT_RCPC2);
354 if (hwcap2 & HWCAP2_LRCPC3)
355 setCPUFeature(FEAT_RCPC3);
356 if (hwcap2 & HWCAP2_BF16)
357 setCPUFeature(FEAT_BF16);
358 if (hwcap2 & HWCAP2_SVEBF16)
359 setCPUFeature(FEAT_SVE_BF16);
360 if (hwcap & HWCAP_SVE)
361 setCPUFeature(FEAT_SVE);
362 if (hwcap2 & HWCAP2_SVE2)
363 setCPUFeature(FEAT_SVE2);
364 if (hwcap & HWCAP_SHA3)
365 setCPUFeature(FEAT_SHA3);
366 setCPUFeature(FEAT_INIT);
367
368 __atomic_store_n (&__aarch64_cpu_features.features, feat, __ATOMIC_RELAXED);
369 }
370
371 void __init_cpu_features_resolver(unsigned long, const __ifunc_arg_t *);
372 void
373 __init_cpu_features_resolver(unsigned long hwcap, const __ifunc_arg_t *arg)
374 {
375 if (__atomic_load_n (&__aarch64_cpu_features.features, __ATOMIC_RELAXED))
376 return;
377 __init_cpu_features_constructor(hwcap, arg);
378 }
379
380 void __init_cpu_features(void);
381 void __attribute__ ((constructor))
382 __init_cpu_features(void)
383 {
384 unsigned long hwcap;
385 unsigned long hwcap2;
386
387 /* CPU features already initialized. */
388 if (__atomic_load_n (&__aarch64_cpu_features.features, __ATOMIC_RELAXED))
389 return;
390 hwcap = getauxval(AT_HWCAP);
391 hwcap2 = getauxval(AT_HWCAP2);
392 __ifunc_arg_t arg;
393 arg._size = sizeof(__ifunc_arg_t);
394 arg._hwcap = hwcap;
395 arg._hwcap2 = hwcap2;
396 __init_cpu_features_constructor(hwcap | _IFUNC_ARG_HWCAP, &arg);
397 #undef extractBits
398 #undef getCPUFeature
399 #undef setCPUFeature
400 }
401 #endif /* __has_include(<asm/hwcap.h>) */
402 #endif /* __has_include(<sys/auxv.h>) */