Debian Codegraph @

lib/aarch64/misc_helpers.S

raw
1 /*
2 * Copyright (c) 2013-2024, Arm Limited and Contributors. All rights reserved.
3 *
4 * SPDX-License-Identifier: BSD-3-Clause
5 */
6
7 #include <arch.h>
8 #include <asm_macros.S>
9 #include <assert_macros.S>
10 #include <common/bl_common.h>
11 #include <lib/xlat_tables/xlat_tables_defs.h>
12
13 .globl smc
14
15 .globl zero_normalmem
16 .globl zeromem
17 .globl memcpy16
18
19 .globl disable_mmu_el1
20 .globl disable_mmu_el3
21 .globl disable_mmu_icache_el1
22 .globl disable_mmu_icache_el3
23 .globl fixup_gdt_reloc
24 #if SUPPORT_VFP
25 .globl enable_vfp
26 #endif
27
28 func smc
29 smc #0
30 endfunc smc
31
32 /* -----------------------------------------------------------------------
33 * void zero_normalmem(void *mem, unsigned int length);
34 *
35 * Initialise a region in normal memory to 0. This functions complies with the
36 * AAPCS and can be called from C code.
37 *
38 * NOTE: MMU must be enabled when using this function as it can only operate on
39 * normal memory. It is intended to be mainly used from C code when MMU
40 * is usually enabled.
41 * -----------------------------------------------------------------------
42 */
43 .equ zero_normalmem, zeromem_dczva
44
45 /* -----------------------------------------------------------------------
46 * void zeromem(void *mem, unsigned int length);
47 *
48 * Initialise a region of device memory to 0. This functions complies with the
49 * AAPCS and can be called from C code.
50 *
51 * NOTE: When data caches and MMU are enabled, zero_normalmem can usually be
52 * used instead for faster zeroing.
53 *
54 * -----------------------------------------------------------------------
55 */
56 func zeromem
57 /* x2 is the address past the last zeroed address */
58 add x2, x0, x1
59 /*
60 * Uses the fallback path that does not use DC ZVA instruction and
61 * therefore does not need enabled MMU
62 */
63 b .Lzeromem_dczva_fallback_entry
64 endfunc zeromem
65
66 /* -----------------------------------------------------------------------
67 * void zeromem_dczva(void *mem, unsigned int length);
68 *
69 * Fill a region of normal memory of size "length" in bytes with null bytes.
70 * MMU must be enabled and the memory be of
71 * normal type. This is because this function internally uses the DC ZVA
72 * instruction, which generates an Alignment fault if used on any type of
73 * Device memory (see section D3.4.9 of the ARMv8 ARM, issue k). When the MMU
74 * is disabled, all memory behaves like Device-nGnRnE memory (see section
75 * D4.2.8), hence the requirement on the MMU being enabled.
76 * NOTE: The code assumes that the block size as defined in DCZID_EL0
77 * register is at least 16 bytes.
78 *
79 * -----------------------------------------------------------------------
80 */
81 func zeromem_dczva
82
83 /*
84 * The function consists of a series of loops that zero memory one byte
85 * at a time, 16 bytes at a time or using the DC ZVA instruction to
86 * zero aligned block of bytes, which is assumed to be more than 16.
87 * In the case where the DC ZVA instruction cannot be used or if the
88 * first 16 bytes loop would overflow, there is fallback path that does
89 * not use DC ZVA.
90 * Note: The fallback path is also used by the zeromem function that
91 * branches to it directly.
92 *
93 * +---------+ zeromem_dczva
94 * | entry |
95 * +----+----+
96 * |
97 * v
98 * +---------+
99 * | checks |>o-------+ (If any check fails, fallback)
100 * +----+----+ |
101 * | |---------------+
102 * v | Fallback path |
103 * +------+------+ |---------------+
104 * | 1 byte loop | |
105 * +------+------+ .Lzeromem_dczva_initial_1byte_aligned_end
106 * | |
107 * v |
108 * +-------+-------+ |
109 * | 16 bytes loop | |
110 * +-------+-------+ |
111 * | |
112 * v |
113 * +------+------+ .Lzeromem_dczva_blocksize_aligned
114 * | DC ZVA loop | |
115 * +------+------+ |
116 * +--------+ | |
117 * | | | |
118 * | v v |
119 * | +-------+-------+ .Lzeromem_dczva_final_16bytes_aligned
120 * | | 16 bytes loop | |
121 * | +-------+-------+ |
122 * | | |
123 * | v |
124 * | +------+------+ .Lzeromem_dczva_final_1byte_aligned
125 * | | 1 byte loop | |
126 * | +-------------+ |
127 * | | |
128 * | v |
129 * | +---+--+ |
130 * | | exit | |
131 * | +------+ |
132 * | |
133 * | +--------------+ +------------------+ zeromem
134 * | | +----------------| zeromem function |
135 * | | | +------------------+
136 * | v v
137 * | +-------------+ .Lzeromem_dczva_fallback_entry
138 * | | 1 byte loop |
139 * | +------+------+
140 * | |
141 * +-----------+
142 */
143
144 /*
145 * Readable names for registers
146 *
147 * Registers x0, x1 and x2 are also set by zeromem which
148 * branches into the fallback path directly, so cursor, length and
149 * stop_address should not be retargeted to other registers.
150 */
151 cursor .req x0 /* Start address and then current address */
152 length .req x1 /* Length in bytes of the region to zero out */
153 /* Reusing x1 as length is never used after block_mask is set */
154 block_mask .req x1 /* Bitmask of the block size read in DCZID_EL0 */
155 stop_address .req x2 /* Address past the last zeroed byte */
156 block_size .req x3 /* Size of a block in bytes as read in DCZID_EL0 */
157 tmp1 .req x4
158 tmp2 .req x5
159
160 #if ENABLE_ASSERTIONS
161 /*
162 * Check for M bit (MMU enabled) of the current SCTLR_EL(1|3)
163 * register value and panic if the MMU is disabled.
164 */
165 #if defined(IMAGE_BL1) || defined(IMAGE_BL31) || (defined(IMAGE_BL2) && \
166 BL2_RUNS_AT_EL3)
167 mrs tmp1, sctlr_el3
168 #else
169 mrs tmp1, sctlr_el1
170 #endif
171
172 tst tmp1, #SCTLR_M_BIT
173 ASM_ASSERT(ne)
174 #endif /* ENABLE_ASSERTIONS */
175
176 /* stop_address is the address past the last to zero */
177 add stop_address, cursor, length
178
179 /*
180 * Get block_size = (log2(<block size>) >> 2) (see encoding of
181 * dczid_el0 reg)
182 */
183 mrs block_size, dczid_el0
184
185 /*
186 * Select the 4 lowest bits and convert the extracted log2(<block size
187 * in words>) to <block size in bytes>
188 */
189 ubfx block_size, block_size, #0, #4
190 mov tmp2, #(1 << 2)
191 lsl block_size, tmp2, block_size
192
193 #if ENABLE_ASSERTIONS
194 /*
195 * Assumes block size is at least 16 bytes to avoid manual realignment
196 * of the cursor at the end of the DCZVA loop.
197 */
198 cmp block_size, #16
199 ASM_ASSERT(hs)
200 #endif
201 /*
202 * Not worth doing all the setup for a region less than a block and
203 * protects against zeroing a whole block when the area to zero is
204 * smaller than that. Also, as it is assumed that the block size is at
205 * least 16 bytes, this also protects the initial aligning loops from
206 * trying to zero 16 bytes when length is less than 16.
207 */
208 cmp length, block_size
209 b.lo .Lzeromem_dczva_fallback_entry
210
211 /*
212 * Calculate the bitmask of the block alignment. It will never
213 * underflow as the block size is between 4 bytes and 2kB.
214 * block_mask = block_size - 1
215 */
216 sub block_mask, block_size, #1
217
218 /*
219 * length alias should not be used after this point unless it is
220 * defined as a register other than block_mask's.
221 */
222 .unreq length
223
224 /*
225 * If the start address is already aligned to zero block size, go
226 * straight to the cache zeroing loop. This is safe because at this
227 * point, the length cannot be smaller than a block size.
228 */
229 tst cursor, block_mask
230 b.eq .Lzeromem_dczva_blocksize_aligned
231
232 /*
233 * Calculate the first block-size-aligned address. It is assumed that
234 * the zero block size is at least 16 bytes. This address is the last
235 * address of this initial loop.
236 */
237 orr tmp1, cursor, block_mask
238 add tmp1, tmp1, #1
239
240 /*
241 * If the addition overflows, skip the cache zeroing loops. This is
242 * quite unlikely however.
243 */
244 cbz tmp1, .Lzeromem_dczva_fallback_entry
245
246 /*
247 * If the first block-size-aligned address is past the last address,
248 * fallback to the simpler code.
249 */
250 cmp tmp1, stop_address
251 b.hi .Lzeromem_dczva_fallback_entry
252
253 /*
254 * If the start address is already aligned to 16 bytes, skip this loop.
255 * It is safe to do this because tmp1 (the stop address of the initial
256 * 16 bytes loop) will never be greater than the final stop address.
257 */
258 tst cursor, #0xf
259 b.eq .Lzeromem_dczva_initial_1byte_aligned_end
260
261 /* Calculate the next address aligned to 16 bytes */
262 orr tmp2, cursor, #0xf
263 add tmp2, tmp2, #1
264 /* If it overflows, fallback to the simple path (unlikely) */
265 cbz tmp2, .Lzeromem_dczva_fallback_entry
266 /*
267 * Next aligned address cannot be after the stop address because the
268 * length cannot be smaller than 16 at this point.
269 */
270
271 /* First loop: zero byte per byte */
272 1:
273 strb wzr, [cursor], #1
274 cmp cursor, tmp2
275 b.ne 1b
276 .Lzeromem_dczva_initial_1byte_aligned_end:
277
278 /*
279 * Second loop: we need to zero 16 bytes at a time from cursor to tmp1
280 * before being able to use the code that deals with block-size-aligned
281 * addresses.
282 */
283 cmp cursor, tmp1
284 b.hs 2f
285 1:
286 stp xzr, xzr, [cursor], #16
287 cmp cursor, tmp1
288 b.lo 1b
289 2:
290
291 /*
292 * Third loop: zero a block at a time using DC ZVA cache block zeroing
293 * instruction.
294 */
295 .Lzeromem_dczva_blocksize_aligned:
296 /*
297 * Calculate the last block-size-aligned address. If the result equals
298 * to the start address, the loop will exit immediately.
299 */
300 bic tmp1, stop_address, block_mask
301
302 cmp cursor, tmp1
303 b.hs 2f
304 1:
305 /* Zero the block containing the cursor */
306 dc zva, cursor
307 /* Increment the cursor by the size of a block */
308 add cursor, cursor, block_size
309 cmp cursor, tmp1
310 b.lo 1b
311 2:
312
313 /*
314 * Fourth loop: zero 16 bytes at a time and then byte per byte the
315 * remaining area
316 */
317 .Lzeromem_dczva_final_16bytes_aligned:
318 /*
319 * Calculate the last 16 bytes aligned address. It is assumed that the
320 * block size will never be smaller than 16 bytes so that the current
321 * cursor is aligned to at least 16 bytes boundary.
322 */
323 bic tmp1, stop_address, #15
324
325 cmp cursor, tmp1
326 b.hs 2f
327 1:
328 stp xzr, xzr, [cursor], #16
329 cmp cursor, tmp1
330 b.lo 1b
331 2:
332
333 /* Fifth and final loop: zero byte per byte */
334 .Lzeromem_dczva_final_1byte_aligned:
335 cmp cursor, stop_address
336 b.eq 2f
337 1:
338 strb wzr, [cursor], #1
339 cmp cursor, stop_address
340 b.ne 1b
341 2:
342 ret
343
344 /* Fallback for unaligned start addresses */
345 .Lzeromem_dczva_fallback_entry:
346 /*
347 * If the start address is already aligned to 16 bytes, skip this loop.
348 */
349 tst cursor, #0xf
350 b.eq .Lzeromem_dczva_final_16bytes_aligned
351
352 /* Calculate the next address aligned to 16 bytes */
353 orr tmp1, cursor, #15
354 add tmp1, tmp1, #1
355 /* If it overflows, fallback to byte per byte zeroing */
356 cbz tmp1, .Lzeromem_dczva_final_1byte_aligned
357 /* If the next aligned address is after the stop address, fall back */
358 cmp tmp1, stop_address
359 b.hs .Lzeromem_dczva_final_1byte_aligned
360
361 /* Fallback entry loop: zero byte per byte */
362 1:
363 strb wzr, [cursor], #1
364 cmp cursor, tmp1
365 b.ne 1b
366
367 b .Lzeromem_dczva_final_16bytes_aligned
368
369 .unreq cursor
370 /*
371 * length is already unreq'ed to reuse the register for another
372 * variable.
373 */
374 .unreq stop_address
375 .unreq block_size
376 .unreq block_mask
377 .unreq tmp1
378 .unreq tmp2
379 endfunc zeromem_dczva
380
381 /* --------------------------------------------------------------------------
382 * void memcpy16(void *dest, const void *src, unsigned int length)
383 *
384 * Copy length bytes from memory area src to memory area dest.
385 * The memory areas should not overlap.
386 * Destination and source addresses must be 16-byte aligned.
387 * --------------------------------------------------------------------------
388 */
389 func memcpy16
390 #if ENABLE_ASSERTIONS
391 orr x3, x0, x1
392 tst x3, #0xf
393 ASM_ASSERT(eq)
394 #endif
395 /* copy 16 bytes at a time */
396 m_loop16:
397 cmp x2, #16
398 b.lo m_loop1
399 ldp x3, x4, [x1], #16
400 stp x3, x4, [x0], #16
401 sub x2, x2, #16
402 b m_loop16
403 /* copy byte per byte */
404 m_loop1:
405 cbz x2, m_end
406 ldrb w3, [x1], #1
407 strb w3, [x0], #1
408 subs x2, x2, #1
409 b.ne m_loop1
410 m_end:
411 ret
412 endfunc memcpy16
413
414 /* ---------------------------------------------------------------------------
415 * Disable the MMU at EL3
416 * ---------------------------------------------------------------------------
417 */
418
419 func disable_mmu_el3
420 mov x1, #(SCTLR_M_BIT | SCTLR_C_BIT)
421 do_disable_mmu_el3:
422 mrs x0, sctlr_el3
423 bic x0, x0, x1
424 msr sctlr_el3, x0
425 isb /* ensure MMU is off */
426 dsb sy
427 ret
428 endfunc disable_mmu_el3
429
430
431 func disable_mmu_icache_el3
432 mov x1, #(SCTLR_M_BIT | SCTLR_C_BIT | SCTLR_I_BIT)
433 b do_disable_mmu_el3
434 endfunc disable_mmu_icache_el3
435
436 /* ---------------------------------------------------------------------------
437 * Disable the MMU at EL1
438 * ---------------------------------------------------------------------------
439 */
440
441 func disable_mmu_el1
442 mov x1, #(SCTLR_M_BIT | SCTLR_C_BIT)
443 do_disable_mmu_el1:
444 mrs x0, sctlr_el1
445 bic x0, x0, x1
446 msr sctlr_el1, x0
447 isb /* ensure MMU is off */
448 dsb sy
449 ret
450 endfunc disable_mmu_el1
451
452
453 func disable_mmu_icache_el1
454 mov x1, #(SCTLR_M_BIT | SCTLR_C_BIT | SCTLR_I_BIT)
455 b do_disable_mmu_el1
456 endfunc disable_mmu_icache_el1
457
458 /* ---------------------------------------------------------------------------
459 * Enable the use of VFP at EL3
460 * ---------------------------------------------------------------------------
461 */
462 #if SUPPORT_VFP
463 func enable_vfp
464 mrs x0, cpacr_el1
465 orr x0, x0, #CPACR_VFP_BITS
466 msr cpacr_el1, x0
467 mrs x0, cptr_el3
468 mov x1, #AARCH64_CPTR_TFP
469 bic x0, x0, x1
470 msr cptr_el3, x0
471 isb
472 ret
473 endfunc enable_vfp
474 #endif
475
476 /* ---------------------------------------------------------------------------
477 * Helper to fixup Global Descriptor table (GDT) and dynamic relocations
478 * (.rela.dyn) at runtime.
479 *
480 * This function is meant to be used when the firmware is compiled with -fpie
481 * and linked with -pie options. We rely on the linker script exporting
482 * appropriate markers for start and end of the section. For GOT, we
483 * expect __GOT_START__ and __GOT_END__. Similarly for .rela.dyn, we expect
484 * __RELA_START__ and __RELA_END__.
485 *
486 * The function takes the limits of the memory to apply fixups to as
487 * arguments (which is usually the limits of the relocable BL image).
488 * x0 - the start of the fixup region
489 * x1 - the limit of the fixup region
490 * These addresses have to be 4KB page aligned.
491 * ---------------------------------------------------------------------------
492 */
493
494 /* Relocation codes */
495 #define R_AARCH64_NONE 0
496 #define R_AARCH64_RELATIVE 1027
497
498 func fixup_gdt_reloc
499 mov x6, x0
500 mov x7, x1
501
502 #if ENABLE_ASSERTIONS
503 /* Test if the limits are 4KB aligned */
504 orr x0, x0, x1
505 tst x0, #(PAGE_SIZE_MASK)
506 ASM_ASSERT(eq)
507 #endif
508 /*
509 * Calculate the offset based on return address in x30.
510 * Assume that this function is called within a page at the start of
511 * fixup region.
512 */
513 and x2, x30, #~(PAGE_SIZE_MASK)
514 subs x0, x2, x6 /* Diff(S) = Current Address - Compiled Address */
515 b.eq 3f /* Diff(S) = 0. No relocation needed */
516
517 adrp x1, __GOT_START__
518 add x1, x1, :lo12:__GOT_START__
519 adrp x2, __GOT_END__
520 add x2, x2, :lo12:__GOT_END__
521
522 /*
523 * GOT is an array of 64_bit addresses which must be fixed up as
524 * new_addr = old_addr + Diff(S).
525 * The new_addr is the address currently the binary is executing from
526 * and old_addr is the address at compile time.
527 */
528 1: ldr x3, [x1]
529
530 /* Skip adding offset if address is < lower limit */
531 cmp x3, x6
532 b.lo 2f
533
534 /* Skip adding offset if address is > upper limit */
535 cmp x3, x7
536 b.hi 2f
537 add x3, x3, x0
538 str x3, [x1]
539
540 2: add x1, x1, #8
541 cmp x1, x2
542 b.lo 1b
543
544 /* Starting dynamic relocations. Use adrp/adr to get RELA_START and END */
545 3: adrp x1, __RELA_START__
546 add x1, x1, :lo12:__RELA_START__
547 adrp x2, __RELA_END__
548 add x2, x2, :lo12:__RELA_END__
549
550 /*
551 * According to ELF-64 specification, the RELA data structure is as
552 * follows:
553 * typedef struct {
554 * Elf64_Addr r_offset;
555 * Elf64_Xword r_info;
556 * Elf64_Sxword r_addend;
557 * } Elf64_Rela;
558 *
559 * r_offset is address of reference
560 * r_info is symbol index and type of relocation (in this case
561 * code 1027 which corresponds to R_AARCH64_RELATIVE).
562 * r_addend is constant part of expression.
563 *
564 * Size of Elf64_Rela structure is 24 bytes.
565 */
566
567 /* Skip R_AARCH64_NONE entry with code 0 */
568 1: ldr x3, [x1, #8]
569 cbz x3, 2f
570
571 #if ENABLE_ASSERTIONS
572 /* Assert that the relocation type is R_AARCH64_RELATIVE */
573 cmp x3, #R_AARCH64_RELATIVE
574 ASM_ASSERT(eq)
575 #endif
576 ldr x3, [x1] /* r_offset */
577 add x3, x0, x3
578 ldr x4, [x1, #16] /* r_addend */
579
580 /* Skip adding offset if r_addend is < lower limit */
581 cmp x4, x6
582 b.lo 2f
583
584 /* Skip adding offset if r_addend entry is > upper limit */
585 cmp x4, x7
586 b.hi 2f
587
588 add x4, x0, x4 /* Diff(S) + r_addend */
589 str x4, [x3]
590
591 2: add x1, x1, #24
592 cmp x1, x2
593 b.lo 1b
594 ret
595 endfunc fixup_gdt_reloc