Debian Codegraph @

libdrgn/arch_aarch64.c

raw
1 // Copyright (c) Meta Platforms, Inc. and affiliates.
2 // SPDX-License-Identifier: LGPL-2.1-or-later
3
4 #include <assert.h>
5 #include <byteswap.h>
6 #include <elf.h>
7 #include <stdlib.h>
8 #include <string.h>
9
10 #include "array.h"
11 #include "error.h"
12 #include "linux_kernel.h"
13 #include "log.h"
14 #include "platform.h" // IWYU pragma: associated
15 #include "program.h"
16 #include "register_state.h"
17 #include "util.h"
18
19 /*
20 * The ABI specification can be found at:
21 * https://developer.arm.com/architectures/system-architectures/software-standards/abi
22 * https://github.com/ARM-software/abi-aa/releases
23 */
24
25 #include "arch_aarch64_defs.inc"
26
27 static_assert(DRGN_AARCH64_RA_SIGN_STATE_REGNO ==
28 DRGN_REGISTER_NUMBER(ra_sign_state),
29 "RA_SIGN_STATE register number is out of sync");
30
31 static const struct drgn_cfi_row default_dwarf_cfi_row_aarch64 = DRGN_CFI_ROW(
32 [DRGN_REGISTER_NUMBER(ra_sign_state)] = {
33 DRGN_CFI_RULE_CONSTANT, .constant = 0
34 },
35 // The psABI defines the CFA as the value of the stack pointer in the
36 // calling frame.
37 [DRGN_REGISTER_NUMBER(sp)] = { DRGN_CFI_RULE_CFA_PLUS_OFFSET },
38 // The psABI defines that callee-saved registers default to
39 // DW_CFA_same_value.
40 DRGN_CFI_SAME_VALUE_INIT(DRGN_REGISTER_NUMBER(x19)),
41 DRGN_CFI_SAME_VALUE_INIT(DRGN_REGISTER_NUMBER(x20)),
42 DRGN_CFI_SAME_VALUE_INIT(DRGN_REGISTER_NUMBER(x21)),
43 DRGN_CFI_SAME_VALUE_INIT(DRGN_REGISTER_NUMBER(x22)),
44 DRGN_CFI_SAME_VALUE_INIT(DRGN_REGISTER_NUMBER(x23)),
45 DRGN_CFI_SAME_VALUE_INIT(DRGN_REGISTER_NUMBER(x24)),
46 DRGN_CFI_SAME_VALUE_INIT(DRGN_REGISTER_NUMBER(x25)),
47 DRGN_CFI_SAME_VALUE_INIT(DRGN_REGISTER_NUMBER(x26)),
48 DRGN_CFI_SAME_VALUE_INIT(DRGN_REGISTER_NUMBER(x27)),
49 DRGN_CFI_SAME_VALUE_INIT(DRGN_REGISTER_NUMBER(x28)),
50 DRGN_CFI_SAME_VALUE_INIT(DRGN_REGISTER_NUMBER(x29)),
51 DRGN_CFI_SAME_VALUE_INIT(DRGN_REGISTER_NUMBER(x30)),
52 );
53
54 // Mask out the pointer authentication code from x30/lr.
55 static void demangle_cfi_registers_aarch64(struct drgn_program *prog,
56 struct drgn_register_state *regs)
57 {
58 struct optional_uint64 ra_sign_state =
59 drgn_register_state_get_u64(prog, regs, ra_sign_state);
60 if (!ra_sign_state.has_value || !(ra_sign_state.value & 1))
61 return;
62 struct optional_uint64 ra =
63 drgn_register_state_get_u64(prog, regs, x30);
64 if (!ra.has_value)
65 return;
66 if (ra.value & (UINT64_C(1) << 55))
67 ra.value |= prog->aarch64_insn_pac_mask;
68 else
69 ra.value &= ~prog->aarch64_insn_pac_mask;
70 drgn_register_state_set_from_u64(prog, regs, x30, ra.value);
71 }
72
73 // elf_gregset_t (in PRSTATUS) and struct user_pt_regs have the same layout.
74 // This layout is a prefix of the in-kernel struct pt_regs (but we don't care
75 // about any of the extra fields).
76 static struct drgn_error *
77 get_initial_registers_from_struct_aarch64(struct drgn_program *prog,
78 const void *buf, size_t size,
79 struct drgn_register_state **ret)
80 {
81 if (size < 272) {
82 return drgn_error_create(DRGN_ERROR_INVALID_ARGUMENT,
83 "registers are truncated");
84 }
85
86 struct drgn_register_state *regs =
87 drgn_register_state_create(pstate, true);
88 if (!regs)
89 return &drgn_enomem;
90
91 drgn_register_state_set_from_buffer(regs, pc, (uint64_t *)buf + 32);
92 drgn_register_state_set_from_buffer(regs, sp, (uint64_t *)buf + 31);
93 drgn_register_state_set_range_from_buffer(regs, x19, x30,
94 (uint64_t *)buf + 19);
95 drgn_register_state_set_range_from_buffer(regs, x0, x18, buf);
96 drgn_register_state_set_from_buffer(regs, pstate, (uint64_t *)buf + 33);
97 drgn_register_state_set_pc_from_register(prog, regs, pc);
98
99 *ret = regs;
100 return NULL;
101 }
102
103
104 static struct drgn_error *
105 get_stackframe_offset(struct drgn_program *prog, uint64_t *ret)
106 {
107 if (!prog->aarch64_stackframe_offset_cached) {
108 struct drgn_error *err;
109 struct drgn_qualified_type pt_regs_type;
110
111 err = drgn_program_find_type(prog, "struct pt_regs", NULL, &pt_regs_type);
112 if (err)
113 return err;
114
115 struct drgn_type_member *stackframe_member;
116 uint64_t bit_offset;
117 err = drgn_type_find_member(pt_regs_type.type, "stackframe",
118 &stackframe_member, &bit_offset);
119 if (err)
120 return err;
121
122 prog->aarch64_stackframe_offset_cached = bit_offset / 8;
123 }
124
125 *ret = prog->aarch64_stackframe_offset_cached;
126 return NULL;
127 }
128
129
130 static struct drgn_error *
131 fallback_unwind_try_pt_regs(struct drgn_program *prog, uint64_t fp,
132 uint64_t frame[2],
133 struct drgn_register_state *regs,
134 struct drgn_register_state **regs_ret,
135 bool *found_ret)
136 {
137 struct drgn_error *err;
138 uint64_t stackframe_offset;
139 const char *method;
140
141 err = get_stackframe_offset(prog, &stackframe_offset);
142 if (err)
143 return err;
144
145 uint64_t pt_regs_addr = fp - stackframe_offset;
146
147 if (!frame[0] && !frame[1]) {
148 // Since c2c6b27b5aa14 ("arm64: stacktrace: unwind exception
149 // boundaries") - v6.13, a frame record with fp and lr both zero
150 // indicates that we can read the next word on the stack for
151 // "metadata" which tells us whether a pt_regs is pushed to the
152 // stack.
153 uint64_t frame_meta_type;
154 err = drgn_program_read_u64(prog, fp + 16, false, &frame_meta_type);
155 if (err)
156 return err;
157
158 // FRAME_META_TYPE_PT_REGS == 2 indicates a pt_regs
159 if (frame_meta_type != 2) {
160 *found_ret = false;
161 return NULL;
162 }
163 method = "frame metadata";
164 } else {
165 // Before this, it's still possible that this stack frame is
166 // embedded within a pt_regs. However, there's no reliable way
167 // to detect it. We can guess what that pt_regs pointer would
168 // be, and check whether any CPU has a __irq_regs value matching
169 // it. If so, we're guaranteed to be in the pt_regs for an IRQ.
170 // If not, it's still possible that this is a pt_regs, but the
171 // __irq_regs accounting has not yet happened (or has been
172 // cleaned up already). So this heuristic shouldn't have any
173 // false positives, but could have some false negatives.
174 bool is_irq_regs;
175 err = drgn_program_is_irq_regs(prog, pt_regs_addr, &is_irq_regs);
176 if (err)
177 return err;
178
179 if (!is_irq_regs) {
180 *found_ret = false;
181 return NULL;
182 }
183 method = "__irq_regs";
184 }
185
186 // Now that we're reasonably confident that there's a pt_regs here, read
187 // it and use it.
188 char pt_regs[272];
189 err = drgn_program_read_memory(prog, &pt_regs, pt_regs_addr,
190 sizeof(pt_regs), false);
191 if (err)
192 return err;
193 err = get_initial_registers_from_struct_aarch64(prog, &pt_regs,
194 sizeof(pt_regs), regs_ret);
195 if (err)
196 return err;
197
198 drgn_log_debug(prog, "aarch64 fallback: discovered pt_regs 0x%"PRIx64" via %s",
199 pt_regs_addr, method);
200 *found_ret = true;
201 return NULL;
202 }
203
204 // Unwind using the frame pointer. Note that leaf functions may not allocate a
205 // stack frame, so this may skip the caller of a leaf function. I don't know of
206 // a good way around that.
207 static struct drgn_error *
208 fallback_unwind_aarch64(struct drgn_program *prog,
209 struct drgn_register_state *regs,
210 struct drgn_register_state **ret)
211 {
212
213 struct drgn_error *err;
214
215 struct optional_uint64 fp =
216 drgn_register_state_get_u64(prog, regs, x29);
217 if (!fp.has_value)
218 return &drgn_stop;
219
220 // The frame pointer points to a frame record of two 64-bit values. The
221 // first (lowest addressed) is the address of the caller's frame record.
222 // The second (highest addressed) is the saved lr.
223 uint64_t frame[2];
224 err = drgn_program_read_memory(prog, frame, fp.value, sizeof(frame),
225 false);
226 if (err) {
227 if (err->code == DRGN_ERROR_FAULT) {
228 drgn_error_destroy(err);
229 err = &drgn_stop;
230 }
231 return err;
232 }
233
234 if (prog->flags & DRGN_PROGRAM_IS_LINUX_KERNEL) {
235 bool found;
236 err = fallback_unwind_try_pt_regs(prog, fp.value, frame, regs, ret, &found);
237 if (err) {
238 // If we encounter any error trying the pt_regs
239 // approach, log it at the debug level and continue.
240 // It's likely an expected error, but could be useful
241 // debugging information.
242 drgn_error_log_debug(prog, err, "encountered error in linux fallback unwinder:");
243 drgn_error_destroy(err);
244 err = NULL;
245 } else if (found) {
246 // We have set the register state based on a pt_regs
247 // frame we found, return!
248 return NULL;
249 }
250 }
251
252 uint64_t unwound_fp =
253 drgn_platform_bswap(&prog->platform) ?
254 bswap_64(frame[0]) : frame[0];
255 if (unwound_fp <= fp.value) {
256 // The unwound stack pointer is either 0, indicating the first
257 // stack frame, or invalid.
258 return &drgn_stop;
259 }
260
261 struct drgn_register_state *unwound =
262 drgn_register_state_create(x30, false);
263 if (!unwound)
264 return &drgn_enomem;
265 drgn_register_state_set_from_buffer(unwound, x30, &frame[1]);
266 drgn_register_state_set_from_buffer(unwound, x29, &frame[0]);
267 // We don't know whether the return address is signed, so just assume
268 // that it is if pointer authentication is enabled. If we're wrong, the
269 // worst that can happen is that we'll "correct" incorrect sign
270 // extension bits or clear an address tag.
271 if (prog->aarch64_insn_pac_mask) {
272 drgn_register_state_set_from_u64(prog, unwound, ra_sign_state,
273 1);
274 demangle_cfi_registers_aarch64(prog, unwound);
275 }
276 drgn_register_state_set_pc_from_register(prog, unwound, x30);
277 // The location of the frame record within the stack frame is not
278 // specified, so we can't determine the stack pointer.
279 *ret = unwound;
280 return NULL;
281 }
282
283 // Unwind a single bl or blr instruction.
284 static struct drgn_error *
285 bad_call_unwind_aarch64(struct drgn_program *prog,
286 struct drgn_register_state *regs,
287 struct drgn_register_state **ret)
288 {
289 struct optional_uint64 lr =
290 drgn_register_state_get_u64(prog, regs, x30);
291 if (!lr.has_value)
292 return &drgn_stop;
293
294 struct drgn_register_state *tmp = drgn_register_state_dup(regs);
295 if (!tmp)
296 return &drgn_enomem;
297
298 // lr contains the the old pc + 4.
299 drgn_register_state_set_pc(prog, tmp, lr.value - 4);
300 // We don't know the old lr.
301 drgn_register_state_unset_has_register(tmp, DRGN_REGISTER_NUMBER(x30));
302 // The interrupted pc is no longer applicable.
303 drgn_register_state_unset_has_register(tmp, DRGN_REGISTER_NUMBER(pc));
304 *ret = tmp;
305 return NULL;
306 }
307
308 static struct drgn_error *
309 pt_regs_get_initial_registers_aarch64(const struct drgn_object *obj,
310 struct drgn_register_state **ret)
311 {
312 return get_initial_registers_from_struct_aarch64(drgn_object_program(obj),
313 drgn_object_buffer(obj),
314 drgn_object_size(obj),
315 ret);
316 }
317
318 static struct drgn_error *
319 prstatus_get_initial_registers_aarch64(struct drgn_program *prog,
320 const void *prstatus, size_t size,
321 struct drgn_register_state **ret)
322 {
323 // offsetof(struct elf_prstatus, pr_reg)
324 static const size_t pr_reg_offset = 112;
325 if (size < pr_reg_offset) {
326 return drgn_error_create(DRGN_ERROR_INVALID_ARGUMENT,
327 "NT_PRSTATUS is truncated");
328 }
329 return get_initial_registers_from_struct_aarch64(prog,
330 (char *)prstatus + pr_reg_offset,
331 size - pr_reg_offset,
332 ret);
333 }
334
335 // The Linux kernel saves the callee-saved registers in
336 // struct task_struct.thread.cpu_context (with type struct cpu_context). See
337 // cpu_switch_to() in arch/arm64/kernel/entry.S (as of Linux v5.19).
338 static struct drgn_error *
339 linux_kernel_get_initial_registers_aarch64(const struct drgn_object *task_obj,
340 struct drgn_register_state **ret)
341 {
342 struct drgn_error *err;
343 struct drgn_program *prog = drgn_object_program(task_obj);
344
345 DRGN_OBJECT(cpu_context_obj, prog);
346
347 err = drgn_object_member_dereference(&cpu_context_obj, task_obj,
348 "thread");
349 if (err)
350 return err;
351 err = drgn_object_member(&cpu_context_obj, &cpu_context_obj,
352 "cpu_context");
353 if (err)
354 return err;
355 if (cpu_context_obj.encoding != DRGN_OBJECT_ENCODING_BUFFER ||
356 drgn_object_size(&cpu_context_obj) < 104) {
357 return drgn_error_create(DRGN_ERROR_INVALID_ARGUMENT,
358 "cpu_context is truncated");
359 }
360 err = drgn_object_read(&cpu_context_obj, &cpu_context_obj);
361 if (err)
362 return err;
363
364 const void *buf = drgn_object_buffer(&cpu_context_obj);
365 struct drgn_register_state *regs =
366 drgn_register_state_create(x30, false);
367 if (!regs)
368 return &drgn_enomem;
369
370 drgn_register_state_set_from_buffer(regs, x30, (uint64_t *)buf + 12);
371 drgn_register_state_set_from_buffer(regs, sp, (uint64_t *)buf + 11);
372 drgn_register_state_set_range_from_buffer(regs, x19, x29, buf);
373 drgn_register_state_set_pc_from_register(prog, regs, x30);
374 *ret = regs;
375 return NULL;
376 }
377
378 static struct drgn_error *
379 apply_elf_reloc_aarch64(const struct drgn_relocating_section *relocating,
380 uint64_t r_offset, uint32_t r_type, const int64_t *r_addend,
381 uint64_t sym_value)
382 {
383 switch (r_type) {
384 case R_AARCH64_NONE:
385 return NULL;
386 case R_AARCH64_ABS64:
387 return drgn_reloc_add64(relocating, r_offset, r_addend,
388 sym_value);
389 case R_AARCH64_ABS32:
390 return drgn_reloc_add32(relocating, r_offset, r_addend,
391 sym_value);
392 case R_AARCH64_ABS16:
393 return drgn_reloc_add16(relocating, r_offset, r_addend,
394 sym_value);
395 case R_AARCH64_PREL64:
396 return drgn_reloc_add64(relocating, r_offset, r_addend,
397 sym_value
398 - (relocating->addr + r_offset));
399 case R_AARCH64_PREL32:
400 return drgn_reloc_add32(relocating, r_offset, r_addend,
401 sym_value
402 - (relocating->addr + r_offset));
403 case R_AARCH64_PREL16:
404 return drgn_reloc_add16(relocating, r_offset, r_addend,
405 sym_value
406 - (relocating->addr + r_offset));
407 default:
408 return DRGN_UNKNOWN_RELOCATION_TYPE(r_type);
409 }
410 }
411
412 struct pgtable_iterator_aarch64 {
413 struct pgtable_iterator it;
414 uint64_t va_bits;
415 // Inclusive range of valid virtual addresses.
416 uint64_t va_range_min, va_range_max;
417 int levels;
418 uint16_t entries_per_level;
419 uint16_t last_level_num_entries;
420 uint64_t cached_virt_addr;
421 uint64_t table[5];
422 uint64_t pa_low_mask;
423 uint64_t pa_high_mask;
424 int pa_high_shift;
425 };
426
427 static struct drgn_error *
428 linux_kernel_pgtable_iterator_create_aarch64(struct drgn_program *prog,
429 struct pgtable_iterator **ret)
430 {
431 const uint64_t page_shift = prog->vmcoreinfo.page_shift;
432 if (page_shift != 12 && page_shift != 14 && page_shift != 16) {
433 return drgn_error_create(DRGN_ERROR_OTHER,
434 "unknown page size for virtual address translation");
435 }
436 const uint64_t pgtable_shift = page_shift - 3;
437
438 // VA_BITS in VMCOREINFO, added in Linux kernel commit 20a166243328
439 // ("arm64: kdump: add VMCOREINFO's for user-space tools") (in v4.12),
440 // is the configured virtual address size. Since Linux kernel commit
441 // b6d00d47e81a ("arm64: mm: Introduce 52-bit Kernel VAs") (in v5.4), it
442 // may be greater than the actual size used at runtime if the kernel is
443 // configured with 52-bit virtual addresses but the hardware does not
444 // support it. 64 - TCR_EL1_T1SZ, added in Linux kernel commit
445 // bbdbc11804ff ("arm64/crash_core: Export TCR_EL1.T1SZ in vmcoreinfo")
446 // (in v5.9), is the size used at runtime.
447 //
448 // With 64k pages, if 52-bit virtual addresses are not supported
449 // (Armv8.2 FEAT_LVA), then the kernel falls back to 48-bit virtual
450 // addresses. Note that the page table has 3 levels with either size. In
451 // this case, swapper_pg_dir is still set up for 52-bit virtual
452 // addresses (an offset is applied when loading swapper_pg_dir into
453 // TTBR1; see Linux kernel commit c812026c54cf ("arm64: mm: Logic to
454 // make offset_ttbr1 conditional") (in v5.4)). So, with 64k pages, we
455 // need to treat kernel addresses as if they have size VA_BITS for the
456 // purposes of translating using swapper_pg_dir. We can also treat user
457 // addresses as if they have size VA_BITS. In the 52- to 48-bit fallback
458 // case, the additional user virtual address bits [51:48] are zero, so
459 // the page table is accessed identically.
460 //
461 // With 4k or 16k pages, if 52-bit virtual addresses are not supported
462 // (Armv8.7 FEAT_LPA2), then the kernel falls back to 48-bit virtual
463 // addresses with 4k pages or 47-bit virtual addresses with 16k pages;
464 // see Linux kernel commit 0dd4f60a2c76 ("arm64: mm: Add support for
465 // folding PUDs at runtime") (in v6.9). This decreases the number of
466 // page table levels (from 5 to 4 for 4k pages and from 4 to 3 for 16k
467 // pages). In this case, we need to use the actual size for both kernel
468 // and user addresses, 64 - TCR_EL1_T1SZ.
469 uint64_t va_bits;
470 if (page_shift == 16 || !prog->vmcoreinfo.tcr_el1_t1sz)
471 va_bits = prog->vmcoreinfo.va_bits;
472 else
473 va_bits = 64 - prog->vmcoreinfo.tcr_el1_t1sz;
474 if (va_bits <= page_shift || va_bits > 52) {
475 return drgn_error_create(DRGN_ERROR_OTHER,
476 "VMCOREINFO does not contain valid TCR_EL1_T1SZ or VA_BITS");
477 }
478
479 struct pgtable_iterator_aarch64 *it = malloc(sizeof(*it));
480 if (!it)
481 return &drgn_enomem;
482
483 it->va_bits = va_bits;
484 it->levels = ((va_bits - page_shift + pgtable_shift - 1) /
485 pgtable_shift);
486 assert(it->levels <= array_size(it->table));
487 it->entries_per_level = 1 << pgtable_shift;
488 it->last_level_num_entries =
489 1 << ((va_bits - page_shift - 1) % pgtable_shift + 1);
490
491 // Descriptor bits [47:PAGE_SHIFT] contain physical address bits
492 // [47:PAGE_SHIFT].
493 //
494 // For 52-bit physical addresses, physical address bits [51:48] come
495 // from elsewhere.
496 //
497 // With 64k pages and 52-bit physical addresses (Armv8.2 FEAT_LPA),
498 // descriptor bits [15:12] contain physical address bits [51:48]. With
499 // 64k pages and 48-bit physical addresses, descriptor bits [15:12] must
500 // be 0. So, if we're using 64k pages, we can always get those bits
501 // without knowing the physical address size.
502 //
503 // With 4k or 16k pages and 52-bit physical addresses (Armv8.7
504 // FEAT_LPA2), descriptor bits [49:48] contain physical address bits
505 // [49:48] and descriptor bits [9:8] contain physical address bits
506 // [51:50]. However, with 4k or 16k pages and 48-bit physical addresses,
507 // descriptor bits [9:8] are used for other purposes. We need to know
508 // the physical address size, but it is not reported in VMCOREINFO.
509 // However, we can check the virtual address size, as with 4k or 16k
510 // pages, the kernel requires that it is equal to the physical address
511 // size; see Linux kernel commit 352b0395b505 ("arm64: Enable 52-bit
512 // virtual addressing for 4k and 16k granule configs") (in v6.9).
513 it->pa_low_mask = (UINT64_C(0x0000ffffffffffff)
514 & ~(prog->vmcoreinfo.page_size - 1));
515 if (page_shift == 16) {
516 it->pa_high_mask = 0xf000;
517 it->pa_high_shift = 36;
518 } else if (va_bits == 52) {
519 it->pa_low_mask |= UINT64_C(0x0003000000000000);
520 it->pa_high_mask = 0x300;
521 it->pa_high_shift = 42;
522 } else {
523 it->pa_high_mask = 0x0;
524 it->pa_high_shift = 0;
525 }
526
527 *ret = &it->it;
528 return NULL;
529 }
530
531 static void linux_kernel_pgtable_iterator_destroy_aarch64(struct pgtable_iterator *_it)
532 {
533 struct pgtable_iterator_aarch64 *it =
534 container_of(_it, struct pgtable_iterator_aarch64, it);
535 free(it);
536 }
537
538 static void linux_kernel_pgtable_iterator_init_aarch64(struct drgn_program *prog,
539 struct pgtable_iterator *_it)
540 {
541 struct pgtable_iterator_aarch64 *it =
542 container_of(_it, struct pgtable_iterator_aarch64, it);
543 if (it->it.pgtable == prog->vmcoreinfo.swapper_pg_dir) {
544 it->va_range_min = UINT64_MAX << it->va_bits;
545 it->va_range_max = UINT64_MAX;
546 } else {
547 it->va_range_min = 0;
548 it->va_range_max = (UINT64_C(1) << it->va_bits) - 1;
549 }
550
551 it->cached_virt_addr = 0;
552 memset(it->table, 0, sizeof(it->table));
553 }
554
555 static struct drgn_error *
556 linux_kernel_pgtable_iterator_next_aarch64(struct drgn_program *prog,
557 struct pgtable_iterator *_it,
558 uint64_t *virt_addr_ret,
559 uint64_t *phys_addr_ret)
560 {
561 struct drgn_error *err;
562 const uint64_t page_shift = prog->vmcoreinfo.page_shift;
563 const uint64_t pgtable_shift = page_shift - 3;
564 const bool bswap = drgn_platform_bswap(&prog->platform);
565 struct pgtable_iterator_aarch64 *it =
566 container_of(_it, struct pgtable_iterator_aarch64, it);
567 const uint64_t virt_addr = it->it.virt_addr;
568
569 if (virt_addr < it->va_range_min || virt_addr > it->va_range_max) {
570 *virt_addr_ret = it->va_range_min;
571 *phys_addr_ret = UINT64_MAX;
572 it->it.virt_addr = it->va_range_max + 1;
573 return NULL;
574 }
575
576 uint16_t num_entries = it->last_level_num_entries;
577 uint64_t table = it->it.pgtable;
578 bool table_physical = false;
579 if (table == prog->vmcoreinfo.swapper_pg_dir
580 && prog->vmcoreinfo.kimage_voffset) {
581 // Avoid recursive address translation on swapper_pg_dir by
582 // directly resolving to a physical address. kimage_voffset has
583 // been present since Linux kernel commit 20a166243328 ("arm64:
584 // kdump: add VMCOREINFO's for user-space tools") (in v4.12).
585 table -= prog->vmcoreinfo.kimage_voffset;
586 table_physical = true;
587 }
588 for (int level = it->levels;; level--) {
589 uint8_t level_shift = page_shift + pgtable_shift * (level - 1);
590 uint16_t index = (virt_addr >> level_shift) & (num_entries - 1);
591 uint16_t cached_index = (it->cached_virt_addr >> level_shift) &
592 (num_entries - 1);
593 if (index != cached_index)
594 memset(it->table, 0, 8 * level);
595 uint64_t *entry_ptr = &it->table[level - 1];
596 if (!*entry_ptr) {
597 err = drgn_program_read_memory(prog, entry_ptr,
598 table + 8 * index, 8,
599 table_physical);
600 if (err)
601 return err;
602 if (bswap)
603 *entry_ptr = bswap_64(*entry_ptr);
604 }
605 uint64_t entry = *entry_ptr;
606
607 num_entries = it->entries_per_level;
608 table = ((entry & it->pa_low_mask) |
609 (entry & it->pa_high_mask) << it->pa_high_shift);
610
611 // Descriptor bits [1:0] identify the descriptor type:
612 //
613 // 0x0, 0x2: invalid
614 // 0x1: lowest level: reserved, invalid
615 // higher levels: block
616 // 0x3: lowest level: page
617 // higher levels: table
618 if ((entry & 0x3) != 0x3 || level == 1) {
619 uint64_t mask = (UINT64_C(1) << level_shift) - 1;
620 *virt_addr_ret = virt_addr & ~mask;
621 if ((entry & 0x3) == (level == 1 ? 0x3 : 0x1))
622 *phys_addr_ret = table & ~mask;
623 else
624 *phys_addr_ret = UINT64_MAX;
625 it->cached_virt_addr = virt_addr;
626 it->it.virt_addr = (virt_addr | mask) + 1;
627 return NULL;
628 }
629 table_physical = true;
630 }
631 }
632
633 static int
634 linux_kernel_section_size_bits_fallback_aarch64(struct drgn_program *prog)
635 {
636 // Since Linux kernel commit f0b13ee23241 ("arm64/sparsemem: reduce
637 // SECTION_SIZE_BITS") (in v5.12), SECTION_SIZE_BITS is 29 with 64k
638 // pages and 27 otherwise. Linux kernel stable commit 70fd2a63fc1c
639 // ("crash_core, vmcoreinfo: append 'SECTION_SIZE_BITS' to vmcoreinfo")
640 // (in v5.12.13) addresses this, but before that there's no good way to
641 // detect it other than the kernel version.
642 char *p = (char *)prog->vmcoreinfo.osrelease;
643 long major = strtol(p, &p, 10), minor = 0;
644 if (*p == '.')
645 minor = strtol(p + 1, &p, 10);
646 if (major > 5 || (major == 5 && minor >= 12))
647 return prog->vmcoreinfo.page_shift == 16 ? 29 : 27;
648 // Before that, it didn't change since Linux kernel commit 4f04d8f00545
649 // ("arm64: MMU definitions") (in v3.7).
650 return 30;
651 }
652
653 static uint64_t untagged_addr_aarch64(uint64_t addr)
654 {
655 /* Apply TBI by sign extending bit 55 into bits 56-63. */
656 return (((int64_t)addr) << 8) >> 8;
657 }
658
659 const struct drgn_architecture_info arch_info_aarch64 = {
660 .name = "AArch64",
661 .arch = DRGN_ARCH_AARCH64,
662 .default_flags = (DRGN_PLATFORM_IS_64_BIT |
663 DRGN_PLATFORM_IS_LITTLE_ENDIAN),
664 .scalar_alignment = { 1, 2, 4, 8, 16 },
665 DRGN_ARCHITECTURE_REGISTERS,
666 .default_dwarf_cfi_row = &default_dwarf_cfi_row_aarch64,
667 .demangle_cfi_registers = demangle_cfi_registers_aarch64,
668 .fallback_unwind = fallback_unwind_aarch64,
669 .bad_call_unwind = bad_call_unwind_aarch64,
670 .pt_regs_get_initial_registers = pt_regs_get_initial_registers_aarch64,
671 .prstatus_get_initial_registers = prstatus_get_initial_registers_aarch64,
672 .linux_kernel_get_initial_registers =
673 linux_kernel_get_initial_registers_aarch64,
674 .apply_elf_reloc = apply_elf_reloc_aarch64,
675 .linux_kernel_pgtable_iterator_create =
676 linux_kernel_pgtable_iterator_create_aarch64,
677 .linux_kernel_pgtable_iterator_destroy =
678 linux_kernel_pgtable_iterator_destroy_aarch64,
679 .linux_kernel_pgtable_iterator_init =
680 linux_kernel_pgtable_iterator_init_aarch64,
681 .linux_kernel_pgtable_iterator_next =
682 linux_kernel_pgtable_iterator_next_aarch64,
683 .linux_kernel_section_size_bits_fallback =
684 linux_kernel_section_size_bits_fallback_aarch64,
685 // MAX_PHYSMEM_BITS depends on CONFIG_ARM64_PA_BITS. Use the default
686 // fallback.
687 .untagged_addr = untagged_addr_aarch64,
688 };