| 1 |
// Copyright (c) Meta Platforms, Inc. and affiliates. |
| 2 |
// SPDX-License-Identifier: LGPL-2.1-or-later |
| 3 |
|
| 4 |
#include <assert.h> |
| 5 |
#include <byteswap.h> |
| 6 |
#include <elf.h> |
| 7 |
#include <stdlib.h> |
| 8 |
#include <string.h> |
| 9 |
|
| 10 |
#include "array.h" |
| 11 |
#include "error.h" |
| 12 |
#include "linux_kernel.h" |
| 13 |
#include "log.h" |
| 14 |
#include "platform.h" // IWYU pragma: associated |
| 15 |
#include "program.h" |
| 16 |
#include "register_state.h" |
| 17 |
#include "util.h" |
| 18 |
|
| 19 |
/* |
| 20 |
* The ABI specification can be found at: |
| 21 |
* https://developer.arm.com/architectures/system-architectures/software-standards/abi |
| 22 |
* https://github.com/ARM-software/abi-aa/releases |
| 23 |
*/ |
| 24 |
|
| 25 |
#include "arch_aarch64_defs.inc" |
| 26 |
|
| 27 |
static_assert(DRGN_AARCH64_RA_SIGN_STATE_REGNO == |
| 28 |
DRGN_REGISTER_NUMBER(ra_sign_state), |
| 29 |
"RA_SIGN_STATE register number is out of sync"); |
| 30 |
|
| 31 |
static const struct drgn_cfi_row default_dwarf_cfi_row_aarch64 = DRGN_CFI_ROW( |
| 32 |
[DRGN_REGISTER_NUMBER(ra_sign_state)] = { |
| 33 |
DRGN_CFI_RULE_CONSTANT, .constant = 0 |
| 34 |
}, |
| 35 |
// The psABI defines the CFA as the value of the stack pointer in the |
| 36 |
// calling frame. |
| 37 |
[DRGN_REGISTER_NUMBER(sp)] = { DRGN_CFI_RULE_CFA_PLUS_OFFSET }, |
| 38 |
// The psABI defines that callee-saved registers default to |
| 39 |
// DW_CFA_same_value. |
| 40 |
DRGN_CFI_SAME_VALUE_INIT(DRGN_REGISTER_NUMBER(x19)), |
| 41 |
DRGN_CFI_SAME_VALUE_INIT(DRGN_REGISTER_NUMBER(x20)), |
| 42 |
DRGN_CFI_SAME_VALUE_INIT(DRGN_REGISTER_NUMBER(x21)), |
| 43 |
DRGN_CFI_SAME_VALUE_INIT(DRGN_REGISTER_NUMBER(x22)), |
| 44 |
DRGN_CFI_SAME_VALUE_INIT(DRGN_REGISTER_NUMBER(x23)), |
| 45 |
DRGN_CFI_SAME_VALUE_INIT(DRGN_REGISTER_NUMBER(x24)), |
| 46 |
DRGN_CFI_SAME_VALUE_INIT(DRGN_REGISTER_NUMBER(x25)), |
| 47 |
DRGN_CFI_SAME_VALUE_INIT(DRGN_REGISTER_NUMBER(x26)), |
| 48 |
DRGN_CFI_SAME_VALUE_INIT(DRGN_REGISTER_NUMBER(x27)), |
| 49 |
DRGN_CFI_SAME_VALUE_INIT(DRGN_REGISTER_NUMBER(x28)), |
| 50 |
DRGN_CFI_SAME_VALUE_INIT(DRGN_REGISTER_NUMBER(x29)), |
| 51 |
DRGN_CFI_SAME_VALUE_INIT(DRGN_REGISTER_NUMBER(x30)), |
| 52 |
); |
| 53 |
|
| 54 |
// Mask out the pointer authentication code from x30/lr. |
| 55 |
static void demangle_cfi_registers_aarch64(struct drgn_program *prog, |
| 56 |
struct drgn_register_state *regs) |
| 57 |
{ |
| 58 |
struct optional_uint64 ra_sign_state = |
| 59 |
drgn_register_state_get_u64(prog, regs, ra_sign_state); |
| 60 |
if (!ra_sign_state.has_value || !(ra_sign_state.value & 1)) |
| 61 |
return; |
| 62 |
struct optional_uint64 ra = |
| 63 |
drgn_register_state_get_u64(prog, regs, x30); |
| 64 |
if (!ra.has_value) |
| 65 |
return; |
| 66 |
if (ra.value & (UINT64_C(1) << 55)) |
| 67 |
ra.value |= prog->aarch64_insn_pac_mask; |
| 68 |
else |
| 69 |
ra.value &= ~prog->aarch64_insn_pac_mask; |
| 70 |
drgn_register_state_set_from_u64(prog, regs, x30, ra.value); |
| 71 |
} |
| 72 |
|
| 73 |
// elf_gregset_t (in PRSTATUS) and struct user_pt_regs have the same layout. |
| 74 |
// This layout is a prefix of the in-kernel struct pt_regs (but we don't care |
| 75 |
// about any of the extra fields). |
| 76 |
static struct drgn_error * |
| 77 |
get_initial_registers_from_struct_aarch64(struct drgn_program *prog, |
| 78 |
const void *buf, size_t size, |
| 79 |
struct drgn_register_state **ret) |
| 80 |
{ |
| 81 |
if (size < 272) { |
| 82 |
return drgn_error_create(DRGN_ERROR_INVALID_ARGUMENT, |
| 83 |
"registers are truncated"); |
| 84 |
} |
| 85 |
|
| 86 |
struct drgn_register_state *regs = |
| 87 |
drgn_register_state_create(pstate, true); |
| 88 |
if (!regs) |
| 89 |
return &drgn_enomem; |
| 90 |
|
| 91 |
drgn_register_state_set_from_buffer(regs, pc, (uint64_t *)buf + 32); |
| 92 |
drgn_register_state_set_from_buffer(regs, sp, (uint64_t *)buf + 31); |
| 93 |
drgn_register_state_set_range_from_buffer(regs, x19, x30, |
| 94 |
(uint64_t *)buf + 19); |
| 95 |
drgn_register_state_set_range_from_buffer(regs, x0, x18, buf); |
| 96 |
drgn_register_state_set_from_buffer(regs, pstate, (uint64_t *)buf + 33); |
| 97 |
drgn_register_state_set_pc_from_register(prog, regs, pc); |
| 98 |
|
| 99 |
*ret = regs; |
| 100 |
return NULL; |
| 101 |
} |
| 102 |
|
| 103 |
|
| 104 |
static struct drgn_error * |
| 105 |
get_stackframe_offset(struct drgn_program *prog, uint64_t *ret) |
| 106 |
{ |
| 107 |
if (!prog->aarch64_stackframe_offset_cached) { |
| 108 |
struct drgn_error *err; |
| 109 |
struct drgn_qualified_type pt_regs_type; |
| 110 |
|
| 111 |
err = drgn_program_find_type(prog, "struct pt_regs", NULL, &pt_regs_type); |
| 112 |
if (err) |
| 113 |
return err; |
| 114 |
|
| 115 |
struct drgn_type_member *stackframe_member; |
| 116 |
uint64_t bit_offset; |
| 117 |
err = drgn_type_find_member(pt_regs_type.type, "stackframe", |
| 118 |
&stackframe_member, &bit_offset); |
| 119 |
if (err) |
| 120 |
return err; |
| 121 |
|
| 122 |
prog->aarch64_stackframe_offset_cached = bit_offset / 8; |
| 123 |
} |
| 124 |
|
| 125 |
*ret = prog->aarch64_stackframe_offset_cached; |
| 126 |
return NULL; |
| 127 |
} |
| 128 |
|
| 129 |
|
| 130 |
static struct drgn_error * |
| 131 |
fallback_unwind_try_pt_regs(struct drgn_program *prog, uint64_t fp, |
| 132 |
uint64_t frame[2], |
| 133 |
struct drgn_register_state *regs, |
| 134 |
struct drgn_register_state **regs_ret, |
| 135 |
bool *found_ret) |
| 136 |
{ |
| 137 |
struct drgn_error *err; |
| 138 |
uint64_t stackframe_offset; |
| 139 |
const char *method; |
| 140 |
|
| 141 |
err = get_stackframe_offset(prog, &stackframe_offset); |
| 142 |
if (err) |
| 143 |
return err; |
| 144 |
|
| 145 |
uint64_t pt_regs_addr = fp - stackframe_offset; |
| 146 |
|
| 147 |
if (!frame[0] && !frame[1]) { |
| 148 |
// Since c2c6b27b5aa14 ("arm64: stacktrace: unwind exception |
| 149 |
// boundaries") - v6.13, a frame record with fp and lr both zero |
| 150 |
// indicates that we can read the next word on the stack for |
| 151 |
// "metadata" which tells us whether a pt_regs is pushed to the |
| 152 |
// stack. |
| 153 |
uint64_t frame_meta_type; |
| 154 |
err = drgn_program_read_u64(prog, fp + 16, false, &frame_meta_type); |
| 155 |
if (err) |
| 156 |
return err; |
| 157 |
|
| 158 |
// FRAME_META_TYPE_PT_REGS == 2 indicates a pt_regs |
| 159 |
if (frame_meta_type != 2) { |
| 160 |
*found_ret = false; |
| 161 |
return NULL; |
| 162 |
} |
| 163 |
method = "frame metadata"; |
| 164 |
} else { |
| 165 |
// Before this, it's still possible that this stack frame is |
| 166 |
// embedded within a pt_regs. However, there's no reliable way |
| 167 |
// to detect it. We can guess what that pt_regs pointer would |
| 168 |
// be, and check whether any CPU has a __irq_regs value matching |
| 169 |
// it. If so, we're guaranteed to be in the pt_regs for an IRQ. |
| 170 |
// If not, it's still possible that this is a pt_regs, but the |
| 171 |
// __irq_regs accounting has not yet happened (or has been |
| 172 |
// cleaned up already). So this heuristic shouldn't have any |
| 173 |
// false positives, but could have some false negatives. |
| 174 |
bool is_irq_regs; |
| 175 |
err = drgn_program_is_irq_regs(prog, pt_regs_addr, &is_irq_regs); |
| 176 |
if (err) |
| 177 |
return err; |
| 178 |
|
| 179 |
if (!is_irq_regs) { |
| 180 |
*found_ret = false; |
| 181 |
return NULL; |
| 182 |
} |
| 183 |
method = "__irq_regs"; |
| 184 |
} |
| 185 |
|
| 186 |
// Now that we're reasonably confident that there's a pt_regs here, read |
| 187 |
// it and use it. |
| 188 |
char pt_regs[272]; |
| 189 |
err = drgn_program_read_memory(prog, &pt_regs, pt_regs_addr, |
| 190 |
sizeof(pt_regs), false); |
| 191 |
if (err) |
| 192 |
return err; |
| 193 |
err = get_initial_registers_from_struct_aarch64(prog, &pt_regs, |
| 194 |
sizeof(pt_regs), regs_ret); |
| 195 |
if (err) |
| 196 |
return err; |
| 197 |
|
| 198 |
drgn_log_debug(prog, "aarch64 fallback: discovered pt_regs 0x%"PRIx64" via %s", |
| 199 |
pt_regs_addr, method); |
| 200 |
*found_ret = true; |
| 201 |
return NULL; |
| 202 |
} |
| 203 |
|
| 204 |
// Unwind using the frame pointer. Note that leaf functions may not allocate a |
| 205 |
// stack frame, so this may skip the caller of a leaf function. I don't know of |
| 206 |
// a good way around that. |
| 207 |
static struct drgn_error * |
| 208 |
fallback_unwind_aarch64(struct drgn_program *prog, |
| 209 |
struct drgn_register_state *regs, |
| 210 |
struct drgn_register_state **ret) |
| 211 |
{ |
| 212 |
|
| 213 |
struct drgn_error *err; |
| 214 |
|
| 215 |
struct optional_uint64 fp = |
| 216 |
drgn_register_state_get_u64(prog, regs, x29); |
| 217 |
if (!fp.has_value) |
| 218 |
return &drgn_stop; |
| 219 |
|
| 220 |
// The frame pointer points to a frame record of two 64-bit values. The |
| 221 |
// first (lowest addressed) is the address of the caller's frame record. |
| 222 |
// The second (highest addressed) is the saved lr. |
| 223 |
uint64_t frame[2]; |
| 224 |
err = drgn_program_read_memory(prog, frame, fp.value, sizeof(frame), |
| 225 |
false); |
| 226 |
if (err) { |
| 227 |
if (err->code == DRGN_ERROR_FAULT) { |
| 228 |
drgn_error_destroy(err); |
| 229 |
err = &drgn_stop; |
| 230 |
} |
| 231 |
return err; |
| 232 |
} |
| 233 |
|
| 234 |
if (prog->flags & DRGN_PROGRAM_IS_LINUX_KERNEL) { |
| 235 |
bool found; |
| 236 |
err = fallback_unwind_try_pt_regs(prog, fp.value, frame, regs, ret, &found); |
| 237 |
if (err) { |
| 238 |
// If we encounter any error trying the pt_regs |
| 239 |
// approach, log it at the debug level and continue. |
| 240 |
// It's likely an expected error, but could be useful |
| 241 |
// debugging information. |
| 242 |
drgn_error_log_debug(prog, err, "encountered error in linux fallback unwinder:"); |
| 243 |
drgn_error_destroy(err); |
| 244 |
err = NULL; |
| 245 |
} else if (found) { |
| 246 |
// We have set the register state based on a pt_regs |
| 247 |
// frame we found, return! |
| 248 |
return NULL; |
| 249 |
} |
| 250 |
} |
| 251 |
|
| 252 |
uint64_t unwound_fp = |
| 253 |
drgn_platform_bswap(&prog->platform) ? |
| 254 |
bswap_64(frame[0]) : frame[0]; |
| 255 |
if (unwound_fp <= fp.value) { |
| 256 |
// The unwound stack pointer is either 0, indicating the first |
| 257 |
// stack frame, or invalid. |
| 258 |
return &drgn_stop; |
| 259 |
} |
| 260 |
|
| 261 |
struct drgn_register_state *unwound = |
| 262 |
drgn_register_state_create(x30, false); |
| 263 |
if (!unwound) |
| 264 |
return &drgn_enomem; |
| 265 |
drgn_register_state_set_from_buffer(unwound, x30, &frame[1]); |
| 266 |
drgn_register_state_set_from_buffer(unwound, x29, &frame[0]); |
| 267 |
// We don't know whether the return address is signed, so just assume |
| 268 |
// that it is if pointer authentication is enabled. If we're wrong, the |
| 269 |
// worst that can happen is that we'll "correct" incorrect sign |
| 270 |
// extension bits or clear an address tag. |
| 271 |
if (prog->aarch64_insn_pac_mask) { |
| 272 |
drgn_register_state_set_from_u64(prog, unwound, ra_sign_state, |
| 273 |
1); |
| 274 |
demangle_cfi_registers_aarch64(prog, unwound); |
| 275 |
} |
| 276 |
drgn_register_state_set_pc_from_register(prog, unwound, x30); |
| 277 |
// The location of the frame record within the stack frame is not |
| 278 |
// specified, so we can't determine the stack pointer. |
| 279 |
*ret = unwound; |
| 280 |
return NULL; |
| 281 |
} |
| 282 |
|
| 283 |
// Unwind a single bl or blr instruction. |
| 284 |
static struct drgn_error * |
| 285 |
bad_call_unwind_aarch64(struct drgn_program *prog, |
| 286 |
struct drgn_register_state *regs, |
| 287 |
struct drgn_register_state **ret) |
| 288 |
{ |
| 289 |
struct optional_uint64 lr = |
| 290 |
drgn_register_state_get_u64(prog, regs, x30); |
| 291 |
if (!lr.has_value) |
| 292 |
return &drgn_stop; |
| 293 |
|
| 294 |
struct drgn_register_state *tmp = drgn_register_state_dup(regs); |
| 295 |
if (!tmp) |
| 296 |
return &drgn_enomem; |
| 297 |
|
| 298 |
// lr contains the the old pc + 4. |
| 299 |
drgn_register_state_set_pc(prog, tmp, lr.value - 4); |
| 300 |
// We don't know the old lr. |
| 301 |
drgn_register_state_unset_has_register(tmp, DRGN_REGISTER_NUMBER(x30)); |
| 302 |
// The interrupted pc is no longer applicable. |
| 303 |
drgn_register_state_unset_has_register(tmp, DRGN_REGISTER_NUMBER(pc)); |
| 304 |
*ret = tmp; |
| 305 |
return NULL; |
| 306 |
} |
| 307 |
|
| 308 |
static struct drgn_error * |
| 309 |
pt_regs_get_initial_registers_aarch64(const struct drgn_object *obj, |
| 310 |
struct drgn_register_state **ret) |
| 311 |
{ |
| 312 |
return get_initial_registers_from_struct_aarch64(drgn_object_program(obj), |
| 313 |
drgn_object_buffer(obj), |
| 314 |
drgn_object_size(obj), |
| 315 |
ret); |
| 316 |
} |
| 317 |
|
| 318 |
static struct drgn_error * |
| 319 |
prstatus_get_initial_registers_aarch64(struct drgn_program *prog, |
| 320 |
const void *prstatus, size_t size, |
| 321 |
struct drgn_register_state **ret) |
| 322 |
{ |
| 323 |
// offsetof(struct elf_prstatus, pr_reg) |
| 324 |
static const size_t pr_reg_offset = 112; |
| 325 |
if (size < pr_reg_offset) { |
| 326 |
return drgn_error_create(DRGN_ERROR_INVALID_ARGUMENT, |
| 327 |
"NT_PRSTATUS is truncated"); |
| 328 |
} |
| 329 |
return get_initial_registers_from_struct_aarch64(prog, |
| 330 |
(char *)prstatus + pr_reg_offset, |
| 331 |
size - pr_reg_offset, |
| 332 |
ret); |
| 333 |
} |
| 334 |
|
| 335 |
// The Linux kernel saves the callee-saved registers in |
| 336 |
// struct task_struct.thread.cpu_context (with type struct cpu_context). See |
| 337 |
// cpu_switch_to() in arch/arm64/kernel/entry.S (as of Linux v5.19). |
| 338 |
static struct drgn_error * |
| 339 |
linux_kernel_get_initial_registers_aarch64(const struct drgn_object *task_obj, |
| 340 |
struct drgn_register_state **ret) |
| 341 |
{ |
| 342 |
struct drgn_error *err; |
| 343 |
struct drgn_program *prog = drgn_object_program(task_obj); |
| 344 |
|
| 345 |
DRGN_OBJECT(cpu_context_obj, prog); |
| 346 |
|
| 347 |
err = drgn_object_member_dereference(&cpu_context_obj, task_obj, |
| 348 |
"thread"); |
| 349 |
if (err) |
| 350 |
return err; |
| 351 |
err = drgn_object_member(&cpu_context_obj, &cpu_context_obj, |
| 352 |
"cpu_context"); |
| 353 |
if (err) |
| 354 |
return err; |
| 355 |
if (cpu_context_obj.encoding != DRGN_OBJECT_ENCODING_BUFFER || |
| 356 |
drgn_object_size(&cpu_context_obj) < 104) { |
| 357 |
return drgn_error_create(DRGN_ERROR_INVALID_ARGUMENT, |
| 358 |
"cpu_context is truncated"); |
| 359 |
} |
| 360 |
err = drgn_object_read(&cpu_context_obj, &cpu_context_obj); |
| 361 |
if (err) |
| 362 |
return err; |
| 363 |
|
| 364 |
const void *buf = drgn_object_buffer(&cpu_context_obj); |
| 365 |
struct drgn_register_state *regs = |
| 366 |
drgn_register_state_create(x30, false); |
| 367 |
if (!regs) |
| 368 |
return &drgn_enomem; |
| 369 |
|
| 370 |
drgn_register_state_set_from_buffer(regs, x30, (uint64_t *)buf + 12); |
| 371 |
drgn_register_state_set_from_buffer(regs, sp, (uint64_t *)buf + 11); |
| 372 |
drgn_register_state_set_range_from_buffer(regs, x19, x29, buf); |
| 373 |
drgn_register_state_set_pc_from_register(prog, regs, x30); |
| 374 |
*ret = regs; |
| 375 |
return NULL; |
| 376 |
} |
| 377 |
|
| 378 |
static struct drgn_error * |
| 379 |
apply_elf_reloc_aarch64(const struct drgn_relocating_section *relocating, |
| 380 |
uint64_t r_offset, uint32_t r_type, const int64_t *r_addend, |
| 381 |
uint64_t sym_value) |
| 382 |
{ |
| 383 |
switch (r_type) { |
| 384 |
case R_AARCH64_NONE: |
| 385 |
return NULL; |
| 386 |
case R_AARCH64_ABS64: |
| 387 |
return drgn_reloc_add64(relocating, r_offset, r_addend, |
| 388 |
sym_value); |
| 389 |
case R_AARCH64_ABS32: |
| 390 |
return drgn_reloc_add32(relocating, r_offset, r_addend, |
| 391 |
sym_value); |
| 392 |
case R_AARCH64_ABS16: |
| 393 |
return drgn_reloc_add16(relocating, r_offset, r_addend, |
| 394 |
sym_value); |
| 395 |
case R_AARCH64_PREL64: |
| 396 |
return drgn_reloc_add64(relocating, r_offset, r_addend, |
| 397 |
sym_value |
| 398 |
- (relocating->addr + r_offset)); |
| 399 |
case R_AARCH64_PREL32: |
| 400 |
return drgn_reloc_add32(relocating, r_offset, r_addend, |
| 401 |
sym_value |
| 402 |
- (relocating->addr + r_offset)); |
| 403 |
case R_AARCH64_PREL16: |
| 404 |
return drgn_reloc_add16(relocating, r_offset, r_addend, |
| 405 |
sym_value |
| 406 |
- (relocating->addr + r_offset)); |
| 407 |
default: |
| 408 |
return DRGN_UNKNOWN_RELOCATION_TYPE(r_type); |
| 409 |
} |
| 410 |
} |
| 411 |
|
| 412 |
struct pgtable_iterator_aarch64 { |
| 413 |
struct pgtable_iterator it; |
| 414 |
uint64_t va_bits; |
| 415 |
// Inclusive range of valid virtual addresses. |
| 416 |
uint64_t va_range_min, va_range_max; |
| 417 |
int levels; |
| 418 |
uint16_t entries_per_level; |
| 419 |
uint16_t last_level_num_entries; |
| 420 |
uint64_t cached_virt_addr; |
| 421 |
uint64_t table[5]; |
| 422 |
uint64_t pa_low_mask; |
| 423 |
uint64_t pa_high_mask; |
| 424 |
int pa_high_shift; |
| 425 |
}; |
| 426 |
|
| 427 |
static struct drgn_error * |
| 428 |
linux_kernel_pgtable_iterator_create_aarch64(struct drgn_program *prog, |
| 429 |
struct pgtable_iterator **ret) |
| 430 |
{ |
| 431 |
const uint64_t page_shift = prog->vmcoreinfo.page_shift; |
| 432 |
if (page_shift != 12 && page_shift != 14 && page_shift != 16) { |
| 433 |
return drgn_error_create(DRGN_ERROR_OTHER, |
| 434 |
"unknown page size for virtual address translation"); |
| 435 |
} |
| 436 |
const uint64_t pgtable_shift = page_shift - 3; |
| 437 |
|
| 438 |
// VA_BITS in VMCOREINFO, added in Linux kernel commit 20a166243328 |
| 439 |
// ("arm64: kdump: add VMCOREINFO's for user-space tools") (in v4.12), |
| 440 |
// is the configured virtual address size. Since Linux kernel commit |
| 441 |
// b6d00d47e81a ("arm64: mm: Introduce 52-bit Kernel VAs") (in v5.4), it |
| 442 |
// may be greater than the actual size used at runtime if the kernel is |
| 443 |
// configured with 52-bit virtual addresses but the hardware does not |
| 444 |
// support it. 64 - TCR_EL1_T1SZ, added in Linux kernel commit |
| 445 |
// bbdbc11804ff ("arm64/crash_core: Export TCR_EL1.T1SZ in vmcoreinfo") |
| 446 |
// (in v5.9), is the size used at runtime. |
| 447 |
// |
| 448 |
// With 64k pages, if 52-bit virtual addresses are not supported |
| 449 |
// (Armv8.2 FEAT_LVA), then the kernel falls back to 48-bit virtual |
| 450 |
// addresses. Note that the page table has 3 levels with either size. In |
| 451 |
// this case, swapper_pg_dir is still set up for 52-bit virtual |
| 452 |
// addresses (an offset is applied when loading swapper_pg_dir into |
| 453 |
// TTBR1; see Linux kernel commit c812026c54cf ("arm64: mm: Logic to |
| 454 |
// make offset_ttbr1 conditional") (in v5.4)). So, with 64k pages, we |
| 455 |
// need to treat kernel addresses as if they have size VA_BITS for the |
| 456 |
// purposes of translating using swapper_pg_dir. We can also treat user |
| 457 |
// addresses as if they have size VA_BITS. In the 52- to 48-bit fallback |
| 458 |
// case, the additional user virtual address bits [51:48] are zero, so |
| 459 |
// the page table is accessed identically. |
| 460 |
// |
| 461 |
// With 4k or 16k pages, if 52-bit virtual addresses are not supported |
| 462 |
// (Armv8.7 FEAT_LPA2), then the kernel falls back to 48-bit virtual |
| 463 |
// addresses with 4k pages or 47-bit virtual addresses with 16k pages; |
| 464 |
// see Linux kernel commit 0dd4f60a2c76 ("arm64: mm: Add support for |
| 465 |
// folding PUDs at runtime") (in v6.9). This decreases the number of |
| 466 |
// page table levels (from 5 to 4 for 4k pages and from 4 to 3 for 16k |
| 467 |
// pages). In this case, we need to use the actual size for both kernel |
| 468 |
// and user addresses, 64 - TCR_EL1_T1SZ. |
| 469 |
uint64_t va_bits; |
| 470 |
if (page_shift == 16 || !prog->vmcoreinfo.tcr_el1_t1sz) |
| 471 |
va_bits = prog->vmcoreinfo.va_bits; |
| 472 |
else |
| 473 |
va_bits = 64 - prog->vmcoreinfo.tcr_el1_t1sz; |
| 474 |
if (va_bits <= page_shift || va_bits > 52) { |
| 475 |
return drgn_error_create(DRGN_ERROR_OTHER, |
| 476 |
"VMCOREINFO does not contain valid TCR_EL1_T1SZ or VA_BITS"); |
| 477 |
} |
| 478 |
|
| 479 |
struct pgtable_iterator_aarch64 *it = malloc(sizeof(*it)); |
| 480 |
if (!it) |
| 481 |
return &drgn_enomem; |
| 482 |
|
| 483 |
it->va_bits = va_bits; |
| 484 |
it->levels = ((va_bits - page_shift + pgtable_shift - 1) / |
| 485 |
pgtable_shift); |
| 486 |
assert(it->levels <= array_size(it->table)); |
| 487 |
it->entries_per_level = 1 << pgtable_shift; |
| 488 |
it->last_level_num_entries = |
| 489 |
1 << ((va_bits - page_shift - 1) % pgtable_shift + 1); |
| 490 |
|
| 491 |
// Descriptor bits [47:PAGE_SHIFT] contain physical address bits |
| 492 |
// [47:PAGE_SHIFT]. |
| 493 |
// |
| 494 |
// For 52-bit physical addresses, physical address bits [51:48] come |
| 495 |
// from elsewhere. |
| 496 |
// |
| 497 |
// With 64k pages and 52-bit physical addresses (Armv8.2 FEAT_LPA), |
| 498 |
// descriptor bits [15:12] contain physical address bits [51:48]. With |
| 499 |
// 64k pages and 48-bit physical addresses, descriptor bits [15:12] must |
| 500 |
// be 0. So, if we're using 64k pages, we can always get those bits |
| 501 |
// without knowing the physical address size. |
| 502 |
// |
| 503 |
// With 4k or 16k pages and 52-bit physical addresses (Armv8.7 |
| 504 |
// FEAT_LPA2), descriptor bits [49:48] contain physical address bits |
| 505 |
// [49:48] and descriptor bits [9:8] contain physical address bits |
| 506 |
// [51:50]. However, with 4k or 16k pages and 48-bit physical addresses, |
| 507 |
// descriptor bits [9:8] are used for other purposes. We need to know |
| 508 |
// the physical address size, but it is not reported in VMCOREINFO. |
| 509 |
// However, we can check the virtual address size, as with 4k or 16k |
| 510 |
// pages, the kernel requires that it is equal to the physical address |
| 511 |
// size; see Linux kernel commit 352b0395b505 ("arm64: Enable 52-bit |
| 512 |
// virtual addressing for 4k and 16k granule configs") (in v6.9). |
| 513 |
it->pa_low_mask = (UINT64_C(0x0000ffffffffffff) |
| 514 |
& ~(prog->vmcoreinfo.page_size - 1)); |
| 515 |
if (page_shift == 16) { |
| 516 |
it->pa_high_mask = 0xf000; |
| 517 |
it->pa_high_shift = 36; |
| 518 |
} else if (va_bits == 52) { |
| 519 |
it->pa_low_mask |= UINT64_C(0x0003000000000000); |
| 520 |
it->pa_high_mask = 0x300; |
| 521 |
it->pa_high_shift = 42; |
| 522 |
} else { |
| 523 |
it->pa_high_mask = 0x0; |
| 524 |
it->pa_high_shift = 0; |
| 525 |
} |
| 526 |
|
| 527 |
*ret = &it->it; |
| 528 |
return NULL; |
| 529 |
} |
| 530 |
|
| 531 |
static void linux_kernel_pgtable_iterator_destroy_aarch64(struct pgtable_iterator *_it) |
| 532 |
{ |
| 533 |
struct pgtable_iterator_aarch64 *it = |
| 534 |
container_of(_it, struct pgtable_iterator_aarch64, it); |
| 535 |
free(it); |
| 536 |
} |
| 537 |
|
| 538 |
static void linux_kernel_pgtable_iterator_init_aarch64(struct drgn_program *prog, |
| 539 |
struct pgtable_iterator *_it) |
| 540 |
{ |
| 541 |
struct pgtable_iterator_aarch64 *it = |
| 542 |
container_of(_it, struct pgtable_iterator_aarch64, it); |
| 543 |
if (it->it.pgtable == prog->vmcoreinfo.swapper_pg_dir) { |
| 544 |
it->va_range_min = UINT64_MAX << it->va_bits; |
| 545 |
it->va_range_max = UINT64_MAX; |
| 546 |
} else { |
| 547 |
it->va_range_min = 0; |
| 548 |
it->va_range_max = (UINT64_C(1) << it->va_bits) - 1; |
| 549 |
} |
| 550 |
|
| 551 |
it->cached_virt_addr = 0; |
| 552 |
memset(it->table, 0, sizeof(it->table)); |
| 553 |
} |
| 554 |
|
| 555 |
static struct drgn_error * |
| 556 |
linux_kernel_pgtable_iterator_next_aarch64(struct drgn_program *prog, |
| 557 |
struct pgtable_iterator *_it, |
| 558 |
uint64_t *virt_addr_ret, |
| 559 |
uint64_t *phys_addr_ret) |
| 560 |
{ |
| 561 |
struct drgn_error *err; |
| 562 |
const uint64_t page_shift = prog->vmcoreinfo.page_shift; |
| 563 |
const uint64_t pgtable_shift = page_shift - 3; |
| 564 |
const bool bswap = drgn_platform_bswap(&prog->platform); |
| 565 |
struct pgtable_iterator_aarch64 *it = |
| 566 |
container_of(_it, struct pgtable_iterator_aarch64, it); |
| 567 |
const uint64_t virt_addr = it->it.virt_addr; |
| 568 |
|
| 569 |
if (virt_addr < it->va_range_min || virt_addr > it->va_range_max) { |
| 570 |
*virt_addr_ret = it->va_range_min; |
| 571 |
*phys_addr_ret = UINT64_MAX; |
| 572 |
it->it.virt_addr = it->va_range_max + 1; |
| 573 |
return NULL; |
| 574 |
} |
| 575 |
|
| 576 |
uint16_t num_entries = it->last_level_num_entries; |
| 577 |
uint64_t table = it->it.pgtable; |
| 578 |
bool table_physical = false; |
| 579 |
if (table == prog->vmcoreinfo.swapper_pg_dir |
| 580 |
&& prog->vmcoreinfo.kimage_voffset) { |
| 581 |
// Avoid recursive address translation on swapper_pg_dir by |
| 582 |
// directly resolving to a physical address. kimage_voffset has |
| 583 |
// been present since Linux kernel commit 20a166243328 ("arm64: |
| 584 |
// kdump: add VMCOREINFO's for user-space tools") (in v4.12). |
| 585 |
table -= prog->vmcoreinfo.kimage_voffset; |
| 586 |
table_physical = true; |
| 587 |
} |
| 588 |
for (int level = it->levels;; level--) { |
| 589 |
uint8_t level_shift = page_shift + pgtable_shift * (level - 1); |
| 590 |
uint16_t index = (virt_addr >> level_shift) & (num_entries - 1); |
| 591 |
uint16_t cached_index = (it->cached_virt_addr >> level_shift) & |
| 592 |
(num_entries - 1); |
| 593 |
if (index != cached_index) |
| 594 |
memset(it->table, 0, 8 * level); |
| 595 |
uint64_t *entry_ptr = &it->table[level - 1]; |
| 596 |
if (!*entry_ptr) { |
| 597 |
err = drgn_program_read_memory(prog, entry_ptr, |
| 598 |
table + 8 * index, 8, |
| 599 |
table_physical); |
| 600 |
if (err) |
| 601 |
return err; |
| 602 |
if (bswap) |
| 603 |
*entry_ptr = bswap_64(*entry_ptr); |
| 604 |
} |
| 605 |
uint64_t entry = *entry_ptr; |
| 606 |
|
| 607 |
num_entries = it->entries_per_level; |
| 608 |
table = ((entry & it->pa_low_mask) | |
| 609 |
(entry & it->pa_high_mask) << it->pa_high_shift); |
| 610 |
|
| 611 |
// Descriptor bits [1:0] identify the descriptor type: |
| 612 |
// |
| 613 |
// 0x0, 0x2: invalid |
| 614 |
// 0x1: lowest level: reserved, invalid |
| 615 |
// higher levels: block |
| 616 |
// 0x3: lowest level: page |
| 617 |
// higher levels: table |
| 618 |
if ((entry & 0x3) != 0x3 || level == 1) { |
| 619 |
uint64_t mask = (UINT64_C(1) << level_shift) - 1; |
| 620 |
*virt_addr_ret = virt_addr & ~mask; |
| 621 |
if ((entry & 0x3) == (level == 1 ? 0x3 : 0x1)) |
| 622 |
*phys_addr_ret = table & ~mask; |
| 623 |
else |
| 624 |
*phys_addr_ret = UINT64_MAX; |
| 625 |
it->cached_virt_addr = virt_addr; |
| 626 |
it->it.virt_addr = (virt_addr | mask) + 1; |
| 627 |
return NULL; |
| 628 |
} |
| 629 |
table_physical = true; |
| 630 |
} |
| 631 |
} |
| 632 |
|
| 633 |
static int |
| 634 |
linux_kernel_section_size_bits_fallback_aarch64(struct drgn_program *prog) |
| 635 |
{ |
| 636 |
// Since Linux kernel commit f0b13ee23241 ("arm64/sparsemem: reduce |
| 637 |
// SECTION_SIZE_BITS") (in v5.12), SECTION_SIZE_BITS is 29 with 64k |
| 638 |
// pages and 27 otherwise. Linux kernel stable commit 70fd2a63fc1c |
| 639 |
// ("crash_core, vmcoreinfo: append 'SECTION_SIZE_BITS' to vmcoreinfo") |
| 640 |
// (in v5.12.13) addresses this, but before that there's no good way to |
| 641 |
// detect it other than the kernel version. |
| 642 |
char *p = (char *)prog->vmcoreinfo.osrelease; |
| 643 |
long major = strtol(p, &p, 10), minor = 0; |
| 644 |
if (*p == '.') |
| 645 |
minor = strtol(p + 1, &p, 10); |
| 646 |
if (major > 5 || (major == 5 && minor >= 12)) |
| 647 |
return prog->vmcoreinfo.page_shift == 16 ? 29 : 27; |
| 648 |
// Before that, it didn't change since Linux kernel commit 4f04d8f00545 |
| 649 |
// ("arm64: MMU definitions") (in v3.7). |
| 650 |
return 30; |
| 651 |
} |
| 652 |
|
| 653 |
static uint64_t untagged_addr_aarch64(uint64_t addr) |
| 654 |
{ |
| 655 |
/* Apply TBI by sign extending bit 55 into bits 56-63. */ |
| 656 |
return (((int64_t)addr) << 8) >> 8; |
| 657 |
} |
| 658 |
|
| 659 |
const struct drgn_architecture_info arch_info_aarch64 = { |
| 660 |
.name = "AArch64", |
| 661 |
.arch = DRGN_ARCH_AARCH64, |
| 662 |
.default_flags = (DRGN_PLATFORM_IS_64_BIT | |
| 663 |
DRGN_PLATFORM_IS_LITTLE_ENDIAN), |
| 664 |
.scalar_alignment = { 1, 2, 4, 8, 16 }, |
| 665 |
DRGN_ARCHITECTURE_REGISTERS, |
| 666 |
.default_dwarf_cfi_row = &default_dwarf_cfi_row_aarch64, |
| 667 |
.demangle_cfi_registers = demangle_cfi_registers_aarch64, |
| 668 |
.fallback_unwind = fallback_unwind_aarch64, |
| 669 |
.bad_call_unwind = bad_call_unwind_aarch64, |
| 670 |
.pt_regs_get_initial_registers = pt_regs_get_initial_registers_aarch64, |
| 671 |
.prstatus_get_initial_registers = prstatus_get_initial_registers_aarch64, |
| 672 |
.linux_kernel_get_initial_registers = |
| 673 |
linux_kernel_get_initial_registers_aarch64, |
| 674 |
.apply_elf_reloc = apply_elf_reloc_aarch64, |
| 675 |
.linux_kernel_pgtable_iterator_create = |
| 676 |
linux_kernel_pgtable_iterator_create_aarch64, |
| 677 |
.linux_kernel_pgtable_iterator_destroy = |
| 678 |
linux_kernel_pgtable_iterator_destroy_aarch64, |
| 679 |
.linux_kernel_pgtable_iterator_init = |
| 680 |
linux_kernel_pgtable_iterator_init_aarch64, |
| 681 |
.linux_kernel_pgtable_iterator_next = |
| 682 |
linux_kernel_pgtable_iterator_next_aarch64, |
| 683 |
.linux_kernel_section_size_bits_fallback = |
| 684 |
linux_kernel_section_size_bits_fallback_aarch64, |
| 685 |
// MAX_PHYSMEM_BITS depends on CONFIG_ARM64_PA_BITS. Use the default |
| 686 |
// fallback. |
| 687 |
.untagged_addr = untagged_addr_aarch64, |
| 688 |
}; |