Debian Codegraph @

libdrgn/arch_x86_64.c

raw
1 // Copyright (c) Meta Platforms, Inc. and affiliates.
2 // SPDX-License-Identifier: LGPL-2.1-or-later
3
4 #include <byteswap.h>
5 #include <elf.h>
6 #include <stdlib.h>
7 #include <string.h>
8
9 #include "array.h"
10 #include "drgn_internal.h"
11 #include "error.h"
12 #include "linux_kernel.h"
13 #include "orc.h"
14 #include "platform.h" // IWYU pragma: associated
15 #include "program.h"
16 #include "register_state.h"
17 #include "util.h"
18
19 #include "arch_x86_64_defs.inc"
20
21 // This is __START_KERNEL_map from the Linux kernel. It has never been modified
22 // since the x86_64 architecture was introduced.
23 #define START_KERNEL_MAP UINT64_C(0xffffffff80000000)
24
25 static const struct drgn_cfi_row default_dwarf_cfi_row_x86_64 = DRGN_CFI_ROW(
26 // The System V psABI defines the CFA as the value of rsp in the calling
27 // frame.
28 [DRGN_REGISTER_NUMBER(rsp)] = { DRGN_CFI_RULE_CFA_PLUS_OFFSET },
29 // Other callee-saved registers default to DW_CFA_same_value. This isn't
30 // explicitly documented in the psABI, but it seems to be the consensus.
31 DRGN_CFI_SAME_VALUE_INIT(DRGN_REGISTER_NUMBER(rbx)),
32 DRGN_CFI_SAME_VALUE_INIT(DRGN_REGISTER_NUMBER(rbp)),
33 DRGN_CFI_SAME_VALUE_INIT(DRGN_REGISTER_NUMBER(r12)),
34 DRGN_CFI_SAME_VALUE_INIT(DRGN_REGISTER_NUMBER(r13)),
35 DRGN_CFI_SAME_VALUE_INIT(DRGN_REGISTER_NUMBER(r14)),
36 DRGN_CFI_SAME_VALUE_INIT(DRGN_REGISTER_NUMBER(r15)),
37 );
38
39 struct drgn_error *
40 drgn_orc_to_cfi_x86_64(const struct drgn_orc_entry *orc,
41 struct drgn_cfi_row **row_ret, bool *interrupted_ret,
42 drgn_register_number *ret_addr_regno_ret)
43 {
44 if (!drgn_cfi_row_copy(row_ret, drgn_empty_cfi_row))
45 return &drgn_enomem;
46
47 if (drgn_orc_type(orc) == DRGN_ORC_TYPE_UNDEFINED)
48 return &drgn_not_found;
49 else if (drgn_orc_type(orc) == DRGN_ORC_TYPE_END_OF_STACK)
50 return NULL;
51
52 struct drgn_cfi_rule rule;
53 switch (drgn_orc_sp_reg(orc)) {
54 case DRGN_ORC_REG_SP:
55 rule.kind = DRGN_CFI_RULE_REGISTER_PLUS_OFFSET;
56 rule.regno = DRGN_REGISTER_NUMBER(rsp);
57 rule.offset = orc->sp_offset;
58 break;
59 case DRGN_ORC_REG_BP:
60 rule.kind = DRGN_CFI_RULE_REGISTER_PLUS_OFFSET;
61 rule.regno = DRGN_REGISTER_NUMBER(rbp);
62 rule.offset = orc->sp_offset;
63 break;
64 case DRGN_ORC_REG_SP_INDIRECT:
65 rule.kind = DRGN_CFI_RULE_AT_REGISTER_ADD_OFFSET;
66 rule.regno = DRGN_REGISTER_NUMBER(rsp);
67 rule.offset = orc->sp_offset;
68 break;
69 case DRGN_ORC_REG_BP_INDIRECT:
70 rule.kind = DRGN_CFI_RULE_AT_REGISTER_PLUS_OFFSET;
71 rule.regno = DRGN_REGISTER_NUMBER(rbp);
72 rule.offset = orc->sp_offset;
73 break;
74 case DRGN_ORC_REG_R10:
75 rule.kind = DRGN_CFI_RULE_REGISTER_PLUS_OFFSET;
76 rule.regno = DRGN_REGISTER_NUMBER(r10);
77 rule.offset = 0;
78 break;
79 case DRGN_ORC_REG_R13:
80 rule.kind = DRGN_CFI_RULE_REGISTER_PLUS_OFFSET;
81 rule.regno = DRGN_REGISTER_NUMBER(r13);
82 rule.offset = 0;
83 break;
84 case DRGN_ORC_REG_DI:
85 rule.kind = DRGN_CFI_RULE_REGISTER_PLUS_OFFSET;
86 rule.regno = DRGN_REGISTER_NUMBER(rdi);
87 rule.offset = 0;
88 break;
89 case DRGN_ORC_REG_DX:
90 rule.kind = DRGN_CFI_RULE_REGISTER_PLUS_OFFSET;
91 rule.regno = DRGN_REGISTER_NUMBER(rdx);
92 rule.offset = 0;
93 break;
94 case DRGN_ORC_REG_AX:
95 rule.kind = DRGN_CFI_RULE_REGISTER_PLUS_OFFSET;
96 rule.regno = DRGN_REGISTER_NUMBER(rax);
97 rule.offset = 0;
98 break;
99 default:
100 return drgn_error_format(DRGN_ERROR_OTHER,
101 "unknown ORC SP base register %d",
102 drgn_orc_sp_reg(orc));
103 }
104 if (!drgn_cfi_row_set_cfa(row_ret, &rule))
105 return &drgn_enomem;
106
107 switch (drgn_orc_type(orc)) {
108 case DRGN_ORC_TYPE_CALL:
109 rule.kind = DRGN_CFI_RULE_AT_CFA_PLUS_OFFSET;
110 rule.offset = -8;
111 if (!drgn_cfi_row_set_register(row_ret,
112 DRGN_REGISTER_NUMBER(rip),
113 &rule))
114 return &drgn_enomem;
115 rule.kind = DRGN_CFI_RULE_CFA_PLUS_OFFSET;
116 rule.offset = 0;
117 if (!drgn_cfi_row_set_register(row_ret,
118 DRGN_REGISTER_NUMBER(rsp),
119 &rule))
120 return &drgn_enomem;
121 break;
122 #define SET_AT_CFA_RULE(reg, cfa_offset) do { \
123 rule.kind = DRGN_CFI_RULE_AT_CFA_PLUS_OFFSET; \
124 rule.offset = cfa_offset; \
125 if (!drgn_cfi_row_set_register(row_ret, DRGN_REGISTER_NUMBER(reg), \
126 &rule)) \
127 return &drgn_enomem; \
128 } while (0)
129 case DRGN_ORC_TYPE_REGS:
130 SET_AT_CFA_RULE(rip, 128);
131 SET_AT_CFA_RULE(rsp, 152);
132 SET_AT_CFA_RULE(r15, 0);
133 SET_AT_CFA_RULE(r14, 8);
134 SET_AT_CFA_RULE(r13, 16);
135 SET_AT_CFA_RULE(r12, 24);
136 SET_AT_CFA_RULE(rbp, 32);
137 SET_AT_CFA_RULE(rbx, 40);
138 SET_AT_CFA_RULE(r11, 48);
139 SET_AT_CFA_RULE(r10, 56);
140 SET_AT_CFA_RULE(r9, 64);
141 SET_AT_CFA_RULE(r8, 72);
142 SET_AT_CFA_RULE(rax, 80);
143 SET_AT_CFA_RULE(rcx, 88);
144 SET_AT_CFA_RULE(rdx, 96);
145 SET_AT_CFA_RULE(rsi, 104);
146 SET_AT_CFA_RULE(rdi, 112);
147 SET_AT_CFA_RULE(cs, 136);
148 SET_AT_CFA_RULE(rflags, 144);
149 SET_AT_CFA_RULE(ss, 160);
150 break;
151 case DRGN_ORC_TYPE_REGS_PARTIAL:
152 SET_AT_CFA_RULE(rip, 0);
153 SET_AT_CFA_RULE(rsp, 24);
154 #undef SET_AT_CFA_RULE
155 #define SET_SAME_VALUE_RULE(reg) do { \
156 rule.kind = DRGN_CFI_RULE_REGISTER_PLUS_OFFSET; \
157 rule.regno = DRGN_REGISTER_NUMBER(reg); \
158 rule.offset = 0; \
159 if (!drgn_cfi_row_set_register(row_ret, DRGN_REGISTER_NUMBER(reg), \
160 &rule)) \
161 return &drgn_enomem; \
162 } while (0)
163 // This ORC entry is for an interrupt handler before it saves
164 // the whole pt_regs. These registers are not clobbered before
165 // they are saved, so they should have the same value. See Linux
166 // kernel commit 81b67439d147 ("x86/unwind/orc: Fix premature
167 // unwind stoppage due to IRET frames").
168 //
169 // This probably also applies to other registers, but to stay on
170 // the safe side we only handle registers used by ORC.
171 SET_SAME_VALUE_RULE(r10);
172 SET_SAME_VALUE_RULE(r13);
173 SET_SAME_VALUE_RULE(rdi);
174 SET_SAME_VALUE_RULE(rdx);
175 #undef SET_SAME_VALUE_RULE
176 break;
177 default:
178 return drgn_error_format(DRGN_ERROR_OTHER,
179 "unknown ORC entry type %d",
180 drgn_orc_type(orc));
181 }
182
183 // For ORC_TYPE_REGS, rbp is already set.
184 if (drgn_orc_type(orc) != DRGN_ORC_TYPE_REGS) {
185 switch (drgn_orc_bp_reg(orc)) {
186 case DRGN_ORC_REG_UNDEFINED:
187 rule.kind = DRGN_CFI_RULE_REGISTER_PLUS_OFFSET;
188 rule.regno = DRGN_REGISTER_NUMBER(rbp);
189 rule.offset = 0;
190 break;
191 case DRGN_ORC_REG_PREV_SP:
192 rule.kind = DRGN_CFI_RULE_AT_CFA_PLUS_OFFSET;
193 rule.offset = orc->bp_offset;
194 break;
195 case DRGN_ORC_REG_BP:
196 rule.kind = DRGN_CFI_RULE_AT_REGISTER_PLUS_OFFSET;
197 rule.regno = DRGN_REGISTER_NUMBER(rbp);
198 rule.offset = orc->bp_offset;
199 break;
200 default:
201 return drgn_error_format(DRGN_ERROR_OTHER,
202 "unknown ORC BP base register %d",
203 drgn_orc_bp_reg(orc));
204 }
205 if (!drgn_cfi_row_set_register(row_ret, DRGN_REGISTER_NUMBER(rbp),
206 &rule))
207 return &drgn_enomem;
208 }
209
210 *interrupted_ret = drgn_orc_signal(orc);
211 *ret_addr_regno_ret = DRGN_REGISTER_NUMBER(rip);
212 return NULL;
213 }
214
215 static struct drgn_error *
216 get_registers_from_frame_pointer(struct drgn_program *prog,
217 uint64_t frame_pointer,
218 struct drgn_register_state **ret)
219 {
220 struct drgn_error *err;
221 uint64_t frame[2];
222 err = drgn_program_read_memory(prog, frame, frame_pointer,
223 sizeof(frame), false);
224 if (err)
225 return err;
226
227 struct drgn_register_state *regs =
228 drgn_register_state_create(rbp, false);
229 if (!regs)
230 return &drgn_enomem;
231 drgn_register_state_set_from_buffer(regs, rip, &frame[1]);
232 drgn_register_state_set_from_u64(prog, regs, rsp,
233 frame_pointer + sizeof(frame));
234 drgn_register_state_set_from_buffer(regs, rbp, &frame[0]);
235 drgn_register_state_set_pc_from_register(prog, regs, rip);
236 *ret = regs;
237 return NULL;
238 }
239
240 // Unwind a single call instruction.
241 static struct drgn_error *
242 bad_call_unwind_x86_64(struct drgn_program *prog,
243 struct drgn_register_state *regs,
244 struct drgn_register_state **ret)
245 {
246 struct drgn_error *err;
247
248 struct optional_uint64 rsp =
249 drgn_register_state_get_u64(prog, regs, rsp);
250 if (!rsp.has_value)
251 return &drgn_stop;
252
253 // Read the return address from the top of the stack.
254 uint64_t ret_addr;
255 err = drgn_program_read_u64(prog, rsp.value, false, &ret_addr);
256 if (err) {
257 if (err->code == DRGN_ERROR_FAULT) {
258 drgn_error_destroy(err);
259 err = &drgn_stop;
260 }
261 return err;
262 }
263
264 // Most of the registers are unchanged.
265 struct drgn_register_state *tmp = drgn_register_state_dup(regs);
266 if (!tmp)
267 return &drgn_enomem;
268
269 // The PC and rip are the return address we just read.
270 drgn_register_state_set_pc(prog, tmp, ret_addr);
271 drgn_register_state_set_from_u64(prog, tmp, rip, ret_addr);
272 // rsp is after the saved return address.
273 drgn_register_state_set_from_u64(prog, tmp, rsp, rsp.value + 8);
274 *ret = tmp;
275 return NULL;
276 }
277
278 // elf_gregset_t (in NT_PRSTATUS) and struct user_regs_struct have the same
279 // layout. struct pt_regs is a prefix of that layout which elides several
280 // segment registers. full_regset tells us which one we were given; true is
281 // elf_gregset_t, false is struct pt_regs.
282 static struct drgn_error *
283 get_initial_registers_from_struct_x86_64(struct drgn_program *prog,
284 const void *buf, size_t size,
285 bool full_regset,
286 struct drgn_register_state **ret)
287 {
288 if (size < (full_regset ? 216 : 168)) {
289 return drgn_error_create(DRGN_ERROR_INVALID_ARGUMENT,
290 "registers are truncated");
291 }
292
293 struct drgn_register_state *regs;
294 if (full_regset)
295 regs = drgn_register_state_create(gs, true);
296 else
297 regs = drgn_register_state_create(ss, true);
298 if (!regs)
299 return &drgn_enomem;
300
301 drgn_register_state_set_from_buffer(regs, rip, (uint64_t *)buf + 16);
302 drgn_register_state_set_from_buffer(regs, rsp, (uint64_t *)buf + 19);
303 drgn_register_state_set_range_from_buffer(regs, r15, rdi, buf);
304 drgn_register_state_set_range_from_buffer(regs, cs, rflags,
305 (uint64_t *)buf + 17);
306 if (full_regset) {
307 drgn_register_state_set_range_from_buffer(regs, ss, gs,
308 (uint64_t *)buf + 20);
309 } else {
310 drgn_register_state_set_from_buffer(regs, ss,
311 (uint64_t *)buf + 20);
312 }
313 drgn_register_state_set_pc_from_register(prog, regs, rip);
314
315 *ret = regs;
316 return NULL;
317 }
318
319 static struct drgn_error *
320 fallback_unwind_x86_64(struct drgn_program *prog,
321 struct drgn_register_state *regs,
322 struct drgn_register_state **ret)
323 {
324 struct drgn_error *err;
325
326 struct optional_uint64 rbp =
327 drgn_register_state_get_u64(prog, regs, rbp);
328 if (!rbp.has_value)
329 return &drgn_stop;
330
331 if ((prog->flags & DRGN_PROGRAM_IS_LINUX_KERNEL) && (rbp.value & 1)) {
332 // A frame pointer value with the LSB set is unaligned, and so
333 // it would likely be garbage. The x86_64 Linux kernel uses this
334 // to signal an "encoded" pt_regs (see decode_frame_pointer()).
335 // Try to treat this as an encoded frame pointer. On failure,
336 // don't bother trying the frame pointer unwind due to the
337 // unaligned RBP.
338 uint64_t regs_addr = rbp.value & ~1UL;
339 char pt_regs[168];
340 err = drgn_program_read_memory(prog, &pt_regs, regs_addr,
341 sizeof(pt_regs), false);
342 if (err) {
343 if (drgn_error_catch(&err, DRGN_ERROR_FAULT))
344 err = &drgn_stop;
345 return err;
346 }
347 err = get_initial_registers_from_struct_x86_64(prog, &pt_regs,
348 sizeof(pt_regs),
349 false, ret);
350 if (err)
351 return err;
352 } else {
353 err = get_registers_from_frame_pointer(prog, rbp.value, ret);
354 if (err) {
355 if (drgn_error_catch(&err, DRGN_ERROR_FAULT))
356 err = &drgn_stop;
357 return err;
358 }
359 drgn_register_state_set_cfa(prog, regs, rbp.value + 16);
360 }
361 return NULL;
362 }
363
364 static struct drgn_error *
365 pt_regs_get_initial_registers_x86_64(const struct drgn_object *obj,
366 struct drgn_register_state **ret)
367 {
368 return get_initial_registers_from_struct_x86_64(drgn_object_program(obj),
369 drgn_object_buffer(obj),
370 drgn_object_size(obj),
371 false, ret);
372 }
373
374 static struct drgn_error *
375 prstatus_get_initial_registers_x86_64(struct drgn_program *prog,
376 const void *prstatus, size_t size,
377 struct drgn_register_state **ret)
378 {
379 // offsetof(struct elf_prstatus, pr_reg)
380 static const size_t pr_reg_offset = 112;
381 if (size < pr_reg_offset) {
382 return drgn_error_create(DRGN_ERROR_INVALID_ARGUMENT,
383 "NT_PRSTATUS is truncated");
384 }
385 return get_initial_registers_from_struct_x86_64(prog,
386 (char *)prstatus + pr_reg_offset,
387 size - pr_reg_offset,
388 true, ret);
389 }
390
391 static struct drgn_error *
392 get_initial_registers_inactive_task_frame(struct drgn_object *frame_obj,
393 struct drgn_register_state **ret)
394 {
395 struct drgn_error *err;
396 struct drgn_program *prog = drgn_object_program(frame_obj);
397
398 uint64_t address = frame_obj->address;
399 err = drgn_object_read(frame_obj, frame_obj);
400 if (err)
401 return err;
402 const char *frame_buf = drgn_object_buffer(frame_obj);
403 size_t frame_size = drgn_object_size(frame_obj);
404
405 struct drgn_register_state *regs =
406 drgn_register_state_create(rbx, false);
407 if (!regs)
408 return &drgn_enomem;
409
410 #define COPY_REGISTER(id, member_name) do { \
411 struct drgn_type_member *member; \
412 uint64_t bit_offset; \
413 err = drgn_type_find_member(frame_obj->type, member_name, &member, \
414 &bit_offset); \
415 if (err) \
416 goto err; \
417 if (bit_offset / 8 + DRGN_REGISTER_SIZE(id) > frame_size) { \
418 err = drgn_error_create(DRGN_ERROR_OUT_OF_BOUNDS, \
419 "out of bounds of value"); \
420 goto err; \
421 } \
422 drgn_register_state_set_from_buffer(regs, id, \
423 frame_buf + bit_offset / 8); \
424 } while (0)
425
426 COPY_REGISTER(rip, "ret_addr");
427 COPY_REGISTER(r15, "r15");
428 COPY_REGISTER(r14, "r14");
429 COPY_REGISTER(r13, "r13");
430 COPY_REGISTER(r12, "r12");
431 COPY_REGISTER(rbp, "bp");
432 COPY_REGISTER(rbx, "bx");
433
434 #undef COPY_REGISTER
435
436 drgn_register_state_set_from_u64(prog, regs, rsp, address + frame_size);
437 drgn_register_state_set_pc_from_register(prog, regs, rip);
438
439 *ret = regs;
440 return NULL;
441
442 err:
443 drgn_register_state_destroy(regs);
444 return err;
445 }
446
447 static struct drgn_error *
448 linux_kernel_get_initial_registers_x86_64(const struct drgn_object *task_obj,
449 struct drgn_register_state **ret)
450 {
451 struct drgn_error *err;
452 struct drgn_program *prog = drgn_object_program(task_obj);
453
454 DRGN_OBJECT(sp_obj, prog);
455
456 err = drgn_object_member_dereference(&sp_obj, task_obj, "thread");
457 if (err)
458 return err;
459 err = drgn_object_member(&sp_obj, &sp_obj, "sp");
460 if (err)
461 return err;
462
463 // Since Linux kernel commit 0100301bfdf5 ("sched/x86: Rewrite the
464 // switch_to() code") (in v4.9), sp points to a struct
465 // inactive_task_frame, which we can use to get the callee-saved
466 // registers. Before that, sp points to bp. As long as frame pointers
467 // are enabled, this in turn points to the previous bp and the return
468 // address.
469 struct drgn_qualified_type frame_type;
470 err = drgn_program_find_type(prog, "struct inactive_task_frame *", NULL,
471 &frame_type);
472 if (!err) {
473 err = drgn_object_cast(&sp_obj, frame_type, &sp_obj);
474 if (err)
475 return err;
476 err = drgn_object_dereference(&sp_obj, &sp_obj);
477 if (err)
478 return err;
479 err = get_initial_registers_inactive_task_frame(&sp_obj, ret);
480 } else if (err->code == DRGN_ERROR_LOOKUP) {
481 drgn_error_destroy(err);
482 err = drgn_program_find_type(prog, "void **", NULL,
483 &frame_type);
484 if (err)
485 return err;
486 err = drgn_object_cast(&sp_obj, frame_type, &sp_obj);
487 if (err)
488 return err;
489 err = drgn_object_dereference(&sp_obj, &sp_obj);
490 if (err)
491 return err;
492 uint64_t frame_pointer;
493 err = drgn_object_read_unsigned(&sp_obj, &frame_pointer);
494 if (err)
495 return err;
496 err = get_registers_from_frame_pointer(prog, frame_pointer,
497 ret);
498 if (err == &drgn_stop) {
499 err = drgn_error_create(DRGN_ERROR_OTHER,
500 "invalid frame pointer");
501 }
502 }
503 return err;
504 }
505
506 static struct drgn_error *
507 apply_elf_reloc_x86_64(const struct drgn_relocating_section *relocating,
508 uint64_t r_offset, uint32_t r_type,
509 const int64_t *r_addend, uint64_t sym_value)
510 {
511 switch (r_type) {
512 case R_X86_64_NONE:
513 return NULL;
514 case R_X86_64_64:
515 return drgn_reloc_add64(relocating, r_offset, r_addend,
516 sym_value);
517 case R_X86_64_PC32:
518 return drgn_reloc_add32(relocating, r_offset, r_addend,
519 sym_value
520 - (relocating->addr + r_offset));
521 // The only difference between 32 and 32S is how overflow is checked,
522 // which we don't do.
523 case R_X86_64_32:
524 case R_X86_64_32S:
525 return drgn_reloc_add32(relocating, r_offset, r_addend,
526 sym_value);
527 case R_X86_64_PC64:
528 return drgn_reloc_add64(relocating, r_offset, r_addend,
529 sym_value
530 - (relocating->addr + r_offset));
531 default:
532 return DRGN_UNKNOWN_RELOCATION_TYPE(r_type);
533 }
534 }
535
536 static struct drgn_error *
537 linux_kernel_live_direct_mapping_fallback_x86_64(struct drgn_program *prog,
538 uint64_t *address_ret,
539 uint64_t *size_ret)
540 {
541 struct drgn_error *err;
542
543 *size_ret = UINT64_C(1) << 46;
544 unsigned long page_offset_base_address;
545 err = proc_kallsyms_symbol_addr("page_offset_base",
546 &page_offset_base_address);
547 if (!err) {
548 return drgn_program_read_word(prog, page_offset_base_address,
549 false, address_ret);
550 } else if (err == &drgn_not_found) {
551 // This is only called for pre-4.11 kernels, so we can assume
552 // the old location.
553 *address_ret = UINT64_C(0xffff880000000000);
554 return NULL;
555 } else {
556 return err;
557 }
558 }
559
560 static struct drgn_error *
561 linux_kernel_direct_mapping_offset_x86_64(struct drgn_program *prog,
562 uint64_t *address_ret)
563 {
564 DRGN_OBJECT(page_offset_base, prog);
565 struct drgn_error *err = drgn_program_find_object(prog, "page_offset_base",
566 NULL, DRGN_FIND_OBJECT_VARIABLE,
567 &page_offset_base);
568 if (err)
569 return err;
570
571 return drgn_object_read_unsigned(&page_offset_base, address_ret);
572 }
573
574 struct pgtable_iterator_x86_64 {
575 struct pgtable_iterator it;
576 uint16_t index[5];
577 uint64_t table[5][512];
578 };
579
580 static struct drgn_error *
581 linux_kernel_pgtable_iterator_create_x86_64(struct drgn_program *prog,
582 struct pgtable_iterator **ret)
583 {
584 struct pgtable_iterator_x86_64 *it = malloc(sizeof(*it));
585 if (!it)
586 return &drgn_enomem;
587 *ret = &it->it;
588 return NULL;
589 }
590
591 static void linux_kernel_pgtable_iterator_destroy_x86_64(struct pgtable_iterator *_it)
592 {
593 free(container_of(_it, struct pgtable_iterator_x86_64, it));
594 }
595
596 static void
597 linux_kernel_pgtable_iterator_init_x86_64(struct drgn_program *prog,
598 struct pgtable_iterator *_it)
599 {
600 struct pgtable_iterator_x86_64 *it =
601 container_of(_it, struct pgtable_iterator_x86_64, it);
602 memset(it->index, 0xff, sizeof(it->index));
603 }
604
605 static struct drgn_error *
606 linux_kernel_pgtable_iterator_next_x86_64(struct drgn_program *prog,
607 struct pgtable_iterator *_it,
608 uint64_t *virt_addr_ret,
609 uint64_t *phys_addr_ret)
610 {
611 static const int PAGE_SHIFT = 12;
612 static const int PGTABLE_SHIFT = 9;
613 static const int PGTABLE_MASK = (1 << PGTABLE_SHIFT) - 1;
614 static const uint64_t PRESENT = 0x1;
615 static const uint64_t PSE = 0x80; // a.k.a. huge page
616 static const uint64_t ADDRESS_MASK = UINT64_C(0xffffffffff000);
617 struct drgn_error *err;
618 bool bswap = drgn_platform_bswap(&prog->platform);
619 struct pgtable_iterator_x86_64 *it =
620 container_of(_it, struct pgtable_iterator_x86_64, it);
621 uint64_t virt_addr = it->it.virt_addr;
622 int levels = prog->vmcoreinfo.pgtable_l5_enabled ? 5 : 4;
623
624 uint64_t start_non_canonical =
625 (UINT64_C(1) <<
626 (PAGE_SHIFT + PGTABLE_SHIFT * levels - 1));
627 uint64_t end_non_canonical =
628 (UINT64_MAX <<
629 (PAGE_SHIFT + PGTABLE_SHIFT * levels - 1));
630 if (virt_addr >= start_non_canonical && virt_addr < end_non_canonical) {
631 *virt_addr_ret = start_non_canonical;
632 *phys_addr_ret = UINT64_MAX;
633 it->it.virt_addr = end_non_canonical;
634 return NULL;
635 }
636
637 // Find the lowest level with cached entries.
638 int level;
639 for (level = 0; level < levels; level++) {
640 if (it->index[level] < array_size(it->table[level]))
641 break;
642 }
643 // For every level below that, refill the cache/return pages.
644 for (;; level--) {
645 uint64_t table;
646 bool table_physical;
647 if (level == levels && prog->vmcoreinfo.have_phys_base &&
648 it->it.pgtable == prog->vmcoreinfo.swapper_pg_dir) {
649 // Avoid recursive address translation on swapper_pg_dir
650 // by directly resolving to a physical address.
651 // phys_base has been present since Linux kernel commit
652 // 401721ecd1dc ("kexec: export the value of phys_base
653 // instead of symbol address") (in v4.10).
654 table = it->it.pgtable + prog->vmcoreinfo.phys_base - START_KERNEL_MAP;
655 table_physical = true;
656 } else if (level == levels) {
657 table = it->it.pgtable;
658 table_physical = false;
659 } else {
660 uint64_t entry = it->table[level][it->index[level]++];
661 if (bswap)
662 entry = bswap_64(entry);
663 table = entry & ADDRESS_MASK;
664 if (!(entry & PRESENT) || (entry & PSE) || level == 0) {
665 uint64_t mask = (UINT64_C(1) <<
666 (PAGE_SHIFT +
667 PGTABLE_SHIFT * level)) - 1;
668 *virt_addr_ret = virt_addr & ~mask;
669 if (entry & PRESENT)
670 *phys_addr_ret = table & ~mask;
671 else
672 *phys_addr_ret = UINT64_MAX;
673 it->it.virt_addr = (virt_addr | mask) + 1;
674 return NULL;
675 }
676 table_physical = true;
677 }
678 uint16_t index = ((virt_addr >>
679 (PAGE_SHIFT + PGTABLE_SHIFT * (level - 1)))
680 & PGTABLE_MASK);
681 // It's only marginally more expensive to read 4096 bytes than 8
682 // bytes, so we always read to the end of the table.
683 err = drgn_program_read_memory(prog,
684 &it->table[level - 1][index],
685 table + 8 * index,
686 sizeof(it->table[0]) - 8 * index,
687 table_physical);
688 if (err)
689 return err;
690 it->index[level - 1] = index;
691 }
692 }
693
694 static int
695 linux_kernel_section_size_bits_fallback_x86_64(struct drgn_program *prog)
696 {
697 // This hasn't changed since Linux kernel commit bbfceef47fb9 ("[PATCH]
698 // add x86-64 specific support for sparsemem") (in v2.6.13).
699 return 27;
700 }
701
702 static int
703 linux_kernel_max_physmem_bits_fallback_x86_64(struct drgn_program *prog)
704 {
705 // This hasn't changed since Linux kernel commits 4c7c44837be7 ("x86/mm:
706 // Define virtual memory map for 5-level paging") (in v4.12) and
707 // c898faf91b3e ("x86: 46 bit physical address support on 64 bits") (in
708 // v2.6.31).
709 return prog->vmcoreinfo.pgtable_l5_enabled ? 52 : 46;
710 }
711
712 const struct drgn_architecture_info arch_info_x86_64 = {
713 .name = "x86-64",
714 .arch = DRGN_ARCH_X86_64,
715 .default_flags = (DRGN_PLATFORM_IS_64_BIT |
716 DRGN_PLATFORM_IS_LITTLE_ENDIAN),
717 .scalar_alignment = { 1, 2, 4, 8, 16 },
718 DRGN_ARCHITECTURE_REGISTERS,
719 .default_dwarf_cfi_row = &default_dwarf_cfi_row_x86_64,
720 .fallback_unwind = fallback_unwind_x86_64,
721 .bad_call_unwind = bad_call_unwind_x86_64,
722 .pt_regs_get_initial_registers = pt_regs_get_initial_registers_x86_64,
723 .prstatus_get_initial_registers = prstatus_get_initial_registers_x86_64,
724 .linux_kernel_get_initial_registers =
725 linux_kernel_get_initial_registers_x86_64,
726 .apply_elf_reloc = apply_elf_reloc_x86_64,
727 .linux_kernel_live_direct_mapping_fallback =
728 linux_kernel_live_direct_mapping_fallback_x86_64,
729 .linux_kernel_direct_mapping_offset =
730 linux_kernel_direct_mapping_offset_x86_64,
731 .linux_kernel_pgtable_iterator_create =
732 linux_kernel_pgtable_iterator_create_x86_64,
733 .linux_kernel_pgtable_iterator_destroy =
734 linux_kernel_pgtable_iterator_destroy_x86_64,
735 .linux_kernel_pgtable_iterator_init =
736 linux_kernel_pgtable_iterator_init_x86_64,
737 .linux_kernel_pgtable_iterator_next =
738 linux_kernel_pgtable_iterator_next_x86_64,
739 .linux_kernel_section_size_bits_fallback =
740 linux_kernel_section_size_bits_fallback_x86_64,
741 .linux_kernel_max_physmem_bits_fallback =
742 linux_kernel_max_physmem_bits_fallback_x86_64,
743 };