Debian Codegraph @

libdrgn/linux_kernel.c

raw
1 // Copyright (c) Meta Platforms, Inc. and affiliates.
2 // SPDX-License-Identifier: LGPL-2.1-or-later
3
4 #include <ctype.h>
5 #include <dirent.h>
6 #include <elf.h>
7 #include <elfutils/libdwelf.h>
8 #include <errno.h>
9 #include <fcntl.h>
10 #include <gelf.h>
11 #include <inttypes.h>
12 #include <libelf.h>
13 #include <stdarg.h>
14 #include <stdio.h>
15 #include <stdlib.h>
16 #include <string.h>
17 #include <sys/mman.h>
18 #include <sys/stat.h>
19 #include <unistd.h>
20
21 #include "array.h"
22 #include "binary_buffer.h"
23 #include "bitops.h"
24 #include "cleanup.h"
25 #include "debug_info.h"
26 #include "drgn_internal.h"
27 #include "elf_file.h"
28 #include "elf_notes.h"
29 #include "error.h"
30 #include "hash_table.h"
31 #include "helpers.h"
32 #include "hexlify.h"
33 #include "io.h"
34 #include "linux_kernel.h"
35 #include "log.h"
36 #include "platform.h"
37 #include "program.h"
38 #include "symbol.h"
39 #include "type.h"
40 #include "util.h"
41
42 #include "drgn_program_parse_vmcoreinfo.inc"
43
44 struct drgn_error *read_memory_via_pgtable(void *buf, uint64_t address,
45 size_t count, uint64_t offset,
46 void *arg, bool physical)
47 {
48 struct drgn_program *prog = arg;
49 return linux_helper_read_vm(prog, prog->vmcoreinfo.swapper_pg_dir,
50 address, buf, count);
51 }
52
53 struct drgn_error *proc_kallsyms_symbol_addr(const char *name,
54 unsigned long *ret)
55 {
56 struct drgn_error *err;
57 FILE *file;
58 char *line = NULL;
59 size_t n = 0;
60
61 file = fopen("/proc/kallsyms", "r");
62 if (!file)
63 return drgn_error_create_os("fopen", errno, "/proc/kallsyms");
64
65 for (;;) {
66 char *addr_str, *sym_str, *saveptr, *end;
67
68 errno = 0;
69 if (getline(&line, &n, file) == -1) {
70 if (errno) {
71 err = drgn_error_create_os("getline", errno,
72 "/proc/kallsyms");
73 } else {
74 err = &drgn_not_found;
75 }
76 break;
77 }
78
79 addr_str = strtok_r(line, "\t ", &saveptr);
80 if (!addr_str || !*addr_str)
81 goto invalid;
82 if (!strtok_r(NULL, "\t ", &saveptr))
83 goto invalid;
84 sym_str = strtok_r(NULL, "\t\n ", &saveptr);
85 if (!sym_str)
86 goto invalid;
87
88 if (strcmp(sym_str, name) != 0)
89 continue;
90
91 errno = 0;
92 *ret = strtoul(line, &end, 16);
93 if (errno || *end) {
94 invalid:
95 err = drgn_error_create(DRGN_ERROR_OTHER,
96 "could not parse /proc/kallsyms");
97 break;
98 }
99 err = NULL;
100 break;
101 }
102 free(line);
103 fclose(file);
104
105 // Depending on sysctl kptr_restrict and perf_event_paranoid, only a
106 // user with CAP_SYSLOG can access the kallsyms addresses. We're not
107 // guaranteed to be running as root even in the live case. Given that
108 // this function is mainly used as a fallback for specific symbols, we
109 // can be reasonably confident that the expected address is not zero.
110 // Catch the case where we find the symbol and read address zero, and
111 // raise an error that will hopefully be useful to the user.
112 if (!err && *ret == 0 && geteuid() != 0)
113 return drgn_error_create(DRGN_ERROR_OTHER,
114 "drgn requires root to access /proc/kallsyms");
115 return err;
116 }
117
118 /*
119 * Before Linux kernel commit 23c85094fe18 ("proc/kcore: add vmcoreinfo note to
120 * /proc/kcore") (in v4.19), /proc/kcore didn't have a VMCOREINFO note. Instead,
121 * we can read from the physical address of the vmcoreinfo note exported in
122 * sysfs.
123 */
124 struct drgn_error *read_vmcoreinfo_fallback(struct drgn_program *prog)
125 {
126 struct drgn_error *err;
127 FILE *file;
128 uint64_t address;
129 size_t size;
130 Elf64_Nhdr *nhdr;
131
132 file = fopen("/sys/kernel/vmcoreinfo", "r");
133 if (!file) {
134 return drgn_error_create_os("fopen", errno,
135 "/sys/kernel/vmcoreinfo");
136 }
137 if (fscanf(file, "%" SCNx64 "%zx", &address, &size) != 2) {
138 fclose(file);
139 return drgn_error_create(DRGN_ERROR_OTHER,
140 "could not parse /sys/kernel/vmcoreinfo");
141 }
142 fclose(file);
143
144 _cleanup_free_ char *buf = malloc(size);
145 if (!buf)
146 return &drgn_enomem;
147
148 err = drgn_program_read_memory(prog, buf, address, size, true);
149 if (err)
150 return err;
151
152 /*
153 * The first 12 bytes are the Elf{32,64}_Nhdr (it's the same in both
154 * formats). The name is padded up to 4 bytes, so the descriptor starts
155 * at byte 24.
156 */
157 nhdr = (Elf64_Nhdr *)buf;
158 if (size < 24 || nhdr->n_namesz != 11 ||
159 memcmp(buf + sizeof(*nhdr), "VMCOREINFO", 10) != 0 ||
160 nhdr->n_descsz > size - 24) {
161 err = drgn_error_create(DRGN_ERROR_OTHER,
162 "VMCOREINFO is invalid");
163 return err;
164 }
165
166 return drgn_program_parse_vmcoreinfo(prog, buf + 24, nhdr->n_descsz);
167 }
168
169 #define LINUX_KERNEL_GET_PRIMITIVE(name, primitive_type, signed_unsigned, expr) \
170 static struct drgn_error *linux_kernel_get_##name(struct drgn_program *prog, \
171 struct drgn_object *ret) \
172 { \
173 struct drgn_error *err; \
174 struct drgn_qualified_type qualified_type; \
175 err = drgn_program_find_primitive_type(prog, (primitive_type), \
176 &qualified_type.type); \
177 if (err) \
178 return err; \
179 qualified_type.qualifiers = 0; \
180 return drgn_object_set_##signed_unsigned(ret, qualified_type, (expr), \
181 0); \
182 }
183
184 #define LINUX_KERNEL_GET_PRIMITIVE_WRAPPER(name, primitive_type) \
185 static struct drgn_error *linux_kernel_get_##name(struct drgn_program *prog, \
186 struct drgn_object *ret) \
187 { \
188 struct drgn_error *err; \
189 typeof(_Generic(&linux_kernel_get_##name##_impl, \
190 struct drgn_error *(*)(struct drgn_program *, \
191 uint64_t *): (uint64_t)0, \
192 struct drgn_error *(*)(struct drgn_program *, \
193 int64_t *): (int64_t)0)) \
194 value; \
195 err = linux_kernel_get_##name##_impl(prog, &value); \
196 if (err) \
197 return err; \
198 struct drgn_qualified_type qualified_type; \
199 err = drgn_program_find_primitive_type(prog, (primitive_type), \
200 &qualified_type.type); \
201 if (err) \
202 return err; \
203 qualified_type.qualifiers = 0; \
204 return _Generic(value, \
205 uint64_t: drgn_object_set_unsigned, \
206 int64_t: drgn_object_set_signed) \
207 (ret, qualified_type, value, 0); \
208 }
209
210 LINUX_KERNEL_GET_PRIMITIVE(page_shift, DRGN_C_TYPE_INT, signed,
211 prog->vmcoreinfo.page_shift)
212
213 LINUX_KERNEL_GET_PRIMITIVE(page_size, DRGN_C_TYPE_UNSIGNED_LONG, unsigned,
214 prog->vmcoreinfo.page_size)
215
216 LINUX_KERNEL_GET_PRIMITIVE(page_mask, DRGN_C_TYPE_UNSIGNED_LONG, unsigned,
217 ~(prog->vmcoreinfo.page_size - 1))
218
219 static struct drgn_error *linux_kernel_get_thread_size(struct drgn_program *prog,
220 struct drgn_object *ret)
221 {
222 struct drgn_error *err;
223 struct drgn_qualified_type qualified_type, thread_union;
224 qualified_type.qualifiers = 0;
225 err = drgn_program_find_primitive_type(prog, DRGN_C_TYPE_UNSIGNED_LONG,
226 &qualified_type.type);
227 if (err)
228 return err;
229
230 if (prog->thread_size_cached)
231 return drgn_object_set_unsigned(ret, qualified_type,
232 prog->thread_size_cached, 0);
233
234 /* Prior to 0500871f21b23 ("Construct init thread stack in the linker
235 * script rather than by union") in v4.16, the file init/init_task.c
236 * defined a variable of type "union thread_union" which contains a
237 * member "stack" whose size is THREAD_SIZE. After that commit, it is
238 * defined via the linker script, and so the variable disappears from
239 * debuginfo, along with its type. Thankfully, the linker script defines
240 * symbols that can also be used to infer THREAD_SIZE.
241 *
242 * Normally, we optimize for recent kernels by putting their cases first
243 * in the code. But in this case, the "__{start,end}_init_task" symbols
244 * do exist on some architectures (e.g. ppc64) prior to v4.16. However,
245 * prior to v4.16, they aren't guaranteed to have the correct value of
246 * THREAD_SIZE. So, we need to check for "union thread_union" first, to
247 * get the most accurate value for those architectures prior to v4.16.
248 */
249 err = drgn_program_find_type(prog, "union thread_union", NULL,
250 &thread_union);
251 if (!err) {
252 struct drgn_type_member *stack_member;
253 uint64_t bit_offset_unused;
254 err = drgn_type_find_member(thread_union.type, "stack",
255 &stack_member, &bit_offset_unused);
256 if (err)
257 return err;
258
259 struct drgn_qualified_type stack_type;
260 err = drgn_member_type(stack_member, &stack_type, NULL);
261 if (err)
262 return err;
263
264 err = drgn_type_sizeof(stack_type.type, &prog->thread_size_cached);
265 if (err)
266 return err;
267
268 return drgn_object_set_unsigned(ret, qualified_type,
269 prog->thread_size_cached, 0);
270 } else if (!drgn_error_catch(&err, DRGN_ERROR_LOOKUP)) {
271 return err;
272 }
273
274 #define SYMBOL_START_END(symname_start, symname_end) do { \
275 struct drgn_symbol _cleanup_symbol_ *sym_start = NULL; \
276 struct drgn_symbol _cleanup_symbol_ *sym_end = NULL; \
277 err = drgn_program_find_symbol_by_name(prog, symname_start, &sym_start); \
278 if (drgn_error_catch(&err, DRGN_ERROR_LOOKUP)) \
279 break; \
280 else if (err) \
281 return err; \
282 err = drgn_program_find_symbol_by_name(prog, symname_end, &sym_end); \
283 if (err) \
284 return err; \
285 prog->thread_size_cached = sym_end->address - sym_start->address; \
286 return drgn_object_set_unsigned(ret, qualified_type, \
287 prog->thread_size_cached, 0); \
288 } while (0)
289
290 /* From Linux 4.16 up to 6.10's commit 8f69cba096b5c ("x86: Rename
291 * __{start,end}_init_task to __{start,end}_init_stack"), the symbols
292 * were named __{start,end}_init_task. Though the commit message
293 * indicates that the init_task is only used on x86, the symbols are
294 * present and accurate on other architectures regardless. */
295 SYMBOL_START_END("__start_init_task", "__end_init_task");
296
297 /* For Linux v6.10 and later, we can observe the stack size by the
298 * __{start,end}_init_stack symbols. */
299 SYMBOL_START_END("__start_init_stack", "__end_init_stack");
300 #undef SYMBOL_START_END
301
302 return &drgn_not_found;
303 }
304
305 static struct drgn_error *
306 linux_kernel_get_uts_release(struct drgn_program *prog, struct drgn_object *ret)
307 {
308 struct drgn_error *err;
309 struct drgn_qualified_type qualified_type;
310 err = drgn_program_find_primitive_type(prog,
311 DRGN_C_TYPE_CHAR,
312 &qualified_type.type);
313 if (err)
314 return err;
315 qualified_type.qualifiers = DRGN_QUALIFIER_CONST;
316 size_t len = strlen(prog->vmcoreinfo.osrelease);
317 err = drgn_array_type_create(prog, qualified_type, len + 1,
318 &drgn_language_c, &qualified_type.type);
319 if (err)
320 return err;
321 qualified_type.qualifiers = 0;
322 return drgn_object_set_from_buffer(ret, qualified_type,
323 prog->vmcoreinfo.osrelease, len + 1,
324 0, 0);
325 }
326
327 // jiffies is defined as an alias of jiffies_64 via the Linux kernel linker
328 // script, so it is not included in debug info.
329 static struct drgn_error *linux_kernel_get_jiffies(struct drgn_program *prog,
330 struct drgn_object *ret)
331 {
332 struct drgn_error *err;
333 DRGN_OBJECT(jiffies_64, prog);
334 err = drgn_program_find_object(prog, "jiffies_64", NULL,
335 DRGN_FIND_OBJECT_VARIABLE, &jiffies_64);
336 if (err) {
337 if (err->code == DRGN_ERROR_LOOKUP) {
338 drgn_error_destroy(err);
339 err = &drgn_not_found;
340 }
341 return err;
342 }
343 if (jiffies_64.kind != DRGN_OBJECT_REFERENCE)
344 return &drgn_not_found;
345 uint64_t address = jiffies_64.address;
346 struct drgn_qualified_type qualified_type;
347 err = drgn_program_find_primitive_type(prog, DRGN_C_TYPE_UNSIGNED_LONG,
348 &qualified_type.type);
349 if (err)
350 return err;
351 qualified_type.qualifiers = DRGN_QUALIFIER_VOLATILE;
352 if (drgn_type_size(qualified_type.type) == 4 &&
353 !drgn_type_little_endian(qualified_type.type))
354 address += 4;
355 return drgn_object_set_reference(ret, qualified_type, address, 0, 0);
356 }
357
358 static struct drgn_error *
359 linux_kernel_get_vmcoreinfo(struct drgn_program *prog, struct drgn_object *ret)
360 {
361 struct drgn_error *err;
362 struct drgn_qualified_type qualified_type;
363 err = drgn_program_find_primitive_type(prog,
364 DRGN_C_TYPE_CHAR,
365 &qualified_type.type);
366 if (err)
367 return err;
368 qualified_type.qualifiers = DRGN_QUALIFIER_CONST;
369 err = drgn_array_type_create(prog, qualified_type, prog->vmcoreinfo.raw_size,
370 &drgn_language_c, &qualified_type.type);
371 if (err)
372 return err;
373 qualified_type.qualifiers = 0;
374 return drgn_object_set_from_buffer(ret, qualified_type, prog->vmcoreinfo.raw,
375 prog->vmcoreinfo.raw_size, 0, 0);
376 }
377
378 // The vmemmap address can vary depending on architecture, kernel version,
379 // configuration options, and KASLR. However, we can get it generically from the
380 // section_mem_map of any valid mem_section.
381 static struct drgn_error *
382 linux_kernel_get_vmemmap_address(struct drgn_program *prog, uint64_t *ret)
383 {
384 static const uint64_t SECTION_HAS_MEM_MAP = 0x2;
385 static const uint64_t SECTION_MAP_MASK = ~((UINT64_C(1) << 6) - 1);
386 struct drgn_error *err;
387
388 DRGN_OBJECT(mem_section, prog);
389 DRGN_OBJECT(root, prog);
390 DRGN_OBJECT(section, prog);
391
392 err = drgn_program_find_object(prog, "vmemmap_populate", NULL,
393 DRGN_FIND_OBJECT_FUNCTION, &mem_section);
394 if (err) {
395 if (err->code == DRGN_ERROR_LOOKUP) {
396 // !CONFIG_SPARSEMEM_VMEMMAP
397 drgn_error_destroy(err);
398 err = &drgn_not_found;
399 }
400 return err;
401 }
402
403 err = drgn_program_find_object(prog, "mem_section", NULL,
404 DRGN_FIND_OBJECT_VARIABLE, &mem_section);
405 if (err)
406 return err;
407
408 const uint64_t nr_section_roots = prog->vmcoreinfo.mem_section_length;
409 uint64_t sections_per_root;
410 if (drgn_type_kind(mem_section.type) == DRGN_TYPE_ARRAY) {
411 // If !CONFIG_SPARSEMEM_EXTREME, mem_section is
412 // struct mem_section mem_section[NR_SECTION_ROOTS][SECTIONS_PER_ROOT],
413 // and SECTIONS_PER_ROOT is 1.
414 sections_per_root = 1;
415 } else {
416 // If CONFIG_SPARSEMEM_EXTREME, mem_section is
417 // struct mem_section **mem_section, and SECTIONS_PER_ROOT is
418 // PAGE_SIZE / sizeof(struct mem_section).
419 struct drgn_type *mem_section_type = mem_section.type;
420 for (int i = 0; i < 2; i++) {
421 if (drgn_type_kind(mem_section_type) != DRGN_TYPE_POINTER) {
422 unrecognized_mem_section_type:
423 return drgn_type_error("mem_section has unrecognized type '%s'",
424 mem_section.type);
425 }
426 mem_section_type = drgn_type_type(mem_section_type).type;
427 }
428 if (drgn_type_kind(mem_section_type) != DRGN_TYPE_STRUCT)
429 goto unrecognized_mem_section_type;
430 uint64_t sizeof_mem_section = drgn_type_size(mem_section_type);
431 if (sizeof_mem_section == 0)
432 goto unrecognized_mem_section_type;
433 sections_per_root =
434 prog->vmcoreinfo.page_size / sizeof_mem_section;
435 }
436
437 // Find a valid section.
438 for (uint64_t i = 0; i < nr_section_roots; i++) {
439 err = drgn_object_subscript(&root, &mem_section, i);
440 if (err)
441 return err;
442 bool truthy;
443 err = drgn_object_bool(&root, &truthy);
444 if (err)
445 return err;
446 if (!truthy)
447 continue;
448
449 for (uint64_t j = 0; j < sections_per_root; j++) {
450 err = drgn_object_subscript(&section, &root, j);
451 if (err)
452 return err;
453 err = drgn_object_member(&section, &section,
454 "section_mem_map");
455 if (err)
456 return err;
457 uint64_t section_mem_map;
458 err = drgn_object_read_unsigned(&section,
459 &section_mem_map);
460 if (err)
461 return err;
462 if (section_mem_map & SECTION_HAS_MEM_MAP) {
463 *ret = section_mem_map & SECTION_MAP_MASK;
464 return NULL;
465 }
466 }
467 }
468 return &drgn_not_found;
469 }
470
471 static struct drgn_error *linux_kernel_get_vmemmap(struct drgn_program *prog,
472 struct drgn_object *ret)
473 {
474 struct drgn_error *err;
475 if (prog->vmemmap.kind == DRGN_OBJECT_ABSENT) {
476 // Silence -Wmaybe-uninitialized false positive last seen with
477 // GCC 13 by initializing to zero.
478 uint64_t address = 0;
479 err = linux_kernel_get_vmemmap_address(prog, &address);
480 if (err)
481 return err;
482 struct drgn_qualified_type qualified_type;
483 err = drgn_program_find_type(prog, "struct page *", NULL,
484 &qualified_type);
485 if (err)
486 return err;
487 err = drgn_object_set_unsigned(&prog->vmemmap, qualified_type,
488 address, 0);
489 if (err)
490 return err;
491 }
492 return drgn_object_copy(ret, &prog->vmemmap);
493 }
494
495 static struct drgn_error *
496 linux_kernel_get_nr_section_roots_impl(struct drgn_program *prog, uint64_t *ret)
497 {
498 if (prog->vmcoreinfo.mem_section_length == 0)
499 return &drgn_not_found;
500 *ret = prog->vmcoreinfo.mem_section_length;
501 return NULL;
502 }
503 LINUX_KERNEL_GET_PRIMITIVE_WRAPPER(nr_section_roots, DRGN_C_TYPE_UNSIGNED_LONG)
504
505 static struct drgn_error *
506 linux_kernel_get_sections_per_root_impl(struct drgn_program *prog, uint64_t *ret)
507 {
508 struct drgn_error *err;
509
510 if (prog->cached_sections_per_root) {
511 *ret = prog->cached_sections_per_root;
512 return NULL;
513 }
514
515 if (!prog->vmcoreinfo.mem_section_length) // !SPARSEMEM
516 return &drgn_not_found;
517
518 DRGN_OBJECT(mem_section, prog);
519 err = drgn_program_find_object(prog, "mem_section", NULL,
520 DRGN_FIND_OBJECT_VARIABLE, &mem_section);
521 if (err)
522 return err;
523
524 // For SPARSEMEM_STATIC, mem_section is always an array of arrays. For
525 // SPARSEMEM_EXTREME, since Linux kernel commit 83e3c48729d9
526 // ("mm/sparsemem: Allocate mem_section at runtime for
527 // CONFIG_SPARSEMEM_EXTREME=y") (in v4.15), it is a pointer to a pointer
528 // to struct mem_section. Before that, it is an array of pointers to
529 // struct mem_section.
530
531 struct drgn_type *outer_type = drgn_underlying_type(mem_section.type);
532 enum drgn_type_kind outer_kind = drgn_type_kind(outer_type);
533 if (outer_kind != DRGN_TYPE_POINTER && outer_kind != DRGN_TYPE_ARRAY) {
534 return drgn_type_error("mem_section has unrecognized type: %s",
535 outer_type);
536 }
537
538 struct drgn_type *inner_type =
539 drgn_underlying_type(drgn_type_type(outer_type).type);
540 enum drgn_type_kind inner_kind = drgn_type_kind(inner_type);
541 if (outer_kind == DRGN_TYPE_ARRAY && inner_kind == DRGN_TYPE_ARRAY) {
542 // SPARSEMEM_STATIC: SECTIONS_PER_ROOT = 1
543 *ret = 1;
544 return NULL;
545 }
546 if (inner_kind != DRGN_TYPE_POINTER) {
547 return drgn_type_error("mem_section[0] has unrecognized type: %s",
548 inner_type);
549 }
550
551 // SPARSEMEM_EXTREME: SECTIONS_PER_ROOT = PAGE_SIZE / sizeof(struct mem_section)
552 uint64_t sizeof_mem_section;
553 err = drgn_type_sizeof(drgn_underlying_type(drgn_type_type(inner_type).type),
554 &sizeof_mem_section);
555 if (err)
556 return err;
557 if (!is_power_of_two(sizeof_mem_section)) {
558 return drgn_error_create(DRGN_ERROR_OTHER,
559 "struct mem_section has invalid size");
560 }
561 *ret = prog->cached_sections_per_root =
562 prog->vmcoreinfo.page_size / sizeof_mem_section;
563 return err;
564 }
565 LINUX_KERNEL_GET_PRIMITIVE_WRAPPER(sections_per_root, DRGN_C_TYPE_UNSIGNED_LONG)
566
567 static struct drgn_error *
568 linux_kernel_get_section_size_bits_impl(struct drgn_program *prog, int64_t *ret)
569 {
570 if (prog->vmcoreinfo.section_size_bits) {
571 *ret = prog->vmcoreinfo.section_size_bits;
572 return NULL;
573 }
574 if (!prog->vmcoreinfo.mem_section_length // !SPARSEMEM
575 || !prog->has_platform
576 || !prog->platform.arch->linux_kernel_section_size_bits_fallback)
577 return &drgn_not_found;
578 // Before Linux kernel commit 4f5aecdff25f ("crash_core, vmcoreinfo:
579 // append 'SECTION_SIZE_BITS' to vmcoreinfo") (in v5.13), we need
580 // architecture- and version-specific logic to determine
581 // SECTION_SIZE_BITS.
582 *ret = prog->vmcoreinfo.section_size_bits =
583 prog->platform.arch->linux_kernel_section_size_bits_fallback(prog);
584 return NULL;
585 }
586 LINUX_KERNEL_GET_PRIMITIVE_WRAPPER(section_size_bits, DRGN_C_TYPE_INT)
587
588 static struct drgn_error *
589 linux_kernel_get_max_physmem_bits_impl(struct drgn_program *prog, int64_t *ret)
590 {
591 struct drgn_error *err;
592 if (prog->vmcoreinfo.max_physmem_bits) {
593 *ret = prog->vmcoreinfo.max_physmem_bits;
594 return NULL;
595 }
596
597 if (!prog->vmcoreinfo.mem_section_length) // !SPARSEMEM
598 return &drgn_not_found;
599
600 // Before Linux kernel commit 1d50e5d0c505 ("crash_core, vmcoreinfo:
601 // Append 'MAX_PHYSMEM_BITS' to vmcoreinfo") (in v5.9), we can compute
602 // MAX_PHYSMEM_BITS from NR_SECTION_ROOTS and SECTION_SIZE_BITS. On
603 // architectures where's it's straightforward to figure out
604 // MAX_PHYSMEM_BITS, we can get it that way, too.
605 if (prog->has_platform
606 && prog->platform.arch->linux_kernel_max_physmem_bits_fallback) {
607 prog->vmcoreinfo.max_physmem_bits =
608 prog->platform.arch->linux_kernel_max_physmem_bits_fallback(prog);
609 } else {
610 // Given:
611 // NR_SECTION_ROOTS = NR_MEM_SECTIONS / SECTIONS_PER_ROOT
612 // NR_MEM_SECTIONS = 1 << SECTIONS_SHIFT
613 // SECTIONS_SHIFT = MAX_PHYSMEM_BITS - SECTION_SIZE_BITS
614 //
615 // Solve for MAX_PHYSMEM_BITS:
616 // => NR_SECTION_ROOTS = (1 << (MAX_PHYSMEM_BITS - SECTION_SIZE_BITS))
617 // / SECTIONS_PER_ROOT
618 //
619 // => NR_SECTION_ROOTS * SECTIONS_PER_ROOT
620 // = (1 << (MAX_PHYSMEM_BITS - SECTION_SIZE_BITS))
621 //
622 // => log2(NR_SECTION_ROOTS * SECTIONS_PER_ROOT)
623 // = MAX_PHYSMEM_BITS - SECTION_SIZE_BITS
624 //
625 // => MAX_PHYSMEM_BITS = log2(NR_SECTION_ROOTS * SECTIONS_PER_ROOT)
626 // + SECTION_SIZE_BITS
627 //
628 // (NR_SECTION_ROOTS and SECTIONS_PER_ROOT are always powers of
629 // two.)
630 //
631 // => MAX_PHYSMEM_BITS = log2(NR_SECTION_ROOTS)
632 // + log2(SECTIONS_PER_ROOT)
633 // + SECTION_SIZE_BITS
634 uint64_t sections_per_root;
635 err = linux_kernel_get_sections_per_root_impl(prog,
636 &sections_per_root);
637 if (err)
638 return err;
639 int64_t section_size_bits;
640 err = linux_kernel_get_section_size_bits_impl(prog,
641 &section_size_bits);
642 if (err)
643 return err;
644 prog->vmcoreinfo.max_physmem_bits =
645 ilog2(prog->vmcoreinfo.mem_section_length)
646 + ilog2(sections_per_root)
647 + section_size_bits;
648 }
649 *ret = prog->vmcoreinfo.max_physmem_bits;
650 return NULL;
651 }
652 LINUX_KERNEL_GET_PRIMITIVE_WRAPPER(max_physmem_bits, DRGN_C_TYPE_INT)
653
654 static struct drgn_error *
655 linux_kernel_get_arch_pfn_offset_impl(struct drgn_program *prog, uint64_t *ret)
656 {
657 struct drgn_error *err;
658 if (prog->arch_pfn_offset_cached) {
659 *ret = prog->arch_pfn_offset;
660 return NULL;
661 }
662 // Note that contig_page_data.node_mem_map is the struct page * for
663 // contig_page_data.node_start_pfn. Therefore:
664 // contig_page_data.node_mem_map = mem_map
665 // + contig_page_data.node_start_pfn
666 // - ARCH_PFN_OFFSET
667 // => ARCH_PFN_OFFSET = mem_map
668 // + contig_page_data.node_start_pfn
669 // - contig_page_data.node_mem_map
670 DRGN_OBJECT(contig_page_data, prog);
671 DRGN_OBJECT(tmp, prog);
672 DRGN_OBJECT(tmp2, prog);
673 err = drgn_program_find_object(prog, "mem_map", NULL,
674 DRGN_FIND_OBJECT_ANY, &tmp);
675 if (err)
676 return err;
677 err = drgn_program_find_object(prog, "contig_page_data", NULL,
678 DRGN_FIND_OBJECT_ANY, &contig_page_data);
679 if (err)
680 return err;
681 err = drgn_object_member(&tmp2, &contig_page_data, "node_start_pfn");
682 if (err)
683 return err;
684 err = drgn_object_add(&tmp, &tmp, &tmp2);
685 if (err)
686 return err;
687 err = drgn_object_member(&tmp2, &contig_page_data, "node_mem_map");
688 if (err)
689 return err;
690 err = drgn_object_sub(&tmp, &tmp, &tmp2);
691 if (err)
692 return err;
693 int64_t arch_pfn_offset;
694 err = drgn_object_read_signed(&tmp, &arch_pfn_offset);
695 if (err)
696 return err;
697 prog->arch_pfn_offset = arch_pfn_offset;
698 prog->arch_pfn_offset_cached = true;
699 *ret = prog->arch_pfn_offset;
700 return NULL;
701 }
702 LINUX_KERNEL_GET_PRIMITIVE_WRAPPER(arch_pfn_offset, DRGN_C_TYPE_UNSIGNED_LONG)
703
704 #include "linux_kernel_object_find.inc" // IWYU pragma: keep
705
706 // Return whether the given kernel is from Fedora. We check whether the release
707 // matches the regular expression /.fc[0-9]+(.|$)/
708 static bool is_fedora_kernel(const char *osrelease)
709 {
710 const char *p = osrelease;
711 while ((p = strstr(p, ".fc"))) {
712 p += sizeof(".fc") - 1;
713 if (isdigit(*p)) {
714 do {
715 p++;
716 } while (isdigit(*p));
717 if (*p == '.' || *p == '\0')
718 return true;
719 }
720 }
721 return false;
722 }
723
724 struct drgn_error *drgn_program_finish_set_kernel(struct drgn_program *prog)
725 {
726 struct drgn_error *err;
727 const struct drgn_object_finder_ops ops = {
728 .find = linux_kernel_object_find,
729 };
730 err = drgn_program_register_object_finder(prog, "linux", &ops, prog, 0);
731 if (err)
732 return err;
733 if (!prog->lang)
734 prog->lang = &drgn_language_c;
735
736 // At the time of writing, only Fedora's debuginfod server provides fast
737 // Linux kernel downloads. It's painfully slow everywhere else, so
738 // disable it.
739 if (!is_fedora_kernel(prog->vmcoreinfo.osrelease)
740 && drgn_handler_list_disable(&prog->dbinfo.debug_info_finders,
741 "debuginfod"))
742 drgn_log_debug(prog, "disabled debuginfod for Linux kernel");
743
744 return NULL;
745 }
746
747 /*
748 * /lib/modules/$(uname -r)/modules.dep.bin maps all installed kernel modules to
749 * their filesystem path (and dependencies, which we don't care about). It is
750 * generated by depmod; the format is a fairly simple serialized radix tree.
751 *
752 * modules.dep(5) contains a warning: "These files are not intended for editing
753 * or use by any additional utilities as their format is subject to change in
754 * the future." But, the format hasn't changed since 2009, and pulling in
755 * libkmod is overkill since we only need a very small subset of its
756 * functionality (plus our minimal parser is more efficient). If the format
757 * changes in the future, we can reevaluate this.
758 */
759
760 static void depmod_index_deinit(struct depmod_index *depmod)
761 {
762 if (depmod->len > 0)
763 munmap(depmod->addr, depmod->len);
764 free(depmod->path);
765 }
766
767 struct depmod_index_buffer {
768 struct binary_buffer bb;
769 struct depmod_index *depmod;
770 };
771
772 static struct drgn_error *depmod_index_buffer_error(struct binary_buffer *bb,
773 const char *pos,
774 const char *message)
775 {
776 struct depmod_index_buffer *buffer =
777 container_of(bb, struct depmod_index_buffer, bb);
778 return drgn_error_format(DRGN_ERROR_OTHER, "%s: %#tx: %s",
779 buffer->depmod->path,
780 pos - (const char *)buffer->depmod->addr,
781 message);
782 }
783
784 static void depmod_index_buffer_init(struct depmod_index_buffer *buffer,
785 struct depmod_index *depmod)
786 {
787 binary_buffer_init(&buffer->bb, depmod->addr, depmod->len, false,
788 depmod_index_buffer_error);
789 buffer->depmod = depmod;
790 }
791
792 static struct drgn_error *depmod_index_validate(struct depmod_index *depmod)
793 {
794 struct drgn_error *err;
795 struct depmod_index_buffer buffer;
796 depmod_index_buffer_init(&buffer, depmod);
797 uint32_t magic;
798 if ((err = binary_buffer_next_u32(&buffer.bb, &magic)))
799 return err;
800 if (magic != 0xb007f457) {
801 return binary_buffer_error(&buffer.bb,
802 "invalid magic 0x%" PRIx32, magic);
803 }
804 uint32_t version;
805 if ((err = binary_buffer_next_u32(&buffer.bb, &version)))
806 return err;
807 if (version != 0x00020001) {
808 return binary_buffer_error(&buffer.bb,
809 "unknown version 0x%" PRIx32,
810 version);
811 }
812 return NULL;
813 }
814
815 static struct drgn_error *depmod_index_init(struct depmod_index *depmod,
816 char *_path, int fd)
817 {
818 struct drgn_error *err;
819 _cleanup_free_ char *path = _path; // Take ownership of path.
820
821 struct stat st;
822 if (fstat(fd, &st) == -1)
823 return drgn_error_create_os("fstat", errno, path);
824
825 if (st.st_size > SIZE_MAX)
826 return &drgn_enomem;
827
828 void *addr = mmap(NULL, st.st_size, PROT_READ, MAP_PRIVATE, fd, 0);
829 if (addr == MAP_FAILED)
830 return drgn_error_create_os("mmap", errno, path);
831
832 depmod->path = no_cleanup_ptr(path);
833 depmod->addr = addr;
834 depmod->len = st.st_size;
835 err = depmod_index_validate(depmod);
836 if (err) {
837 depmod_index_deinit(depmod);
838 depmod->path = NULL;
839 depmod->len = 0;
840 }
841 return err;
842 }
843
844 /*
845 * Look up the path of the kernel module with the given name.
846 *
847 * @param[in] name Name of the kernel module.
848 * @param[out] path_ret Returned path of the kernel module, relative to
849 * /lib/modules/$(uname -r). This is @em not null-terminated. @c NULL if not
850 * found.
851 * @param[out] len_ret Returned length of @p path_ret.
852 */
853 static struct drgn_error *depmod_index_find(struct depmod_index *depmod,
854 const char *name,
855 const char **path_ret,
856 size_t *len_ret)
857 {
858 static const uint32_t INDEX_NODE_MASK = UINT32_C(0x0fffffff);
859 static const uint32_t INDEX_NODE_CHILDS = UINT32_C(0x20000000);
860 static const uint32_t INDEX_NODE_VALUES = UINT32_C(0x40000000);
861 static const uint32_t INDEX_NODE_PREFIX = UINT32_C(0x80000000);
862
863 struct drgn_error *err;
864 struct depmod_index_buffer buffer;
865 depmod_index_buffer_init(&buffer, depmod);
866
867 /* depmod_index_validate() already checked that this is within bounds. */
868 buffer.bb.pos += 8;
869 uint32_t offset;
870 for (;;) {
871 if ((err = binary_buffer_next_u32(&buffer.bb, &offset)))
872 return err;
873 if ((offset & INDEX_NODE_MASK) > depmod->len) {
874 return binary_buffer_error(&buffer.bb,
875 "offset is out of bounds");
876 }
877 buffer.bb.pos = (const char *)depmod->addr + (offset & INDEX_NODE_MASK);
878
879 if (offset & INDEX_NODE_PREFIX) {
880 const char *prefix;
881 size_t prefix_len;
882 if ((err = binary_buffer_next_string(&buffer.bb,
883 &prefix,
884 &prefix_len)))
885 return err;
886 if (strncmp(name, prefix, prefix_len) != 0)
887 goto not_found;
888 name += prefix_len;
889 }
890
891 if (offset & INDEX_NODE_CHILDS) {
892 uint8_t first, last;
893 if ((err = binary_buffer_next_u8(&buffer.bb, &first)) ||
894 (err = binary_buffer_next_u8(&buffer.bb, &last)))
895 return err;
896 if (*name) {
897 uint8_t cur = *name;
898 if (cur < first || cur > last)
899 goto not_found;
900 if ((err = binary_buffer_skip(&buffer.bb,
901 4 * (cur - first))))
902 return err;
903 name++;
904 continue;
905 } else {
906 if ((err = binary_buffer_skip(&buffer.bb,
907 4 * (last - first + 1))))
908 return err;
909 break;
910 }
911 } else if (*name) {
912 goto not_found;
913 } else {
914 break;
915 }
916 }
917 if (!(offset & INDEX_NODE_VALUES))
918 goto not_found;
919
920 uint32_t value_count;
921 if ((err = binary_buffer_next_u32(&buffer.bb, &value_count)))
922 return err;
923 if (!value_count)
924 goto not_found; /* Or is this malformed? */
925
926 /* Skip over priority. */
927 if ((err = binary_buffer_skip(&buffer.bb, 4)))
928 return err;
929
930 const char *colon = memchr(buffer.bb.pos, ':',
931 buffer.bb.end - buffer.bb.pos);
932 if (!colon) {
933 return binary_buffer_error(&buffer.bb,
934 "expected string containing ':'");
935 }
936 *path_ret = buffer.bb.pos;
937 *len_ret = colon - buffer.bb.pos;
938 return NULL;
939
940 not_found:
941 *path_ret = NULL;
942 return NULL;
943 }
944
945 DEFINE_VECTOR_FUNCTIONS(char_p_vector);
946
947 DEFINE_HASH_MAP_FUNCTIONS(drgn_kmod_walk_module_map, c_string_key_hash_pair,
948 c_string_key_eq);
949
950 struct drgn_kmod_walk_stack_entry {
951 DIR *dir;
952 size_t path_len;
953 };
954
955 DEFINE_VECTOR_FUNCTIONS(drgn_kmod_walk_stack);
956
957 static inline struct hash_pair
958 drgn_kmod_walk_inode_hash_pair(const struct drgn_kmod_walk_inode *entry)
959 {
960 return hash_pair_from_avalanching_hash(hash_combine(entry->dev, entry->ino));
961 }
962
963 static inline bool
964 drgn_kmod_walk_inode_eq(const struct drgn_kmod_walk_inode *a,
965 const struct drgn_kmod_walk_inode *b)
966 {
967 return a->dev == b->dev && a->ino == b->ino;
968 }
969
970 DEFINE_HASH_SET_FUNCTIONS(drgn_kmod_walk_inode_set,
971 drgn_kmod_walk_inode_hash_pair,
972 drgn_kmod_walk_inode_eq);
973
974 static void
975 drgn_kmod_walk_module_map_entry_deinit(struct drgn_kmod_walk_module_map_entry *entry)
976 {
977 vector_for_each(char_p_vector, path, &entry->value)
978 free(*path);
979 char_p_vector_deinit(&entry->value);
980 }
981
982 static void
983 drgn_kmod_walk_state_deinit(struct drgn_kmod_walk_state *state)
984 {
985 drgn_kmod_walk_inode_set_deinit(&state->visited_dirs);
986 string_builder_deinit(&state->path);
987 vector_for_each(drgn_kmod_walk_stack, entry, &state->stack)
988 closedir(entry->dir);
989 drgn_kmod_walk_stack_deinit(&state->stack);
990 hash_table_for_each(drgn_kmod_walk_module_map, it, &state->modules)
991 drgn_kmod_walk_module_map_entry_deinit(it.entry);
992 drgn_kmod_walk_module_map_deinit(&state->modules);
993 }
994
995 void
996 drgn_standard_debug_info_find_state_deinit(struct drgn_standard_debug_info_find_state *state)
997 {
998 drgn_kmod_walk_state_deinit(&state->kmod_walk);
999 depmod_index_deinit(&state->modules_dep);
1000 }
1001
1002 static struct drgn_error *
1003 drgn_module_try_vmlinux_in_debug_directories(struct drgn_module *module,
1004 const struct drgn_debug_info_options *options,
1005 struct string_builder *sb)
1006 {
1007 struct drgn_error *err;
1008 // Paths relative to the debug directory where vmlinux might be
1009 // installed.
1010 static const char * const debug_dir_paths[] = {
1011 // Debian, Ubuntu:
1012 "/boot/vmlinux-%s",
1013 // Fedora, CentOS:
1014 "/lib/modules/%s/vmlinux",
1015 // SUSE:
1016 "/lib/modules/%s/vmlinux.debug",
1017 };
1018 for (size_t i = 0; options->directories[i]; i++) {
1019 const char *debug_dir = options->directories[i];
1020 sb->len = 0;
1021 if (!string_builder_append(sb, debug_dir))
1022 return &drgn_enomem;
1023 size_t debug_dir_len = sb->len;
1024 array_for_each(format, debug_dir_paths) {
1025 sb->len = debug_dir_len;
1026 if (!string_builder_appendf(sb, *format,
1027 module->prog->vmcoreinfo.osrelease)
1028 || !string_builder_null_terminate(sb))
1029 return &drgn_enomem;
1030 err = drgn_module_try_standard_file(module, options,
1031 sb->str, -1, true,
1032 NULL);
1033 if (err || !drgn_module_wants_file(module))
1034 return err;
1035 }
1036 }
1037 return NULL;
1038 }
1039
1040 struct drgn_error *
1041 drgn_module_try_vmlinux_files(struct drgn_module *module,
1042 const struct drgn_debug_info_options *options)
1043 {
1044 struct drgn_error *err;
1045 struct drgn_program *prog = module->prog;
1046
1047 const char *osrelease = prog->vmcoreinfo.osrelease;
1048 STRING_BUILDER(sb);
1049 for (size_t i = 0; options->kernel_directories[i]; i++) {
1050 const char *kernel_dir = options->kernel_directories[i];
1051
1052 if (kernel_dir[0]) {
1053 sb.len = 0;
1054 if (!string_builder_append(&sb, kernel_dir))
1055 return &drgn_enomem;
1056 } else {
1057 // Empty path. Try under the debug directories first.
1058 err = drgn_module_try_vmlinux_in_debug_directories(module,
1059 options,
1060 &sb);
1061 if (err || !drgn_module_wants_file(module))
1062 return err;
1063
1064 // Try /boot/vmlinux-$osrelease.
1065 sb.len = 0;
1066 if (!string_builder_append(&sb, "/boot/vmlinux-")
1067 || !string_builder_append(&sb, osrelease)
1068 || !string_builder_null_terminate(&sb))
1069 return &drgn_enomem;
1070 err = drgn_module_try_standard_file(module, options,
1071 sb.str, -1, true,
1072 NULL);
1073 if (err || !drgn_module_wants_file(module))
1074 return err;
1075
1076 // Try /lib/modules/$osrelease as the kernel directory.
1077 sb.len = 0;
1078 if (!string_builder_append(&sb, "/lib/modules/")
1079 || !string_builder_append(&sb, osrelease))
1080 return &drgn_enomem;
1081 }
1082
1083 // Paths relative to the kernel directory where vmlinux might be
1084 // installed.
1085 static const char * const kernel_dir_paths[] = {
1086 "/build/vmlinux",
1087 "/vmlinux",
1088 };
1089 size_t kernel_dir_len = sb.len;
1090 array_for_each(path, kernel_dir_paths) {
1091 if (!string_builder_append(&sb, *path)
1092 || !string_builder_null_terminate(&sb))
1093 return &drgn_enomem;
1094 err = drgn_module_try_standard_file(module, options,
1095 sb.str, -1, true,
1096 NULL);
1097 if (err || !drgn_module_wants_file(module))
1098 return err;
1099 sb.len = kernel_dir_len;
1100 }
1101 }
1102
1103 return NULL;
1104 }
1105
1106 static struct drgn_error *
1107 drgn_open_modules_dep(struct drgn_program *prog,
1108 const struct drgn_debug_info_options *options,
1109 struct depmod_index *modules_dep)
1110 {
1111 struct drgn_error *err;
1112
1113 if (modules_dep->addr)
1114 return NULL;
1115
1116 STRING_BUILDER(sb);
1117 _cleanup_close_ int fd = -1;
1118 for (size_t i = 0; options->kernel_directories[i]; i++) {
1119 const char *kernel_dir = options->kernel_directories[i];
1120
1121 sb.len = 0;
1122 if (kernel_dir[0]) {
1123 if (!string_builder_append(&sb, kernel_dir))
1124 return &drgn_enomem;
1125 } else {
1126 // Empty path. Try /lib/modules/$osrelease.
1127 if (!string_builder_append(&sb, "/lib/modules/")
1128 || !string_builder_append(&sb,
1129 prog->vmcoreinfo.osrelease))
1130 return &drgn_enomem;
1131 }
1132 if (!string_builder_append(&sb, "/modules.dep.bin")
1133 || !string_builder_null_terminate(&sb))
1134 return &drgn_enomem;
1135 fd = open(sb.str, O_RDONLY);
1136 if (fd >= 0)
1137 break;
1138 drgn_log_debug(prog, "%s: %m", sb.str);
1139 }
1140 if (fd < 0) {
1141 drgn_log_debug(prog, "couldn't find depmod index");
1142 fail:
1143 // Set addr so that we don't try again.
1144 modules_dep->addr = MAP_FAILED;
1145 return NULL;
1146 }
1147
1148 err = depmod_index_init(modules_dep, string_builder_steal(&sb), fd);
1149 if (err) {
1150 if (drgn_error_is_fatal(err))
1151 return err;
1152 drgn_error_log_warning(prog, err,
1153 "couldn't open depmod index: ");
1154 drgn_error_destroy(err);
1155 goto fail;
1156 }
1157 drgn_log_debug(prog, "found depmod index %s", modules_dep->path);
1158 return NULL;
1159 }
1160
1161 static struct drgn_error *
1162 drgn_module_try_depmod_in_debug_directories(struct drgn_module *module,
1163 const struct drgn_debug_info_options *options,
1164 struct string_builder *sb,
1165 const char *depmod_path, size_t ko_len)
1166 {
1167 struct drgn_error *err;
1168 for (size_t i = 0; options->directories[i]; i++) {
1169 const char *debug_dir = options->directories[i];
1170 sb->len = 0;
1171 // Debian, Ubuntu:
1172 // $debug_dir/lib/modules/$(uname -r)/$ko_name
1173 if (!string_builder_append(sb, debug_dir)
1174 || !string_builder_append(sb, "/lib/modules/")
1175 || !string_builder_append(sb,
1176 module->prog->vmcoreinfo.osrelease)
1177 || !string_builder_appendc(sb, '/')
1178 || !string_builder_appendn(sb, depmod_path, ko_len)
1179 || !string_builder_null_terminate(sb))
1180 return &drgn_enomem;
1181 err = drgn_module_try_standard_file(module, options, sb->str,
1182 -1, true, NULL);
1183 if (err || !drgn_module_wants_file(module))
1184 return err;
1185
1186 // Fedora, CentOS, SUSE:
1187 // $debug_dir/lib/modules/$(uname -r)/$ko_name.debug
1188 if (!string_builder_append(sb, ".debug")
1189 || !string_builder_null_terminate(sb))
1190 return &drgn_enomem;
1191 err = drgn_module_try_standard_file(module, options, sb->str,
1192 -1, true, NULL);
1193 if (err || !drgn_module_wants_file(module))
1194 return err;
1195 }
1196 return NULL;
1197 }
1198
1199 static struct drgn_error *
1200 drgn_module_try_linux_kmod_depmod(struct drgn_module *module,
1201 const struct drgn_debug_info_options *options,
1202 struct drgn_standard_debug_info_find_state *state)
1203 {
1204 struct drgn_error *err;
1205 struct drgn_program *prog = module->prog;
1206
1207 const char *depmod_path;
1208 size_t depmod_path_len;
1209 err = depmod_index_find(&state->modules_dep, module->name, &depmod_path,
1210 &depmod_path_len);
1211 if (err) {
1212 drgn_error_log_warning(prog, err,
1213 "couldn't parse depmod index: ");
1214 drgn_error_destroy(err);
1215 return NULL;
1216 }
1217 if (!depmod_path) {
1218 drgn_log_debug(prog, "couldn't find %s in depmod index",
1219 module->name);
1220 return NULL;
1221 }
1222 drgn_log_debug(prog, "found %.*s in depmod index",
1223 depmod_path_len > INT_MAX
1224 ? INT_MAX : (int)depmod_path_len,
1225 depmod_path);
1226
1227 // Get the length of the path with one extension after ".ko" removed if
1228 // present (e.g., ".gz", ".xz", or ".zst").
1229 const char *name = memrchr(depmod_path, '/', depmod_path_len);
1230 if (name)
1231 name = name + 1;
1232 else
1233 name = depmod_path;
1234 const char *name_end = depmod_path + depmod_path_len;
1235 size_t ko_len = depmod_path_len;
1236 for (int j = 0; j < 2; j++) {
1237 char *dot = memrchr(name, '.', name_end - name);
1238 if (!dot)
1239 break;
1240 if (name_end - dot == 3
1241 && dot[1] == 'k' && dot[2] == 'o') {
1242 ko_len = name_end - depmod_path;
1243 break;
1244 }
1245 name_end = dot;
1246 }
1247
1248 STRING_BUILDER(sb);
1249 for (size_t i = 0; options->kernel_directories[i]; i++) {
1250 const char *kernel_dir = options->kernel_directories[i];
1251
1252 if (kernel_dir[0]) {
1253 sb.len = 0;
1254 if (!string_builder_append(&sb, kernel_dir))
1255 return &drgn_enomem;
1256 } else {
1257 // Empty path. Try under the debug directories first.
1258 err = drgn_module_try_depmod_in_debug_directories(module,
1259 options,
1260 &sb,
1261 depmod_path,
1262 ko_len);
1263 if (err || !drgn_module_wants_file(module))
1264 return err;
1265
1266 // Try /lib/modules/$osrelease as the kernel directory.
1267 sb.len = 0;
1268 if (!string_builder_append(&sb, "/lib/modules/")
1269 || !string_builder_append(&sb,
1270 prog->vmcoreinfo.osrelease))
1271 return &drgn_enomem;
1272 }
1273 if (!string_builder_appendc(&sb, '/')
1274 || !string_builder_appendn(&sb, depmod_path, depmod_path_len)
1275 || !string_builder_null_terminate(&sb))
1276 return &drgn_enomem;
1277 err = drgn_module_try_standard_file(module, options, sb.str, -1,
1278 true, NULL);
1279 if (err || !drgn_module_wants_file(module))
1280 return err;
1281 }
1282 return NULL;
1283 }
1284
1285 static struct drgn_error *
1286 drgn_kmod_walk_next_dir(struct drgn_program *prog,
1287 const struct drgn_debug_info_options *options,
1288 struct drgn_kmod_walk_state *state)
1289 {
1290 struct string_builder *path = &state->path;
1291 for (;;) {
1292 if (state->next_debug_dir) {
1293 const char *debug_dir = *state->next_debug_dir++;
1294 path->len = 0;
1295 if (debug_dir) {
1296 if (!string_builder_append(path, debug_dir))
1297 return &drgn_enomem;
1298 } else {
1299 state->next_debug_dir = NULL;
1300 }
1301 if (!string_builder_append(path, "/lib/modules/")
1302 || !string_builder_append(path,
1303 prog->vmcoreinfo.osrelease))
1304 return &drgn_enomem;
1305 } else {
1306 const char *kernel_dir = *state->next_kernel_dir;
1307 if (!kernel_dir)
1308 return &drgn_stop;
1309 state->next_kernel_dir++;
1310 if (kernel_dir[0]) {
1311 path->len = 0;
1312 if (!string_builder_append(path, kernel_dir))
1313 return &drgn_enomem;
1314 } else {
1315 state->next_debug_dir = options->directories;
1316 continue;
1317 }
1318 }
1319
1320 if (!string_builder_null_terminate(path))
1321 return &drgn_enomem;
1322 struct drgn_kmod_walk_stack_entry entry = {
1323 .dir = opendir(path->str),
1324 .path_len = path->len,
1325 };
1326 if (!entry.dir) {
1327 drgn_log_debug(prog, "opendir: %s: %m", path->str);
1328 continue;
1329 }
1330 if (!drgn_kmod_walk_stack_append(&state->stack, &entry)) {
1331 closedir(entry.dir);
1332 return &drgn_enomem;
1333 }
1334 drgn_log_debug(prog, "searching for kernel modules in %s",
1335 path->str);
1336 return NULL;
1337 }
1338 }
1339
1340 static struct drgn_error *
1341 drgn_kmod_walk(struct drgn_program *prog,
1342 const struct drgn_debug_info_options *options,
1343 struct drgn_kmod_walk_state *state,
1344 struct drgn_kmod_walk_module_map_entry *current)
1345 {
1346 struct drgn_error *err;
1347 struct string_builder *path = &state->path;
1348
1349 for (;;) {
1350 if (drgn_kmod_walk_stack_empty(&state->stack)) {
1351 err = drgn_kmod_walk_next_dir(prog, options, state);
1352 if (err)
1353 return err;
1354 }
1355
1356 struct drgn_kmod_walk_stack_entry *top =
1357 drgn_kmod_walk_stack_last(&state->stack);
1358 errno = 0;
1359 struct dirent *ent = readdir(top->dir);
1360 if (!ent) {
1361 if (errno) {
1362 path->str[top->path_len] = '\0';
1363 drgn_log_debug(prog, "%s: readdir: %m",
1364 path->str);
1365 }
1366 closedir(top->dir);
1367 drgn_kmod_walk_stack_pop(&state->stack);
1368 continue;
1369 }
1370
1371 // Skip "." and "..".
1372 if (ent->d_name[0] == '.'
1373 && (!ent->d_name[1]
1374 || (ent->d_name[1] == '.' && !ent->d_name[2])))
1375 continue;
1376
1377 bool is_directory = false;
1378 if (ent->d_type == DT_LNK || ent->d_type == DT_UNKNOWN) {
1379 struct stat st;
1380 if (fstatat(dirfd(top->dir), ent->d_name, &st, 0) < 0) {
1381 path->str[top->path_len] = '\0';
1382 drgn_log_debug(prog, "%s/%s: fstatat: %m",
1383 path->str, ent->d_name);
1384 continue;
1385 }
1386 if (S_ISDIR(st.st_mode))
1387 is_directory = true;
1388 else if (!S_ISREG(st.st_mode))
1389 continue;
1390 } else if (ent->d_type == DT_DIR) {
1391 is_directory = true;
1392 } else if (ent->d_type != DT_REG) {
1393 continue;
1394 }
1395
1396 if (is_directory) {
1397 path->len = top->path_len;
1398 if (!string_builder_appendc(path, '/')
1399 || !string_builder_append(path, ent->d_name)
1400 || !string_builder_null_terminate(path))
1401 return &drgn_enomem;
1402
1403 _cleanup_close_ int fd =
1404 openat(dirfd(top->dir), ent->d_name,
1405 O_RDONLY | O_DIRECTORY);
1406 if (fd < 0) {
1407 drgn_log_debug(prog, "openat: %s: %m",
1408 path->str);
1409 continue;
1410 }
1411
1412 struct stat st;
1413 if (fstat(fd, &st) < 0) {
1414 drgn_log_debug(prog, "fstat: %s: %m",
1415 path->str);
1416 continue;
1417 }
1418 struct drgn_kmod_walk_inode inode = {
1419 .dev = st.st_dev,
1420 .ino = st.st_ino,
1421 };
1422 int r = drgn_kmod_walk_inode_set_insert(&state->visited_dirs,
1423 &inode, NULL);
1424 if (r < 0)
1425 return &drgn_enomem;
1426 if (r == 0) {
1427 drgn_log_debug(prog,
1428 "%s is cycle or duplicate; skipping",
1429 path->str);
1430 continue;
1431 }
1432
1433 struct drgn_kmod_walk_stack_entry entry = {
1434 .dir = fdopendir(fd),
1435 .path_len = path->len,
1436 };
1437 if (!entry.dir) {
1438 drgn_log_debug(prog, "fdopendir: %s: %m",
1439 path->str);
1440 continue;
1441 }
1442 fd = -1; // entry.dir owns fd now.
1443 if (!drgn_kmod_walk_stack_append(&state->stack,
1444 &entry)) {
1445 closedir(entry.dir);
1446 return &drgn_enomem;
1447 }
1448 } else {
1449 // Match anything where the first extension is ".ko".
1450 char *dot = strchr(ent->d_name, '.');
1451 if (!dot || dot[1] != 'k' || dot[2] != 'o'
1452 || (dot[3] != '\0' && dot[3] != '.'))
1453 continue;
1454
1455 // Borrow the path string builder to build the module
1456 // name (removing extensions and replacing '-' with
1457 // '_').
1458 path->len = top->path_len;
1459 if (!string_builder_appendn(path, ent->d_name,
1460 dot - ent->d_name)
1461 || !string_builder_null_terminate(path))
1462 return &drgn_enomem;
1463 char *dash = &path->str[top->path_len];
1464 while ((dash = strchr(dash, '-')))
1465 *dash++ = '_';
1466
1467 // Find the module (if wanted).
1468 const char *module_name = &path->str[top->path_len];
1469 auto it = drgn_kmod_walk_module_map_search(&state->modules,
1470 &module_name);
1471 if (!it.entry)
1472 continue;
1473
1474 size_t name_len = strlen(ent->d_name);
1475 size_t path_len;
1476 if (__builtin_add_overflow(top->path_len, name_len,
1477 &path_len)
1478 || __builtin_add_overflow(path_len, 2, &path_len))
1479 return &drgn_enomem;
1480 _cleanup_free_ char *file_path = malloc(path_len);
1481 if (!file_path)
1482 return &drgn_enomem;
1483 memcpy(file_path, path->str, top->path_len);
1484 file_path[top->path_len] = '/';
1485 memcpy(&file_path[top->path_len + 1], ent->d_name,
1486 name_len + 1);
1487 drgn_log_debug(prog, "found kernel module %s", file_path);
1488
1489 if (!char_p_vector_append(&it.entry->value, &file_path))
1490 return &drgn_enomem;
1491 file_path = NULL; // it.entry->value owns file_path now.
1492
1493 // If the file matches the current module, return it.
1494 // Otherwise, keep going.
1495 if (it.entry == current)
1496 return NULL;
1497 }
1498 }
1499 }
1500
1501 struct drgn_error *
1502 drgn_module_try_linux_kmod_files(struct drgn_module *module,
1503 const struct drgn_debug_info_options *options,
1504 struct drgn_standard_debug_info_find_state *state)
1505 {
1506 struct drgn_error *err;
1507
1508 if (options->try_kmod == DRGN_KMOD_SEARCH_NONE)
1509 return NULL;
1510
1511 if (options->try_kmod != DRGN_KMOD_SEARCH_WALK) {
1512 err = drgn_open_modules_dep(module->prog, options,
1513 &state->modules_dep);
1514 if (err)
1515 return err;
1516 if (state->modules_dep.len > 0) {
1517 err = drgn_module_try_linux_kmod_depmod(module, options,
1518 state);
1519 if (err
1520 || options->try_kmod != DRGN_KMOD_SEARCH_DEPMOD_AND_WALK
1521 || !drgn_module_wants_file(module))
1522 return err;
1523 }
1524 if (options->try_kmod == DRGN_KMOD_SEARCH_DEPMOD)
1525 return NULL;
1526 }
1527
1528 if (drgn_kmod_walk_module_map_empty(&state->kmod_walk.modules)) {
1529 for (size_t i = 0; i < state->num_modules; i++) {
1530 if (!drgn_module_wants_file(state->modules[i]))
1531 continue;
1532 struct drgn_kmod_walk_module_map_entry entry = {
1533 .key = state->modules[i]->name,
1534 .value = VECTOR_INIT,
1535 };
1536 if (drgn_kmod_walk_module_map_insert(&state->kmod_walk.modules,
1537 &entry, NULL) < 0)
1538 return &drgn_enomem;
1539 }
1540 }
1541
1542 const char *module_name = module->name;
1543 auto it = drgn_kmod_walk_module_map_search(&state->kmod_walk.modules,
1544 &module_name);
1545 size_t i = 0;
1546 for (;;) {
1547 if (i >= char_p_vector_size(&it.entry->value)) {
1548 // No matches remaining for this module. Clear the old
1549 // matches and find another one.
1550 vector_for_each(char_p_vector, path, &it.entry->value)
1551 free(*path);
1552 char_p_vector_clear(&it.entry->value);
1553 i = 0;
1554
1555 err = drgn_kmod_walk(module->prog, options,
1556 &state->kmod_walk, it.entry);
1557 if (err == &drgn_stop)
1558 break;
1559 else if (err)
1560 return err;
1561 }
1562 char *path = *char_p_vector_at(&it.entry->value, i++);
1563 err = drgn_module_try_standard_file(module, options, path, -1,
1564 true, NULL);
1565 if (err)
1566 return err;
1567 if (!drgn_module_wants_file(module))
1568 break;
1569 }
1570 // We won't need any more matches for this module.
1571 drgn_kmod_walk_module_map_entry_deinit(it.entry);
1572 drgn_kmod_walk_module_map_delete_iterator(&state->kmod_walk.modules,
1573 it);
1574 return NULL;
1575 }
1576
1577 // This has a weird calling convention so that the caller can call
1578 // drgn_error_format_os() itself.
1579 static const char *get_gnu_build_id_from_note_file(int fd,
1580 void **bufp,
1581 size_t *buf_capacityp,
1582 const void **build_id_ret,
1583 size_t *build_id_len_ret)
1584 {
1585 struct stat st;
1586 if (fstat(fd, &st) < 0)
1587 return "fstat";
1588
1589 if (st.st_size > SSIZE_MAX
1590 || !alloc_or_reuse(bufp, buf_capacityp, st.st_size))
1591 return "";
1592
1593 ssize_t r = read_all(fd, *bufp, st.st_size);
1594 if (r < 0)
1595 return "read";
1596 *build_id_len_ret = parse_gnu_build_id_from_notes(*bufp, r, 4, false,
1597 build_id_ret);
1598 return NULL;
1599 }
1600
1601 static struct drgn_error *
1602 get_build_id_from_sys_kernel_notes(void **buf_ret,
1603 const void **build_id_ret,
1604 size_t *build_id_len_ret)
1605 {
1606 static const char path[] = "/sys/kernel/notes";
1607 _cleanup_close_ int fd = open(path, O_RDONLY);
1608 if (fd == -1)
1609 return drgn_error_create_os("open", errno, path);
1610
1611 _cleanup_free_ void *buf = NULL;
1612 size_t buf_capacity = 0;
1613 const char *message = get_gnu_build_id_from_note_file(fd, &buf,
1614 &buf_capacity,
1615 build_id_ret,
1616 build_id_len_ret);
1617 if (message && message[0])
1618 return drgn_error_create_os(message, errno, path);
1619 else if (message)
1620 return &drgn_enomem;
1621 *buf_ret = no_cleanup_ptr(buf);
1622 return NULL;
1623 }
1624
1625 // Arbitrary limit on the number iterations to make through the modules list in
1626 // order to avoid getting stuck in a cycle.
1627 static const int MAX_MODULE_LIST_ITERATIONS = 10000;
1628
1629 struct linux_kernel_loaded_module_iterator {
1630 struct drgn_module_iterator it;
1631 bool yielded_vmlinux;
1632 int module_list_iterations_remaining;
1633 // `struct module` type.
1634 struct drgn_qualified_type module_type;
1635 // `struct list_head *` in next module to yield.
1636 struct drgn_object node;
1637 // Address of `struct list_head modules`.
1638 uint64_t modules_head;
1639 };
1640
1641 static void
1642 linux_kernel_loaded_module_iterator_destroy(struct drgn_module_iterator *_it)
1643 {
1644 struct linux_kernel_loaded_module_iterator *it =
1645 container_of(_it, struct linux_kernel_loaded_module_iterator, it);
1646 drgn_object_deinit(&it->node);
1647 free(it);
1648 }
1649
1650 static struct drgn_error *
1651 yield_vmlinux(struct linux_kernel_loaded_module_iterator *it,
1652 struct drgn_module **ret, bool *new_ret)
1653 {
1654 struct drgn_error *err;
1655 struct drgn_program *prog = it->it.prog;
1656
1657 _cleanup_(drgn_module_deletep) struct drgn_module *module = NULL;
1658 bool new;
1659 err = drgn_module_find_or_create_main(prog, "kernel", &module, &new);
1660 if (err)
1661 return err;
1662 if (!new) {
1663 *ret = no_cleanup_ptr(module);
1664 if (new_ret)
1665 *new_ret = new;
1666 return NULL;
1667 }
1668
1669 if (prog->vmcoreinfo.build_id_len > 0) {
1670 // Since Linux kernel commit 0935288c6e00 ("kdump: append kernel
1671 // build-id string to VMCOREINFO") (in v5.9), we can get the
1672 // build ID from VMCOREINFO.
1673 err = drgn_module_set_build_id(module, prog->vmcoreinfo.build_id,
1674 prog->vmcoreinfo.build_id_len);
1675 if (err)
1676 return err;
1677 drgn_log_debug(prog,
1678 "found kernel build ID %s in VMCOREINFO",
1679 module->build_id_str);
1680 } else if ((prog->flags & (DRGN_PROGRAM_IS_LIVE | DRGN_PROGRAM_IS_LOCAL))
1681 == (DRGN_PROGRAM_IS_LIVE | DRGN_PROGRAM_IS_LOCAL)) {
1682 // Before that, on the live kernel, we can get the build ID from
1683 // /sys/kernel/notes.
1684 _cleanup_free_ void *build_id_buf = NULL;
1685 const void *build_id;
1686 size_t build_id_len;
1687 err = get_build_id_from_sys_kernel_notes(&build_id_buf,
1688 &build_id,
1689 &build_id_len);
1690 if (err)
1691 return err;
1692 if (build_id_len > 0) {
1693 err = drgn_module_set_build_id(module, build_id,
1694 build_id_len);
1695 if (err)
1696 return err;
1697 drgn_log_debug(prog,
1698 "found kernel build ID %s in /sys/kernel/notes",
1699 module->build_id_str);
1700 } else {
1701 drgn_log_debug(prog,
1702 "couldn't find kernel build ID in /sys/kernel/notes");
1703 }
1704 } else {
1705 // Otherwise, we can't get the build ID.
1706 drgn_log_debug(prog, "couldn't find kernel build ID");
1707 }
1708 *ret = no_cleanup_ptr(module);
1709 if (new_ret)
1710 *new_ret = new;
1711 return NULL;
1712 }
1713
1714 enum kernel_module_address_ranges_version {
1715 // Since Linux kernel commit ac3b43283923 ("module: replace
1716 // module_layout with module_memory") (in v6.4), `struct module`
1717 // contains an array, `struct module_memory mem[]`, of discontiguous
1718 // allocations per memory type (`module->mem[type].base` and
1719 // `module->mem[type].size`). The module address is
1720 // `module->mem[MOD_TEXT].base`.
1721 MODULE_MEMORY,
1722 // Between that and Linux kernel commit 7523e4dc5057 ("module: use a
1723 // structure to encapsulate layout.") (in v4.5), `struct module`
1724 // contains a `struct module_layout core_layout` member with the base
1725 // address (`module->core_layout.base`) and contiguous size
1726 // (`module->core_layout.size`).
1727 MODULE_LAYOUT,
1728 // Before that, `struct module` contains the base address
1729 // (`module->module_core`) and contiguous size (`module->core_size`)
1730 // directly.
1731 IN_MODULE,
1732 };
1733
1734 static struct drgn_error *
1735 kernel_module_address(const struct drgn_object *module_obj,
1736 struct drgn_object *mem,
1737 enum kernel_module_address_ranges_version *version_ret,
1738 uint64_t *address_ret)
1739 {
1740 struct drgn_program *prog = drgn_object_program(module_obj);
1741 struct drgn_error *err;
1742
1743 DRGN_OBJECT(tmp, prog);
1744 err = drgn_object_member(mem, module_obj, "mem");
1745 if (!err) {
1746 *version_ret = MODULE_MEMORY;
1747 if (!prog->mod_text_cached) {
1748 err = drgn_program_find_object(prog, "MOD_TEXT", NULL,
1749 DRGN_FIND_OBJECT_CONSTANT,
1750 &tmp);
1751 if (err)
1752 return err;
1753 union drgn_value mod_text_value;
1754 err = drgn_object_read_integer(&tmp, &mod_text_value);
1755 if (err)
1756 return err;
1757 prog->mod_text = mod_text_value.uvalue;
1758 prog->mod_text_cached = true;
1759 }
1760 err = drgn_object_subscript(&tmp, mem, prog->mod_text);
1761 if (err)
1762 return err;
1763 err = drgn_object_member(&tmp, &tmp, "base");
1764 } else if (drgn_error_catch(&err, DRGN_ERROR_LOOKUP)) {
1765 err = drgn_object_member(mem, module_obj, "core_layout");
1766 if (!err) {
1767 *version_ret = MODULE_LAYOUT;
1768 err = drgn_object_member(&tmp, mem, "base");
1769 } else if (drgn_error_catch(&err, DRGN_ERROR_LOOKUP)) {
1770 *version_ret = IN_MODULE;
1771 err = drgn_object_member(&tmp, module_obj,
1772 "module_core");
1773 }
1774 }
1775 if (err)
1776 return err;
1777 return drgn_object_read_unsigned(&tmp, address_ret);
1778 }
1779
1780 // If version is MODULE_MEMORY, mem is struct module::mem. If version is
1781 // MODULE_LAYOUT, mem is struct module::core_layout.
1782 static struct drgn_error *
1783 kernel_module_set_address_ranges(struct drgn_module *module,
1784 enum kernel_module_address_ranges_version version,
1785 const struct drgn_object *module_obj,
1786 const struct drgn_object *mem,
1787 uint64_t address)
1788 {
1789 struct drgn_program *prog = module->prog;
1790 struct drgn_error *err;
1791
1792 DRGN_OBJECT(tmp, prog);
1793 if (version != MODULE_MEMORY) {
1794 if (version == IN_MODULE)
1795 err = drgn_object_member(&tmp, module_obj, "core_size");
1796 else
1797 err = drgn_object_member(&tmp, mem, "size");
1798 if (err)
1799 return err;
1800 uint64_t size;
1801 err = drgn_object_read_unsigned(&tmp, &size);
1802 if (err)
1803 return err;
1804 drgn_log_debug(prog, "module size is %" PRIu64, size);
1805 return drgn_module_set_address_range(module, address,
1806 address + size);
1807 }
1808
1809 struct drgn_type *mem_array_type = drgn_underlying_type(mem->type);
1810 if (drgn_type_kind(mem_array_type) != DRGN_TYPE_ARRAY) {
1811 return drgn_error_create(DRGN_ERROR_TYPE,
1812 "struct module::mem is not an array");
1813 }
1814 uint64_t length = drgn_type_length(mem_array_type);
1815
1816 if (length > SIZE_MAX)
1817 return &drgn_enomem;
1818 _cleanup_free_ uint64_t (*ranges)[2] =
1819 malloc_array(length, sizeof(*ranges));
1820 if (!ranges)
1821 return &drgn_enomem;
1822
1823 DRGN_OBJECT(element, prog);
1824 size_t num_ranges = 0;
1825 for (size_t i = 0; i < length; i++) {
1826 err = drgn_object_subscript(&element, mem, i);
1827 if (err)
1828 return err;
1829
1830 err = drgn_object_member(&tmp, &element, "size");
1831 if (err)
1832 return err;
1833 uint64_t size;
1834 err = drgn_object_read_unsigned(&tmp, &size);
1835 if (err)
1836 return err;
1837 if (!size)
1838 continue;
1839
1840 err = drgn_object_member(&tmp, &element, "base");
1841 if (err)
1842 return err;
1843 uint64_t base;
1844 err = drgn_object_read_unsigned(&tmp, &base);
1845 if (err)
1846 return err;
1847
1848 drgn_log_debug(prog, "module has address range %" PRIu64 "-%" PRIu64,
1849 base, base + size);
1850 ranges[num_ranges][0] = base;
1851 ranges[num_ranges][1] = base + size;
1852 num_ranges++;
1853 }
1854 return drgn_module_set_address_ranges(module, ranges, num_ranges);
1855 }
1856
1857 static struct drgn_error *
1858 kernel_module_set_build_id_live(struct drgn_module *module)
1859 {
1860 struct drgn_error *err;
1861 struct drgn_program *prog = module->prog;
1862
1863 _cleanup_free_ char *path;
1864 if (asprintf(&path, "/sys/module/%s/notes", module->name) < 0) {
1865 path = NULL;
1866 return &drgn_enomem;
1867 }
1868 _cleanup_closedir_ DIR *dir = opendir(path);
1869 if (!dir) {
1870 if (errno == ENOENT) {
1871 drgn_log_debug(prog, "opendir: %s: %m", path);
1872 return NULL;
1873 } else {
1874 return drgn_error_create_os("opendir", errno, path);
1875 }
1876 }
1877
1878 _cleanup_free_ void *buf = NULL;
1879 size_t capacity = 0;
1880
1881 struct dirent *ent;
1882 while ((errno = 0, ent = readdir(dir))) {
1883 if (ent->d_type == DT_DIR)
1884 continue;
1885
1886 _cleanup_close_ int fd = openat(dirfd(dir), ent->d_name,
1887 O_RDONLY);
1888 if (fd < 0) {
1889 return drgn_error_format_os("openat", errno, "%s/%s",
1890 path, ent->d_name);
1891 }
1892
1893 const void *build_id;
1894 size_t build_id_len;
1895 const char *message =
1896 get_gnu_build_id_from_note_file(fd, &buf, &capacity,
1897 &build_id,
1898 &build_id_len);
1899 if (message && message[0]) {
1900 return drgn_error_format_os(message, errno, "%s/%s",
1901 path, ent->d_name);
1902 } else if (message) {
1903 return &drgn_enomem;
1904 }
1905 if (build_id_len > 0) {
1906 err = drgn_module_set_build_id(module, build_id,
1907 build_id_len);
1908 if (!err) {
1909 drgn_log_debug(prog,
1910 "found build ID %s in %s/%s",
1911 module->build_id_str, path,
1912 ent->d_name);
1913 }
1914 return err;
1915 }
1916 }
1917 if (errno)
1918 return drgn_error_create_os("readdir", errno, path);
1919 drgn_log_debug(prog, "couldn't find build ID in %s", path);
1920 return NULL;
1921 }
1922
1923 static struct drgn_error *
1924 kernel_module_set_build_id(struct drgn_module *module,
1925 const struct drgn_object *module_obj,
1926 bool use_sys_module)
1927 {
1928 if (use_sys_module)
1929 return kernel_module_set_build_id_live(module);
1930
1931 struct drgn_error *err;
1932 struct drgn_program *prog = module->prog;
1933 const bool bswap = drgn_platform_bswap(&prog->platform);
1934
1935 DRGN_OBJECT(attrs, prog);
1936 DRGN_OBJECT(attr, prog);
1937 DRGN_OBJECT(tmp, prog);
1938 _cleanup_free_ void *buf = NULL;
1939 size_t capacity = 0;
1940
1941 err = drgn_object_member(&attrs, module_obj, "notes_attrs");
1942 if (err)
1943 return err;
1944
1945 bool group = true;
1946 uint64_t n;
1947 err = drgn_object_member_dereference(&attrs, &attrs, "grp");
1948 if (!err) {
1949 // Since Linux kernel commit 4723f16de64e ("module: sysfs: Add
1950 // notes attributes through attribute_group") (in v6.14), we
1951 // have to iterate over struct attribute_group::bin_attrs, a
1952 // null-terminated array of struct bin_attribute pointers.
1953
1954 // attr = mod->notes_attrs->grp.bin_attrs
1955 err = drgn_object_member(&attrs, &attrs, "bin_attrs");
1956 if (err)
1957 return err;
1958 } else if (drgn_error_catch(&err, DRGN_ERROR_LOOKUP)) {
1959 // Before that, there was no struct attribute_group for notes,
1960 // so we iterate over struct module_notes_attrs::attrs, an array
1961 // of struct bin_attribute with a length given by struct
1962 // module_notes_attrs::notes.
1963 group = false;
1964 // n = mod->notes_attrs->notes
1965 err = drgn_object_member_dereference(&tmp, &attrs, "notes");
1966 if (err)
1967 return err;
1968 err = drgn_object_read_unsigned(&tmp, &n);
1969 if (err)
1970 return err;
1971
1972 // attrs = mod->notes_attrs->attrs
1973 err = drgn_object_member_dereference(&attrs, &attrs, "attrs");
1974 if (err)
1975 return err;
1976 } else {
1977 return err;
1978 }
1979
1980 // If we're not using struct attribute_group, we know how many
1981 // attributes there are.
1982 for (uint64_t i = 0; group || i < n; i++) {
1983 // attr = attrs[i]
1984 err = drgn_object_subscript(&attr, &attrs, i);
1985 if (err)
1986 return err;
1987
1988 if (group) {
1989 // If we're using struct attribute_group, we stop when
1990 // we hit a NULL pointer.
1991 err = drgn_object_read(&attr, &attr);
1992 if (err)
1993 return err;
1994 bool truthy;
1995 err = drgn_object_bool(&attr, &truthy);
1996 if (err)
1997 return err;
1998 if (!truthy)
1999 break;
2000 } else {
2001 // attr = &attrs[i]
2002 err = drgn_object_address_of(&attr, &attr);
2003 if (err)
2004 return err;
2005 }
2006
2007 // address = attr->private
2008 err = drgn_object_member_dereference(&tmp, &attr, "private");
2009 if (err)
2010 return err;
2011 uint64_t address;
2012 err = drgn_object_read_unsigned(&tmp, &address);
2013 if (err)
2014 return err;
2015
2016 // size = attr->size
2017 err = drgn_object_member_dereference(&tmp, &attr, "size");
2018 if (err)
2019 return err;
2020 uint64_t size;
2021 err = drgn_object_read_unsigned(&tmp, &size);
2022 if (err)
2023 return err;
2024
2025 if (size > SIZE_MAX || !alloc_or_reuse(&buf, &capacity, size))
2026 return &drgn_enomem;
2027
2028 err = drgn_program_read_memory(prog, buf, address, size, false);
2029 if (err)
2030 return err;
2031
2032 const void *build_id;
2033 size_t build_id_len =
2034 parse_gnu_build_id_from_notes(buf, size, 4, bswap,
2035 &build_id);
2036 if (build_id_len > 0) {
2037 err = drgn_module_set_build_id(module, build_id,
2038 build_id_len);
2039 if (!err) {
2040 drgn_log_debug(prog,
2041 "found build ID %s in notes_attrs",
2042 module->build_id_str);
2043 }
2044 return err;
2045 }
2046 }
2047 drgn_log_debug(prog,
2048 "couldn't find build ID in notes_attrs");
2049 return NULL;
2050 }
2051
2052 static struct drgn_error *
2053 kernel_module_set_section_addresses_live(struct drgn_module *module)
2054 {
2055 struct drgn_error *err;
2056 struct drgn_program *prog = module->prog;
2057 bool logged = false;
2058
2059 _cleanup_free_ char *path;
2060 if (asprintf(&path, "/sys/module/%s/sections", module->name) < 0) {
2061 path = NULL;
2062 return &drgn_enomem;
2063 }
2064 _cleanup_closedir_ DIR *dir = opendir(path);
2065 if (!dir)
2066 return drgn_error_create_os("opendir", errno, path);
2067
2068 struct dirent *ent;
2069 while ((errno = 0, ent = readdir(dir))) {
2070 if (ent->d_type == DT_DIR)
2071 continue;
2072
2073 _cleanup_close_ int fd = openat(dirfd(dir), ent->d_name,
2074 O_RDONLY);
2075 if (fd < 0) {
2076 return drgn_error_format_os("openat", errno, "%s/%s",
2077 path, ent->d_name);
2078 }
2079
2080 _cleanup_fclose_ FILE *file = fdopen(fd, "r");
2081 if (!file)
2082 return drgn_error_create_os("fdopen", errno, NULL);
2083 uint64_t address;
2084 if (fscanf(file, "%" SCNx64, &address) != 1) {
2085 return drgn_error_format(DRGN_ERROR_OTHER,
2086 "could not parse %s/%s",
2087 path, ent->d_name);
2088 }
2089
2090 if (!logged) {
2091 drgn_log_debug(prog,
2092 "getting section addresses from %s",
2093 path);
2094 logged = true;
2095 }
2096 err = drgn_module_set_section_address(module, ent->d_name,
2097 address);
2098 if (err)
2099 return err;
2100 }
2101 if (errno)
2102 return drgn_error_create_os("readdir", errno, path);
2103 return NULL;
2104 }
2105
2106 static struct drgn_error *
2107 kernel_module_set_section_addresses(struct drgn_module *module,
2108 const struct drgn_object *module_obj,
2109 bool use_sys_module)
2110 {
2111 struct drgn_error *err;
2112 struct drgn_program *prog = module->prog;
2113
2114 DRGN_OBJECT(tmp, prog);
2115
2116 // As of Linux 6.0, the .data..percpu section is not included in the
2117 // section attributes. (kernel/module/sysfs.c:add_sect_attrs() only
2118 // creates attributes for sections with the SHF_ALLOC flag set, but
2119 // kernel/module/main.c:layout_and_allocate() clears the SHF_ALLOC flag
2120 // for the .data..percpu section.) However, we need this address so that
2121 // global per-CPU variables will be relocated correctly. Get it from
2122 // `struct module`.
2123 err = drgn_object_member(&tmp, module_obj, "percpu");
2124 if (!err) {
2125 uint64_t address;
2126 err = drgn_object_read_unsigned(&tmp, &address);
2127 if (err)
2128 return err;
2129 drgn_log_debug(prog, "module percpu is 0x%" PRIx64, address);
2130 // struct module::percpu is NULL if the module doesn't have any
2131 // per-CPU data.
2132 if (address) {
2133 err = drgn_module_set_section_address(module,
2134 ".data..percpu",
2135 address);
2136 if (err)
2137 return err;
2138 }
2139 } else if (err->code == DRGN_ERROR_LOOKUP) {
2140 // struct module::percpu doesn't exist if !SMP.
2141 drgn_error_destroy(err);
2142 } else {
2143 return err;
2144 }
2145
2146 if (use_sys_module) {
2147 err = kernel_module_set_section_addresses_live(module);
2148 // We could be debugging /proc/kcore without root privileges via
2149 // an fd that we were passed. If we didn't have permission to
2150 // access the files in /sys/module/$module/sections, fall back
2151 // to the non-live path.
2152 if (!err || err->code != DRGN_ERROR_OS || err->errnum != EACCES)
2153 return err;
2154 drgn_error_log_debug(prog, err,
2155 "falling back to section addresses from sect_attrs: ");
2156 drgn_error_destroy(err);
2157 } else {
2158 drgn_log_debug(prog,
2159 "getting section addresses from sect_attrs");
2160 }
2161
2162 DRGN_OBJECT(attrs, prog);
2163 DRGN_OBJECT(attr, prog);
2164
2165 err = drgn_object_member(&attrs, module_obj, "sect_attrs");
2166 if (err)
2167 return err;
2168
2169 bool group = true;
2170 uint64_t nsections;
2171 err = drgn_object_member_dereference(&tmp, &attrs, "nsections");
2172 if (drgn_error_catch(&err, DRGN_ERROR_LOOKUP)) {
2173 // Since Linux kernel commit d8959b947a8d ("module: sysfs: Drop
2174 // member 'module_sect_attrs::nsections'") (in v6.14), we have
2175 // to iterate over struct attribute_group::bin_attrs, a
2176 // null-terminated array of struct bin_attribute pointers.
2177
2178 // attrs = mod->sect_attrs->grp.bin_attrs
2179 err = drgn_object_member_dereference(&attrs, &attrs, "grp");
2180 if (err)
2181 return err;
2182 err = drgn_object_member(&attrs, &attrs, "bin_attrs");
2183 if (err)
2184 return err;
2185 } else if (!err) {
2186 // Before that, struct module_sect_attrs::grp still exists.
2187 // However, since Linux kernel commit ed66f991bb19 ("module:
2188 // Refactor section attr into bin attribute") (in v5.8), the
2189 // sections are in struct attribute_group::bin_attrs, and before
2190 // that, they're in struct attribute_group::attrs. Additionally,
2191 // we'd then have to get the containing struct module_sect_attr
2192 // to get the section address.
2193 //
2194 // Instead, it's easier to iterate over struct
2195 // module_sect_attrs::attrs, an array of struct module_sect_attr
2196 // with a length given by struct module_sect_attrs::nsections.
2197 group = false;
2198 // nsections = mod->sect_attrs->nsections
2199 err = drgn_object_read_unsigned(&tmp, &nsections);
2200 if (err)
2201 return err;
2202
2203 // attrs = mod->sect_attrs->attrs
2204 err = drgn_object_member_dereference(&attrs, &attrs, "attrs");
2205 if (err)
2206 return err;
2207 } else {
2208 return err;
2209 }
2210
2211 // If we're not using struct attribute_group, we know how many
2212 // attributes there are.
2213 for (uint64_t i = 0; group || i < nsections; i++) {
2214 // attr = attrs[i]
2215 err = drgn_object_subscript(&attr, &attrs, i);
2216 if (err)
2217 return err;
2218
2219 if (group) {
2220 // If we're using struct attribute_group, we stop when
2221 // we hit a NULL pointer.
2222 err = drgn_object_read(&attr, &attr);
2223 if (err)
2224 return err;
2225 bool truthy;
2226 err = drgn_object_bool(&attr, &truthy);
2227 if (err)
2228 return err;
2229 if (!truthy)
2230 break;
2231 // Since Linux kernel commit 4b2c11e4aaf7 ("module:
2232 // sysfs: Drop member 'module_sect_attr::address'") (in
2233 // v6.14), the section address is in struct
2234 // bin_attribute::private.
2235 err = drgn_object_member_dereference(&tmp, &attr,
2236 "private");
2237 } else {
2238 // Before that, the section address is in struct
2239 // module_sect_attr::address.
2240 err = drgn_object_member(&tmp, &attr, "address");
2241 if (err)
2242 return err;
2243 }
2244 uint64_t address;
2245 err = drgn_object_read_unsigned(&tmp, &address);
2246 if (err)
2247 return err;
2248
2249 if (group) {
2250 // attr = attr->attr
2251 err = drgn_object_member_dereference(&attr, &attr,
2252 "attr");
2253 if (err)
2254 return err;
2255 } else {
2256 // Since Linux kernel commit ed66f991bb19 ("module:
2257 // Refactor section attr into bin attribute") (in v5.8),
2258 // the section name is module_sect_attr.battr.attr.name.
2259 // Before that, it is simply module_sect_attr.name.
2260
2261 // attr = attr.battr.attr
2262 err = drgn_object_member(&attr, &attr, "battr");
2263 if (!err) {
2264 err = drgn_object_member(&attr, &attr, "attr");
2265 if (err)
2266 return err;
2267 } else if (!drgn_error_catch(&err, DRGN_ERROR_LOOKUP)) {
2268 return err;
2269 }
2270 }
2271 err = drgn_object_member(&tmp, &attr, "name");
2272 if (err)
2273 return err;
2274 _cleanup_free_ char *name = NULL;
2275 err = drgn_object_read_c_string(&tmp, &name);
2276 if (err)
2277 return err;
2278
2279 err = drgn_module_set_section_address(module, name, address);
2280 if (err)
2281 return err;
2282 }
2283 return NULL;
2284 }
2285
2286 static struct drgn_error *
2287 kernel_module_find_or_create_internal(const struct drgn_object *module_ptr,
2288 const struct drgn_object *module_obj,
2289 struct drgn_module **ret, bool *new_ret,
2290 bool create, bool log)
2291 {
2292 struct drgn_error *err;
2293 struct drgn_program *prog = drgn_object_program(module_obj);
2294
2295 uint64_t name_offset;
2296 err = drgn_type_offsetof(module_obj->type, "name", &name_offset);
2297 if (err)
2298 return err;
2299 if (name_offset >= drgn_object_size(module_obj)
2300 || !memchr(drgn_object_buffer(module_obj) + name_offset, '\0',
2301 drgn_object_size(module_obj) - name_offset)) {
2302 return drgn_error_create(DRGN_ERROR_OTHER,
2303 "couldn't read module name");
2304 }
2305 const char *name = drgn_object_buffer(module_obj) + name_offset;
2306
2307 DRGN_OBJECT(mem, prog);
2308 enum kernel_module_address_ranges_version version;
2309 uint64_t address;
2310 err = kernel_module_address(module_obj, &mem, &version, &address);
2311 if (err)
2312 return err;
2313
2314 if (log) {
2315 drgn_log_debug(prog, "found loaded kernel module %s@0x%" PRIx64,
2316 name, address);
2317 }
2318
2319 if (!create) {
2320 *ret = drgn_module_find_relocatable(prog, name, address);
2321 if (new_ret)
2322 *new_ret = false;
2323 return NULL;
2324 }
2325
2326 _cleanup_(drgn_module_deletep) struct drgn_module *module = NULL;
2327 bool new;
2328 err = drgn_module_find_or_create_relocatable(prog, name, address,
2329 &module, &new);
2330 if (err)
2331 return err;
2332 if (!new) {
2333 *ret = no_cleanup_ptr(module);
2334 if (new_ret)
2335 *new_ret = new;
2336 return NULL;
2337 }
2338
2339 err = drgn_module_set_object(module, module_ptr);
2340 if (err)
2341 return err;
2342
2343 err = kernel_module_set_address_ranges(module, version, module_obj,
2344 &mem, address);
2345 if (err)
2346 return err;
2347
2348 // If we're debugging the running kernel, we can use
2349 // /sys/module/$module/notes and /sys/module/$module/sections instead of
2350 // getting the equivalent information from the core dump. This fast path
2351 // can be disabled via an environment variable for testing. It may also
2352 // be disabled if we encounter permission issues using
2353 // /sys/module/$module/sections.
2354 bool use_sys_module = false;
2355 if ((prog->flags & (DRGN_PROGRAM_IS_LIVE | DRGN_PROGRAM_IS_LOCAL))
2356 == (DRGN_PROGRAM_IS_LIVE | DRGN_PROGRAM_IS_LOCAL)) {
2357 char *env = getenv("DRGN_USE_SYS_MODULE");
2358 use_sys_module = !env || atoi(env);
2359 }
2360 err = kernel_module_set_build_id(module, module_obj, use_sys_module);
2361 if (err)
2362 return err;
2363 err = kernel_module_set_section_addresses(module, module_obj,
2364 use_sys_module);
2365 if (err)
2366 return err;
2367
2368 *ret = no_cleanup_ptr(module);
2369 if (new_ret)
2370 *new_ret = new;
2371 return NULL;
2372 }
2373
2374 static struct drgn_error *
2375 drgn_module_find_or_create_linux_kernel_loadable_internal(const struct drgn_object *module_ptr,
2376 struct drgn_module **ret,
2377 bool *new_ret,
2378 bool create)
2379 {
2380 struct drgn_error *err;
2381 struct drgn_program *prog = drgn_object_program(module_ptr);
2382
2383 if (drgn_type_kind(drgn_underlying_type(module_ptr->type))
2384 != DRGN_TYPE_POINTER)
2385 return drgn_error_create(DRGN_ERROR_TYPE,
2386 "struct module * is required");
2387
2388 DRGN_OBJECT(module_obj, prog);
2389 err = drgn_object_dereference(&module_obj, module_ptr);
2390 if (err)
2391 return err;
2392
2393 err = drgn_object_read(&module_obj, &module_obj);
2394 if (err)
2395 return err;
2396
2397 return kernel_module_find_or_create_internal(module_ptr, &module_obj, ret, new_ret,
2398 create, false);
2399 }
2400
2401 LIBDRGN_PUBLIC struct drgn_error *
2402 drgn_module_find_linux_kernel_loadable(const struct drgn_object *module_ptr,
2403 struct drgn_module **ret)
2404 {
2405 return drgn_module_find_or_create_linux_kernel_loadable_internal(module_ptr, ret,
2406 NULL, false);
2407 }
2408
2409 LIBDRGN_PUBLIC struct drgn_error *
2410 drgn_module_find_or_create_linux_kernel_loadable(const struct drgn_object *module_ptr,
2411 struct drgn_module **ret,
2412 bool *new_ret)
2413 {
2414 return drgn_module_find_or_create_linux_kernel_loadable_internal(module_ptr, ret,
2415 new_ret, true);
2416 }
2417
2418 static struct drgn_error *
2419 yield_kernel_module(struct linux_kernel_loaded_module_iterator *it,
2420 struct drgn_module **ret, bool *new_ret)
2421 {
2422 struct drgn_error *err;
2423 struct drgn_program *prog = it->it.prog;
2424
2425 DRGN_OBJECT(mod, prog);
2426 DRGN_OBJECT(mod_ptr, prog);
2427 for (;;) {
2428 uint64_t addr;
2429 err = drgn_object_read_unsigned(&it->node, &addr);
2430 if (err) {
2431 list_walk_err:
2432 if (!drgn_error_is_fatal(err)) {
2433 drgn_error_log_warning(prog, err,
2434 "can't find remaining kernel modules: "
2435 "couldn't read next module: ");
2436 drgn_error_destroy(err);
2437 *ret = NULL;
2438 err = NULL;
2439 }
2440 return err;
2441 }
2442 if (addr == it->modules_head) {
2443 drgn_log_debug(prog,
2444 "found end of loaded kernel module list");
2445 *ret = NULL;
2446 return NULL;
2447 }
2448
2449 if (it->module_list_iterations_remaining == 0) {
2450 drgn_log_warning(prog,
2451 "can't find remaining kernel modules: "
2452 "too many entries or cycle in modules list");
2453 *ret = NULL;
2454 return NULL;
2455 }
2456 it->module_list_iterations_remaining--;
2457
2458 err = drgn_object_container_of(&mod_ptr, &it->node, it->module_type,
2459 "list");
2460 if (err)
2461 goto list_walk_err;
2462
2463 err = drgn_object_dereference(&mod, &mod_ptr);
2464 if (err)
2465 goto list_walk_err;
2466 // We need several fields from the `struct module`. Especially
2467 // for /proc/kcore, it is faster to read the entire structure
2468 // (which is <2kB as of Linux 6.5) from the core dump all at
2469 // once than it is to read each field individually.
2470 err = drgn_object_read(&mod, &mod);
2471 if (err)
2472 goto list_walk_err;
2473
2474 err = drgn_object_member(&it->node, &mod, "list");
2475 if (err)
2476 goto list_walk_err;
2477 err = drgn_object_member(&it->node, &it->node, "next");
2478 if (err)
2479 goto list_walk_err;
2480
2481 err = kernel_module_find_or_create_internal(&mod_ptr, &mod, ret,
2482 new_ret, true, true);
2483 if (err && !drgn_error_is_fatal(err)) {
2484 drgn_error_log_warning(prog, err, "ignoring module: ");
2485 drgn_error_destroy(err);
2486 continue;
2487 }
2488 return err;
2489 }
2490 }
2491
2492 static struct drgn_error *
2493 linux_kernel_loaded_module_iterator_next(struct drgn_module_iterator *_it,
2494 struct drgn_module **ret,
2495 bool *new_ret)
2496 {
2497 struct drgn_error *err;
2498 struct linux_kernel_loaded_module_iterator *it =
2499 container_of(_it, struct linux_kernel_loaded_module_iterator, it);
2500 struct drgn_program *prog = it->it.prog;
2501
2502 if (!it->yielded_vmlinux) {
2503 it->yielded_vmlinux = true;
2504 return yield_vmlinux(it, ret, new_ret);
2505 }
2506
2507 // Start the module list walk if we haven't yet.
2508 if (!it->module_type.type) {
2509 for (int attempt = 1; attempt <= 2; attempt++) {
2510 err = drgn_program_find_type(prog, "struct module",
2511 NULL, &it->module_type);
2512 if (!err) {
2513 err = drgn_program_find_object(prog, "modules",
2514 NULL,
2515 DRGN_FIND_OBJECT_VARIABLE,
2516 &it->node);
2517 }
2518 if (err && err->code == DRGN_ERROR_LOOKUP) {
2519 drgn_error_destroy(err);
2520 if (attempt == 1 && prog->dbinfo.main_module) {
2521 struct drgn_module *module =
2522 prog->dbinfo.main_module;
2523 if (module->debug_file_status
2524 == DRGN_MODULE_FILE_DONT_WANT) {
2525 module->debug_file_status =
2526 DRGN_MODULE_FILE_WANT;
2527 }
2528 if (drgn_module_wants_debug_file(module)) {
2529 err = drgn_load_module_debug_info(&module,
2530 &(size_t){1});
2531 if (err)
2532 return err;
2533 continue;
2534 }
2535 }
2536 if (!prog->dbinfo.main_module
2537 || drgn_module_wants_debug_file(prog->dbinfo.main_module)) {
2538 drgn_log(it->it.for_load_debug_info
2539 ? DRGN_LOG_DEBUG
2540 : DRGN_LOG_WARNING,
2541 prog,
2542 "can't find loaded modules without kernel debug info");
2543 } else {
2544 drgn_log_debug(prog,
2545 "kernel does not have loadable module support");
2546 }
2547 *ret = NULL;
2548 return NULL;
2549 } else if (err) {
2550 return err;
2551 }
2552 }
2553 if (it->node.kind != DRGN_OBJECT_REFERENCE) {
2554 drgn_log_warning(prog,
2555 "can't find kernel modules: "
2556 "can't get address of modules list");
2557 *ret = NULL;
2558 return NULL;
2559 }
2560 it->modules_head = it->node.address;
2561 err = drgn_object_member(&it->node, &it->node, "next");
2562 if (!err)
2563 err = drgn_object_read(&it->node, &it->node);
2564 if (err) {
2565 if (drgn_error_is_fatal(err))
2566 return err;
2567 drgn_error_log_warning(prog, err,
2568 "can't find kernel modules: "
2569 "couldn't read modules list: ");
2570 drgn_error_destroy(err);
2571 *ret = NULL;
2572 return NULL;
2573 }
2574 }
2575
2576 return yield_kernel_module(it, ret, new_ret);
2577 }
2578
2579 struct drgn_error *
2580 linux_kernel_loaded_module_iterator_create(struct drgn_program *prog,
2581 struct drgn_module_iterator **ret)
2582 {
2583 struct linux_kernel_loaded_module_iterator *it = calloc(1, sizeof(*it));
2584 if (!it)
2585 return &drgn_enomem;
2586 drgn_module_iterator_init(&it->it, prog,
2587 linux_kernel_loaded_module_iterator_destroy,
2588 linux_kernel_loaded_module_iterator_next);
2589 it->module_list_iterations_remaining = MAX_MODULE_LIST_ITERATIONS;
2590 drgn_object_init(&it->node, prog);
2591 *ret = &it->it;
2592 return NULL;
2593 }
2594
2595 static struct drgn_error *
2596 linux_cpu_present_mask(struct drgn_program *prog, uint64_t **bitmap_ret,
2597 size_t *size_ret)
2598 {
2599 // This C implementation is not exposed as a drgn helper because it doesn't
2600 // really provide any benefit over the Python implementation. We would need
2601 // to wrap this in another Python function in order to provide the existing
2602 // "default program" behavior that cpu_present_mask() already provides.
2603 // Rather than add the extra boilerplate and complexity, have this copy for
2604 // internal use.
2605 struct drgn_error *err;
2606
2607 DRGN_OBJECT(tmp, prog);
2608 err = drgn_program_find_object(prog, "__cpu_present_mask", NULL,
2609 DRGN_FIND_OBJECT_VARIABLE, &tmp);
2610 if (drgn_error_catch(&err, DRGN_ERROR_LOOKUP)) {
2611 err = drgn_program_find_object(prog, "cpu_present_mask", NULL,
2612 DRGN_FIND_OBJECT_VARIABLE, &tmp);
2613 if (err)
2614 return err;
2615 err = drgn_object_dereference(&tmp, &tmp);
2616 }
2617 if (err)
2618 return err;
2619
2620 err = drgn_object_member(&tmp, &tmp, "bits");
2621 if (err)
2622 return err;
2623
2624 uint64_t bitmap_addr = tmp.address;
2625 DRGN_OBJECT(nr_cpu_ids_obj, prog);
2626 uint64_t nr_cpu_ids;
2627 err = drgn_program_find_object(prog, "nr_cpu_ids", NULL,
2628 DRGN_FIND_OBJECT_VARIABLE, &nr_cpu_ids_obj);
2629 if (err) {
2630 if (drgn_error_catch(&err, DRGN_ERROR_LOOKUP))
2631 nr_cpu_ids = 1;
2632 else
2633 return err;
2634 } else {
2635 err = drgn_object_read_unsigned(&nr_cpu_ids_obj, &nr_cpu_ids);
2636 if (err)
2637 return err;
2638 }
2639 // A bogus value could overflow a 32-bit size_t, validate it here
2640 if (nr_cpu_ids > SIZE_MAX)
2641 return &drgn_enomem;
2642
2643 bool is_64_bit;
2644 err = drgn_program_is_64_bit(prog, &is_64_bit);
2645 if (err)
2646 return err;
2647
2648 bool bswap;
2649 err = drgn_program_bswap(prog, &bswap);
2650 if (err)
2651 return err;
2652
2653 size_t size = nr_cpu_ids / 64 + ((nr_cpu_ids % 64) ? 1 : 0);
2654 _cleanup_free_ uint64_t *bitmap = calloc(size, sizeof(*bitmap));
2655 if (!bitmap)
2656 return &drgn_enomem;
2657
2658 if (is_64_bit) {
2659 err = drgn_program_read_memory(prog, bitmap, bitmap_addr, size * 8, false);
2660 if (err)
2661 return err;
2662
2663 if (bswap)
2664 for (size_t i = 0; i < size; i++)
2665 bitmap[i] = bswap_64(bitmap[i]);
2666 } else {
2667 size_t nr_words = nr_cpu_ids / 32 + ((nr_cpu_ids % 32) ? 1 : 0);
2668 _cleanup_free_ uint32_t *orig = malloc_array(nr_words, sizeof(*orig));
2669 if (!bitmap)
2670 return &drgn_enomem;
2671
2672 err = drgn_program_read_memory(prog, orig, bitmap_addr, nr_words * 4, false);
2673 if (err)
2674 return err;
2675
2676 for (size_t i = 0; i < nr_words; i++) {
2677 uint32_t value = bswap ? bswap_32(orig[i]) : orig[i];
2678 if (i & 1)
2679 bitmap[i >> 1] |= (uint64_t) value << 32;
2680 else
2681 bitmap[i >> 1] |= value;
2682 }
2683 }
2684
2685 *bitmap_ret = no_cleanup_ptr(bitmap);
2686 *size_ret = nr_cpu_ids;
2687 return NULL;
2688 }
2689
2690 DEFINE_VECTOR(uint64_vector, uint64_t);
2691
2692 struct drgn_error *
2693 drgn_program_is_irq_regs(struct drgn_program *prog, uint64_t addr,
2694 bool *ret)
2695 {
2696 if (!(prog->flags & DRGN_PROGRAM_IS_LINUX_KERNEL) ||
2697 (prog->flags & DRGN_PROGRAM_IS_LIVE)) {
2698 *ret = false;
2699 return NULL;
2700 }
2701
2702 if (!prog->irq_regs_cached) {
2703 struct drgn_error *err;
2704 DRGN_OBJECT(irq_regs_obj, prog);
2705
2706 err = drgn_program_find_object(prog, "__irq_regs", NULL,
2707 DRGN_FIND_OBJECT_VARIABLE, &irq_regs_obj);
2708 if (err)
2709 return err;
2710
2711 err = drgn_object_address_of(&irq_regs_obj, &irq_regs_obj);
2712 if (err)
2713 return err;
2714
2715 _cleanup_free_ uint64_t *cpumask = NULL;
2716 size_t nr_cpus;
2717 err = linux_cpu_present_mask(prog, &cpumask, &nr_cpus);
2718 if (err)
2719 return err;
2720
2721 DRGN_OBJECT(percpu_regs, prog);
2722 VECTOR(uint64_vector, addresses);
2723 uint64_t regs_ptr;
2724 for (size_t word_index = 0; word_index * 64 < nr_cpus; word_index++) {
2725 uint64_t word = cpumask[word_index];
2726 unsigned int i;
2727 for_each_bit(i, word) {
2728 size_t cpu = word_index * 64 + i;
2729 err = linux_helper_per_cpu_ptr(&percpu_regs, &irq_regs_obj, cpu);
2730 if (err)
2731 return err;
2732 err = drgn_object_dereference(&percpu_regs, &percpu_regs);
2733 if (err)
2734 return err;
2735 err = drgn_object_read_unsigned(&percpu_regs, &regs_ptr);
2736 if (err)
2737 return err;
2738 if (!regs_ptr)
2739 continue;
2740 drgn_log_debug(prog, "irq_regs: cpu %zu: %"PRIx64, cpu, regs_ptr);
2741 if (!uint64_vector_append(&addresses, &regs_ptr))
2742 return &drgn_enomem;
2743 }
2744 }
2745 drgn_log_debug(prog, "irq_regs: loaded %zu per-cpu __irq_regs",
2746 uint64_vector_size(&addresses));
2747
2748 // A signal value so we don't need to store length in the
2749 // program. The array won't need random access, so this is a
2750 // reasonable tradeoff.
2751 regs_ptr = 0;
2752 if (!uint64_vector_append(&addresses, &regs_ptr))
2753 return &drgn_enomem;
2754
2755 uint64_vector_shrink_to_fit(&addresses);
2756 uint64_vector_steal(&addresses, &prog->irq_regs_cached, NULL);
2757 }
2758
2759 for (size_t i = 0; prog->irq_regs_cached[i]; i++) {
2760 if (prog->irq_regs_cached[i] == addr) {
2761 *ret = true;
2762 return NULL;
2763 }
2764 }
2765 *ret = false;
2766 return NULL;
2767 }