Debian Codegraph @

libdrgn/linux_kernel_helpers.c

raw
1 // Copyright (c) Meta Platforms, Inc. and affiliates.
2 // SPDX-License-Identifier: LGPL-2.1-or-later
3
4 #include <inttypes.h>
5 #include <stdio.h>
6
7 #include "drgn_internal.h"
8 #include "error.h"
9 #include "helpers.h"
10 #include "minmax.h"
11 #include "platform.h"
12 #include "program.h"
13 #include "util.h"
14
15 static void end_virtual_address_translation(struct drgn_program *prog)
16 {
17 prog->in_address_translation = false;
18 }
19
20 static struct drgn_error *
21 begin_virtual_address_translation(struct drgn_program *prog, uint64_t pgtable,
22 uint64_t virt_addr)
23 {
24 struct drgn_error *err;
25
26 if (prog->in_address_translation) {
27 return drgn_error_create_fault("recursive address translation; "
28 "page table may be missing from core dump",
29 virt_addr);
30 }
31 prog->in_address_translation = true;
32 if (!prog->pgtable_it) {
33 if (!(prog->flags & DRGN_PROGRAM_IS_LINUX_KERNEL)) {
34 err = drgn_error_create(DRGN_ERROR_INVALID_ARGUMENT,
35 "virtual address translation is only available for the Linux kernel");
36 goto err;
37 }
38 if (!prog->has_platform) {
39 err = drgn_error_create(DRGN_ERROR_INVALID_ARGUMENT,
40 "cannot do virtual address translation without platform");
41 goto err;
42 }
43 if (!prog->platform.arch->linux_kernel_pgtable_iterator_next) {
44 err = drgn_error_format(DRGN_ERROR_NOT_IMPLEMENTED,
45 "virtual address translation is not implemented for %s architecture",
46 prog->platform.arch->name);
47 goto err;
48 }
49 err = prog->platform.arch->linux_kernel_pgtable_iterator_create(prog,
50 &prog->pgtable_it);
51 if (err) {
52 prog->pgtable_it = NULL;
53 goto err;
54 }
55 }
56 prog->pgtable_it->pgtable = pgtable;
57 prog->pgtable_it->virt_addr = virt_addr;
58 prog->platform.arch->linux_kernel_pgtable_iterator_init(prog, prog->pgtable_it);
59 return NULL;
60
61 err:
62 end_virtual_address_translation(prog);
63 return err;
64 }
65
66 struct drgn_error *linux_helper_direct_mapping_offset(struct drgn_program *prog,
67 uint64_t *ret)
68 {
69 struct drgn_error *err;
70
71 if (prog->direct_mapping_offset_cached) {
72 *ret = prog->direct_mapping_offset;
73 return NULL;
74 }
75
76 if (prog->platform.arch->linux_kernel_direct_mapping_offset) {
77 err = prog->platform.arch->linux_kernel_direct_mapping_offset(
78 prog, &prog->direct_mapping_offset);
79 if (drgn_error_catch(&err, DRGN_ERROR_LOOKUP)) {
80 // Fall back to address translation.
81 } else if (err) {
82 return err;
83 } else {
84 prog->direct_mapping_offset_cached = true;
85 *ret = prog->direct_mapping_offset;
86 return NULL;
87 }
88 }
89
90 // The direct mapping offset can vary depending on architecture, kernel
91 // version, configuration options, and KASLR. For architectures that
92 // don't handle all of that, get a virtual address in the direct mapping
93 // and translate it to a physical address via the page table. The
94 // difference is the offset.
95 //
96 // The virtual address we pick doesn't matter as long as:
97 //
98 // 1. It is in the direct mapping on all configurations of all supported
99 // kernel versions on all architectures.
100 // 2. That is unlikely to change in the future.
101 //
102 // This is our current arbitrary choice.
103 static const char direct_mapping_variable[] = "saved_command_line";
104
105 struct drgn_object tmp;
106 drgn_object_init(&tmp, prog);
107 err = drgn_program_find_object(prog, direct_mapping_variable, NULL,
108 DRGN_FIND_OBJECT_VARIABLE, &tmp);
109 uint64_t virt_addr;
110 if (!err) {
111 err = drgn_object_read_unsigned(&tmp, &virt_addr);
112 } else if (drgn_error_catch(&err, DRGN_ERROR_LOOKUP)) {
113 // Avoid a confusing error message with our arbitrary variable
114 // name.
115 err = drgn_error_create(DRGN_ERROR_OTHER,
116 "could not find variable in direct mapping");
117 }
118 drgn_object_deinit(&tmp);
119 if (err)
120 return err;
121
122 err = begin_virtual_address_translation(prog,
123 prog->vmcoreinfo.swapper_pg_dir,
124 virt_addr);
125 if (err)
126 return err;
127 uint64_t start_virt_addr, start_phys_addr;
128 err = prog->platform.arch->linux_kernel_pgtable_iterator_next(prog,
129 prog->pgtable_it,
130 &start_virt_addr,
131 &start_phys_addr);
132 if (err)
133 goto out;
134 if (start_phys_addr == UINT64_MAX) {
135 err = drgn_error_create(DRGN_ERROR_OTHER,
136 "could not determine direct mapping offset");
137 goto out;
138 }
139 prog->direct_mapping_offset = start_virt_addr - start_phys_addr;
140 prog->direct_mapping_offset_cached = true;
141 *ret = prog->direct_mapping_offset;
142 err = NULL;
143 out:
144 end_virtual_address_translation(prog);
145 return err;
146 }
147
148 struct drgn_error *linux_helper_read_vm(struct drgn_program *prog,
149 uint64_t pgtable, uint64_t virt_addr,
150 void *buf, size_t count)
151 {
152 struct drgn_error *err;
153
154 err = begin_virtual_address_translation(prog, pgtable, virt_addr);
155 if (err)
156 return err;
157 if (!count) {
158 err = NULL;
159 goto out;
160 }
161
162 struct pgtable_iterator *it = prog->pgtable_it;
163 pgtable_iterator_next_fn *next =
164 prog->platform.arch->linux_kernel_pgtable_iterator_next;
165 uint64_t read_addr = 0;
166 size_t read_size = 0;
167 do {
168 uint64_t start_virt_addr, start_phys_addr;
169 err = next(prog, it, &start_virt_addr, &start_phys_addr);
170 if (err)
171 break;
172 if (start_phys_addr == UINT64_MAX) {
173 err = drgn_error_create_fault("address is not mapped",
174 virt_addr);
175 break;
176 }
177
178 uint64_t phys_addr =
179 start_phys_addr + (virt_addr - start_virt_addr);
180 size_t n = min(it->virt_addr - virt_addr, (uint64_t)count);
181 if (read_size && phys_addr == read_addr + read_size) {
182 read_size += n;
183 } else {
184 if (read_size) {
185 err = drgn_program_read_memory(prog, buf,
186 read_addr,
187 read_size, true);
188 if (err)
189 break;
190 buf = (char *)buf + read_size;
191 }
192 read_addr = phys_addr;
193 read_size = n;
194 }
195 virt_addr = it->virt_addr;
196 count -= n;
197 } while (count);
198 if (!err) {
199 err = drgn_program_read_memory(prog, buf, read_addr, read_size,
200 true);
201 }
202 out:
203 end_virtual_address_translation(prog);
204 return err;
205 }
206
207 struct drgn_error *linux_helper_follow_phys(struct drgn_program *prog,
208 uint64_t pgtable,
209 uint64_t virt_addr, uint64_t *ret)
210 {
211 struct drgn_error *err;
212
213 err = begin_virtual_address_translation(prog, pgtable, virt_addr);
214 if (err)
215 return err;
216
217 struct pgtable_iterator *it = prog->pgtable_it;
218 pgtable_iterator_next_fn *next =
219 prog->platform.arch->linux_kernel_pgtable_iterator_next;
220 uint64_t start_virt_addr, start_phys_addr;
221 err = next(prog, it, &start_virt_addr, &start_phys_addr);
222 if (err)
223 goto out;
224 if (start_phys_addr == UINT64_MAX) {
225 err = drgn_error_create_fault("address is not mapped",
226 virt_addr);
227 goto out;
228 }
229 *ret = start_phys_addr + (virt_addr - start_virt_addr);
230 err = NULL;
231 out:
232 end_virtual_address_translation(prog);
233 return err;
234 }
235
236 struct drgn_error *linux_helper_per_cpu_ptr(struct drgn_object *res,
237 const struct drgn_object *ptr,
238 uint64_t cpu)
239 {
240 struct drgn_error *err;
241 struct drgn_program *prog = drgn_object_program(ptr);
242
243 DRGN_OBJECT(tmp, prog);
244 err = drgn_program_find_object(prog, "__per_cpu_offset", NULL,
245 DRGN_FIND_OBJECT_ANY, &tmp);
246 if (!err) {
247 err = drgn_object_subscript(&tmp, &tmp, cpu);
248 if (err)
249 return err;
250 union drgn_value per_cpu_offset;
251 err = drgn_object_read_integer(&tmp, &per_cpu_offset);
252 if (err)
253 return err;
254
255 uint64_t ptr_value;
256 err = drgn_object_read_unsigned(ptr, &ptr_value);
257 if (err)
258 return err;
259
260 return drgn_object_set_unsigned(res,
261 drgn_object_qualified_type(ptr),
262 ptr_value + per_cpu_offset.uvalue,
263 0);
264 } else if (drgn_error_catch(&err, DRGN_ERROR_LOOKUP)) {
265 return drgn_object_copy(res, ptr);
266 } else {
267 return err;
268 }
269 }
270
271 static struct drgn_error *cpu_rq_member(struct drgn_object *res, uint64_t cpu,
272 const char *member_name)
273 {
274 struct drgn_error *err;
275 struct drgn_program *prog = drgn_object_program(res);
276
277 DRGN_OBJECT(tmp, prog);
278 err = drgn_program_find_object(prog, "runqueues", NULL,
279 DRGN_FIND_OBJECT_ANY, &tmp);
280 if (err)
281 return err;
282 err = drgn_object_address_of(&tmp, &tmp);
283 if (err)
284 return err;
285 err = linux_helper_per_cpu_ptr(&tmp, &tmp, cpu);
286 if (err)
287 return err;
288 err = drgn_object_member_dereference(&tmp, &tmp, member_name);
289 if (err)
290 return err;
291 return drgn_object_read(res, &tmp);
292 }
293
294 struct drgn_error *linux_helper_cpu_curr(struct drgn_object *res, uint64_t cpu)
295 {
296 return cpu_rq_member(res, cpu, "curr");
297 }
298
299 struct drgn_error *linux_helper_idle_task(struct drgn_object *res, uint64_t cpu)
300 {
301 return cpu_rq_member(res, cpu, "idle");
302 }
303
304 struct drgn_error *linux_helper_task_thread_info(struct drgn_object *res,
305 const struct drgn_object *task)
306 {
307 struct drgn_error *err;
308 DRGN_OBJECT(tmp, drgn_object_program(task));
309
310 err = drgn_object_member_dereference(&tmp, task, "thread_info");
311 if (!err) {
312 // CONFIG_THREAD_INFO_IN_TASK=y
313 return drgn_object_address_of(res, &tmp);
314 } else if (drgn_error_catch(&err, DRGN_ERROR_LOOKUP)) {
315 // CONFIG_THREAD_INFO_IN_TASK=n
316 err = drgn_object_member_dereference(&tmp, task, "stack");
317 if (err)
318 return err;
319 struct drgn_qualified_type thread_info_type;
320 err = drgn_program_find_type(drgn_object_program(task),
321 "struct thread_info *", NULL,
322 &thread_info_type);
323 if (err)
324 return err;
325 return drgn_object_cast(res, thread_info_type, &tmp);
326 } else {
327 return err;
328 }
329 }
330
331 struct drgn_error *linux_helper_task_cpu(const struct drgn_object *task,
332 uint64_t *ret)
333 {
334 struct drgn_error *err;
335 DRGN_OBJECT(tmp, drgn_object_program(task));
336
337 // The CPU may be task_thread_info(task)->cpu or task->cpu depending on
338 // the kernel version. If neither exists, then the kernel must be !SMP.
339 //
340 // Since Linux kernel commit bcf9033e5449 ("sched: move CPU field back
341 // into thread_info if THREAD_INFO_IN_TASK=y") (in v5.16), or if
342 // CONFIG_THREAD_INFO_IN_TASK=n, or before Linux kernel commit
343 // c65eacbe290b ("sched/core: Allow putting thread_info into
344 // task_struct") (in v4.9), the CPU is task_thread_info(task)->cpu.
345 //
346 // Between Linux kernel commits bcf9033e5449 ("sched: move CPU field
347 // back into thread_info if THREAD_INFO_IN_TASK=y") (in v5.16) and
348 // c65eacbe290b ("sched/core: Allow putting thread_info into
349 // task_struct") (in v4.9), if CONFIG_THREAD_INFO_IN_TASK=y, then the
350 // CPU is task->cpu.
351 //
352 // Note that between Linux kernel commit bcf9033e5449 ("sched: move CPU
353 // field back into thread_info if THREAD_INFO_IN_TASK=y") and commits
354 // 001430c1910d ("arm64: add CPU field to struct thread_info"),
355 // 5443f98fb9e0 ("x86: add CPU field to struct thread_info"),
356 // bd2e2632556a ("s390: add CPU field to struct thread_info"), and
357 // 227d735d889e ("powerpc: add CPU field to struct thread_info") (all in
358 // v5.16-rc1), if CONFIG_THREAD_INFO_IN_TASK=y, then
359 // struct thread_info::cpu may exist but task->cpu is still used.
360 // Therefore, we must check for task->cpu first. (Normally we don't care
361 // about commits in the middle of a release candidate, but CentOS Stream
362 // 9 and its derivatives apparently backported commit 5443f98fb9e0
363 // without commit bcf9033e5449:
364 // https://gitlab.com/redhat/centos-stream/src/kernel/centos-stream-9/-/commit/6d09fbd042c8d99009e16ddba62af09c89358f80.)
365 err = drgn_object_member_dereference(&tmp, task, "cpu");
366 if (drgn_error_catch(&err, DRGN_ERROR_LOOKUP)) {
367 err = linux_helper_task_thread_info(&tmp, task);
368 if (err)
369 return err;
370 err = drgn_object_member_dereference(&tmp, &tmp, "cpu");
371 }
372 if (!err) {
373 union drgn_value value;
374 err = drgn_object_read_integer(&tmp, &value);
375 if (!err)
376 *ret = value.uvalue;
377 } else if (drgn_error_catch(&err, DRGN_ERROR_LOOKUP)) {
378 // CONFIG_SMP=n
379 *ret = 0;
380 }
381 return err;
382 }
383
384 struct drgn_error *linux_helper_task_on_cpu(const struct drgn_object *task,
385 bool *ret)
386 {
387 struct drgn_error *err;
388 DRGN_OBJECT(tmp, drgn_object_program(task));
389
390 err = drgn_object_member_dereference(&tmp, task, "on_cpu");
391 if (!err)
392 return drgn_object_bool(&tmp, ret);
393
394 if (!drgn_error_catch(&err, DRGN_ERROR_LOOKUP))
395 return err;
396
397 // The kernel must be !SMP. We have to check cpu_curr(0) instead.
398 err = linux_helper_cpu_curr(&tmp, 0);
399 if (err)
400 return err;
401 int cmp;
402 err = drgn_object_cmp(&tmp, task, &cmp);
403 if (err)
404 return err;
405 *ret = cmp == 0;
406 return NULL;
407 }
408
409 struct drgn_error *
410 linux_helper_xa_load(struct drgn_object *res,
411 const struct drgn_object *xa, uint64_t index)
412 {
413 struct drgn_error *err;
414
415 struct drgn_qualified_type node_type;
416 uint64_t internal_flag, node_min;
417
418 DRGN_OBJECT(entry, drgn_object_program(res));
419 DRGN_OBJECT(node, drgn_object_program(res));
420 DRGN_OBJECT(tmp, drgn_object_program(res));
421
422 // See xa_for_each() in drgn/helpers/linux/xarray.py for a description
423 // of the cases we have to handle.
424 // entry = xa->xa_head
425 err = drgn_object_member_dereference(&entry, xa, "xa_head");
426 if (!err) {
427 err = drgn_object_read(&entry, &entry);
428 if (err)
429 return err;
430 // node_type = struct xa_node *
431 err = drgn_program_find_type(drgn_object_program(res),
432 "struct xa_node *", NULL,
433 &node_type);
434 if (err)
435 return err;
436 internal_flag = 2;
437 node_min = 4097;
438 } else if (drgn_error_catch(&err, DRGN_ERROR_LOOKUP)) {
439 // entry = (void *)xa->rnode
440 err = drgn_object_member_dereference(&entry, xa, "rnode");
441 if (err)
442 return err;
443 // node_type = typeof(xa->rnode)
444 node_type = drgn_object_qualified_type(&entry);
445 struct drgn_qualified_type voidp_type;
446 err = drgn_program_find_type(drgn_object_program(res), "void *",
447 NULL, &voidp_type);
448 if (err)
449 return err;
450 err = drgn_object_cast(&entry, voidp_type, &entry);
451 if (err)
452 return err;
453 internal_flag = 1;
454 node_min = 0;
455 } else {
456 return err;
457 }
458
459 // xa_is_node() or radix_tree_is_internal_node()
460 #define is_node(entry_value) \
461 (((entry_value) & 3) == internal_flag && (entry_value) >= node_min)
462
463 struct drgn_type_member *member;
464 uint64_t member_bit_offset;
465 err = drgn_type_find_member(drgn_type_type(node_type.type).type,
466 "slots", &member, &member_bit_offset);
467 if (err)
468 return err;
469 struct drgn_qualified_type member_type;
470 err = drgn_member_type(member, &member_type, NULL);
471 if (err)
472 return err;
473 if (drgn_type_kind(member_type.type) != DRGN_TYPE_ARRAY) {
474 return drgn_error_create(DRGN_ERROR_TYPE,
475 "struct xa_node slots member is not an array");
476 }
477 uint64_t XA_CHUNK_MASK = drgn_type_length(member_type.type) - 1;
478 uint64_t sizeof_slots;
479 if (node_min == 0) { // !xarray
480 err = drgn_type_sizeof(member_type.type, &sizeof_slots);
481 if (err)
482 return err;
483 }
484
485 uint64_t entry_value;
486 err = drgn_object_read_unsigned(&entry, &entry_value);
487 if (err)
488 return err;
489 if (is_node(entry_value)) {
490 // node = xa_to_node(entry)
491 // or
492 // node = entry_to_node(entry)
493 err = drgn_object_set_unsigned(&node, node_type,
494 entry_value - internal_flag, 0);
495 if (err)
496 return err;
497 // node_shift = node->shift
498 err = drgn_object_member_dereference(&tmp, &node, "shift");
499 if (err)
500 return err;
501 union drgn_value node_shift;
502 err = drgn_object_read_integer(&tmp, &node_shift);
503 if (err)
504 return err;
505
506 uint64_t offset;
507 if (node_shift.uvalue >= 64) // Avoid undefined behavior.
508 offset = 0;
509 else
510 offset = index >> node_shift.uvalue;
511 if (offset > XA_CHUNK_MASK)
512 goto null;
513
514 for (;;) {
515 // entry = node->slots[offset]
516 err = drgn_object_member_dereference(&tmp, &node,
517 "slots");
518 if (err)
519 return err;
520 err = drgn_object_subscript(&entry, &tmp, offset);
521 if (err)
522 return err;
523 err = drgn_object_read(&entry, &entry);
524 if (err)
525 return err;
526 err = drgn_object_read_unsigned(&entry, &entry_value);
527 if (err)
528 return err;
529
530 if ((entry_value & 3) == internal_flag) {
531 if (node_min != 0 && // xarray
532 entry_value < 256) { // xa_is_sibling()
533 // entry = node->slots[xa_to_sibling(entry)]
534 err = drgn_object_subscript(&entry,
535 &tmp,
536 entry_value >> 2);
537 if (err)
538 return err;
539 err = drgn_object_read(&entry, &entry);
540 if (err)
541 return err;
542 err = drgn_object_read_unsigned(&entry,
543 &entry_value);
544 if (err)
545 return err;
546 } else if (node_min == 0 && // !xarray
547 tmp.address <= entry_value &&
548 entry_value < tmp.address + sizeof_slots) { // is_sibling_entry()
549 // entry = *(void **)entry_to_node(entry)
550 struct drgn_qualified_type voidpp_type;
551 err = drgn_program_find_type(drgn_object_program(res),
552 "void **",
553 NULL,
554 &voidpp_type);
555 if (err)
556 return err;
557 err = drgn_object_set_unsigned(&entry,
558 voidpp_type,
559 entry_value - 1,
560 0);
561 if (err)
562 return err;
563 err = drgn_object_dereference(&entry,
564 &entry);
565 if (err)
566 return err;
567 err = drgn_object_read(&entry, &entry);
568 if (err)
569 return err;
570 err = drgn_object_read_unsigned(&entry,
571 &entry_value);
572 if (err)
573 return err;
574 }
575 }
576
577 if (node_shift.uvalue == 0 || !is_node(entry_value))
578 break;
579
580 // node = xa_to_node(entry)
581 // or
582 // node = entry_to_node(entry)
583 err = drgn_object_set_unsigned(&node, node_type,
584 entry_value - internal_flag,
585 0);
586 if (err)
587 return err;
588 // node_shift = node->shift
589 err = drgn_object_member_dereference(&tmp, &node,
590 "shift");
591 if (err)
592 return err;
593 err = drgn_object_read_integer(&tmp, &node_shift);
594 if (err)
595 return err;
596
597 if (node_shift.uvalue >= 64) // Avoid undefined behavior.
598 offset = 0;
599 else
600 offset = (index >> node_shift.uvalue) & XA_CHUNK_MASK;
601 }
602 } else if (index) {
603 goto null;
604 }
605
606 return drgn_object_copy(res, &entry);
607
608 null:
609 return drgn_object_set_unsigned(res, drgn_object_qualified_type(&entry),
610 0, 0);
611
612 #undef is_node
613 }
614
615 // Note that this only works since Linux kernel commit 0a835c4f090a
616 // ("Reimplement IDR and IDA using the radix tree") (in v4.11). We only need
617 // this since Linux kernel commit 95846ecf9dac ("pid: replace pid bitmap
618 // implementation with IDR API") (in v4.15) (see find_pid_in_pid_hash()), so
619 // that's okay.
620 struct drgn_error *linux_helper_idr_find(struct drgn_object *res,
621 const struct drgn_object *idr,
622 uint64_t id)
623 {
624 struct drgn_error *err;
625
626 DRGN_OBJECT(tmp, drgn_object_program(res));
627
628 /* id -= idr->idr_base */
629 err = drgn_object_member_dereference(&tmp, idr, "idr_base");
630 if (!err) {
631 union drgn_value idr_base;
632
633 err = drgn_object_read_integer(&tmp, &idr_base);
634 if (err)
635 return err;
636 id -= idr_base.uvalue;
637 } else if (!drgn_error_catch(&err, DRGN_ERROR_LOOKUP)) {
638 /* idr_base was added in v4.16. */
639 return err;
640 }
641
642 /* radix_tree_lookup(&idr->idr_rt, id) */
643 err = drgn_object_member_dereference(&tmp, idr, "idr_rt");
644 if (err)
645 return err;
646 err = drgn_object_address_of(&tmp, &tmp);
647 if (err)
648 return err;
649 return linux_helper_xa_load(res, &tmp, id);
650 }
651
652 /*
653 * Before Linux kernel commit 95846ecf9dac ("pid: replace pid bitmap
654 * implementation with IDR API") (in v4.15), (struct pid_namespace).idr does not
655 * exist, so we have to search pid_hash. We could implement pid_hashfn() and
656 * only search that bucket, but it's different for 32-bit and 64-bit systems,
657 * and it has changed at least once, in v4.7. Searching the whole hash table is
658 * slower but foolproof.
659 */
660 static struct drgn_error *
661 find_pid_in_pid_hash(struct drgn_object *res, const struct drgn_object *ns,
662 const struct drgn_object *pid_hash, uint64_t pid)
663 {
664 struct drgn_error *err;
665
666 struct drgn_qualified_type pidp_type;
667 err = drgn_program_find_type(drgn_object_program(res), "struct pid *",
668 NULL, &pidp_type);
669 if (err)
670 return err;
671
672 struct drgn_qualified_type upid_type;
673 err = drgn_program_find_type(drgn_object_program(res), "struct upid",
674 NULL, &upid_type);
675 if (err)
676 return err;
677
678 struct drgn_type_member *pid_chain_member;
679 uint64_t pid_chain_bit_offset;
680 err = drgn_type_find_member(upid_type.type, "pid_chain",
681 &pid_chain_member, &pid_chain_bit_offset);
682 if (err)
683 return err;
684
685 struct drgn_type_member *nr_member;
686 uint64_t nr_bit_offset;
687 err = drgn_type_find_member(upid_type.type, "nr", &nr_member,
688 &nr_bit_offset);
689 if (err)
690 return err;
691 struct drgn_qualified_type nr_type;
692 err = drgn_member_type(nr_member, &nr_type, NULL);
693 if (err)
694 return err;
695
696 struct drgn_type_member *ns_member;
697 uint64_t ns_bit_offset;
698 err = drgn_type_find_member(upid_type.type, "ns", &ns_member,
699 &ns_bit_offset);
700 if (err)
701 return err;
702 struct drgn_qualified_type ns_type;
703 err = drgn_member_type(ns_member, &ns_type, NULL);
704 if (err)
705 return err;
706
707 DRGN_OBJECT(node, drgn_object_program(res));
708 DRGN_OBJECT(tmp, drgn_object_program(res));
709
710 err = drgn_object_read(&tmp, ns);
711 if (err)
712 return err;
713 uint64_t ns_addr;
714 err = drgn_object_read_unsigned(&tmp, &ns_addr);
715 if (err)
716 return err;
717 union drgn_value ns_level;
718 err = drgn_object_member_dereference(&tmp, &tmp, "level");
719 if (err)
720 return err;
721 err = drgn_object_read_integer(&tmp, &ns_level);
722 if (err)
723 return err;
724
725 /* i = 1 << pidhash_shift */
726 err = drgn_program_find_object(drgn_object_program(res),
727 "pidhash_shift", NULL,
728 DRGN_FIND_OBJECT_ANY, &tmp);
729 if (err)
730 return err;
731 union drgn_value pidhash_shift;
732 err = drgn_object_read_integer(&tmp, &pidhash_shift);
733 if (err)
734 return err;
735 uint64_t i;
736 if (pidhash_shift.uvalue >= 64)
737 i = 0;
738 else
739 i = UINT64_C(1) << pidhash_shift.uvalue;
740 while (i--) {
741 /* for (node = pid_hash[i].first; node; node = node->next) */
742 err = drgn_object_subscript(&node, pid_hash, i);
743 if (err)
744 return err;
745 err = drgn_object_member(&node, &node, "first");
746 if (err)
747 return err;
748 for (;;) {
749 uint64_t addr, tmp_addr;
750 union drgn_value node_nr;
751 uint64_t node_ns;
752
753 err = drgn_object_read(&node, &node);
754 if (err)
755 return err;
756 err = drgn_object_read_unsigned(&node, &addr);
757 if (err)
758 return err;
759 if (!addr)
760 break;
761 addr -= pid_chain_bit_offset / 8;
762
763 /* tmp = container_of(node, struct upid, pid_chain)->nr */
764 tmp_addr = addr + nr_bit_offset / 8;
765 err = drgn_object_set_reference(&tmp, nr_type, tmp_addr,
766 0, 0);
767 if (err)
768 return err;
769 err = drgn_object_read_integer(&tmp, &node_nr);
770 if (err)
771 return err;
772 if (node_nr.uvalue != pid)
773 goto next;
774
775 /* tmp = container_of(node, struct upid, pid_chain)->ns */
776 tmp_addr = addr + ns_bit_offset / 8;
777 err = drgn_object_set_reference(&tmp, ns_type, tmp_addr,
778 0, 0);
779 if (err)
780 return err;
781
782 err = drgn_object_read_unsigned(&tmp, &node_ns);
783 if (err)
784 return err;
785 if (node_ns != ns_addr)
786 goto next;
787
788 #define FORMAT "numbers[%" PRIu64 "].pid_chain"
789 char member[sizeof(FORMAT)
790 - sizeof("%" PRIu64)
791 + max_decimal_length(uint64_t)
792 + 1];
793 snprintf(member, sizeof(member), FORMAT,
794 ns_level.uvalue);
795 #undef FORMAT
796 return drgn_object_container_of(res, &node,
797 drgn_type_type(pidp_type.type),
798 member);
799
800 next:
801 err = drgn_object_member_dereference(&node, &node, "next");
802 if (err)
803 return err;
804 }
805 }
806
807 return drgn_object_set_unsigned(res, pidp_type, 0, 0);
808 }
809
810 struct drgn_error *linux_helper_find_pid(struct drgn_object *res,
811 const struct drgn_object *ns,
812 uint64_t pid)
813 {
814 struct drgn_error *err;
815
816 DRGN_OBJECT(tmp, drgn_object_program(res));
817
818 /* (struct pid *)idr_find(&ns->idr, pid) */
819 err = drgn_object_member_dereference(&tmp, ns, "idr");
820 if (!err) {
821 struct drgn_qualified_type qualified_type;
822
823 err = drgn_object_address_of(&tmp, &tmp);
824 if (err)
825 return err;
826 err = linux_helper_idr_find(&tmp, &tmp, pid);
827 if (err)
828 return err;
829 err = drgn_program_find_type(drgn_object_program(res),
830 "struct pid *", NULL,
831 &qualified_type);
832 if (err)
833 return err;
834 return drgn_object_cast(res, qualified_type, &tmp);
835 } else if (drgn_error_catch(&err, DRGN_ERROR_LOOKUP)) {
836 err = drgn_program_find_object(drgn_object_program(res),
837 "pid_hash", NULL,
838 DRGN_FIND_OBJECT_ANY, &tmp);
839 if (err)
840 return err;
841 return find_pid_in_pid_hash(res, ns, &tmp, pid);
842 } else {
843 return err;
844 }
845 }
846
847 struct drgn_error *linux_helper_pid_task(struct drgn_object *res,
848 const struct drgn_object *pid,
849 uint64_t pid_type)
850 {
851 struct drgn_error *err;
852 struct drgn_qualified_type task_structp_type;
853 struct drgn_qualified_type task_struct_type;
854 bool truthy;
855
856 DRGN_OBJECT(first, drgn_object_program(res));
857
858 err = drgn_program_find_type(drgn_object_program(res),
859 "struct task_struct *", NULL,
860 &task_structp_type);
861 if (err)
862 return err;
863 task_struct_type = drgn_type_type(task_structp_type.type);
864
865 err = drgn_object_bool(pid, &truthy);
866 if (err)
867 return err;
868 if (!truthy)
869 goto null;
870
871 /* first = &pid->tasks[pid_type].first */
872 err = drgn_object_member_dereference(&first, pid, "tasks");
873 if (err)
874 return err;
875 err = drgn_object_subscript(&first, &first, pid_type);
876 if (err)
877 return err;
878 err = drgn_object_member(&first, &first, "first");
879 if (err)
880 return err;
881
882 err = drgn_object_bool(&first, &truthy);
883 if (err)
884 return err;
885 if (!truthy)
886 goto null;
887
888 /* container_of(first, struct task_struct, pid_links[pid_type]) */
889 #define PID_LINKS_FORMAT "pid_links[%" PRIu64 "]"
890 #define PIDS_NODE_FORMAT "pids[%" PRIu64 "].node"
891 char member[max_iconst(sizeof(PID_LINKS_FORMAT),
892 sizeof(PIDS_NODE_FORMAT))
893 - sizeof("%" PRIu64)
894 + max_decimal_length(uint64_t)
895 + 1];
896 snprintf(member, sizeof(member), PID_LINKS_FORMAT, pid_type);
897 err = drgn_object_container_of(res, &first, task_struct_type, member);
898 if (drgn_error_catch(&err, DRGN_ERROR_LOOKUP)) {
899 /* container_of(first, struct task_struct, pids[pid_type].node) */
900 snprintf(member, sizeof(member), PIDS_NODE_FORMAT, pid_type);
901 #undef PID_LINKS_FORMAT
902 #undef PIDS_NODE_FORMAT
903 err = drgn_object_container_of(res, &first, task_struct_type,
904 member);
905 }
906 return err;
907
908 null:
909 return drgn_object_set_unsigned(res, task_structp_type, 0, 0);
910 }
911
912 struct drgn_error *linux_helper_find_task(struct drgn_object *res,
913 const struct drgn_object *ns,
914 uint64_t pid)
915 {
916 struct drgn_error *err;
917 union drgn_value pid_type;
918
919 DRGN_OBJECT(pid_obj, drgn_object_program(res));
920 DRGN_OBJECT(pid_type_obj, drgn_object_program(res));
921
922 err = linux_helper_find_pid(&pid_obj, ns, pid);
923 if (err)
924 return err;
925 err = drgn_program_find_object(drgn_object_program(res), "PIDTYPE_PID",
926 NULL, DRGN_FIND_OBJECT_CONSTANT,
927 &pid_type_obj);
928 if (err)
929 return err;
930 err = drgn_object_read_integer(&pid_type_obj, &pid_type);
931 if (err)
932 return err;
933 return linux_helper_pid_task(res, &pid_obj, pid_type.uvalue);
934 }
935
936 static inline struct drgn_error *
937 linux_helper_task_iterator_set_thread_node(struct linux_helper_task_iterator *it)
938 {
939 struct drgn_error *err;
940 err = drgn_object_container_of(&it->thread_node, &it->tasks_node,
941 it->task_struct_type, "tasks");
942 if (err)
943 return err;
944 err = drgn_object_member_dereference(&it->thread_node, &it->thread_node,
945 "signal");
946 if (err)
947 return err;
948 err = drgn_object_member_dereference(&it->thread_node, &it->thread_node,
949 "thread_head");
950 if (err)
951 return err;
952 err = drgn_object_address_of(&it->thread_node, &it->thread_node);
953 if (err)
954 return err;
955 return drgn_object_read_unsigned(&it->thread_node, &it->thread_head);
956 }
957
958 struct drgn_error *
959 linux_helper_task_iterator_init(struct linux_helper_task_iterator *it,
960 struct drgn_program *prog)
961 {
962 struct drgn_error *err;
963 drgn_object_init(&it->tasks_node, prog);
964 drgn_object_init(&it->thread_node, prog);
965
966 err = drgn_program_find_object(prog, "init_task", NULL,
967 DRGN_FIND_OBJECT_VARIABLE,
968 &it->tasks_node);
969 if (err)
970 goto err;
971 it->task_struct_type = drgn_object_qualified_type(&it->tasks_node);
972 err = drgn_object_member(&it->tasks_node, &it->tasks_node, "tasks");
973 if (err)
974 goto err;
975 if (it->tasks_node.kind != DRGN_OBJECT_REFERENCE) {
976 err = drgn_error_create(DRGN_ERROR_OTHER,
977 "can't get address of tasks list");
978 goto err;
979 }
980 it->tasks_head = it->tasks_node.address;
981 err = drgn_object_member(&it->tasks_node, &it->tasks_node, "next");
982 if (err)
983 goto err;
984 err = drgn_object_read(&it->tasks_node, &it->tasks_node);
985 if (err)
986 goto err;
987 uint64_t tasks_node_value;
988 err = drgn_object_read_unsigned(&it->tasks_node, &tasks_node_value);
989 if (err)
990 goto err;
991 if (tasks_node_value == it->tasks_head) {
992 it->done = true;
993 } else {
994 it->done = false;
995 err = linux_helper_task_iterator_set_thread_node(it);
996 if (err)
997 goto err;
998 }
999 return NULL;
1000
1001 err:
1002 linux_helper_task_iterator_deinit(it);
1003 return err;
1004 }
1005
1006 void linux_helper_task_iterator_deinit(struct linux_helper_task_iterator *it)
1007 {
1008 drgn_object_deinit(&it->thread_node);
1009 drgn_object_deinit(&it->tasks_node);
1010 }
1011
1012 struct drgn_error *
1013 linux_helper_task_iterator_next(struct linux_helper_task_iterator *it,
1014 struct drgn_object *ret)
1015 {
1016 struct drgn_error *err;
1017
1018 if (it->done)
1019 return &drgn_stop;
1020
1021 for (;;) {
1022 err = drgn_object_member_dereference(&it->thread_node,
1023 &it->thread_node, "next");
1024 if (err)
1025 return err;
1026 err = drgn_object_read(&it->thread_node, &it->thread_node);
1027 if (err)
1028 return err;
1029 uint64_t thread_node_value;
1030 err = drgn_object_read_unsigned(&it->thread_node, &thread_node_value);
1031 if (err)
1032 return err;
1033 if (thread_node_value != it->thread_head)
1034 break;
1035
1036 err = drgn_object_member_dereference(&it->tasks_node,
1037 &it->tasks_node, "next");
1038 if (err)
1039 return err;
1040 err = drgn_object_read(&it->tasks_node, &it->tasks_node);
1041 if (err)
1042 return err;
1043 uint64_t tasks_node_value;
1044 err = drgn_object_read_unsigned(&it->tasks_node,
1045 &tasks_node_value);
1046 if (err)
1047 return err;
1048 if (tasks_node_value == it->tasks_head) {
1049 it->done = true;
1050 return &drgn_stop;
1051 }
1052 err = linux_helper_task_iterator_set_thread_node(it);
1053 if (err)
1054 return err;
1055 }
1056 return drgn_object_container_of(ret, &it->thread_node,
1057 it->task_struct_type, "thread_node");
1058 }