Debian Codegraph @

tests/linux_kernel/kmod/drgn_test.c

raw
1 // Copyright (c) Meta Platforms, Inc. and affiliates.
2 // SPDX-License-Identifier: GPL-2.0-or-later
3
4 // Linux kernel module for testing drgn helpers and kernel support. For now,
5 // this is all in one file for simplicity and to keep the compilation fast
6 // (since this is compiled for every kernel version in CI).
7 //
8 // This is intended to be used with drgn's vmtest framework, but in theory it
9 // can be used with any kernel that has debug info enabled (at your own risk).
10
11 #include <linux/version.h>
12
13 #include <linux/blkdev.h>
14 #include <linux/blk-mq.h>
15 #include <linux/completion.h>
16 #include <linux/hrtimer.h>
17 #include <linux/io.h>
18 #include <linux/irq_work.h>
19 #include <linux/kernel.h>
20 #include <linux/kexec.h>
21 #include <linux/kthread.h>
22 #include <linux/list.h>
23 #include <linux/llist.h>
24 #if LINUX_VERSION_CODE >= KERNEL_VERSION(6, 1, 0)
25 #define HAVE_MAPLE_TREE 1
26 #include <linux/maple_tree.h>
27 #else
28 #define HAVE_MAPLE_TREE 0
29 #endif
30 #include <linux/memory.h>
31 #include <linux/mm.h>
32 #include <linux/mmzone.h>
33 #include <linux/module.h>
34 #include <linux/mutex.h>
35 #include <linux/netdevice.h>
36 #include <linux/nodemask.h>
37 #include <linux/plist.h>
38 #include <linux/radix-tree.h>
39 #include <linux/rbtree.h>
40 #include <linux/rbtree_augmented.h>
41 #include <linux/rwsem.h>
42 #include <linux/sbitmap.h>
43 #include <linux/skbuff.h>
44 #include <linux/slab.h>
45 #include <linux/stacktrace.h>
46 #ifdef CONFIG_STACKDEPOT
47 #include <linux/stackdepot.h>
48 #endif
49 #include <linux/sysfs.h>
50 #include <linux/timekeeping.h>
51 #include <linux/timer.h>
52 #include <linux/vmalloc.h>
53 #include <linux/wait.h>
54 #include <linux/swait.h>
55
56 // Before Linux kernel commit b3dae109fa89 ("sched/swait: Rename to exclusive")
57 // (in v4.19), prepare_to_swait_exclusive() was named prepare_to_swait().
58 #if LINUX_VERSION_CODE < KERNEL_VERSION(4, 19, 0)
59 #define prepare_to_swait_exclusive prepare_to_swait
60 #endif
61
62 #if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 20, 0)
63 #define HAVE_XARRAY 1
64 #include <linux/xarray.h>
65 #else
66 #define HAVE_XARRAY 0
67 #endif
68
69 // Page pools were added in Linux kernel commit ff7d6b27f894 ("page_pool:
70 // refurbish version of page_pool code") (in v4.18) and may not be enabled.
71 #ifdef CONFIG_PAGE_POOL
72 #define HAVE_PAGE_POOL 1
73 // The header file was moved in Linux kernel commit a9ca9f9ceff3 ("page_pool:
74 // split types and declarations from page_pool.h") (in v6.6).
75 #if LINUX_VERSION_CODE >= KERNEL_VERSION(6, 6, 0)
76 #include <net/page_pool/helpers.h>
77 #else
78 #include <net/page_pool.h>
79 #endif
80 #else
81 #define HAVE_PAGE_POOL 0
82 #endif
83
84 // Architecture-specific includes for triggering NMIs for stack traces
85 #ifdef __x86_64__
86 #include <asm/apic.h>
87 #include <asm/nmi.h>
88 #endif
89
90 #if LINUX_VERSION_CODE < KERNEL_VERSION(4, 18, 0)
91 // These were added in b9ff604cff11 ("timekeeping: Add
92 // ktime_get_coarse_with_offset") (in v4.18-rc1).
93 static inline ktime_t ktime_get_coarse_boottime(void)
94 {
95 struct timespec64 ts = get_monotonic_coarse64();
96
97 return ktime_mono_to_any(timespec64_to_ktime(ts), TK_OFFS_BOOT);
98 }
99
100 static inline ktime_t ktime_get_coarse_clocktai(void)
101 {
102 struct timespec64 ts = get_monotonic_coarse64();
103
104 return ktime_mono_to_any(timespec64_to_ktime(ts), TK_OFFS_TAI);
105 }
106
107 // These were added in Linux kernel commit 06aa376903b6 ("timekeeping: Add more
108 // coarse clocktai/boottime interfaces") (in v4.18).
109 static inline time64_t ktime_get_boottime_seconds(void)
110 {
111 return ktime_divns(ktime_get_coarse_boottime(), NSEC_PER_SEC);
112 }
113
114 static inline time64_t ktime_get_clocktai_seconds(void)
115 {
116 return ktime_divns(ktime_get_coarse_clocktai(), NSEC_PER_SEC);
117 }
118 #endif
119
120 #if LINUX_VERSION_CODE < KERNEL_VERSION(5, 3, 0)
121 // These were added in 4c54294d01e6 ("timekeeping: Add missing _ns functions for
122 // coarse accessors") (in v5.3).
123 static inline u64 ktime_get_coarse_boottime_ns(void)
124 {
125 return ktime_to_ns(ktime_get_coarse_boottime());
126 }
127
128 static inline u64 ktime_get_coarse_clocktai_ns(void)
129 {
130 return ktime_to_ns(ktime_get_coarse_clocktai());
131 }
132 #endif
133
134
135 // Convert a 4-character string to a seed for drgn_test_prng32().
136 static inline u32 drgn_test_prng32_seed(const char *s)
137 {
138 BUG_ON(strlen(s) != 4);
139 return ((u32)s[0] << 24) | ((u32)s[1] << 16) | ((u32)s[2] << 8) | (u32)s[3];
140 }
141
142 // x must not be 0; the return value is never 0.
143 static u32 drgn_test_prng32(u32 x)
144 {
145 // Xorshift RNG with a period of 2^32 - 1.
146 x ^= x << 13;
147 x ^= x >> 17;
148 x ^= x << 5;
149 return x;
150 }
151
152 // constants
153
154 unsigned long drgn_test_THREAD_SIZE;
155 #ifdef NR_SECTION_ROOTS
156 unsigned long drgn_test_NR_SECTION_ROOTS;
157 #endif
158 #ifdef SECTIONS_PER_ROOT
159 unsigned long drgn_test_SECTIONS_PER_ROOT;
160 #endif
161 #ifdef SECTION_SIZE_BITS
162 unsigned long drgn_test_SECTION_SIZE_BITS;
163 #endif
164 #ifdef MAX_PHYSMEM_BITS
165 unsigned long drgn_test_MAX_PHYSMEM_BITS;
166 #endif
167 #ifdef CONFIG_FLATMEM
168 unsigned long drgn_test_ARCH_PFN_OFFSET;
169 #endif
170
171 int drgn_test_MEM_ONLINE = MEM_ONLINE;
172 int drgn_test_MEM_GOING_OFFLINE = MEM_GOING_OFFLINE;
173 int drgn_test_MEM_OFFLINE = MEM_OFFLINE;
174 int drgn_test_MEM_GOING_ONLINE = MEM_GOING_ONLINE;
175 int drgn_test_MEM_CANCEL_ONLINE = MEM_CANCEL_ONLINE;
176 int drgn_test_MEM_CANCEL_OFFLINE = MEM_CANCEL_OFFLINE;
177 #ifdef MEM_PREPARE_ONLINE
178 int drgn_test_MEM_PREPARE_ONLINE = MEM_PREPARE_ONLINE;
179 #endif
180 #ifdef MEM_FINISH_OFFLINE
181 int drgn_test_MEM_FINISH_OFFLINE = MEM_FINISH_OFFLINE;
182 #endif
183
184 static void drgn_test_constants_init(void)
185 {
186 // Some of these aren't actually compile-time constants, so we
187 // initialize all of them at runtime.
188 drgn_test_THREAD_SIZE = THREAD_SIZE;
189 #ifdef NR_SECTION_ROOTS
190 drgn_test_NR_SECTION_ROOTS = NR_SECTION_ROOTS;
191 #endif
192 #ifdef SECTIONS_PER_ROOT
193 drgn_test_SECTIONS_PER_ROOT = SECTIONS_PER_ROOT;
194 #endif
195 #ifdef SECTION_SIZE_BITS
196 drgn_test_SECTION_SIZE_BITS = SECTION_SIZE_BITS;
197 #endif
198 #ifdef MAX_PHYSMEM_BITS
199 drgn_test_MAX_PHYSMEM_BITS = MAX_PHYSMEM_BITS;
200 #endif
201 #ifdef CONFIG_FLATMEM
202 drgn_test_ARCH_PFN_OFFSET = ARCH_PFN_OFFSET;
203 #endif
204 }
205
206 // block
207
208 // blk_mode_t, BLK_OPEN_READ, and BLK_OPEN_WRITE were added in Linux kernel
209 // commit 05bdb9965305 ("block: replace fmode_t with a block-specific type for
210 // block open flags") (in v6.5).
211 #if LINUX_VERSION_CODE < KERNEL_VERSION(6, 5, 0)
212 #define blk_mode_t fmode_t
213 #define BLK_OPEN_READ FMODE_READ
214 #define BLK_OPEN_WRITE FMODE_WRITE
215 #endif
216
217 // blk_status_t and BLK_STS_OK were added in Linux kernel commit 2a842acab109
218 // ("block: introduce new block status code type") (in v4.13).
219 #ifndef BLK_STS_OK
220 #define blk_status_t int
221 #define BLK_STS_OK 0
222 #endif
223
224 // blk_mq_alloc_disk() was added in Linux kernel commit b461dfc49eb6 ("blk-mq:
225 // add the blk_mq_alloc_disk APIs") (in v5.14).
226 #if LINUX_VERSION_CODE < KERNEL_VERSION(5, 14, 0)
227 static struct gendisk *blk_mq_alloc_disk(struct blk_mq_tag_set *set, void *queuedata)
228 {
229 struct request_queue *q;
230 struct gendisk *disk;
231
232 // Not blk_mq_init_queue_data() because that was added in Linux kernel
233 // commit 2f227bb99934 ("block: add a blk_mq_init_queue_data helper") (in
234 // v5.7).
235 q = blk_mq_init_queue(set);
236 if (IS_ERR(q))
237 return ERR_CAST(q);
238 q->queuedata = queuedata;
239
240 disk = alloc_disk_node(0, set->numa_node);
241 if (!disk) {
242 blk_cleanup_queue(q);
243 return ERR_PTR(-ENOMEM);
244 }
245 disk->queue = q;
246 return disk;
247 }
248 #endif
249
250 static int drgn_test_blkdev_major;
251
252 struct drgn_test_blkdev {
253 struct blk_mq_tag_set tag_set;
254 struct gendisk *disk;
255 spinlock_t lock;
256 int truant;
257 struct list_head loafing_requests;
258 } drgn_test_blkdevs[2];
259
260 static void drgn_test_blkdev_complete_rq(struct request *rq)
261 {
262 struct bio *bio;
263
264 if (req_op(rq) == REQ_OP_READ) {
265 __rq_for_each_bio(bio, rq)
266 zero_fill_bio(bio);
267 }
268 blk_mq_end_request(rq, BLK_STS_OK);
269 }
270
271 static ssize_t drgn_test_blkdev_truant_show(struct device *dev,
272 struct device_attribute *attr,
273 char *buf)
274 {
275 struct drgn_test_blkdev *data = dev_to_disk(dev)->private_data;
276 return sprintf(buf, "%d\n", READ_ONCE(data->truant));
277 }
278
279 static ssize_t drgn_test_blkdev_truant_store(struct device *dev,
280 struct device_attribute *attr,
281 const char *buf, size_t count)
282 {
283 struct drgn_test_blkdev *data = dev_to_disk(dev)->private_data;
284 LIST_HEAD(to_complete);
285 struct list_head *pos;
286 int ret, val;
287
288 ret = kstrtoint(buf, 0, &val);
289 if (ret < 0)
290 return ret;
291 if (val != 0 && val != 1)
292 return -EINVAL;
293
294 spin_lock(&data->lock);
295 if (!val)
296 list_splice_init(&data->loafing_requests, &to_complete);
297 data->truant = val;
298 spin_unlock(&data->lock);
299
300 list_for_each(pos, &to_complete)
301 drgn_test_blkdev_complete_rq(blk_mq_rq_from_pdu(pos));
302
303 return count;
304 }
305
306 static ssize_t drgn_test_blkdev_num_loafing_show(struct device *dev,
307 struct device_attribute *attr,
308 char *buf)
309 {
310 struct drgn_test_blkdev *data = dev_to_disk(dev)->private_data;
311 struct list_head *pos;
312 size_t n = 0;
313
314 spin_lock(&data->lock);
315 list_for_each(pos, &data->loafing_requests)
316 n++;
317 spin_unlock(&data->lock);
318 return sprintf(buf, "%zu\n", n);
319 }
320
321 static struct device_attribute drgn_test_blkdev_attr_truant =
322 __ATTR(truant, 0600, drgn_test_blkdev_truant_show,
323 drgn_test_blkdev_truant_store);
324
325 static struct device_attribute drgn_test_blkdev_attr_num_loafing =
326 __ATTR(num_loafing, 0400, drgn_test_blkdev_num_loafing_show, NULL);
327
328 static const struct attribute_group drgn_test_blkdev_attr_group = {
329 .attrs = (struct attribute *[]){
330 &drgn_test_blkdev_attr_truant.attr,
331 &drgn_test_blkdev_attr_num_loafing.attr,
332 NULL,
333 },
334 };
335
336 static const struct attribute_group *drgn_test_blkdev_attr_groups[] = {
337 &drgn_test_blkdev_attr_group,
338 NULL,
339 };
340
341 static blk_status_t drgn_test_queue_rq(struct blk_mq_hw_ctx *hctx,
342 const struct blk_mq_queue_data *bd)
343 {
344 struct request *rq = bd->rq;
345 struct drgn_test_blkdev *data = rq->q->queuedata;
346
347 blk_mq_start_request(rq);
348
349 spin_lock(&data->lock);
350 if (data->truant) {
351 list_add_tail(blk_mq_rq_to_pdu(rq), &data->loafing_requests);
352 spin_unlock(&data->lock);
353 } else {
354 spin_unlock(&data->lock);
355 drgn_test_blkdev_complete_rq(rq);
356 }
357
358 return BLK_STS_OK;
359 }
360
361 // For testing flush requests, we want an interface that synchronously queues a
362 // flush request. Unfortunately, aio IOCB_CMD_FSYNC queues the request
363 // asynchronously (with schedule_work()). Instead, we expose this via an ioctl.
364 static int drgn_test_blkdev_ioctl(struct block_device *bdev, blk_mode_t mode,
365 unsigned cmd, unsigned long arg)
366 {
367 struct bio *bio;
368
369 // We co-opt the number for LOOP_SET_FD so that the test code doesn't
370 // need to worry about ioctl number encoding.
371 if (cmd != 0x4c00)
372 return -ENOTTY;
373
374 if (!(mode & (BLK_OPEN_READ | BLK_OPEN_WRITE)))
375 return -EBADF;
376
377 // The bdev and opf parameters to bio_alloc() were added in Linux kernel
378 // commit 07888c665b40 ("block: pass a block_device and opf to
379 // bio_alloc") (in v5.18). Before that, we have to set the block device
380 // and operation manually.
381 #if LINUX_VERSION_CODE >= KERNEL_VERSION(5, 18, 0)
382 bio = bio_alloc(bdev, 0, REQ_OP_WRITE | REQ_PREFLUSH, GFP_KERNEL);
383 if (!bio)
384 return -ENOMEM;
385 #else
386 bio = bio_alloc(GFP_KERNEL, 0);
387 if (!bio)
388 return -ENOMEM;
389 // bio_set_dev() was added in Linux kernel commit 74d46992e0d9 ("block:
390 // replace bi_bdev with a gendisk pointer and partitions index") (in
391 // v4.14). Before that, we have to set bi_bdev manually.
392 #if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 14, 0)
393 bio_set_dev(bio, bdev);
394 #else
395 bio->bi_bdev = bdev;
396 #endif
397 bio->bi_opf = REQ_OP_WRITE | REQ_PREFLUSH;
398 #endif
399 bio->bi_end_io = bio_put;
400 submit_bio(bio);
401 // We shouldn't be poking at internals like this, but flush requests are
402 // added to the requeue list and queued asynchronously. We need them to
403 // be queued before we return.
404 flush_delayed_work(&bdev->bd_disk->queue->requeue_work);
405 return 0;
406 }
407
408 static const struct blk_mq_ops drgn_test_blk_mq_ops = {
409 .queue_rq = drgn_test_queue_rq,
410 };
411
412 static const struct block_device_operations drgn_test_blk_fops = {
413 .ioctl = drgn_test_blkdev_ioctl,
414 .owner = THIS_MODULE,
415 };
416
417 static int drgn_test_blkdev_create(struct drgn_test_blkdev *bdev,
418 int index, int nr_hw_queues, int queue_depth,
419 unsigned int tag_set_flags)
420 {
421 int ret;
422
423 spin_lock_init(&bdev->lock);
424 INIT_LIST_HEAD(&bdev->loafing_requests);
425
426 // Before Linux kernel commit f8a5b12247fe ("blk-mq: make mq_ops a const
427 // pointer") (in v4.11), struct blk_mq_tag_set::ops wasn't marked const.
428 bdev->tag_set.ops =
429 (struct blk_mq_ops *)&drgn_test_blk_mq_ops;
430 bdev->tag_set.cmd_size = sizeof(struct list_head);
431 bdev->tag_set.nr_hw_queues = nr_hw_queues;
432 bdev->tag_set.queue_depth = queue_depth;
433 bdev->tag_set.numa_node = NUMA_NO_NODE;
434 bdev->tag_set.flags = tag_set_flags;
435 ret = blk_mq_alloc_tag_set(&bdev->tag_set);
436 if (ret) {
437 bdev->tag_set.ops = NULL;
438 return ret;
439 }
440
441 // We need write cache support to test flush requests.
442 // BLK_FEAT_WRITE_CACHE was added in Linux kernel commit 1122c0c1cc71
443 // ("block: move cache control settings out of queue->flags") (in
444 // v6.11). Before that, we have to call blk_queue_write_cache().
445 //
446 // The lim parameter was added to blk_mq_alloc_disk() in Linux kernel
447 // commit 27e32cd23fed ("block: pass a queue_limits argument to
448 // blk_mq_alloc_disk") (in v6.9).
449 bdev->disk = blk_mq_alloc_disk(&bdev->tag_set,
450 #if LINUX_VERSION_CODE >= KERNEL_VERSION(6, 11, 0)
451 (&(struct queue_limits){
452 .features = BLK_FEAT_WRITE_CACHE,
453 }),
454 #elif LINUX_VERSION_CODE >= KERNEL_VERSION(6, 9, 0)
455 NULL,
456 #endif
457 bdev);
458 if (IS_ERR(bdev->disk))
459 return PTR_ERR(bdev->disk);
460 #if LINUX_VERSION_CODE < KERNEL_VERSION(6, 11, 0)
461 blk_queue_write_cache(bdev->disk->queue, true, false);
462 #endif
463 bdev->disk->major = drgn_test_blkdev_major;
464 bdev->disk->first_minor = index;
465 bdev->disk->minors = 1;
466 bdev->disk->fops = &drgn_test_blk_fops;
467 sprintf(bdev->disk->disk_name, "drgntestb%d", index);
468 set_capacity(bdev->disk, SZ_1G >> SECTOR_SHIFT);
469 bdev->disk->private_data = bdev;
470
471 // The groups parameter was added to device_add_disk() in Linux kernel
472 // commit fef912bf860e ("block: genhd: add 'groups' argument to
473 // device_add_disk") (in v4.20). Before that, we have to call
474 // sysfs_create_groups().
475 #if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 20, 0)
476 // Before Linux kernel commit 83cbce957446 ("block: add error handling
477 // for device_add_disk / add_disk") (in v5.15), device_add_disk() didn't
478 // return anything.
479 ret =
480 #if LINUX_VERSION_CODE < KERNEL_VERSION(5, 15, 0)
481 0;
482 #endif
483 device_add_disk(NULL, bdev->disk, drgn_test_blkdev_attr_groups);
484 if (ret)
485 return ret;
486 #else
487 add_disk(bdev->disk);
488 ret = sysfs_create_groups(&disk_to_dev(bdev->disk)->kobj,
489 drgn_test_blkdev_attr_groups);
490 if (ret)
491 return ret;
492 #endif
493
494 return 0;
495 }
496
497 static void drgn_test_blkdev_destroy(struct drgn_test_blkdev *bdev)
498 {
499 if (!IS_ERR_OR_NULL(bdev->disk))
500 put_disk(bdev->disk);
501 if (bdev->tag_set.ops)
502 blk_mq_free_tag_set(&bdev->tag_set);
503 }
504
505 static int drgn_test_block_init(void)
506 {
507 int ret;
508
509 drgn_test_blkdev_major = register_blkdev(0, "drgntest");
510 if (drgn_test_blkdev_major < 0)
511 return drgn_test_blkdev_major;
512
513 ret = drgn_test_blkdev_create(&drgn_test_blkdevs[0], 0, 2, 2, 0);
514 if (ret)
515 return ret;
516 // BLK_MQ_F_TAG_HCTX_SHARED was added in Linux kernel commit
517 // 32bc15afed04 ("blk-mq: Facilitate a shared sbitmap per tagset") (in
518 // v5.10).
519 #if LINUX_VERSION_CODE >= KERNEL_VERSION(5, 10, 0)
520 ret = drgn_test_blkdev_create(&drgn_test_blkdevs[1], 1, 2, 4,
521 BLK_MQ_F_TAG_HCTX_SHARED);
522 if (ret)
523 return ret;
524 #endif
525
526 return 0;
527 }
528
529 static void drgn_test_block_exit(void)
530 {
531 int i;
532
533 for (i = 0; i < ARRAY_SIZE(drgn_test_blkdevs); i++)
534 drgn_test_blkdev_destroy(&drgn_test_blkdevs[i]);
535 if (drgn_test_blkdev_major > 0)
536 unregister_blkdev(drgn_test_blkdev_major, "drgntest");
537 }
538
539 // list
540
541 LIST_HEAD(drgn_test_empty_list);
542 LIST_HEAD(drgn_test_full_list);
543 LIST_HEAD(drgn_test_singular_list);
544 LIST_HEAD(drgn_test_corrupted_list);
545 // Corrupted list where entry 1 points back to entry 0.
546 LIST_HEAD(drgn_test_list_cycle1);
547 // Corrupted list where entry 2 points back to entry 1.
548 LIST_HEAD(drgn_test_list_cycle2);
549 // Corrupted list where entry 3 points back to entry 2.
550 LIST_HEAD(drgn_test_list_cycle3);
551 // Corrupted list where entry 2 points to itself.
552 LIST_HEAD(drgn_test_list_self_cycle);
553 // Corrupted list where entry 0 points to NULL.
554 LIST_HEAD(drgn_test_list_null);
555
556 struct drgn_test_list_entry {
557 int value;
558 struct list_head node;
559 };
560
561 struct drgn_test_list_entry drgn_test_circular_list = {
562 .value = 1,
563 .node = LIST_HEAD_INIT(drgn_test_circular_list.node),
564 };
565
566 struct drgn_test_list_anchor {
567 long x, y;
568 struct list_head list;
569 } drgn_test_anchored_list = {
570 .x = 101,
571 .y = 13,
572 .list = LIST_HEAD_INIT(drgn_test_anchored_list.list),
573 };
574
575 struct drgn_test_list_entry drgn_test_list_entries[3];
576 struct drgn_test_list_entry drgn_test_singular_list_entry;
577 struct drgn_test_list_entry drgn_test_circular_list_entries[2];
578 struct drgn_test_list_entry drgn_test_corrupted_list_entries[2];
579 struct drgn_test_list_entry drgn_test_list_cycle1_entries[2];
580 struct drgn_test_list_entry drgn_test_list_cycle2_entries[3];
581 struct drgn_test_list_entry drgn_test_list_cycle3_entries[4];
582 struct drgn_test_list_entry drgn_test_list_self_cycle_entries[3];
583 struct drgn_test_list_entry drgn_test_list_null_entry;
584 struct drgn_test_list_entry drgn_test_anchored_list_entries[3];
585
586 HLIST_HEAD(drgn_test_empty_hlist);
587 HLIST_HEAD(drgn_test_full_hlist);
588
589 struct drgn_test_hlist_entry {
590 int value;
591 struct hlist_node node;
592 };
593
594 struct drgn_test_hlist_entry drgn_test_hlist_entries[3];
595
596 struct drgn_test_custom_list_entry {
597 int value;
598 struct drgn_test_custom_list_entry *next;
599 };
600
601 struct drgn_test_custom_list_entry drgn_test_custom_list;
602 struct drgn_test_custom_list_entry drgn_test_custom_list_entries[2];
603 // Custom list where entry 4 points back to entry 2.
604 struct drgn_test_custom_list_entry drgn_test_custom_list_cycle[5];
605 // Custom list where entry 2 points to itself.
606 struct drgn_test_custom_list_entry drgn_test_custom_list_self_cycle[3];
607
608 // Emulate a race condition between two threads calling list_add() at the same
609 // time.
610 static void init_corrupted_list(void)
611 {
612 struct list_head *prev = &drgn_test_corrupted_list;
613 struct list_head *next = drgn_test_corrupted_list.next;
614 struct list_head *new1 = &drgn_test_corrupted_list_entries[0].node;
615 struct list_head *new2 = &drgn_test_corrupted_list_entries[1].node;
616
617 // Thread 1 starts list_add().
618 next->prev = new1;
619
620 // Thread 2 races in and does its own list_add().
621 next->prev = new2;
622 new2->next = next;
623 new2->prev = prev;
624 WRITE_ONCE(prev->next, new2);
625
626 // Thread 1 finishes list_add().
627 new1->next = next;
628 new1->prev = prev;
629 WRITE_ONCE(prev->next, new1);
630 }
631
632 static void drgn_test_list_init(void)
633 {
634 size_t i;
635
636 for (i = 0; i < ARRAY_SIZE(drgn_test_list_entries); i++) {
637 drgn_test_list_entries[i].value = i + 1;
638 list_add_tail(&drgn_test_list_entries[i].node,
639 &drgn_test_full_list);
640 }
641 list_add(&drgn_test_singular_list_entry.node, &drgn_test_singular_list);
642
643 for (i = ARRAY_SIZE(drgn_test_hlist_entries); i-- > 0;) {
644 hlist_add_head(&drgn_test_hlist_entries[i].node,
645 &drgn_test_full_hlist);
646 }
647
648 for (i = 0; i < ARRAY_SIZE(drgn_test_circular_list_entries); i++) {
649 drgn_test_circular_list_entries[i].value = i + 2;
650 list_add_tail(&drgn_test_circular_list_entries[i].node,
651 &drgn_test_circular_list.node);
652 }
653
654 init_corrupted_list();
655
656 #define init_list_cycle(n) \
657 do { \
658 for (i = 0; i < ARRAY_SIZE(drgn_test_list_cycle##n##_entries); i++) {\
659 list_add_tail(&drgn_test_list_cycle##n##_entries[i].node,\
660 &drgn_test_list_cycle##n); \
661 } \
662 drgn_test_list_cycle##n##_entries[ARRAY_SIZE(drgn_test_list_cycle##n##_entries) - 1].node.next =\
663 &drgn_test_list_cycle##n##_entries[ARRAY_SIZE(drgn_test_list_cycle##n##_entries) - 2].node;\
664 drgn_test_list_cycle##n##_entries[ARRAY_SIZE(drgn_test_list_cycle##n##_entries) - 2].node.prev =\
665 &drgn_test_list_cycle##n##_entries[ARRAY_SIZE(drgn_test_list_cycle##n##_entries) - 1].node;\
666 } while (0)
667
668 init_list_cycle(1);
669 init_list_cycle(2);
670 init_list_cycle(3);
671 #undef init_list_cycle
672
673 for (i = 0; i < ARRAY_SIZE(drgn_test_list_self_cycle_entries); i++) {
674 list_add_tail(&drgn_test_list_self_cycle_entries[i].node,
675 &drgn_test_list_self_cycle);
676 }
677 INIT_LIST_HEAD(&drgn_test_list_self_cycle_entries[ARRAY_SIZE(drgn_test_list_self_cycle_entries) - 1].node);
678
679 list_add_tail(&drgn_test_list_null_entry.node, &drgn_test_list_null);
680 drgn_test_list_null_entry.node.next = NULL;
681
682 for (i = 0; i < ARRAY_SIZE(drgn_test_anchored_list_entries); i++) {
683 drgn_test_anchored_list_entries[i].value = i + 1;
684 list_add_tail(&drgn_test_anchored_list_entries[i].node,
685 &drgn_test_anchored_list.list);
686 }
687
688 drgn_test_custom_list.value = 1;
689 for (i = 0; i < ARRAY_SIZE(drgn_test_custom_list_entries); i++) {
690 drgn_test_custom_list_entries[i].value = i + 2;
691 if (i == 0) {
692 drgn_test_custom_list.next = &drgn_test_custom_list_entries[i];
693 } else {
694 drgn_test_custom_list_entries[i - 1].next =
695 &drgn_test_custom_list_entries[i];
696 }
697 }
698
699 drgn_test_custom_list_cycle[0].value = 1;
700 for (i = 1; i < ARRAY_SIZE(drgn_test_custom_list_cycle); i++) {
701 drgn_test_custom_list_cycle[i].value = i + 1;
702 drgn_test_custom_list_cycle[i - 1].next =
703 &drgn_test_custom_list_cycle[i];
704 }
705 drgn_test_custom_list_cycle[4].next = &drgn_test_custom_list_cycle[2];
706
707 drgn_test_custom_list_self_cycle[0].value = 1;
708 for (i = 1; i < ARRAY_SIZE(drgn_test_custom_list_self_cycle); i++) {
709 drgn_test_custom_list_self_cycle[i].value = i + 1;
710 drgn_test_custom_list_self_cycle[i - 1].next =
711 &drgn_test_custom_list_self_cycle[i];
712 }
713 drgn_test_custom_list_self_cycle[2].next =
714 &drgn_test_custom_list_self_cycle[2];
715 }
716
717 // llist
718
719 LLIST_HEAD(drgn_test_empty_llist);
720 LLIST_HEAD(drgn_test_full_llist);
721 LLIST_HEAD(drgn_test_singular_llist);
722
723 struct drgn_test_llist_entry {
724 int value;
725 struct llist_node node;
726 };
727
728 struct drgn_test_llist_entry drgn_test_llist_entries[3];
729 struct drgn_test_llist_entry drgn_test_singular_llist_entry;
730
731 static void drgn_test_llist_init(void)
732 {
733 size_t i;
734
735 for (i = 0; i < ARRAY_SIZE(drgn_test_llist_entries); i++) {
736 llist_add(&drgn_test_llist_entries[i].node,
737 &drgn_test_full_llist);
738 }
739
740 llist_add(&drgn_test_singular_llist_entry.node, &drgn_test_singular_llist);
741 }
742
743 // plist
744
745 PLIST_HEAD(drgn_test_empty_plist);
746 PLIST_HEAD(drgn_test_full_plist);
747 struct plist_node drgn_test_empty_plist_node =
748 PLIST_NODE_INIT(drgn_test_empty_plist_node, 50);
749
750 struct drgn_test_plist_entry {
751 char c;
752 struct plist_node node;
753 };
754
755 struct drgn_test_plist_entry drgn_test_plist_entries[3];
756
757 // Copy of plist_add() (minus debugging code) since it's not exported.
758 static void drgn_plist_add(struct plist_node *node, struct plist_head *head)
759 {
760 struct plist_node *first, *iter, *prev = NULL;
761 struct list_head *node_next = &head->node_list;
762
763 WARN_ON(!plist_node_empty(node));
764 WARN_ON(!list_empty(&node->prio_list));
765
766 if (plist_head_empty(head))
767 goto ins_node;
768
769 first = iter = plist_first(head);
770
771 do {
772 if (node->prio < iter->prio) {
773 node_next = &iter->node_list;
774 break;
775 }
776
777 prev = iter;
778 iter = list_entry(iter->prio_list.next,
779 struct plist_node, prio_list);
780 } while (iter != first);
781
782 if (!prev || prev->prio != node->prio)
783 list_add_tail(&node->prio_list, &iter->prio_list);
784 ins_node:
785 list_add_tail(&node->node_list, node_next);
786 }
787
788 static void drgn_test_plist_init(void)
789 {
790 plist_node_init(&drgn_test_plist_entries[0].node, 10);
791 drgn_test_plist_entries[0].c = 'H';
792 plist_node_init(&drgn_test_plist_entries[1].node, 20);
793 drgn_test_plist_entries[1].c = 'I';
794 plist_node_init(&drgn_test_plist_entries[2].node, 30);
795 drgn_test_plist_entries[2].c = '!';
796
797 drgn_plist_add(&drgn_test_plist_entries[1].node, &drgn_test_full_plist);
798 drgn_plist_add(&drgn_test_plist_entries[0].node, &drgn_test_full_plist);
799 drgn_plist_add(&drgn_test_plist_entries[2].node, &drgn_test_full_plist);
800 }
801
802 // locking
803
804 static DECLARE_COMPLETION(drgn_test_locking_kthread_ready);
805 struct task_struct *drgn_test_locking_kthread;
806 struct task_struct *drgn_test_locking_kthread2;
807 DEFINE_MUTEX(drgn_test_mutex_locked);
808 DEFINE_MUTEX(drgn_test_mutex_unlocked);
809 DECLARE_RWSEM(drgn_test_rwsem_read_locked);
810 DECLARE_RWSEM(drgn_test_rwsem_write_locked);
811 DECLARE_RWSEM(drgn_test_rwsem_previously_read_locked);
812 DECLARE_RWSEM(drgn_test_rwsem_previously_write_locked);
813 DECLARE_RWSEM(drgn_test_rwsem_never_locked);
814 DECLARE_RWSEM(drgn_test_rwsem_writer_waiting);
815
816 static int drgn_test_locking_kthread_fn(void *arg)
817 {
818 mutex_lock(&drgn_test_mutex_locked);
819
820 down_read(&drgn_test_rwsem_read_locked);
821 down_write(&drgn_test_rwsem_write_locked);
822
823 down_read(&drgn_test_rwsem_previously_read_locked);
824 up_read(&drgn_test_rwsem_previously_read_locked);
825
826 down_write(&drgn_test_rwsem_previously_write_locked);
827 up_write(&drgn_test_rwsem_previously_write_locked);
828
829 down_read(&drgn_test_rwsem_writer_waiting);
830
831 complete(&drgn_test_locking_kthread_ready);
832 for (;;) {
833 set_current_state(TASK_INTERRUPTIBLE);
834 if (kthread_should_stop()) {
835 __set_current_state(TASK_RUNNING);
836 break;
837 }
838 if (kthread_should_park()) {
839 __set_current_state(TASK_RUNNING);
840 kthread_parkme();
841 continue;
842 }
843 schedule();
844 __set_current_state(TASK_RUNNING);
845 }
846
847 up_read(&drgn_test_rwsem_writer_waiting);
848 up_write(&drgn_test_rwsem_write_locked);
849 up_read(&drgn_test_rwsem_read_locked);
850
851 mutex_unlock(&drgn_test_mutex_locked);
852 return 0;
853 }
854
855 static int drgn_test_locking_kthread_fn2(void *arg)
856 {
857 down_write(&drgn_test_rwsem_writer_waiting);
858 up_write(&drgn_test_rwsem_writer_waiting);
859 return 0;
860 }
861
862 static int drgn_test_locking_init(void)
863 {
864 drgn_test_locking_kthread = kthread_create(drgn_test_locking_kthread_fn,
865 NULL,
866 "drgn_test_locking_kthread");
867 if (!drgn_test_locking_kthread)
868 return -1;
869 wake_up_process(drgn_test_locking_kthread);
870 wait_for_completion(&drgn_test_locking_kthread_ready);
871
872 drgn_test_locking_kthread2 =
873 kthread_create(drgn_test_locking_kthread_fn2, NULL,
874 "drgn_test_locking_kthread2");
875 if (!drgn_test_locking_kthread2)
876 return -1;
877 wake_up_process(drgn_test_locking_kthread2);
878
879 return kthread_park(drgn_test_locking_kthread);
880 }
881
882 static void drgn_test_locking_exit(void)
883 {
884 // This one needs to exit first to unblock the second one.
885 if (drgn_test_locking_kthread) {
886 kthread_stop(drgn_test_locking_kthread);
887 drgn_test_locking_kthread = NULL;
888 }
889 if (drgn_test_locking_kthread2) {
890 kthread_stop(drgn_test_locking_kthread2);
891 drgn_test_locking_kthread2 = NULL;
892 }
893 }
894
895 // mapletree
896
897 const int drgn_test_have_maple_tree = HAVE_MAPLE_TREE;
898 #if HAVE_MAPLE_TREE
899 const int drgn_test_maple_range64_slots = MAPLE_RANGE64_SLOTS;
900 const int drgn_test_maple_arange64_slots = MAPLE_ARANGE64_SLOTS;
901
902 #define DRGN_TEST_MAPLE_TREES \
903 X(empty) \
904 X(one) \
905 X(one_range) \
906 X(one_at_zero) \
907 X(one_range_at_zero) \
908 X(zero_entry) \
909 X(zero_entry_at_zero) \
910 X(dense) \
911 X(dense_ranges) \
912 X(sparse) \
913 X(sparse_ranges) \
914 X(three_levels_dense_1) \
915 X(three_levels_dense_2) \
916 X(three_levels_ranges_1) \
917 X(three_levels_ranges_2)
918
919 #define X(name) \
920 DEFINE_MTREE(drgn_test_maple_tree_##name); \
921 struct maple_tree drgn_test_maple_tree_arange_##name = \
922 MTREE_INIT(drgn_test_maple_tree_arange_##name, \
923 MT_FLAGS_ALLOC_RANGE);
924 DRGN_TEST_MAPLE_TREES
925 #undef X
926
927 static int drgn_test_maple_tree_init(void)
928 {
929 int ret;
930 unsigned int arange, i;
931 #define X(name) struct maple_tree *name = &drgn_test_maple_tree_##name;
932 DRGN_TEST_MAPLE_TREES
933 #undef X
934
935 for (arange = 0; arange < 2; arange++) {
936 int node_slots = arange ? MAPLE_ARANGE64_SLOTS : MAPLE_RANGE64_SLOTS;
937
938 ret = mtree_insert(one, 666, (void *)0xdeadb00, GFP_KERNEL);
939 if (ret)
940 return ret;
941
942 ret = mtree_insert_range(one_range, 616, 666,
943 (void *)0xdeadb000, GFP_KERNEL);
944 if (ret)
945 return ret;
946
947 ret = mtree_insert(one_at_zero, 0, (void *)0x1234, GFP_KERNEL);
948 if (ret)
949 return ret;
950
951 ret = mtree_insert_range(one_range_at_zero, 0, 0x1337,
952 (void *)0x5678, GFP_KERNEL);
953 if (ret)
954 return ret;
955
956 ret = mtree_insert(zero_entry, 666, XA_ZERO_ENTRY, GFP_KERNEL);
957 if (ret)
958 return ret;
959
960 ret = mtree_insert(zero_entry_at_zero, 0, XA_ZERO_ENTRY,
961 GFP_KERNEL);
962 if (ret)
963 return ret;
964
965 for (i = 0; i < 5; i++) {
966 ret = mtree_insert(dense, i,
967 (void *)(uintptr_t)(0xb0ba000 | i),
968 GFP_KERNEL);
969 if (ret)
970 return ret;
971 }
972
973 for (i = 0; i < 5; i++) {
974 ret = mtree_insert_range(dense_ranges, i * i,
975 (i + 1) * (i + 1) - 1,
976 (void *)(uintptr_t)(0xb0ba000 | i),
977 GFP_KERNEL);
978 if (ret)
979 return ret;
980 }
981
982 for (i = 0; i < 5; i++) {
983 ret = mtree_insert(sparse, (i + 1) * (i + 1),
984 (void *)(uintptr_t)(0xb0ba000 | i),
985 GFP_KERNEL);
986 if (ret)
987 return ret;
988 }
989
990 for (i = 0; i < 5; i++) {
991 ret = mtree_insert_range(sparse_ranges,
992 (2 * i + 1) * (2 * i + 1),
993 (2 * i + 2) * (2 * i + 2),
994 (void *)(uintptr_t)(0xb0ba000 | i),
995 GFP_KERNEL);
996 if (ret)
997 return ret;
998 }
999
1000 // In theory, a leaf can reference up to MAPLE_RANGE64_SLOTS
1001 // entries, and a level 1 node can reference up to node_slots *
1002 // MAPLE_RANGE64_SLOTS entries. In practice, as of Linux 6.6,
1003 // the maple tree code only fully packs nodes with a maximum of
1004 // ULONG_MAX. We create and test trees with both the observed
1005 // and theoretical limits.
1006 for (i = 0;
1007 i < 2 * (node_slots - 1) * (MAPLE_RANGE64_SLOTS - 1) + (MAPLE_RANGE64_SLOTS - 1);
1008 i++) {
1009 ret = mtree_insert(three_levels_dense_1, i,
1010 (void *)(uintptr_t)(0xb0ba000 | i),
1011 GFP_KERNEL);
1012 if (ret)
1013 return ret;
1014 }
1015
1016 for (i = 0; i < 2 * node_slots * MAPLE_RANGE64_SLOTS; i++) {
1017 ret = mtree_insert(three_levels_dense_2, i,
1018 (void *)(uintptr_t)(0xb0ba000 | i),
1019 GFP_KERNEL);
1020 if (ret)
1021 return ret;
1022 }
1023
1024 for (i = 0;
1025 i < 2 * (node_slots - 1) * (MAPLE_RANGE64_SLOTS - 1) + (MAPLE_RANGE64_SLOTS - 1);
1026 i++) {
1027 ret = mtree_insert_range(three_levels_ranges_1, 2 * i,
1028 2 * i + 1,
1029 (void *)(uintptr_t)(0xb0ba000 | i),
1030 GFP_KERNEL);
1031 if (ret)
1032 return ret;
1033 }
1034 ret = mtree_insert_range(three_levels_ranges_1, 2 * i,
1035 ULONG_MAX,
1036 (void *)(uintptr_t)(0xb0ba000 | i),
1037 GFP_KERNEL);
1038 if (ret)
1039 return ret;
1040
1041 for (i = 0; i < 2 * node_slots * MAPLE_RANGE64_SLOTS; i++) {
1042 ret = mtree_insert_range(three_levels_ranges_2, 2 * i,
1043 2 * i + 1,
1044 (void *)(uintptr_t)(0xb0ba000 | i),
1045 GFP_KERNEL);
1046 if (ret)
1047 return ret;
1048 }
1049 ret = mtree_insert_range(three_levels_ranges_2, 2 * i,
1050 ULONG_MAX,
1051 (void *)(uintptr_t)(0xb0ba000 | i),
1052 GFP_KERNEL);
1053 if (ret)
1054 return ret;
1055
1056 #define X(name) name = &drgn_test_maple_tree_arange_##name;
1057 DRGN_TEST_MAPLE_TREES
1058 #undef X
1059 }
1060 return 0;
1061 }
1062
1063 static void drgn_test_maple_tree_exit(void)
1064 {
1065 #define X(name) \
1066 mtree_destroy(&drgn_test_maple_tree_##name); \
1067 mtree_destroy(&drgn_test_maple_tree_arange_##name);
1068 DRGN_TEST_MAPLE_TREES
1069 #undef X
1070 }
1071 #else
1072 static int drgn_test_maple_tree_init(void) { return 0; }
1073 static void drgn_test_maple_tree_exit(void) {}
1074 #endif
1075
1076 // mm
1077
1078 const int drgn_test_vmap_stack_enabled = IS_ENABLED(CONFIG_VMAP_STACK);
1079 const int drgn_test_slab_stack_enabled =
1080 !IS_ENABLED(CONFIG_VMAP_STACK) && THREAD_SIZE < PAGE_SIZE;
1081 void *drgn_test_va;
1082 phys_addr_t drgn_test_pa;
1083 unsigned long drgn_test_pfn;
1084 struct page *drgn_test_page;
1085 struct page *drgn_test_compound_page;
1086 void *drgn_test_vmalloc_va;
1087 phys_addr_t drgn_test_vmalloc_pa;
1088 unsigned long drgn_test_vmalloc_pfn;
1089 struct page *drgn_test_vmalloc_page;
1090
1091 #ifdef CONFIG_SPARSEMEM
1092 unsigned long drgn_test_section_nr;
1093 unsigned long drgn_test_section_pfn;
1094 struct mem_section *drgn_test_mem_section;
1095 struct page *drgn_test_section_mem_map;
1096 struct page *drgn_test_section_decoded_mem_map;
1097
1098 unsigned long drgn_test_SECTION_MARKED_PRESENT = SECTION_MARKED_PRESENT;
1099 struct mem_section *drgn_test_present_section = &(struct mem_section){
1100 .section_mem_map = SECTION_MARKED_PRESENT,
1101 };
1102
1103 unsigned long drgn_test_SECTION_HAS_MEM_MAP = SECTION_HAS_MEM_MAP;
1104 struct mem_section *drgn_test_valid_section = &(struct mem_section){
1105 .section_mem_map = SECTION_HAS_MEM_MAP,
1106 };
1107
1108 #ifdef SECTION_IS_ONLINE
1109 unsigned long drgn_test_SECTION_IS_ONLINE = SECTION_IS_ONLINE;
1110 struct mem_section *drgn_test_online_section = &(struct mem_section){
1111 .section_mem_map = SECTION_IS_ONLINE,
1112 };
1113 #endif
1114
1115 #ifdef SECTION_IS_EARLY
1116 unsigned long drgn_test_SECTION_IS_EARLY = SECTION_IS_EARLY;
1117 struct mem_section *drgn_test_early_section = &(struct mem_section){
1118 .section_mem_map = SECTION_IS_EARLY,
1119 };
1120 #endif
1121
1122 #ifdef SECTION_TAINT_ZONE_DEVICE
1123 unsigned long drgn_test_SECTION_TAINT_ZONE_DEVICE = SECTION_TAINT_ZONE_DEVICE;
1124 #endif
1125
1126 struct {
1127 unsigned long nr;
1128 struct mem_section *section;
1129 } *drgn_test_present_sections;
1130 size_t drgn_test_num_present_sections;
1131 #endif
1132
1133 #ifdef CONFIG_SPARSEMEM
1134 // __highest_present_section_nr is not exported, so we can't use
1135 // for_each_present_section_nr(). We can derive __highest_present_section_nr
1136 // from max_pfn, but that's also not exported. So, we derive max_pfn from the
1137 // maximum zone end PFN.
1138 static unsigned long drgn_test_highest_present_section_nr(void)
1139 {
1140 unsigned long max_pfn = 0;
1141 unsigned long nr;
1142 int nid;
1143
1144 for_each_online_node(nid) {
1145 pg_data_t *pgdat = NODE_DATA(nid);
1146 int i;
1147
1148 for (i = 0; i < MAX_NR_ZONES; i++) {
1149 unsigned long end_pfn = zone_end_pfn(&pgdat->node_zones[i]);
1150 if (end_pfn > max_pfn)
1151 max_pfn = end_pfn;
1152 }
1153 }
1154
1155 nr = pfn_to_section_nr(max_pfn - 1);
1156 for (;;) {
1157 if (present_section_nr(nr))
1158 return nr;
1159 if (nr == 0)
1160 break;
1161 nr--;
1162 }
1163 return 0;
1164 }
1165 #endif
1166
1167 static int drgn_test_mm_init(void)
1168 {
1169 u32 fill;
1170 size_t i;
1171 #ifdef CONFIG_SPARSEMEM
1172 unsigned long highest_present_section_nr;
1173 size_t n;
1174 #endif
1175
1176 drgn_test_page = alloc_page(GFP_KERNEL);
1177 if (!drgn_test_page)
1178 return -ENOMEM;
1179 drgn_test_compound_page = alloc_pages(GFP_KERNEL | __GFP_COMP, 1);
1180 if (!drgn_test_compound_page)
1181 return -ENOMEM;
1182 drgn_test_va = page_address(drgn_test_page);
1183 // Fill the page with a PRNG sequence.
1184 fill = drgn_test_prng32_seed("PAGE");
1185 for (i = 0; i < PAGE_SIZE / sizeof(fill); i++) {
1186 fill = drgn_test_prng32(fill);
1187 ((u32 *)drgn_test_va)[i] = fill;
1188 }
1189 drgn_test_pa = virt_to_phys(drgn_test_va);
1190 drgn_test_pfn = PHYS_PFN(drgn_test_pa);
1191 drgn_test_vmalloc_va = vmalloc(PAGE_SIZE);
1192 if (!drgn_test_vmalloc_va)
1193 return -ENOMEM;
1194 drgn_test_vmalloc_pfn = vmalloc_to_pfn(drgn_test_vmalloc_va);
1195 drgn_test_vmalloc_page = vmalloc_to_page(drgn_test_vmalloc_va);
1196 drgn_test_vmalloc_pa = page_to_phys(drgn_test_vmalloc_page)
1197 + offset_in_page(drgn_test_vmalloc_va);
1198
1199 #ifdef CONFIG_SPARSEMEM
1200 drgn_test_section_nr = pfn_to_section_nr(drgn_test_pfn);
1201 drgn_test_section_pfn = section_nr_to_pfn(drgn_test_section_nr);
1202 drgn_test_mem_section = __nr_to_section(drgn_test_section_nr);
1203 drgn_test_section_mem_map = __section_mem_map_addr(drgn_test_mem_section);
1204 // sparse_decode_mem_map() isn't exported, so we do the equivalent
1205 // ourselves.
1206 drgn_test_section_decoded_mem_map =
1207 drgn_test_section_mem_map + drgn_test_section_pfn;
1208
1209 highest_present_section_nr = drgn_test_highest_present_section_nr();
1210
1211 for (i = 0, n = 0; i <= highest_present_section_nr; i++) {
1212 if (present_section_nr(i))
1213 n++;
1214 }
1215
1216 drgn_test_present_sections =
1217 kmalloc_array(n, sizeof(drgn_test_present_sections[0]),
1218 GFP_KERNEL);
1219 if (!drgn_test_present_sections)
1220 return -ENOMEM;
1221
1222 for (i = 0; i <= highest_present_section_nr; i++) {
1223 if (present_section_nr(i)) {
1224 drgn_test_present_sections[drgn_test_num_present_sections].nr = i;
1225 drgn_test_present_sections[drgn_test_num_present_sections].section =
1226 __nr_to_section(i);
1227 drgn_test_num_present_sections++;
1228 if (drgn_test_num_present_sections >= n)
1229 break;
1230 }
1231 }
1232 #endif
1233
1234 return 0;
1235 }
1236
1237 static void drgn_test_mm_exit(void)
1238 {
1239 vfree(drgn_test_vmalloc_va);
1240 if (drgn_test_compound_page)
1241 __free_pages(drgn_test_compound_page, 1);
1242 if (drgn_test_page)
1243 __free_pages(drgn_test_page, 0);
1244 }
1245
1246 // mmzone
1247
1248 int drgn_test_nid;
1249 struct pglist_data *drgn_test_pgdat;
1250
1251 static void drgn_test_mmzone_init(void)
1252 {
1253 drgn_test_nid = first_online_node;
1254 drgn_test_pgdat = NODE_DATA(drgn_test_nid);
1255 }
1256
1257 // net
1258
1259 struct net_device *drgn_test_netdev;
1260 void *drgn_test_netdev_priv;
1261 struct sk_buff *drgn_test_skb;
1262 struct skb_shared_info *drgn_test_skb_shinfo;
1263
1264 static int drgn_test_net_init(void)
1265 {
1266 drgn_test_netdev = dev_get_by_name(&init_net, "lo");
1267 if (!drgn_test_netdev)
1268 return -ENODEV;
1269 // The loopback device doesn't actually have private data, but we just
1270 // need to compare the pointer.
1271 drgn_test_netdev_priv = netdev_priv(drgn_test_netdev);
1272 drgn_test_skb = alloc_skb(64, GFP_KERNEL);
1273 if (!drgn_test_skb)
1274 return -ENOMEM;
1275 drgn_test_skb_shinfo = skb_shinfo(drgn_test_skb);
1276 return 0;
1277 }
1278
1279 static void drgn_test_net_exit(void)
1280 {
1281 kfree_skb(drgn_test_skb);
1282 dev_put(drgn_test_netdev);
1283 }
1284
1285 // page_pool
1286
1287 const int drgn_test_have_page_pool = HAVE_PAGE_POOL;
1288
1289 #if HAVE_PAGE_POOL
1290 struct page_pool *drgn_test_page_pool;
1291 struct page *drgn_test_page_pool_page;
1292 #endif
1293
1294 static int drgn_test_page_pool_init(void)
1295 {
1296 #if HAVE_PAGE_POOL
1297 struct page_pool_params params = {
1298 .order = 0,
1299 .flags = 0,
1300 .pool_size = 1,
1301 .nid = NUMA_NO_NODE,
1302 };
1303 struct page_pool *pool;
1304
1305 pool = page_pool_create(&params);
1306 if (IS_ERR(pool))
1307 return PTR_ERR(pool);
1308 drgn_test_page_pool = pool;
1309
1310 drgn_test_page_pool_page = page_pool_alloc_pages(pool, GFP_KERNEL);
1311 if (!drgn_test_page_pool_page)
1312 return -ENOMEM;
1313 #endif
1314 return 0;
1315 }
1316
1317 static void drgn_test_page_pool_exit(void)
1318 {
1319 #if HAVE_PAGE_POOL
1320 if (drgn_test_page_pool_page) {
1321 // page_pool_put_page() changed in Linux kernel commit
1322 // 458de8a97f10 ("net: page_pool: API cleanup and comments") (in
1323 // v5.7).
1324 page_pool_put_page(drgn_test_page_pool,
1325 drgn_test_page_pool_page,
1326 #if LINUX_VERSION_CODE >= KERNEL_VERSION(5, 7, 0)
1327 0,
1328 #endif
1329 true);
1330 }
1331 page_pool_destroy(drgn_test_page_pool);
1332 #endif
1333 }
1334
1335 // percpu
1336
1337 DEFINE_PER_CPU(u32, drgn_test_percpu_static);
1338 u32 __percpu *drgn_test_percpu_dynamic;
1339 struct percpu_counter drgn_test_percpu_counter;
1340 struct percpu_counter drgn_test_percpu_counter_negative;
1341
1342 struct drgn_test_percpu_struct {
1343 int cpu;
1344 int i;
1345 };
1346
1347 DEFINE_PER_CPU(struct drgn_test_percpu_struct, drgn_test_percpu_structs);
1348
1349 typedef struct drgn_test_percpu_struct drgn_test_percpu_array[3];
1350
1351 DEFINE_PER_CPU(drgn_test_percpu_array, drgn_test_percpu_arrays);
1352
1353 static int drgn_test_percpu_init(void)
1354 {
1355 int ret;
1356 int cpu;
1357 u32 static_seed = drgn_test_prng32_seed("PCPU");
1358 u32 dynamic_seed = drgn_test_prng32_seed("pcpu");
1359
1360 drgn_test_percpu_dynamic = alloc_percpu(u32);
1361 if (!drgn_test_percpu_dynamic)
1362 return -ENOMEM;
1363 // Initialize the per-cpu variables with a PRNG sequence.
1364 for_each_possible_cpu(cpu) {
1365 int i;
1366
1367 static_seed = drgn_test_prng32(static_seed);
1368 per_cpu(drgn_test_percpu_static, cpu) = static_seed;
1369 dynamic_seed = drgn_test_prng32(dynamic_seed);
1370 *per_cpu_ptr(drgn_test_percpu_dynamic, cpu) = dynamic_seed;
1371
1372 per_cpu(drgn_test_percpu_structs, cpu) =
1373 (struct drgn_test_percpu_struct){
1374 .cpu = cpu,
1375 };
1376
1377 for (i = 0; i < 3; i++) {
1378 per_cpu(drgn_test_percpu_arrays, cpu)[i] =
1379 (struct drgn_test_percpu_struct){
1380 .cpu = cpu,
1381 .i = i,
1382 };
1383 }
1384 }
1385
1386 ret = percpu_counter_init(&drgn_test_percpu_counter,
1387 10, GFP_KERNEL);
1388 if (ret)
1389 return ret;
1390 percpu_counter_add(&drgn_test_percpu_counter, 3);
1391
1392 ret = percpu_counter_init(&drgn_test_percpu_counter_negative,
1393 33, GFP_KERNEL);
1394 if (ret)
1395 return ret;
1396 percpu_counter_sub(&drgn_test_percpu_counter_negative, 99);
1397
1398 return 0;
1399 }
1400
1401 static void drgn_test_percpu_exit(void)
1402 {
1403 percpu_counter_destroy(&drgn_test_percpu_counter_negative);
1404 percpu_counter_destroy(&drgn_test_percpu_counter);
1405 free_percpu(drgn_test_percpu_dynamic);
1406 }
1407
1408 // rbtree
1409
1410 struct rb_root drgn_test_empty_rb_root = RB_ROOT;
1411 struct rb_root drgn_test_rb_root = RB_ROOT;
1412
1413 struct drgn_test_rb_entry {
1414 struct rb_node node;
1415 int value;
1416 };
1417
1418 struct drgn_test_rb_entry drgn_test_rb_entries[4];
1419
1420 struct rb_node drgn_test_empty_rb_node;
1421
1422 struct drgn_test_rbtree_container_struct {
1423 struct drgn_test_rb_entry entries[2];
1424 struct rb_root root;
1425 } drgn_test_rbtree_container;
1426
1427 struct rb_root drgn_test_rbtree_with_equal = RB_ROOT;
1428 struct drgn_test_rb_entry drgn_test_rb_entries_with_equal[4];
1429
1430 struct rb_root drgn_test_rbtree_out_of_order = RB_ROOT;
1431 struct drgn_test_rb_entry drgn_test_rb_entries_out_of_order[4];
1432
1433 struct rb_root drgn_test_rbtree_with_bad_root_parent = RB_ROOT;
1434 struct drgn_test_rb_entry drgn_test_rb_entry_bad_root_parent;
1435
1436 struct rb_root drgn_test_rbtree_with_red_root = RB_ROOT;
1437 struct drgn_test_rb_entry drgn_test_rb_entry_red_root;
1438
1439 struct rb_root drgn_test_rbtree_with_inconsistent_parents = RB_ROOT;
1440 struct drgn_test_rb_entry drgn_test_rb_entries_with_inconsistent_parents[2];
1441
1442 struct rb_root drgn_test_rbtree_with_red_violation = RB_ROOT;
1443 struct drgn_test_rb_entry drgn_test_rb_entries_with_red_violation[3];
1444
1445 struct rb_root drgn_test_rbtree_with_black_violation = RB_ROOT;
1446 struct drgn_test_rb_entry drgn_test_rb_entries_with_black_violation[2];
1447
1448 static void drgn_test_rbtree_insert(struct rb_root *root,
1449 struct drgn_test_rb_entry *entry)
1450 {
1451 struct rb_node **new = &root->rb_node, *parent = NULL;
1452
1453 while (*new) {
1454 struct drgn_test_rb_entry *this =
1455 container_of(*new, struct drgn_test_rb_entry, node);
1456
1457 parent = *new;
1458 if (entry->value <= this->value)
1459 new = &(*new)->rb_left;
1460 else
1461 new = &(*new)->rb_right;
1462 }
1463
1464 rb_link_node(&entry->node, parent, new);
1465 rb_insert_color(&entry->node, root);
1466 }
1467
1468 static void drgn_test_rbtree_init(void)
1469 {
1470 struct rb_node *node;
1471 size_t i;
1472
1473 for (i = 0; i < ARRAY_SIZE(drgn_test_rb_entries); i++) {
1474 drgn_test_rb_entries[i].value = i;
1475 drgn_test_rbtree_insert(&drgn_test_rb_root,
1476 &drgn_test_rb_entries[i]);
1477 }
1478 RB_CLEAR_NODE(&drgn_test_empty_rb_node);
1479
1480 for (i = 0; i < ARRAY_SIZE(drgn_test_rbtree_container.entries); i++) {
1481 drgn_test_rbtree_container.entries[i].value = i;
1482 drgn_test_rbtree_insert(&drgn_test_rbtree_container.root,
1483 &drgn_test_rbtree_container.entries[i]);
1484 }
1485
1486 // Red-black tree with entries that compare equal to each other.
1487 for (i = 0; i < ARRAY_SIZE(drgn_test_rb_entries_with_equal); i++) {
1488 drgn_test_rb_entries_with_equal[i].value = i / 2;
1489 drgn_test_rbtree_insert(&drgn_test_rbtree_with_equal,
1490 &drgn_test_rb_entries_with_equal[i]);
1491 }
1492
1493 // Bad red-black tree whose entries are out of order.
1494 for (i = 0; i < ARRAY_SIZE(drgn_test_rb_entries_out_of_order); i++) {
1495 drgn_test_rb_entries_out_of_order[i].value = i;
1496 drgn_test_rbtree_insert(&drgn_test_rbtree_out_of_order,
1497 &drgn_test_rb_entries_out_of_order[i]);
1498 }
1499 drgn_test_rb_entries_out_of_order[0].value = 99;
1500
1501 // Bad red-black tree with a root node that has a non-NULL parent.
1502 drgn_test_rbtree_insert(&drgn_test_rbtree_with_bad_root_parent,
1503 &drgn_test_rb_entry_bad_root_parent);
1504 rb_set_parent(&drgn_test_rb_entry_bad_root_parent.node,
1505 &drgn_test_empty_rb_node);
1506
1507 // Bad red-black tree with a red root node.
1508 rb_link_node(&drgn_test_rb_entry_red_root.node, NULL,
1509 &drgn_test_rbtree_with_red_root.rb_node);
1510
1511 // Bad red-black tree with inconsistent rb_parent.
1512 for (i = 0; i < ARRAY_SIZE(drgn_test_rb_entries_with_inconsistent_parents); i++) {
1513 drgn_test_rb_entries_with_inconsistent_parents[i].value = i;
1514 drgn_test_rbtree_insert(&drgn_test_rbtree_with_inconsistent_parents,
1515 &drgn_test_rb_entries_with_inconsistent_parents[i]);
1516 }
1517 node = drgn_test_rbtree_with_inconsistent_parents.rb_node;
1518 rb_set_parent(node->rb_left ? node->rb_left : node->rb_right,
1519 &drgn_test_empty_rb_node);
1520
1521 // Bad red-black tree with red node with red child.
1522 for (i = 0; i < ARRAY_SIZE(drgn_test_rb_entries_with_red_violation); i++)
1523 drgn_test_rb_entries_with_red_violation[i].value = i;
1524 drgn_test_rbtree_insert(&drgn_test_rbtree_with_red_violation,
1525 &drgn_test_rb_entries_with_red_violation[0]);
1526 rb_link_node(&drgn_test_rb_entries_with_red_violation[1].node,
1527 &drgn_test_rb_entries_with_red_violation[0].node,
1528 &drgn_test_rb_entries_with_red_violation[0].node.rb_right);
1529 rb_link_node(&drgn_test_rb_entries_with_red_violation[2].node,
1530 &drgn_test_rb_entries_with_red_violation[1].node,
1531 &drgn_test_rb_entries_with_red_violation[1].node.rb_right);
1532
1533 // Bad red-black tree with unequal number of black nodes in paths from
1534 // root to leaves.
1535 for (i = 0; i < ARRAY_SIZE(drgn_test_rb_entries_with_black_violation); i++)
1536 drgn_test_rb_entries_with_black_violation[i].value = i;
1537 drgn_test_rbtree_insert(&drgn_test_rbtree_with_black_violation,
1538 &drgn_test_rb_entries_with_black_violation[0]);
1539 rb_link_node(&drgn_test_rb_entries_with_black_violation[1].node,
1540 &drgn_test_rb_entries_with_black_violation[0].node,
1541 &drgn_test_rb_entries_with_black_violation[0].node.rb_right);
1542 drgn_test_rb_entries_with_black_violation[1].node.__rb_parent_color |= RB_BLACK;
1543 }
1544
1545 // sbitmap
1546
1547 #if LINUX_VERSION_CODE < KERNEL_VERSION(5, 0, 0)
1548 // sbitmap_deferred_clear_bit() was added in Linux kernel commit ea86ea2cdced
1549 // ("sbitmap: ammortize cost of clearing bits") (in v5.0).
1550 #define sbitmap_deferred_clear_bit sbitmap_clear_bit
1551 #endif
1552
1553 struct sbitmap drgn_test_sbitmap;
1554
1555 static int drgn_test_sbitmap_init(void)
1556 {
1557 int ret;
1558
1559 ret = sbitmap_init_node(&drgn_test_sbitmap, 128, 4, GFP_KERNEL,
1560 NUMA_NO_NODE
1561 // The round_robin and alloc_hint parameters were added in Linux kernel commits
1562 // efe1f3a1d583 ("scsi: sbitmap: Maintain allocation round_robin in sbitmap"),
1563 // and c548e62bcf6a ("scsi: sbitmap: Move allocation hint into sbitmap") (both
1564 // in v5.13), respectively.
1565 #if LINUX_VERSION_CODE >= KERNEL_VERSION(5, 13, 0)
1566 , false , false
1567 #endif
1568 );
1569 if (ret)
1570 return ret;
1571
1572 sbitmap_set_bit(&drgn_test_sbitmap, 13);
1573 sbitmap_set_bit(&drgn_test_sbitmap, 23);
1574 sbitmap_set_bit(&drgn_test_sbitmap, 24);
1575 sbitmap_set_bit(&drgn_test_sbitmap, 97);
1576 sbitmap_set_bit(&drgn_test_sbitmap, 98);
1577 sbitmap_set_bit(&drgn_test_sbitmap, 99);
1578 sbitmap_set_bit(&drgn_test_sbitmap, 123);
1579
1580 sbitmap_clear_bit(&drgn_test_sbitmap, 97);
1581 // Resize to smaller than a set bit to test that the depth is enforced
1582 // properly.
1583 sbitmap_resize(&drgn_test_sbitmap, 100);
1584 sbitmap_deferred_clear_bit(&drgn_test_sbitmap, 98);
1585 return 0;
1586 }
1587
1588 static void drgn_test_sbitmap_exit(void)
1589 {
1590 if (drgn_test_sbitmap.map)
1591 sbitmap_free(&drgn_test_sbitmap);
1592 }
1593
1594 // slab
1595
1596 const int drgn_test_slob = IS_ENABLED(CONFIG_SLOB);
1597 struct kmem_cache *drgn_test_small_kmem_cache;
1598 struct kmem_cache *drgn_test_big_kmem_cache;
1599
1600 struct drgn_test_small_slab_object {
1601 int padding[11];
1602 int value;
1603 };
1604
1605 struct drgn_test_big_slab_object {
1606 unsigned long padding[PAGE_SIZE / 3 * 4 / sizeof(unsigned long) - 1];
1607 unsigned long value;
1608 };
1609
1610 struct drgn_test_small_slab_object *drgn_test_small_slab_objects[5];
1611 struct drgn_test_big_slab_object *drgn_test_big_slab_objects[5];
1612
1613 static void drgn_test_slab_exit(void)
1614 {
1615 size_t i;
1616
1617 if (drgn_test_big_kmem_cache) {
1618 for (i = 0; i < ARRAY_SIZE(drgn_test_big_slab_objects); i++) {
1619 if (drgn_test_big_slab_objects[i]) {
1620 kmem_cache_free(drgn_test_big_kmem_cache,
1621 drgn_test_big_slab_objects[i]);
1622 }
1623 }
1624 kmem_cache_destroy(drgn_test_big_kmem_cache);
1625 }
1626 if (drgn_test_small_kmem_cache) {
1627 for (i = 0; i < ARRAY_SIZE(drgn_test_small_slab_objects); i++) {
1628 if (drgn_test_small_slab_objects[i]) {
1629 kmem_cache_free(drgn_test_small_kmem_cache,
1630 drgn_test_small_slab_objects[i]);
1631 }
1632 }
1633 kmem_cache_destroy(drgn_test_small_kmem_cache);
1634 }
1635 }
1636
1637 // Dummy constructor so test slab caches won't get merged.
1638 static void drgn_test_slab_ctor(void *arg)
1639 {
1640 }
1641
1642 static int drgn_test_slab_init(void)
1643 {
1644 size_t num_tmp_objs;
1645 size_t i, j;
1646
1647 // We want objects in the drgn_test_small cache to be spread out over
1648 // multiple slabs. To accomplish that, we allocate a bunch of temporary
1649 // objects in the middle of the allocations we intend to keep, then free
1650 // the temporary objects.
1651 num_tmp_objs = PAGE_SIZE / sizeof(struct drgn_test_small_slab_object);
1652
1653 drgn_test_small_kmem_cache =
1654 kmem_cache_create("drgn_test_small",
1655 sizeof(struct drgn_test_small_slab_object),
1656 __alignof__(struct drgn_test_small_slab_object),
1657 0, drgn_test_slab_ctor);
1658 if (!drgn_test_small_kmem_cache)
1659 return -ENOMEM;
1660 for (i = 0; i < ARRAY_SIZE(drgn_test_small_slab_objects); i++) {
1661 const bool alloc_tmp_objs =
1662 i == ARRAY_SIZE(drgn_test_small_slab_objects) / 2;
1663 void **tmp_objs;
1664 int error = 0;
1665
1666 if (alloc_tmp_objs) {
1667 tmp_objs = kmalloc_array(num_tmp_objs,
1668 sizeof(*tmp_objs), GFP_KERNEL);
1669 if (!tmp_objs)
1670 return -ENOMEM;
1671 for (j = 0; j < num_tmp_objs; j++) {
1672 tmp_objs[j] = kmem_cache_alloc(drgn_test_small_kmem_cache,
1673 GFP_KERNEL);
1674 // We check for allocation failures below to
1675 // avoid duplicating cleanup code.
1676 }
1677 }
1678
1679 drgn_test_small_slab_objects[i] =
1680 kmem_cache_alloc(drgn_test_small_kmem_cache,
1681 GFP_KERNEL);
1682 if (!drgn_test_small_slab_objects[i])
1683 error = -ENOMEM;
1684
1685 if (alloc_tmp_objs) {
1686 for (j = 0; j < num_tmp_objs; j++) {
1687 if (!tmp_objs[j])
1688 error = -ENOMEM;
1689 kmem_cache_free(drgn_test_small_kmem_cache,
1690 tmp_objs[j]);
1691 }
1692 kfree(tmp_objs);
1693 }
1694 if (error)
1695 return -ENOMEM;
1696
1697 drgn_test_small_slab_objects[i]->value = i;
1698 }
1699 drgn_test_big_kmem_cache =
1700 kmem_cache_create("drgn_test_big",
1701 sizeof(struct drgn_test_big_slab_object),
1702 __alignof__(struct drgn_test_big_slab_object),
1703 0, drgn_test_slab_ctor);
1704 if (!drgn_test_big_kmem_cache)
1705 return -ENOMEM;
1706 for (i = 0; i < ARRAY_SIZE(drgn_test_big_slab_objects); i++) {
1707 drgn_test_big_slab_objects[i] =
1708 kmem_cache_alloc(drgn_test_big_kmem_cache, GFP_KERNEL);
1709 if (!drgn_test_big_slab_objects[i])
1710 return -ENOMEM;
1711 drgn_test_big_slab_objects[i]->value = i;
1712 }
1713 return 0;
1714 }
1715
1716 // timekeeping
1717
1718 ktime_t drgn_test_ktime;
1719 s64 drgn_test_ktime_ns;
1720
1721 static void drgn_test_timekeeping_init(void)
1722 {
1723 drgn_test_ktime = ktime_get();
1724 drgn_test_ktime_ns = ktime_to_ns(drgn_test_ktime);
1725 }
1726
1727 // timer
1728
1729 static void drgn_test_timer_fn(struct timer_list *timer)
1730 {
1731 mod_timer(timer, jiffies + HZ * 60 * 60 * 24);
1732 }
1733
1734 // Before Linux kernel commits 1d27e3e2252b ("timer: Remove expires and data
1735 // arguments from DEFINE_TIMER") and 354b46b1a0ad ("timer: Switch callback
1736 // prototype to take struct timer_list * argument") (in v4.15), the timer
1737 // callback took an unsigned long that had to be passed explicitly.
1738 #if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 15, 0)
1739 DEFINE_TIMER(drgn_test_timer, drgn_test_timer_fn);
1740 #else
1741 static void drgn_test_timer_fn_old(unsigned long timer)
1742 {
1743 drgn_test_timer_fn((struct timer_list *)timer);
1744 }
1745
1746 DEFINE_TIMER(drgn_test_timer, drgn_test_timer_fn_old, 0,
1747 (unsigned long)&drgn_test_timer);
1748 #endif
1749
1750 #define drgn_test_hrtimer_interval ms_to_ktime(1000 * 60 * 60 * 24)
1751
1752 static enum hrtimer_restart drgn_test_hrtimer_fn(struct hrtimer *hrtimer)
1753 {
1754 hrtimer_forward_now(hrtimer, drgn_test_hrtimer_interval);
1755 return HRTIMER_RESTART;
1756 }
1757
1758 struct hrtimer drgn_test_hrtimer;
1759 bool drgn_test_hrtimer_started;
1760
1761 static void drgn_test_timer_init(void)
1762 {
1763 drgn_test_timer_fn(&drgn_test_timer);
1764 // hrtimer initialization was changed in Linux kernel commit
1765 // 908a1d775422 ("hrtimers: Introduce hrtimer_setup() to replace
1766 // hrtimer_init()") (in v6.13).
1767 #if LINUX_VERSION_CODE >= KERNEL_VERSION(6, 13, 0)
1768 hrtimer_setup(&drgn_test_hrtimer, drgn_test_hrtimer_fn, CLOCK_MONOTONIC,
1769 HRTIMER_MODE_REL);
1770 #else
1771 hrtimer_init(&drgn_test_hrtimer, CLOCK_MONOTONIC,
1772 HRTIMER_MODE_REL);
1773 drgn_test_hrtimer.function = drgn_test_hrtimer_fn;
1774 #endif
1775 hrtimer_start(&drgn_test_hrtimer, drgn_test_hrtimer_interval,
1776 HRTIMER_MODE_REL);
1777 drgn_test_hrtimer_started = true;
1778 }
1779
1780 static void drgn_test_timer_exit(void)
1781 {
1782 // This was renamed in Linux kernel commit 9b13df3fb64e ("timers: Rename
1783 // del_timer_sync() to timer_delete_sync()") (in v6.2).
1784 #if LINUX_VERSION_CODE >= KERNEL_VERSION(6, 2, 0)
1785 timer_delete_sync(&drgn_test_timer);
1786 #else
1787 del_timer_sync(&drgn_test_timer);
1788 #endif
1789 if (drgn_test_hrtimer_started)
1790 hrtimer_cancel(&drgn_test_hrtimer);
1791 }
1792
1793 // kthread for stack trace
1794
1795 struct task_struct *drgn_test_kthread;
1796 struct thread_info *drgn_test_kthread_info;
1797
1798 const int drgn_test_have_stacktrace = IS_ENABLED(CONFIG_STACKTRACE);
1799 #ifdef CONFIG_STACKTRACE
1800 unsigned long drgn_test_stack_entries[16];
1801 unsigned int drgn_test_num_stack_entries;
1802
1803 #if LINUX_VERSION_CODE < KERNEL_VERSION(5, 2, 0)
1804 // stack_trace_save() was added in Linux kernel commit 214d8ca6ee85
1805 // ("stacktrace: Provide common infrastructure") (in v5.2). Wrap the old
1806 // save_stack_trace() interface. save_stack_trace() skips the caller, so we also
1807 // need the extra frame.
1808 static noinline unsigned int
1809 drgn_test_stack_trace_save(unsigned long *store, unsigned int size,
1810 unsigned int skipnr)
1811 {
1812 struct stack_trace trace = {
1813 .entries = store,
1814 .max_entries = size,
1815 .skip = skipnr,
1816 };
1817 save_stack_trace(&trace);
1818 return trace.nr_entries;
1819 }
1820 #elif defined(__arm__) && LINUX_VERSION_CODE < KERNEL_VERSION(6, 2, 0)
1821 // Before Linux kernel commit 9fbed16c3f4f ("ARM: 9259/1: stacktrace: Convert
1822 // stacktrace to generic ARCH_STACKWALK") (in v6.2), stack_trace_save() skips
1823 // the caller on Arm. Wrap it in an extra frame.
1824 static noinline unsigned int
1825 drgn_test_stack_trace_save(unsigned long *store, unsigned int size,
1826 unsigned int skipnr)
1827 {
1828 unsigned int ret = stack_trace_save(store, size, skipnr);
1829 barrier(); // Prevent tail call optimization.
1830 return ret;
1831 }
1832 #else
1833 #define drgn_test_stack_trace_save stack_trace_save
1834 #endif
1835 #endif
1836
1837 const int drgn_test_have_stackdepot = IS_ENABLED(CONFIG_STACKDEPOT);
1838 #ifdef CONFIG_STACKDEPOT
1839 depot_stack_handle_t drgn_test_stack_handle;
1840 #endif
1841
1842 // Completion indicating that the kthread has set up its stack frames and is
1843 // ready to be parked.
1844 static DECLARE_COMPLETION(drgn_test_kthread_ready);
1845 struct pt_regs drgn_test_kthread_pt_regs;
1846 static inline void drgn_test_get_pt_regs(struct pt_regs *regs)
1847 {
1848 #if defined(__aarch64__)
1849 // Copied from crash_setup_regs() in arch/arm64/include/asm/kexec.h as
1850 // of Linux v6.1.
1851 u64 tmp1, tmp2;
1852
1853 __asm__ __volatile__ (
1854 "stp x0, x1, [%2, #16 * 0]\n"
1855 "stp x2, x3, [%2, #16 * 1]\n"
1856 "stp x4, x5, [%2, #16 * 2]\n"
1857 "stp x6, x7, [%2, #16 * 3]\n"
1858 "stp x8, x9, [%2, #16 * 4]\n"
1859 "stp x10, x11, [%2, #16 * 5]\n"
1860 "stp x12, x13, [%2, #16 * 6]\n"
1861 "stp x14, x15, [%2, #16 * 7]\n"
1862 "stp x16, x17, [%2, #16 * 8]\n"
1863 "stp x18, x19, [%2, #16 * 9]\n"
1864 "stp x20, x21, [%2, #16 * 10]\n"
1865 "stp x22, x23, [%2, #16 * 11]\n"
1866 "stp x24, x25, [%2, #16 * 12]\n"
1867 "stp x26, x27, [%2, #16 * 13]\n"
1868 "stp x28, x29, [%2, #16 * 14]\n"
1869 "mov %0, sp\n"
1870 "stp x30, %0, [%2, #16 * 15]\n"
1871
1872 "/* faked current PSTATE */\n"
1873 "mrs %0, CurrentEL\n"
1874 "mrs %1, SPSEL\n"
1875 "orr %0, %0, %1\n"
1876 "mrs %1, DAIF\n"
1877 "orr %0, %0, %1\n"
1878 "mrs %1, NZCV\n"
1879 "orr %0, %0, %1\n"
1880 /* pc */
1881 "adr %1, 1f\n"
1882 "1:\n"
1883 "stp %1, %0, [%2, #16 * 16]\n"
1884 : "=&r" (tmp1), "=&r" (tmp2)
1885 : "r" (regs)
1886 : "memory"
1887 );
1888 #elif defined(__arm__)
1889 // Copied from crash_setup_regs() in arch/arm/include/asm/kexec.h as of
1890 // Linux v6.11.
1891 __asm__ __volatile__ (
1892 "stmia %[regs_base], {r0-r12}\n\t"
1893 "mov %[_ARM_sp], sp\n\t"
1894 "str lr, %[_ARM_lr]\n\t"
1895 "adr %[_ARM_pc], 1f\n\t"
1896 "mrs %[_ARM_cpsr], cpsr\n\t"
1897 "1:"
1898 : [_ARM_pc] "=r" (regs->ARM_pc),
1899 [_ARM_cpsr] "=r" (regs->ARM_cpsr),
1900 [_ARM_sp] "=r" (regs->ARM_sp),
1901 [_ARM_lr] "=o" (regs->ARM_lr)
1902 : [regs_base] "r" (&regs->ARM_r0)
1903 : "memory"
1904 );
1905 #elif defined(__powerpc64__)
1906 unsigned long link;
1907 unsigned long ccr;
1908
1909 asm volatile("std 0,%0" : "=m"(regs->gpr[0]));
1910 asm volatile("std 1,%0" : "=m"(regs->gpr[1]));
1911 asm volatile("std 2,%0" : "=m"(regs->gpr[2]));
1912 asm volatile("std 3,%0" : "=m"(regs->gpr[3]));
1913 asm volatile("std 4,%0" : "=m"(regs->gpr[4]));
1914 asm volatile("std 5,%0" : "=m"(regs->gpr[5]));
1915 asm volatile("std 6,%0" : "=m"(regs->gpr[6]));
1916 asm volatile("std 7,%0" : "=m"(regs->gpr[7]));
1917 asm volatile("std 8,%0" : "=m"(regs->gpr[8]));
1918 asm volatile("std 9,%0" : "=m"(regs->gpr[9]));
1919 asm volatile("std 10,%0" : "=m"(regs->gpr[10]));
1920 asm volatile("std 11,%0" : "=m"(regs->gpr[11]));
1921 asm volatile("std 12,%0" : "=m"(regs->gpr[12]));
1922 asm volatile("std 13,%0" : "=m"(regs->gpr[13]));
1923 asm volatile("std 14,%0" : "=m"(regs->gpr[14]));
1924 asm volatile("std 15,%0" : "=m"(regs->gpr[15]));
1925 asm volatile("std 16,%0" : "=m"(regs->gpr[16]));
1926 asm volatile("std 17,%0" : "=m"(regs->gpr[17]));
1927 asm volatile("std 18,%0" : "=m"(regs->gpr[18]));
1928 asm volatile("std 19,%0" : "=m"(regs->gpr[19]));
1929 asm volatile("std 20,%0" : "=m"(regs->gpr[20]));
1930 asm volatile("std 21,%0" : "=m"(regs->gpr[21]));
1931 asm volatile("std 22,%0" : "=m"(regs->gpr[22]));
1932 asm volatile("std 23,%0" : "=m"(regs->gpr[23]));
1933 asm volatile("std 24,%0" : "=m"(regs->gpr[24]));
1934 asm volatile("std 25,%0" : "=m"(regs->gpr[25]));
1935 asm volatile("std 26,%0" : "=m"(regs->gpr[26]));
1936 asm volatile("std 27,%0" : "=m"(regs->gpr[27]));
1937 asm volatile("std 28,%0" : "=m"(regs->gpr[28]));
1938 asm volatile("std 29,%0" : "=m"(regs->gpr[29]));
1939 asm volatile("std 30,%0" : "=m"(regs->gpr[30]));
1940 asm volatile("std 31,%0" : "=m"(regs->gpr[31]));
1941 asm volatile("mflr %0" : "=r"(link));
1942 asm volatile("std %1,%0" : "=m"(regs->link) : "r"(link));
1943 asm volatile("mfcr %0" : "=r"(ccr));
1944 asm volatile("std %1,%0" : "=m"(regs->ccr) : "r"(ccr));
1945 regs->nip = _THIS_IP_;
1946 #elif defined(__s390x__)
1947 regs->psw.mask = __extract_psw();
1948 regs->psw.addr = _THIS_IP_;
1949 asm volatile("stmg 0,15,%0\n" : "=S" (regs->gprs) : : "memory");
1950 #elif defined(__x86_64__)
1951 // Copied from crash_setup_regs() in arch/x86/include/asm/kexec.h as of
1952 // Linux v6.1.
1953 asm volatile("movq %%rbx,%0" : "=m"(regs->bx));
1954 asm volatile("movq %%rcx,%0" : "=m"(regs->cx));
1955 asm volatile("movq %%rdx,%0" : "=m"(regs->dx));
1956 asm volatile("movq %%rsi,%0" : "=m"(regs->si));
1957 asm volatile("movq %%rdi,%0" : "=m"(regs->di));
1958 asm volatile("movq %%rbp,%0" : "=m"(regs->bp));
1959 asm volatile("movq %%rax,%0" : "=m"(regs->ax));
1960 asm volatile("movq %%rsp,%0" : "=m"(regs->sp));
1961 asm volatile("movq %%r8,%0" : "=m"(regs->r8));
1962 asm volatile("movq %%r9,%0" : "=m"(regs->r9));
1963 asm volatile("movq %%r10,%0" : "=m"(regs->r10));
1964 asm volatile("movq %%r11,%0" : "=m"(regs->r11));
1965 asm volatile("movq %%r12,%0" : "=m"(regs->r12));
1966 asm volatile("movq %%r13,%0" : "=m"(regs->r13));
1967 asm volatile("movq %%r14,%0" : "=m"(regs->r14));
1968 asm volatile("movq %%r15,%0" : "=m"(regs->r15));
1969 asm volatile("movl %%ss, %%eax;" :"=a"(regs->ss));
1970 asm volatile("movl %%cs, %%eax;" :"=a"(regs->cs));
1971 asm volatile("pushfq; popq %0" :"=m"(regs->flags));
1972 regs->ip = _THIS_IP_;
1973 #endif
1974 }
1975
1976 __attribute__((__noipa__))
1977 static void drgn_test_kthread_fn3(void)
1978 {
1979 // Create some local variables for the test cases to use. Use volatile
1980 // to prevent them from being optimized out.
1981 volatile int a, b, c;
1982 volatile struct drgn_test_small_slab_object *slab_object;
1983
1984 a = 1;
1985 b = 2;
1986 c = 3;
1987 slab_object = drgn_test_small_slab_objects[0];
1988
1989 // Force slab_object onto the stack.
1990 __asm__ __volatile__ ("" : : "r" (&slab_object) : "memory");
1991
1992 #ifdef CONFIG_STACKTRACE
1993 drgn_test_num_stack_entries =
1994 drgn_test_stack_trace_save(drgn_test_stack_entries,
1995 ARRAY_SIZE(drgn_test_stack_entries),
1996 0);
1997 #endif
1998 #ifdef CONFIG_STACKDEPOT
1999 stack_depot_init();
2000 drgn_test_stack_handle = stack_depot_save(drgn_test_stack_entries,
2001 drgn_test_num_stack_entries,
2002 GFP_KERNEL);
2003 #endif
2004
2005 complete(&drgn_test_kthread_ready);
2006 for (;;) {
2007 set_current_state(TASK_INTERRUPTIBLE);
2008 if (kthread_should_stop()) {
2009 __set_current_state(TASK_RUNNING);
2010 break;
2011 }
2012 if (kthread_should_park()) {
2013 __set_current_state(TASK_RUNNING);
2014 drgn_test_get_pt_regs(&drgn_test_kthread_pt_regs);
2015 kthread_parkme();
2016 continue;
2017 }
2018 schedule();
2019 __set_current_state(TASK_RUNNING);
2020 }
2021
2022 // Make sure slab_object stays on the stack.
2023 __asm__ __volatile__ ("" : : "r" (&slab_object) : "memory");
2024 }
2025
2026 __attribute__((__noipa__))
2027 static void drgn_test_kthread_fn2(void)
2028 {
2029 drgn_test_kthread_fn3();
2030 barrier(); // Prevent tail call.
2031 }
2032
2033 __attribute__((__noipa__))
2034 static noinline int drgn_test_kthread_fn(void *arg)
2035 {
2036 drgn_test_kthread_fn2();
2037 return 0;
2038 }
2039
2040 static void drgn_test_stack_trace_exit(void)
2041 {
2042 if (drgn_test_kthread) {
2043 kthread_stop(drgn_test_kthread);
2044 drgn_test_kthread = NULL;
2045 }
2046 }
2047
2048 static int drgn_test_stack_trace_init(void)
2049 {
2050 drgn_test_kthread = kthread_create(drgn_test_kthread_fn,
2051 (void *)0xb0ba000,
2052 "drgn_test_kthread");
2053 if (!drgn_test_kthread)
2054 return -1;
2055 drgn_test_kthread_info = task_thread_info(drgn_test_kthread);
2056 wake_up_process(drgn_test_kthread);
2057 wait_for_completion(&drgn_test_kthread_ready);
2058 return kthread_park(drgn_test_kthread);
2059 }
2060
2061 // radixtree
2062
2063 RADIX_TREE(drgn_test_radix_tree_empty, GFP_KERNEL);
2064 RADIX_TREE(drgn_test_radix_tree_one, GFP_KERNEL);
2065 RADIX_TREE(drgn_test_radix_tree_one_at_zero, GFP_KERNEL);
2066 RADIX_TREE(drgn_test_radix_tree_sparse, GFP_KERNEL);
2067 #ifdef CONFIG_RADIX_TREE_MULTIORDER
2068 RADIX_TREE(drgn_test_radix_tree_multi_order, GFP_KERNEL);
2069 #endif
2070
2071 static int drgn_test_radix_tree_init(void)
2072 {
2073 int ret;
2074
2075 ret = radix_tree_insert(&drgn_test_radix_tree_one, 666,
2076 (void *)0xdeadb00);
2077 if (ret)
2078 return ret;
2079
2080 ret = radix_tree_insert(&drgn_test_radix_tree_one_at_zero, 0,
2081 (void *)0x1234);
2082 if (ret)
2083 return ret;
2084
2085 ret = radix_tree_insert(&drgn_test_radix_tree_sparse, 1,
2086 (void *)0x1234);
2087 if (ret)
2088 return ret;
2089
2090 ret = radix_tree_insert(&drgn_test_radix_tree_sparse, 0x80808080,
2091 (void *)0x5678);
2092 if (ret)
2093 return ret;
2094
2095 ret = radix_tree_insert(&drgn_test_radix_tree_sparse, 0xffffffff,
2096 (void *)0x9abc);
2097 if (ret)
2098 return ret;
2099
2100 #ifdef CONFIG_RADIX_TREE_MULTIORDER
2101 ret = __radix_tree_insert(&drgn_test_radix_tree_multi_order, 0x80808000,
2102 9, (void *)0x1234);
2103 if (ret)
2104 return ret;
2105 #endif
2106
2107 return 0;
2108 }
2109
2110 static void drgn_test_radix_tree_destroy(struct radix_tree_root *root)
2111 {
2112 struct radix_tree_iter iter;
2113 void __rcu **slot;
2114
2115 radix_tree_for_each_slot(slot, root, &iter, 0)
2116 radix_tree_delete(root, iter.index);
2117 }
2118
2119 static void drgn_test_radix_tree_exit(void)
2120 {
2121 drgn_test_radix_tree_destroy(&drgn_test_radix_tree_one);
2122 drgn_test_radix_tree_destroy(&drgn_test_radix_tree_one_at_zero);
2123 drgn_test_radix_tree_destroy(&drgn_test_radix_tree_sparse);
2124 #ifdef CONFIG_RADIX_TREE_MULTIORDER
2125 drgn_test_radix_tree_destroy(&drgn_test_radix_tree_multi_order);
2126 #endif
2127 }
2128
2129 // xarray
2130 const int drgn_test_have_xarray = HAVE_XARRAY;
2131 #if HAVE_XARRAY
2132 DEFINE_XARRAY(drgn_test_xarray_empty);
2133 DEFINE_XARRAY(drgn_test_xarray_one);
2134 DEFINE_XARRAY(drgn_test_xarray_one_at_zero);
2135 DEFINE_XARRAY(drgn_test_xarray_sparse);
2136 DEFINE_XARRAY(drgn_test_xarray_multi_index);
2137 DEFINE_XARRAY(drgn_test_xarray_zero_entry);
2138 DEFINE_XARRAY(drgn_test_xarray_zero_entry_at_zero);
2139 DEFINE_XARRAY(drgn_test_xarray_value);
2140 DEFINE_XARRAY(drgn_test_xarray_pointers);
2141 void *drgn_test_xa_zero_entry;
2142
2143 struct drgn_test_xarray_entry {
2144 int value;
2145 };
2146
2147 struct drgn_test_xarray_entry drgn_test_xarray_entries[4];
2148
2149 static int drgn_test_xa_store_order(struct xarray *xa, unsigned long index,
2150 unsigned order, void *entry, gfp_t gfp)
2151 {
2152 XA_STATE_ORDER(xas, xa, index, order);
2153
2154 do {
2155 xas_lock(&xas);
2156 xas_store(&xas, entry);
2157 xas_unlock(&xas);
2158 } while (xas_nomem(&xas, gfp));
2159 return xas_error(&xas);
2160 }
2161 #endif
2162
2163 static int drgn_test_xarray_init(void)
2164 {
2165 #if HAVE_XARRAY
2166 void *entry;
2167 int ret;
2168 size_t i;
2169
2170 drgn_test_xa_zero_entry = XA_ZERO_ENTRY;
2171
2172 entry = xa_store(&drgn_test_xarray_one, 666, (void *)0xdeadb00,
2173 GFP_KERNEL);
2174 if (xa_is_err(entry))
2175 return xa_err(entry);
2176
2177 entry = xa_store(&drgn_test_xarray_one_at_zero, 0, (void *)0x1234,
2178 GFP_KERNEL);
2179 if (xa_is_err(entry))
2180 return xa_err(entry);
2181
2182 entry = xa_store(&drgn_test_xarray_sparse, 1, (void *)0x1234,
2183 GFP_KERNEL);
2184 if (xa_is_err(entry))
2185 return xa_err(entry);
2186 entry = xa_store(&drgn_test_xarray_sparse, 0x80808080, (void *)0x5678,
2187 GFP_KERNEL);
2188 if (xa_is_err(entry))
2189 return xa_err(entry);
2190 entry = xa_store(&drgn_test_xarray_sparse, 0xffffffffUL, (void *)0x9abc,
2191 GFP_KERNEL);
2192 if (xa_is_err(entry))
2193 return xa_err(entry);
2194
2195 ret = drgn_test_xa_store_order(&drgn_test_xarray_multi_index,
2196 0x80808000, 9, (void *)0x1234,
2197 GFP_KERNEL);
2198 if (ret)
2199 return ret;
2200
2201 ret = xa_reserve(&drgn_test_xarray_zero_entry, 666, GFP_KERNEL);
2202 if (ret)
2203 return ret;
2204
2205 ret = xa_reserve(&drgn_test_xarray_zero_entry_at_zero, 0, GFP_KERNEL);
2206 if (ret)
2207 return ret;
2208
2209 entry = xa_store(&drgn_test_xarray_value, 0, xa_mk_value(1337),
2210 GFP_KERNEL);
2211 if (xa_is_err(entry))
2212 return xa_err(entry);
2213
2214 for (i = 0; i < ARRAY_SIZE(drgn_test_xarray_entries); i++) {
2215 drgn_test_xarray_entries[i].value = i;
2216 entry = xa_store(&drgn_test_xarray_pointers, i,
2217 &drgn_test_xarray_entries[i], GFP_KERNEL);
2218 if (xa_is_err(entry))
2219 return xa_err(entry);
2220 }
2221
2222 #endif
2223
2224 return 0;
2225 }
2226
2227 static void drgn_test_xarray_exit(void)
2228 {
2229 #if HAVE_XARRAY
2230 xa_destroy(&drgn_test_xarray_one);
2231 xa_destroy(&drgn_test_xarray_one_at_zero);
2232 xa_destroy(&drgn_test_xarray_sparse);
2233 xa_destroy(&drgn_test_xarray_multi_index);
2234 xa_destroy(&drgn_test_xarray_zero_entry);
2235 xa_destroy(&drgn_test_xarray_zero_entry_at_zero);
2236 xa_destroy(&drgn_test_xarray_value);
2237 #endif
2238 }
2239
2240 // idr
2241
2242 DEFINE_IDR(drgn_test_idr_empty);
2243 DEFINE_IDR(drgn_test_idr_one);
2244 DEFINE_IDR(drgn_test_idr_one_at_zero);
2245 DEFINE_IDR(drgn_test_idr_sparse);
2246
2247 static int drgn_test_idr_init(void)
2248 {
2249 int ret;
2250
2251 ret = idr_alloc(&drgn_test_idr_one, (void *)0xdeadb00, 66, 67,
2252 GFP_KERNEL);
2253 if (ret < 0)
2254 return ret;
2255
2256 ret = idr_alloc(&drgn_test_idr_one_at_zero, (void *)0x1234, 0, 1,
2257 GFP_KERNEL);
2258 if (ret < 0)
2259 return ret;
2260
2261 ret = idr_alloc(&drgn_test_idr_sparse, (void *)0x1234, 1, 2,
2262 GFP_KERNEL);
2263 if (ret < 0)
2264 return ret;
2265
2266 ret = idr_alloc(&drgn_test_idr_sparse, (void *)0x5678, 0x80, 0x81,
2267 GFP_KERNEL);
2268 if (ret < 0)
2269 return ret;
2270
2271 ret = idr_alloc(&drgn_test_idr_sparse, (void *)0x9abc, 0xee, 0xef,
2272 GFP_KERNEL);
2273 if (ret < 0)
2274 return ret;
2275
2276 return 0;
2277 }
2278
2279 static void drgn_test_idr_exit(void)
2280 {
2281 idr_destroy(&drgn_test_idr_one);
2282 idr_destroy(&drgn_test_idr_one_at_zero);
2283 idr_destroy(&drgn_test_idr_sparse);
2284 }
2285
2286 // wait-queue
2287 static struct task_struct *drgn_test_waitq_kthread;
2288 static wait_queue_head_t drgn_test_waitq;
2289 static wait_queue_head_t drgn_test_empty_waitq;
2290
2291 struct drgn_test_waitq_container_struct {
2292 int x;
2293 wait_queue_head_t waitq;
2294 } drgn_test_waitq_container;
2295
2296 static int drgn_test_waitq_kthread_fn(void *arg)
2297 {
2298 wait_event_interruptible(drgn_test_waitq, kthread_should_stop());
2299 return 0;
2300 }
2301
2302 static int drgn_test_waitq_init(void)
2303 {
2304 init_waitqueue_head(&drgn_test_waitq);
2305 init_waitqueue_head(&drgn_test_empty_waitq);
2306 init_waitqueue_head(&drgn_test_waitq_container.waitq);
2307
2308 drgn_test_waitq_kthread = kthread_create(drgn_test_waitq_kthread_fn,
2309 NULL,
2310 "drgn_test_waitq_kthread");
2311 if (!drgn_test_waitq_kthread)
2312 return -1;
2313
2314 wake_up_process(drgn_test_waitq_kthread);
2315 return 0;
2316 }
2317
2318 static void drgn_test_waitq_exit(void)
2319 {
2320 if (drgn_test_waitq_kthread) {
2321 kthread_stop(drgn_test_waitq_kthread);
2322 drgn_test_waitq_kthread = NULL;
2323 }
2324 }
2325
2326 // simple-wait-queue
2327 static struct task_struct *drgn_test_swaitq_kthread;
2328 static struct swait_queue_head drgn_test_swaitq;
2329 static struct swait_queue_head drgn_test_empty_swaitq;
2330
2331 static int drgn_test_swaitq_kthread_fn(void *arg)
2332 {
2333 DECLARE_SWAITQUEUE(swait);
2334 for (;;) {
2335 prepare_to_swait_exclusive(&drgn_test_swaitq, &swait, TASK_UNINTERRUPTIBLE);
2336 if (kthread_should_stop())
2337 break;
2338 schedule();
2339 }
2340 finish_swait(&drgn_test_swaitq, &swait);
2341 return 0;
2342 }
2343
2344 static int drgn_test_swaitq_init(void)
2345 {
2346 init_swait_queue_head(&drgn_test_swaitq);
2347 init_swait_queue_head(&drgn_test_empty_swaitq);
2348
2349 drgn_test_swaitq_kthread = kthread_create(drgn_test_swaitq_kthread_fn,
2350 NULL,
2351 "drgn_test_swaitq_kthread");
2352 if (!drgn_test_swaitq_kthread)
2353 return -1;
2354
2355 wake_up_process(drgn_test_swaitq_kthread);
2356 return 0;
2357 }
2358
2359 static void drgn_test_swaitq_exit(void)
2360 {
2361 if (drgn_test_swaitq_kthread) {
2362 kthread_stop(drgn_test_swaitq_kthread);
2363 drgn_test_swaitq_kthread = NULL;
2364 }
2365 }
2366
2367 // completion variable
2368 static struct task_struct *drgn_test_completion_kthread;
2369 static struct completion drgn_test_completion;
2370 static struct completion drgn_test_done_completion;
2371
2372 static int drgn_test_completion_kthread_fn(void *arg)
2373 {
2374 complete(&drgn_test_done_completion);
2375 wait_for_completion(&drgn_test_completion);
2376 return 0;
2377 }
2378
2379 static int drgn_test_completion_init(void)
2380 {
2381 drgn_test_completion_kthread = kthread_create(drgn_test_completion_kthread_fn,
2382 NULL,
2383 "drgn_test_completion_kthread");
2384 if (!drgn_test_completion_kthread)
2385 return -1;
2386
2387 init_completion(&drgn_test_completion);
2388 init_completion(&drgn_test_done_completion);
2389 wake_up_process(drgn_test_completion_kthread);
2390 return 0;
2391 }
2392
2393 static void drgn_test_completion_exit(void)
2394 {
2395 if (drgn_test_completion_kthread) {
2396 complete(&drgn_test_completion);
2397 drgn_test_completion_kthread = NULL;
2398 }
2399 }
2400
2401 // Dummy function symbol.
2402 int drgn_test_function(int x); // Silence -Wmissing-prototypes.
2403 int drgn_test_function(int x)
2404 {
2405 return x + 1;
2406 }
2407
2408 char drgn_test_data[] = "abc";
2409 const char drgn_test_rodata[] = "def";
2410
2411 // search
2412
2413 struct {
2414 unsigned long l;
2415 unsigned long l2;
2416 unsigned int i;
2417 unsigned short s;
2418 char c[64];
2419 char c2[64];
2420 int (*func)(int);
2421 } drgn_test_search_subject = {
2422 .l = 0xdeadbeef,
2423 .l2 = 0xfee1dead,
2424 .i = 0xdeadb00,
2425 .s = 0xb0ba,
2426 .c = "hello, world!",
2427 .c2 = "goodbye, world!",
2428 .func = drgn_test_function,
2429 };
2430 unsigned long *drgn_test_search_subject2;
2431
2432 static int __init drgn_test_search_init(void)
2433 {
2434 // Module memory can be in high memory, which makes it impossible to
2435 // test physical memory searches, so we kmalloc() this.
2436 drgn_test_search_subject2 = kmalloc(sizeof(*drgn_test_search_subject2),
2437 GFP_KERNEL);
2438 if (!drgn_test_search_subject2)
2439 return -ENOMEM;
2440 *drgn_test_search_subject2 = 0x12345678;
2441 return 0;
2442 }
2443
2444 static void drgn_test_search_exit(void)
2445 {
2446 kfree(drgn_test_search_subject2);
2447 }
2448
2449 // kmodify
2450
2451 #ifdef __x86_64__
2452 enum drgn_kmodify_enum {
2453 DRGN_KMODIFY_ONE = 1,
2454 DRGN_KMODIFY_TWO,
2455 DRGN_KMODIFY_THREE,
2456 };
2457
2458 struct drgn_kmodify_test_struct {
2459 void *v;
2460 int *i;
2461 };
2462 struct drgn_kmodify_test_struct *drgn_kmodify_test_ptr =
2463 &(struct drgn_kmodify_test_struct){};
2464
2465 char drgn_kmodify_test_memory[16];
2466
2467 int drgn_kmodify_test_int;
2468 int *drgn_kmodify_test_int_ptr;
2469
2470 unsigned long drgn_kmodify_test_bitmap[2];
2471 module_param_array_named(kmodify_bitmap, drgn_kmodify_test_bitmap, ulong, NULL,
2472 0600);
2473
2474 struct {
2475 unsigned int expect0_1 : 8;
2476 unsigned int byte_aligned : 8;
2477 unsigned int expect1_1 : 1;
2478 unsigned int bit : 1;
2479 unsigned int expect1_2 : 1;
2480 unsigned int expect0_2 : 1;
2481 unsigned int two_bits : 2;
2482 unsigned int unaligned : 8;
2483 } drgn_kmodify_test_bit_field = {
2484 .expect1_1 = 1,
2485 .expect1_2 = 1,
2486 };
2487
2488 #define DEFINE_KMODIFY_TEST_RETURN(name, return_type, return_value) \
2489 int drgn_kmodify_test_##name##_called = 0; \
2490 return_type drgn_kmodify_test_##name(void); \
2491 return_type drgn_kmodify_test_##name(void) \
2492 { \
2493 drgn_kmodify_test_##name##_called++; \
2494 return (return_value); \
2495 }
2496
2497 #define DEFINE_KMODIFY_TEST_ARGS(name, parameters, condition) \
2498 int drgn_kmodify_test_##name##_called = 0; \
2499 void drgn_kmodify_test_##name parameters; \
2500 void drgn_kmodify_test_##name parameters \
2501 { \
2502 if (condition) \
2503 drgn_kmodify_test_##name##_called++; \
2504 }
2505
2506 DEFINE_KMODIFY_TEST_ARGS(void_return, (void), 1)
2507 DEFINE_KMODIFY_TEST_RETURN(signed_char_return, signed char, -66)
2508 DEFINE_KMODIFY_TEST_RETURN(unsigned_char_return, unsigned char, 200)
2509 DEFINE_KMODIFY_TEST_RETURN(short_return, short, -666)
2510 DEFINE_KMODIFY_TEST_RETURN(unsigned_short_return, unsigned short, 7777)
2511 DEFINE_KMODIFY_TEST_RETURN(int_return, int, -12345)
2512 DEFINE_KMODIFY_TEST_RETURN(unsigned_int_return, unsigned int, 54321U)
2513 DEFINE_KMODIFY_TEST_RETURN(long_return, long, -2468013579L)
2514 DEFINE_KMODIFY_TEST_RETURN(unsigned_long_return, unsigned long, 4000000000UL)
2515 DEFINE_KMODIFY_TEST_RETURN(long_long_return, long long, -9080706050403020100LL)
2516 DEFINE_KMODIFY_TEST_RETURN(unsigned_long_long_return, unsigned long long,
2517 12345678909876543210ULL)
2518 DEFINE_KMODIFY_TEST_RETURN(pointer_return, struct drgn_kmodify_test_struct *,
2519 drgn_kmodify_test_ptr)
2520 DEFINE_KMODIFY_TEST_RETURN(enum_return, enum drgn_kmodify_enum,
2521 DRGN_KMODIFY_TWO)
2522
2523 DEFINE_KMODIFY_TEST_ARGS(
2524 signed_args,
2525 (signed char c, short s, int i, long l, long long ll),
2526 (c == -66 && s == -666 && i == -12345 && l == -2468013579L && ll == -9080706050403020100LL)
2527 )
2528 DEFINE_KMODIFY_TEST_ARGS(
2529 unsigned_args,
2530 (unsigned char c, unsigned short s, unsigned int i, unsigned long l, unsigned long long ll),
2531 (c == 200 && s == 7777 && i == 54321 && l == 4000000000UL && ll == 12345678909876543210ULL)
2532 )
2533
2534 DEFINE_KMODIFY_TEST_ARGS(
2535 many_args,
2536 (char c,
2537 signed char sc, short ss, int si, long sl, long long sll,
2538 unsigned char uc, unsigned short us, unsigned int ui, unsigned long ul, unsigned long long ull),
2539 (c == 48
2540 && sc == -66 && ss == -666 && si == -12345 && sl == -2468013579L && sll == -9080706050403020100LL
2541 && uc == 200 && us == 7777 && ui == 54321 && ul == 4000000000UL && ull == 12345678909876543210ULL)
2542 )
2543
2544 DEFINE_KMODIFY_TEST_ARGS(
2545 enum_args,
2546 (enum drgn_kmodify_enum a1, enum drgn_kmodify_enum *a2),
2547 ({
2548 int match = a1 == DRGN_KMODIFY_ONE && *a2 == DRGN_KMODIFY_TWO;
2549 *a2 = DRGN_KMODIFY_THREE;
2550 match;
2551 })
2552 )
2553
2554 DEFINE_KMODIFY_TEST_ARGS(
2555 pointer_args,
2556 (struct drgn_kmodify_test_struct *ptr),
2557 (ptr == drgn_kmodify_test_ptr)
2558 )
2559
2560 char *drgn_kmodify_test_char_str = "Hello";
2561 signed char *drgn_kmodify_test_signed_char_str = ", ";
2562 unsigned char *drgn_kmodify_test_unsigned_char_str = "world";
2563 const char *drgn_kmodify_test_const_char_str = "!";
2564 DEFINE_KMODIFY_TEST_ARGS(
2565 string_args,
2566 (char *c, signed char *sc, unsigned char *uc, const char *cc),
2567 (strcmp(c, drgn_kmodify_test_char_str) == 0 &&
2568 strcmp(sc, drgn_kmodify_test_signed_char_str) == 0 &&
2569 strcmp(uc, drgn_kmodify_test_unsigned_char_str) == 0 &&
2570 strcmp(cc, drgn_kmodify_test_const_char_str) == 0)
2571 )
2572
2573 DEFINE_KMODIFY_TEST_ARGS(
2574 integer_out_params,
2575 (signed char *c, short *s, int *i, long *l, long long *ll),
2576 ({
2577 int match = *c == -66 && *s == -666 && *i == -12345 && *l == -2468013579L && *ll == -9080706050403020100LL;
2578 *c = 33;
2579 *s = 333;
2580 *i = 23456;
2581 *l = 2222222222L;
2582 *ll = 9090909090909090909LL;
2583 match;
2584 })
2585 )
2586
2587 DEFINE_KMODIFY_TEST_ARGS(
2588 array_out_params,
2589 (long arr[3]),
2590 ({
2591 int match = arr[0] == 1 && arr[1] == 2 && arr[2] == 3;
2592 arr[0] = 2;
2593 arr[1] = 3;
2594 arr[2] = 5;
2595 match;
2596 })
2597 )
2598
2599 DEFINE_KMODIFY_TEST_ARGS(
2600 many_out_params,
2601 (char *c,
2602 signed char *sc, short *ss, int *si, long *sl, long long *sll,
2603 unsigned char *uc, unsigned short *us, unsigned int *ui, unsigned long *ul, unsigned long long *ull),
2604 ({
2605 int match = (*c == 48
2606 && *sc == -66 && *ss == -666 && *si == -12345 && *sl == -2468013579L && *sll == -9080706050403020100LL
2607 && *uc == 200 && *us == 7777 && *ui == 54321 && *ul == 4000000000UL && *ull == 12345678909876543210ULL);
2608 *c /= 3;
2609 *sc /= 3;
2610 *ss /= 3;
2611 *si /= 3;
2612 *sl /= 3;
2613 *sll /= 3;
2614 *uc /= 3;
2615 *us /= 3;
2616 *ui /= 3;
2617 *ul /= 3;
2618 *ull /= 3;
2619 match;
2620 })
2621 )
2622 #endif
2623
2624 #ifdef CONFIG_SYSFS
2625
2626 // Crash from an NMI + IRQ handler on architectures where drgn supports
2627 // unwinding through them.
2628 #ifdef __x86_64__
2629 #define DRGN_TEST_NMI_CRASH
2630 #elif defined(__aarch64__) || defined(__powerpc64__) || defined (__arm__)
2631 #define DRGN_TEST_IRQ_CRASH
2632 #endif
2633
2634
2635 // NMI test depends on IRQ
2636 #ifdef DRGN_TEST_NMI_CRASH
2637 #define DRGN_TEST_IRQ_CRASH
2638 #endif
2639
2640 static __noreturn noinline_for_stack void drgn_test_crash_func(void)
2641 {
2642 panic("drgn_test\n");
2643 }
2644
2645
2646 #ifdef DRGN_TEST_NMI_CRASH
2647 static bool drgn_panic = false;
2648
2649 static noinline_for_stack int drgn_test_nmi_handler(unsigned int cmd, struct pt_regs *regs)
2650 {
2651 if (drgn_panic) {
2652 drgn_test_crash_func();
2653 return NMI_HANDLED; /* lol */
2654 }
2655
2656 return NMI_DONE;
2657 }
2658
2659 static noinline_for_stack void drgn_test_crash_irq_work_fn(struct irq_work *work)
2660 {
2661 drgn_panic = true;
2662 apic->send_IPI_mask(cpumask_of(smp_processor_id()), NMI_VECTOR);
2663 }
2664 #elif defined(DRGN_TEST_IRQ_CRASH)
2665 static noinline_for_stack void drgn_test_crash_irq_work_fn(struct irq_work *work)
2666 {
2667 drgn_test_crash_func();
2668 }
2669 #endif
2670
2671 #ifdef DRGN_TEST_IRQ_CRASH
2672 static DEFINE_IRQ_WORK(drgn_test_crash_irq_work, drgn_test_crash_irq_work_fn);
2673 #endif
2674
2675 static ssize_t drgn_test_crash_store(struct kobject *kobj,
2676 struct kobj_attribute *attr,
2677 const char *buf, size_t count)
2678 {
2679 int ret, val;
2680
2681 ret = kstrtoint(buf, 0, &val);
2682 if (ret < 0)
2683 return ret;
2684 if (val != 1)
2685 return -EINVAL;
2686
2687 #ifdef DRGN_TEST_IRQ_CRASH
2688 preempt_disable();
2689 irq_work_queue(&drgn_test_crash_irq_work);
2690 // Spin until we get interrupted and crash.
2691 while (1);
2692 #else
2693 drgn_test_crash_func();
2694 #endif
2695 }
2696
2697 static int drgn_test_crash_init(void)
2698 {
2699 #ifdef DRGN_TEST_NMI_CRASH
2700 return register_nmi_handler(NMI_LOCAL, drgn_test_nmi_handler,
2701 0, "drgn_panic");
2702 #else
2703 return 0;
2704 #endif
2705 }
2706
2707 static struct kobj_attribute drgn_test_crash_attr =
2708 __ATTR(crash, 0200, NULL, drgn_test_crash_store);
2709
2710 static ssize_t drgn_test_boottime_seconds_show(struct kobject *kobj,
2711 struct kobj_attribute *attr,
2712 char *buf)
2713 {
2714 return sprintf(buf, "%lld\n", ktime_get_boottime_seconds());
2715 }
2716
2717 static struct kobj_attribute drgn_test_boottime_seconds_attr =
2718 __ATTR(boottime_seconds, 0444, drgn_test_boottime_seconds_show, NULL);
2719
2720 static ssize_t drgn_test_coarse_boottime_ns_show(struct kobject *kobj,
2721 struct kobj_attribute *attr,
2722 char *buf)
2723 {
2724 return sprintf(buf, "%llu\n", ktime_get_coarse_boottime_ns());
2725 }
2726
2727 static struct kobj_attribute drgn_test_coarse_boottime_ns_attr =
2728 __ATTR(coarse_boottime_ns, 0444, drgn_test_coarse_boottime_ns_show,
2729 NULL);
2730
2731 static ssize_t drgn_test_clocktai_seconds_show(struct kobject *kobj,
2732 struct kobj_attribute *attr,
2733 char *buf)
2734 {
2735 return sprintf(buf, "%lld\n", ktime_get_clocktai_seconds());
2736 }
2737
2738 static struct kobj_attribute drgn_test_clocktai_seconds_attr =
2739 __ATTR(clocktai_seconds, 0444, drgn_test_clocktai_seconds_show, NULL);
2740
2741 static ssize_t drgn_test_coarse_clocktai_ns_show(struct kobject *kobj,
2742 struct kobj_attribute *attr,
2743 char *buf)
2744 {
2745 return sprintf(buf, "%llu\n", ktime_get_coarse_clocktai_ns());
2746 }
2747
2748 static struct kobj_attribute drgn_test_coarse_clocktai_ns_attr =
2749 __ATTR(coarse_clocktai_ns, 0444, drgn_test_coarse_clocktai_ns_show,
2750 NULL);
2751
2752 static struct attribute_group drgn_test_attr_group = {
2753 .attrs = (struct attribute *[]){
2754 &drgn_test_crash_attr.attr,
2755 &drgn_test_boottime_seconds_attr.attr,
2756 &drgn_test_coarse_boottime_ns_attr.attr,
2757 &drgn_test_clocktai_seconds_attr.attr,
2758 &drgn_test_coarse_clocktai_ns_attr.attr,
2759 NULL,
2760 },
2761 };
2762
2763 static struct kobject *drgn_test_kobj;
2764
2765 static int __init drgn_test_sysfs_init(void)
2766 {
2767 drgn_test_kobj = kobject_create_and_add("drgn_test", kernel_kobj);
2768 if (!drgn_test_kobj)
2769 return -ENOMEM;
2770
2771 return sysfs_create_group(drgn_test_kobj, &drgn_test_attr_group);
2772 }
2773
2774 static void drgn_test_sysfs_exit(void)
2775 {
2776 kobject_put(drgn_test_kobj);
2777 }
2778 #else
2779 static inline int drgn_test_sysfs_init(void) { return 0; }
2780 static inline void drgn_test_sysfs_exit(void) {}
2781 #endif
2782
2783 // types
2784
2785 union drgn_test_union {
2786 u32 u;
2787 s32 s;
2788 };
2789 union drgn_test_union drgn_test_union_var;
2790
2791 typedef union {
2792 u64 u;
2793 s64 s;
2794 } drgn_test_anonymous_union;
2795 drgn_test_anonymous_union drgn_test_anonymous_union_var;
2796
2797 static void drgn_test_exit(void)
2798 {
2799 drgn_test_sysfs_exit();
2800 drgn_test_slab_exit();
2801 drgn_test_search_exit();
2802 drgn_test_sbitmap_exit();
2803 drgn_test_percpu_exit();
2804 drgn_test_maple_tree_exit();
2805 drgn_test_mm_exit();
2806 drgn_test_net_exit();
2807 drgn_test_page_pool_exit();
2808 drgn_test_stack_trace_exit();
2809 drgn_test_timer_exit();
2810 drgn_test_radix_tree_exit();
2811 drgn_test_xarray_exit();
2812 drgn_test_waitq_exit();
2813 drgn_test_locking_exit();
2814 drgn_test_idr_exit();
2815 drgn_test_block_exit();
2816 drgn_test_swaitq_exit();
2817 drgn_test_completion_exit();
2818 }
2819
2820 static int __init drgn_test_init(void)
2821 {
2822 int ret;
2823
2824 drgn_test_constants_init();
2825 ret = drgn_test_block_init();
2826 if (ret)
2827 goto out;
2828 drgn_test_list_init();
2829 drgn_test_llist_init();
2830 drgn_test_plist_init();
2831 ret = drgn_test_locking_init();
2832 if (ret)
2833 goto out;
2834 ret = drgn_test_maple_tree_init();
2835 if (ret)
2836 goto out;
2837 ret = drgn_test_mm_init();
2838 if (ret)
2839 goto out;
2840 drgn_test_mmzone_init();
2841 ret = drgn_test_net_init();
2842 if (ret)
2843 goto out;
2844 ret = drgn_test_page_pool_init();
2845 if (ret)
2846 goto out;
2847 ret = drgn_test_percpu_init();
2848 if (ret)
2849 goto out;
2850 drgn_test_rbtree_init();
2851 ret = drgn_test_sbitmap_init();
2852 if (ret)
2853 goto out;
2854 ret = drgn_test_search_init();
2855 if (ret)
2856 goto out;
2857 ret = drgn_test_slab_init();
2858 if (ret)
2859 goto out;
2860 ret = drgn_test_stack_trace_init();
2861 if (ret)
2862 goto out;
2863 ret = drgn_test_radix_tree_init();
2864 if (ret)
2865 goto out;
2866 drgn_test_timekeeping_init();
2867 drgn_test_timer_init();
2868 ret = drgn_test_xarray_init();
2869 if (ret)
2870 goto out;
2871
2872 ret = drgn_test_waitq_init();
2873 if (ret)
2874 goto out;
2875 ret = drgn_test_idr_init();
2876 if (ret)
2877 goto out;
2878 ret = drgn_test_sysfs_init();
2879 if (ret)
2880 goto out;
2881 ret = drgn_test_crash_init();
2882 if (ret)
2883 goto out;
2884
2885 ret = drgn_test_swaitq_init();
2886 if (ret)
2887 goto out;
2888
2889 ret = drgn_test_completion_init();
2890 out:
2891 if (ret)
2892 drgn_test_exit();
2893 return ret;
2894 }
2895
2896 module_init(drgn_test_init);
2897 module_exit(drgn_test_exit);
2898
2899 MODULE_LICENSE("GPL");
2900 MODULE_DESCRIPTION("Module for testing drgn");