Debian Codegraph @

runtime/linux/kprobes.c

raw
1 /* -*- linux-c -*-
2 * Common functions for using kprobes
3 * Copyright (C) 2014-2015 Red Hat Inc.
4 *
5 * This file is part of systemtap, and is free software. You can
6 * redistribute it and/or modify it under the terms of the GNU General
7 * Public License (GPL); either version 2, or (at your option) any
8 * later version.
9 */
10
11 #ifndef _KPROBES_C_
12 #define _KPROBES_C_
13
14 // Warn of misconfigured kernels
15 #if !defined(CONFIG_KPROBES)
16 #error "Need CONFIG_KPROBES!"
17 #endif
18
19 // PR26074: if kallsyms_on_each_symbol() is not exported we try to get it via relocations.
20 // Need to define the runtime.h variable here to check if it's defined from kprobes.c.
21 // (This code would go in a header if it were used anywhere else.)
22 //
23 // PR11514: kallsyms_on_each_symbol() should not be used on PPC64.
24 #ifndef CONFIG_PPC64
25 #define STAPCONF_KALLSYMS_ON_EACH_SYMBOL
26 #if !defined(STAPCONF_KALLSYMS_ON_EACH_SYMBOL_EXPORTED)
27 extern void *_stp_kallsyms_on_each_symbol;
28 #endif
29 #endif
30
31 // No export check and gates. This one seems simply like a non-export.
32 extern void *_stp_module_kallsyms_on_each_symbol;
33
34 #if defined(STAPCONF_KALLSYMS_ON_EACH_SYMBOL) && defined(STAPCONF_KALLSYMS_ON_EACH_SYMBOL_EXPORTED)
35 #define USE_KALLSYMS_ON_EACH_SYMBOL (1)
36 #elif defined(STAPCONF_KALLSYMS_ON_EACH_SYMBOL)
37 // XXX May not be available; if available, is passed through relocation:
38 #define USE_KALLSYMS_ON_EACH_SYMBOL (_stp_kallsyms_on_each_symbol != NULL)
39 #else
40 #define USE_KALLSYMS_ON_EACH_SYMBOL (0)
41 #endif
42
43 #include <linux/kprobes.h>
44 #include <linux/module.h>
45
46 #ifdef DEBUG_KPROBES
47 #define dbug_stapkp(args...) do { \
48 _stp_dbug(__FUNCTION__, __LINE__, args); \
49 } while (0)
50 #define dbug_stapkp_cond(cond, args...) do { \
51 if (cond) \
52 dbug_stapkp(args); \
53 } while (0)
54 #else
55 #define dbug_stapkp(args...) do { } while (0)
56 #define dbug_stapkp_cond(cond, args...) do { } while (0)
57 #endif
58
59 #ifndef KRETACTIVE
60 #define KRETACTIVE (max(15, 6 * (int)num_possible_cpus()))
61 #endif
62
63 // This shouldn't happen, but check as a precaution. If we're on kver >= 2.6.30,
64 // then we must also have STP_ON_THE_FLY_TIMER_ENABLE (which is turned on for
65 // kver >= 2.6.17, see translate_pass()). This indicates that the background
66 // timer is available and thus that kprobes can be armed/disarmed on-the-fly.
67 #if LINUX_VERSION_CODE >= KERNEL_VERSION(2,6,30) \
68 && !defined(STP_ON_THE_FLY_TIMER_ENABLE)
69 #error "STP_ON_THE_FLY_TIMER_ENABLE undefined"
70 #endif
71
72 // NB: this struct is set up by the stapkp_prepare_* functions prior to
73 // registering and zero'ed out again after each unregister
74 struct stap_kprobe {
75 union { struct kprobe kp; struct kretprobe krp; } u;
76 #ifdef __ia64__
77 // PR6028: We register a second dummy probe at the same address so that the
78 // kernel uses aggr_kprobe. This is needed ensure that the bspcache is always
79 // valid.
80 struct kprobe dummy;
81 #endif
82 };
83
84
85 struct stap_kprobe_probe {
86 const unsigned return_p:1;
87 const unsigned maxactive_p:1;
88 const unsigned optional_p:1;
89 unsigned registered_p:1;
90 const unsigned short maxactive_val;
91
92 // data saved in the kretprobe_instance packet
93 const unsigned short saved_longs;
94 const unsigned short saved_strings;
95
96 // These macros declare the module and section strings as either const char[]
97 // or const char * const. Their actual types are determined at translate-time
98 // in dwarf_derived_probe_group::emit_module_decls().
99 STAP_KPROBE_PROBE_STR_module;
100 STAP_KPROBE_PROBE_STR_section;
101
102 // For the majority of dwarf-based kprobes, we'll use address-based
103 // probing. But, for dwarf-based kprobes in modules, we need to
104 // switch to using symbol_name+offset (on kernels that support
105 // symbol_name+offset probing).
106 const unsigned long address;
107
108 // Note we can't really check for STAPCONF_KPROBE_SYMBOL_NAME here,
109 // since that complicates the init logic too much.
110 const char *symbol_name;
111 unsigned int offset;
112
113 const struct stap_probe * const probe;
114 const struct stap_probe * const entry_probe;
115 struct stap_kprobe * const kprobe;
116 };
117
118
119 // Forward declare the main entry functions (stap-generated)
120 static int
121 enter_kprobe_probe(struct kprobe *inst,
122 struct pt_regs *regs);
123 static int
124 enter_kretprobe_common(struct kretprobe_instance *inst,
125 struct pt_regs *regs, int entry);
126
127 // Helper entry functions for kretprobes
128 static int
129 enter_kretprobe_probe(struct kretprobe_instance *inst,
130 struct pt_regs *regs)
131 {
132 return enter_kretprobe_common(inst, regs, 0);
133 }
134
135 static int
136 enter_kretprobe_entry_probe(struct kretprobe_instance *inst,
137 struct pt_regs *regs)
138 {
139 return enter_kretprobe_common(inst, regs, 1);
140 }
141
142
143 static unsigned long
144 stapkp_relocate_addr(struct stap_kprobe_probe *skp)
145 {
146 return _stp_kmodule_relocate(skp->module, skp->section, skp->address);
147 }
148
149
150 static int
151 stapkp_prepare_kprobe(struct stap_kprobe_probe *skp)
152 {
153 struct kprobe *kp = &skp->kprobe->u.kp;
154 unsigned long addr = 0;
155
156 // PR28557 Try a pass resolving the address now with the currently
157 // known module/section addresses within our own stap-symbols tables.
158 addr = stapkp_relocate_addr(skp);
159 if (addr != 0) {
160 kp->addr = (void*) addr;
161 }
162
163 // fall back to kallsyms-based or kernel kprobes-delegated symbolic
164 // registration
165 else if (skp->symbol_name) {
166 // If we're doing symbolic name + offset probing (that gets
167 // converted to an address), it doesn't really matter if the
168 // symbol is in a module and the module isn't loaded right
169 // now. The registration will fail, but will get tried again
170 // when the module is loaded.
171 if (USE_KALLSYMS_ON_EACH_SYMBOL && kp->addr == 0)
172 return 4;
173 else if (!USE_KALLSYMS_ON_EACH_SYMBOL) {
174 // If we don't have kallsyms_on_each_symbol(), we'll use
175 // symbol_name+offset probing and let
176 // register_kprobe()/register_kretprobe() call
177 // kallsyms_lookup_name() for us. However, on kernels < 3.11,
178 // module_kallsyms_lookup_name() (called from
179 // kallsyms_lookup_name()) has a bug where it modifies its
180 // argument. So, for those kernels we'll workaround the bug by
181 // duplicating the string (so we go from read-only memory in the
182 // initialized struct data to read-write allocated memory). The
183 // memory gets freed when the probe is unregistered.
184 //
185 // This bug was fixed in kernel 3.11+ by the following commit:
186 //
187 // commit 4f6de4d51f4a3ab06a85e91e708cc89a513ef30c
188 // Author: Mathias Krause <[email protected]>
189 // Date: Tue Jul 2 15:35:11 2013 +0930
190 //
191 // module: don't modify argument of module_kallsyms_lookup_name()
192 #if LINUX_VERSION_CODE < KERNEL_VERSION(3,11,0)
193 if (kp->symbol_name == NULL)
194 kp->symbol_name = kstrdup(skp->symbol_name, STP_ALLOC_FLAGS);
195 #else
196 kp->symbol_name = (typeof(kp->symbol_name))skp->symbol_name;
197 #endif
198 kp->offset = skp->offset;
199 }
200 }
201
202 // NB: success paths need to come here, to set the pre_handler
203 kp->pre_handler = &enter_kprobe_probe;
204
205 #ifdef __ia64__ // PR6028
206 skp->kprobe->dummy.addr = kp->addr;
207 skp->kprobe->dummy.pre_handler = NULL;
208 skp->kprobe->dummy.symbol_name = kp->symbol_name;
209 #endif
210
211 #if LINUX_VERSION_CODE >= KERNEL_VERSION(2,6,30)
212 if (!skp->probe->cond_enabled) {
213 kp->flags |= KPROBE_FLAG_DISABLED;
214 dbug_otf("registering as disabled (kprobe) pidx %zu\n",
215 skp->probe->index);
216 }
217 #endif
218
219 return 0;
220 }
221
222
223 static int
224 stapkp_arch_register_kprobe(struct stap_kprobe_probe *skp)
225 {
226 int ret = 0;
227 struct kprobe *kp = &skp->kprobe->u.kp;
228
229 #ifndef __ia64__
230 ret = register_kprobe(kp);
231 if (ret == 0) {
232 if (skp->symbol_name)
233 dbug_stapkp("+kprobe %s+%u\n", kp->symbol_name, kp->offset);
234 else
235 dbug_stapkp("+kprobe %p\n", kp->addr);
236 }
237 #else // PR6028
238 ret = register_kprobe(&skp->kprobe->dummy);
239 if (ret == 0) {
240 ret = register_kprobe(kp);
241 if (ret != 0)
242 unregister_kprobe(&skp->kprobe->dummy);
243 }
244 dbug_stapkp_cond(ret == 0, "+kprobe %p\n", skp->kprobe->dummy.addr);
245 dbug_stapkp_cond(ret == 0, "+kprobe %p\n", kp->addr);
246 #endif
247
248 skp->registered_p = (ret ? 0 : 1);
249
250 return ret;
251 }
252
253
254 static int
255 stapkp_register_kprobe(struct stap_kprobe_probe *skp)
256 {
257 int ret = stapkp_prepare_kprobe(skp);
258 if (ret == 0)
259 ret = stapkp_arch_register_kprobe(skp);
260
261 dbug_stapkp("stapkp_register_kprobe module %s addr 0x%lx rc %d\n",
262 skp->module, (unsigned long)skp->address,
263 ret);
264
265 return ret;
266 }
267
268
269 static int
270 stapkp_prepare_kretprobe(struct stap_kprobe_probe *skp)
271 {
272 struct kretprobe *krp = &skp->kprobe->u.krp;
273 unsigned long addr = 0;
274
275 // PR28557 Try a pass resolving the address now with the currently
276 // known module/section addresses within our own stap-symbols tables.
277 addr = stapkp_relocate_addr(skp);
278 if (addr != 0) {
279 krp->kp.addr = (void*) addr;
280 }
281 // fall back to kallsyms-based or kernel kprobes-delegated symbolic
282 // registration
283 else if (skp->symbol_name) {
284 if (USE_KALLSYMS_ON_EACH_SYMBOL && krp->kp.addr == 0)
285 return 4;
286 else if (!USE_KALLSYMS_ON_EACH_SYMBOL) {
287 #if LINUX_VERSION_CODE < KERNEL_VERSION(3,11,0)
288 if (krp->kp.symbol_name == NULL)
289 krp->kp.symbol_name = kstrdup(skp->symbol_name, STP_ALLOC_FLAGS);
290 #else
291 krp->kp.symbol_name = (typeof(krp->kp.symbol_name))skp->symbol_name;
292 #endif
293 krp->kp.offset = skp->offset;
294 }
295 }
296
297 if (skp->maxactive_p)
298 krp->maxactive = skp->maxactive_val;
299 else
300 krp->maxactive = KRETACTIVE;
301
302 krp->handler = &enter_kretprobe_probe;
303
304 #if LINUX_VERSION_CODE >= KERNEL_VERSION(2,6,25)
305 if (skp->entry_probe) {
306 krp->entry_handler = &enter_kretprobe_entry_probe;
307 krp->data_size = skp->saved_longs * sizeof(int64_t) +
308 skp->saved_strings * MAXSTRINGLEN;
309 }
310 #endif
311
312 #ifdef __ia64__ // PR6028
313 skp->kprobe->dummy.addr = krp->kp.addr;
314 skp->kprobe->dummy.pre_handler = NULL;
315 skp->kprobe->dummy.symbol_name = krp->kp.symbol_name;
316 #endif
317
318 #if LINUX_VERSION_CODE >= KERNEL_VERSION(2,6,30)
319 if (!skp->probe->cond_enabled) {
320 krp->kp.flags |= KPROBE_FLAG_DISABLED;
321 dbug_otf("registering as disabled (kretprobe) pidx %zu\n",
322 skp->probe->index);
323 }
324 #endif
325
326 return 0;
327 }
328
329
330 static int
331 stapkp_arch_register_kretprobe(struct stap_kprobe_probe *skp)
332 {
333 int ret = 0;
334 struct kretprobe *krp = &skp->kprobe->u.krp;
335
336 #ifndef __ia64__
337 ret = register_kretprobe(krp);
338 dbug_stapkp_cond(ret == 0, "+kretprobe %p\n", krp->kp.addr);
339 #else // PR6028
340 ret = register_kprobe(&skp->kprobe->dummy);
341 if (ret == 0) {
342 ret = register_kretprobe(krp);
343 if (ret != 0)
344 unregister_kprobe(&skp->kprobe->dummy);
345 }
346 dbug_stapkp_cond(ret == 0, "+kprobe %p\n", skp->kprobe->dummy.addr);
347 dbug_stapkp_cond(ret == 0, "+kretprobe %p\n", krp->kp.addr);
348 #endif
349
350 skp->registered_p = (ret ? 0 : 1);
351
352 return ret;
353 }
354
355
356 static int
357 stapkp_register_kretprobe(struct stap_kprobe_probe *skp)
358 {
359 int ret = stapkp_prepare_kretprobe(skp);
360 if (ret == 0)
361 ret = stapkp_arch_register_kretprobe(skp);
362 return ret;
363 }
364
365
366 static int
367 stapkp_register_probe(struct stap_kprobe_probe *skp)
368 {
369 dbug_stapkp("stapkp_register_probe module %s addr 0x%lx reg %d\n",
370 skp->module, (unsigned long)skp->address,
371 skp->registered_p);
372
373 if (skp->registered_p)
374 return 0;
375
376 return skp->return_p ? stapkp_register_kretprobe(skp)
377 : stapkp_register_kprobe(skp);
378 }
379
380
381 static void
382 stapkp_add_missed(struct stap_kprobe_probe *skp)
383 {
384 if (skp->return_p) {
385
386 struct kretprobe *krp = &skp->kprobe->u.krp;
387
388 atomic_add(krp->nmissed, skipped_count());
389 #ifdef STP_TIMING
390 if (krp->nmissed)
391 _stp_warn ("Skipped due to missed kretprobe/1 on '%s': %d\n",
392 skp->probe->pp, krp->nmissed);
393 #endif
394
395 atomic_add(krp->kp.nmissed, skipped_count());
396 #ifdef STP_TIMING
397 if (krp->kp.nmissed)
398 _stp_warn ("Skipped due to missed kretprobe/2 on '%s': %lu\n",
399 skp->probe->pp, krp->kp.nmissed);
400 #endif
401
402 } else {
403
404 struct kprobe *kp = &skp->kprobe->u.kp;
405
406 atomic_add (kp->nmissed, skipped_count());
407 #ifdef STP_TIMING
408 if (kp->nmissed)
409 _stp_warn ("Skipped due to missed kprobe on '%s': %lu\n",
410 skp->probe->pp, kp->nmissed);
411 #endif
412 }
413 }
414
415
416 static void
417 stapkp_unregister_probe(struct stap_kprobe_probe *skp)
418 {
419 struct stap_kprobe *sk = skp->kprobe;
420
421 if (!skp->registered_p)
422 return;
423
424 if (skp->return_p) {
425 unregister_kretprobe (&sk->u.krp);
426 if (skp->symbol_name)
427 dbug_stapkp("-kretprobe %s:%d\n", sk->u.krp.kp.symbol_name,
428 sk->u.krp.kp.offset);
429 else
430 dbug_stapkp("-kretprobe %p\n", sk->u.krp.kp.addr);
431 } else {
432 unregister_kprobe (&sk->u.kp);
433 if (skp->symbol_name)
434 dbug_stapkp("-kprobe %s:%u\n", sk->u.kp.symbol_name,
435 sk->u.kp.offset);
436 else
437 dbug_stapkp("-kprobe %p\n", sk->u.kp.addr);
438 }
439
440 #if defined(__ia64__)
441 unregister_kprobe (&sk->dummy);
442 dbug_stapkp("-kprobe %p\n", sk->dummy.addr);
443 #endif
444
445 skp->registered_p = 0;
446
447 stapkp_add_missed(skp);
448
449 #if LINUX_VERSION_CODE < KERNEL_VERSION(3,11,0)
450 if (skp->symbol_name != NULL) {
451 if (skp->return_p) {
452 if (sk->u.krp.kp.symbol_name != NULL)
453 kfree(sk->u.krp.kp.symbol_name);
454 }
455 else {
456 if (sk->u.kp.symbol_name != NULL)
457 kfree(sk->u.kp.symbol_name);
458 }
459 }
460 #endif
461
462 // PR16861: kprobes may have left some things in the k[ret]probe struct.
463 // Let's reset it to be sure it's safe for re-use.
464 memset(sk, 0, sizeof(struct stap_kprobe));
465 }
466
467
468 #if defined(STAPCONF_UNREGISTER_KPROBES)
469
470 // The actual size is set later on in
471 // generic_kprobe_derived_probe_group::emit_module_decls().
472 static void * stap_unreg_kprobes[];
473
474 enum collect_type {
475 COLLECT_KPROBES,
476 #if defined(__ia64__)
477 COLLECT_DUMMYS,
478 #endif
479 COLLECT_KRETPROBES
480 };
481
482 static size_t
483 stapkp_collect_registered_probes(struct stap_kprobe_probe *probes,
484 size_t nprobes, enum collect_type type)
485 {
486 size_t i, j;
487
488 j = 0;
489 for (i = 0; i < nprobes; i++) {
490
491 struct stap_kprobe_probe *skp = &probes[i];
492 struct stap_kprobe *sk = skp->kprobe;
493
494 if (!skp->registered_p)
495 continue;
496
497 if (type == COLLECT_KPROBES && !skp->return_p)
498 stap_unreg_kprobes[j++] = &sk->u.kp;
499 else if (type == COLLECT_KRETPROBES && skp->return_p)
500 stap_unreg_kprobes[j++] = &sk->u.krp;
501 #if defined(__ia64__)
502 else if (type == COLLECT_DUMMYS)
503 stap_unreg_kprobes[j++] = &sk->dummy;
504 #endif
505 }
506
507 return j;
508 }
509
510 static void
511 stapkp_batch_unregister_probes(struct stap_kprobe_probe *probes,
512 size_t nprobes)
513 {
514 size_t i, n;
515
516 n = stapkp_collect_registered_probes(probes,
517 nprobes, COLLECT_KPROBES);
518 unregister_kprobes((struct kprobe **)stap_unreg_kprobes, n);
519 dbug_stapkp_cond(n > 0, "-kprobe * %zd\n", n);
520
521 n = stapkp_collect_registered_probes(probes,
522 nprobes, COLLECT_KRETPROBES);
523 unregister_kretprobes((struct kretprobe **)stap_unreg_kprobes, n);
524 dbug_stapkp_cond(n > 0, "-kretprobe * %zd\n", n);
525
526 #ifdef __ia64__
527 n = stapkp_collect_registered_probes(probes,
528 nprobes, COLLECT_DUMMYS);
529 unregister_kprobes((struct kprobe **)stap_unreg_kprobes, n);
530 dbug_stapkp_cond(n > 0, "-kprobe * %zd\n", n);
531 #endif
532
533 // Now for all of those we just unregistered, we need to update registered_p
534 // and account for (and possibly report) missed hits.
535 for (i = 0; i < nprobes; i++) {
536
537 struct stap_kprobe_probe *skp = &probes[i];
538
539 if (!skp->registered_p)
540 continue;
541
542 skp->registered_p = 0;
543
544 stapkp_add_missed(skp);
545
546 #if LINUX_VERSION_CODE < KERNEL_VERSION(3,11,0)
547 if (skp->symbol_name != NULL) {
548 if (skp->return_p) {
549 if (skp->kprobe->u.krp.kp.symbol_name != NULL)
550 kfree(skp->kprobe->u.krp.kp.symbol_name);
551 }
552 else {
553 if (skp->kprobe->u.kp.symbol_name != NULL)
554 kfree(skp->kprobe->u.kp.symbol_name);
555 }
556 }
557 #endif
558
559 // PR16861: kprobes may have left some things in the k[ret]probe struct.
560 // Let's reset it to be sure it's safe for re-use.
561 memset(skp->kprobe, 0, sizeof(struct stap_kprobe));
562 }
563 }
564
565 #endif /* STAPCONF_UNREGISTER_KPROBES */
566
567
568 static void
569 stapkp_unregister_probes(struct stap_kprobe_probe *probes,
570 size_t nprobes)
571 {
572 #if defined(STAPCONF_UNREGISTER_KPROBES)
573
574 // Unregister using batch mode
575 stapkp_batch_unregister_probes(probes, nprobes);
576
577 #else
578
579 // We'll have to unregister them one by one
580 size_t i;
581 for (i = 0; i < nprobes; i++) {
582
583 struct stap_kprobe_probe *skp = &probes[i];
584
585 if (!skp->registered_p)
586 continue;
587
588 stapkp_unregister_probe(skp);
589 }
590
591 #endif
592 }
593
594
595 #if LINUX_VERSION_CODE >= KERNEL_VERSION(2,6,30)
596
597 static int
598 stapkp_enabled(struct stap_kprobe_probe *skp)
599 {
600 if (!skp->registered_p)
601 return 0;
602
603 return skp->return_p ? !kprobe_disabled(&skp->kprobe->u.krp.kp)
604 : !kprobe_disabled(&skp->kprobe->u.kp);
605 }
606
607
608 static int
609 stapkp_should_enable_probe(struct stap_kprobe_probe *skp)
610 {
611 return skp->registered_p
612 && !stapkp_enabled(skp)
613 && skp->probe->cond_enabled;
614 }
615
616
617 static int
618 stapkp_enable_probe(struct stap_kprobe_probe *skp)
619 {
620 int ret = 0;
621
622 dbug_otf("enabling (k%sprobe) pidx %zu\n",
623 skp->return_p ? "ret" : "", skp->probe->index);
624
625 ret = skp->return_p ? enable_kretprobe(&skp->kprobe->u.krp)
626 : enable_kprobe(&skp->kprobe->u.kp);
627
628 if (ret != 0) {
629 stapkp_unregister_probe(skp);
630 dbug_otf("failed to enable (k%sprobe) pidx %zu (rc %d)\n",
631 skp->return_p ? "ret" : "", skp->probe->index, ret);
632 }
633
634 return ret;
635 }
636
637
638 static int
639 stapkp_should_disable_probe(struct stap_kprobe_probe *skp)
640 {
641 return skp->registered_p
642 && stapkp_enabled(skp)
643 && !skp->probe->cond_enabled;
644 }
645
646
647 static int
648 stapkp_disable_probe(struct stap_kprobe_probe *skp)
649 {
650 int ret = 0;
651
652 dbug_otf("disabling (k%sprobe) pidx %zu\n",
653 skp->return_p ? "ret" : "", skp->probe->index);
654
655 ret = skp->return_p ? disable_kretprobe(&skp->kprobe->u.krp)
656 : disable_kprobe(&skp->kprobe->u.kp);
657
658 if (ret != 0) {
659 stapkp_unregister_probe(skp);
660 dbug_otf("failed to disable (k%sprobe) pidx %zu (rc %d)\n",
661 skp->return_p ? "ret" : "", skp->probe->index, ret);
662 }
663
664 return ret;
665 }
666
667
668 static int
669 stapkp_refresh_probe(struct stap_kprobe_probe *skp)
670 {
671 if (stapkp_should_enable_probe(skp))
672 return stapkp_enable_probe(skp);
673 if (stapkp_should_disable_probe(skp))
674 return stapkp_disable_probe(skp);
675 return 0;
676 }
677
678 #endif /* LINUX_VERSION_CODE >= 2.6.30 */
679
680
681 struct stapkp_symbol_data {
682 struct stap_kprobe_probe *probes;
683 size_t nprobes; /* number of probes in "probes" */
684 size_t probe_max; /* number of probes to process */
685 const char *modname;
686 };
687
688
689 static int
690 __stapkp_symbol_callback(void *data, const char *name,
691 struct module *mod, unsigned long addr)
692 {
693 struct stapkp_symbol_data *sd = data;
694 size_t i;
695
696 if ((mod && sd->modname && strcmp(mod->name, sd->modname) != 0)
697 || (!mod && sd->modname))
698 return 0;
699
700 for (i = 0; i < sd->nprobes; i++) {
701 struct stap_kprobe_probe *skp = &sd->probes[i];
702 int update_addr = 0;
703
704 if (! skp->symbol_name)
705 continue;
706
707 // If (1) We're probing a module symbol and we're in that module
708 // and the names match; or (2) we're probing a symbol in the
709 // kernel and the names match, then update the k[ret]probe
710 // address.
711 if (mod && skp->module && strcmp(mod->name, skp->module) == 0) {
712 char *colon = strchr(skp->symbol_name, ':');
713
714 if (colon != NULL && strcmp(name, colon+1) == 0)
715 update_addr = 1;
716 }
717 else if (!mod && (skp->module == NULL || skp->module[0] == '\0')
718 && strcmp(name, skp->symbol_name) == 0)
719 update_addr = 1;
720 if (update_addr) {
721
722 if (skp->return_p)
723 skp->kprobe->u.krp.kp.addr = (void *)(addr + skp->offset);
724 else
725 skp->kprobe->u.kp.addr = (void *)(addr + skp->offset);
726 // Note that we could have more than 1 probe at the same
727 // symbol (with the same or differing offsets), so we can't
728 // return here.
729 //
730 // But we can quit if we've processed all the needed probes.
731 --sd->probe_max;
732 if (sd->probe_max == 0)
733 return -1;
734 }
735 }
736 return 0;
737 }
738
739 static int
740 #if defined(STAPCONF_KALLSYMS_6_4)
741 stapkp_symbol_callback(void *data, const char *name,
742 unsigned long addr)
743 {
744 struct module *mod = NULL;
745 #else
746 stapkp_symbol_callback(void *data, const char *name,
747 struct module *mod, unsigned long addr)
748 {
749 #endif
750 return __stapkp_symbol_callback(data, name, mod, addr);
751 }
752
753 static int
754 stapkp_init(struct stap_kprobe_probe *probes,
755 size_t nprobes)
756 {
757 size_t i;
758
759 if (USE_KALLSYMS_ON_EACH_SYMBOL) {
760 // If we have any symbol_name+offset probes, we need to try to
761 // convert those into address-based probes.
762 size_t probe_max = 0;
763 for (i = 0; i < nprobes; i++) {
764 struct stap_kprobe_probe *skp = &probes[i];
765
766 if (! skp->symbol_name)
767 continue;
768 ++probe_max;
769 }
770 if (probe_max > 0) {
771 // Here we're going to try to convert any symbol_name+offset
772 // probes into address probes.
773 struct stapkp_symbol_data sd;
774 dbug_stapkp("looking up %lu probes\n", probe_max);
775 sd.probes = probes;
776 sd.nprobes = nprobes;
777 sd.probe_max = probe_max;
778 sd.modname = NULL;
779 #ifdef STAPCONF_MODULE_MUTEX
780 mutex_lock(&module_mutex);
781 #endif
782 kallsyms_on_each_symbol(stapkp_symbol_callback, &sd);
783 #if defined(STAPCONF_KALLSYMS_6_3) || defined(STAPCONF_KALLSYMS_6_4)
784 module_kallsyms_on_each_symbol(sd.modname, stapkp_symbol_callback, &sd);
785 #elif LINUX_VERSION_CODE >= KERNEL_VERSION(5,12,0)
786 module_kallsyms_on_each_symbol(stapkp_symbol_callback, &sd);
787 #endif
788 #ifdef STAPCONF_MODULE_MUTEX
789 mutex_unlock(&module_mutex);
790 #endif
791 dbug_stapkp("found %lu probes\n", sd.probe_max);
792 }
793 }
794
795 for (i = 0; i < nprobes; i++) {
796 struct stap_kprobe_probe *skp = &probes[i];
797 int rc = 0;
798
799 rc = stapkp_register_probe(skp);
800 if (rc == 1) // failed to relocate addr?
801 continue; // don't fuss about it, module probably not loaded
802
803 // PR32695 gcc might statically determine the strcmp will nevery match
804 #pragma GCC diagnostic ignored "-Wstring-compare"
805 // NB: We keep going even if a probe failed to register (PR6749). We only
806 // warn about it if it wasn't optional and isn't in a module.
807 if (rc && !skp->optional_p
808 && ((skp->module == NULL) || skp->module[0] == '\0'
809 || strcmp(skp->module, "kernel") == 0)) {
810 if (skp->symbol_name)
811 _stp_warn("probe %s (%s+%u) registration error [man warning::pass5] (rc %d)",
812 skp->probe->pp, skp->symbol_name, skp->offset, rc);
813 else
814 _stp_warn("probe %s (address 0x%lx) registration error [man warning::pass5] (rc %d)",
815 skp->probe->pp, stapkp_relocate_addr(skp), rc);
816 }
817 }
818
819 return 0;
820 }
821
822
823 /* stapkp_refresh is called for two reasons: either a kprobe needs to be
824 * enabled/disabled (modname is NULL), or a module has been loaded/unloaded and
825 * kprobes need to be registered/unregistered (modname is !NULL). */
826 static void
827 stapkp_refresh(const char *modname,
828 struct stap_kprobe_probe *probes,
829 size_t nprobes)
830 {
831 size_t i;
832
833 dbug_stapkp("refresh %lu probes with module %s\n", nprobes, modname ?: "?");
834
835 if (USE_KALLSYMS_ON_EACH_SYMBOL) {
836 if (modname) {
837 size_t probe_max = 0;
838 for (i = 0; i < nprobes; i++) {
839 struct stap_kprobe_probe *skp = &probes[i];
840
841 // If this probe is in the same module that is being
842 // loaded/unloaded and the probe is symbol_name+offset based
843 // and it isn't registered (so the module must be loaded),
844 // try to convert all probes in the same module to
845 // address-based probes.
846 if (skp->module && strcmp(modname, skp->module) == 0
847 && skp->symbol_name && skp->registered_p == 0)
848 ++probe_max;
849 }
850 if (probe_max > 0) {
851 struct stapkp_symbol_data sd;
852 sd.probes = probes;
853 sd.nprobes = nprobes;
854 sd.probe_max = probe_max;
855 sd.modname = modname;
856 #ifdef STAPCONF_MODULE_MUTEX
857 mutex_lock(&module_mutex);
858 #endif
859 kallsyms_on_each_symbol(stapkp_symbol_callback, &sd);
860 #if defined(STAPCONF_KALLSYMS_6_3) || defined(STAPCONF_KALLSYMS_6_4)
861 module_kallsyms_on_each_symbol(sd.modname, stapkp_symbol_callback, &sd);
862 #elif LINUX_VERSION_CODE >= KERNEL_VERSION(5,12,0)
863 module_kallsyms_on_each_symbol(stapkp_symbol_callback, &sd);
864 #endif
865 #ifdef STAPCONF_MODULE_MUTEX
866 mutex_unlock(&module_mutex);
867 #endif
868 }
869 }
870 }
871
872 for (i = 0; i < nprobes; i++) {
873
874 struct stap_kprobe_probe *skp = &probes[i];
875
876 // was this probe's target module loaded/unloaded
877 if (modname && skp->module
878 && strcmp(modname, skp->module) == 0) {
879 int rc;
880 unsigned long addr = (! skp->symbol_name
881 ? stapkp_relocate_addr(skp) : 0);
882
883 // module being loaded?
884 if (skp->registered_p == 0 && (addr != 0 || skp->symbol_name))
885 stapkp_register_probe(skp);
886 // module/section being unloaded?
887 else if (skp->registered_p == 1 && addr == 0)
888 stapkp_unregister_probe(skp);
889
890 }
891 #if LINUX_VERSION_CODE >= KERNEL_VERSION(2,6,30)
892 else if (stapkp_should_enable_probe(skp)
893 || stapkp_should_disable_probe(skp)) {
894 stapkp_refresh_probe(skp);
895 }
896 #endif
897 }
898 }
899
900
901 static void
902 stapkp_exit(struct stap_kprobe_probe *probes,
903 size_t nprobes)
904 {
905 stapkp_unregister_probes(probes, nprobes);
906 }
907
908
909 #endif /* _KPROBES_C_ */