| 1 |
/* -*- linux-c -*- |
| 2 |
* Common functions for using kprobes |
| 3 |
* Copyright (C) 2014-2015 Red Hat Inc. |
| 4 |
* |
| 5 |
* This file is part of systemtap, and is free software. You can |
| 6 |
* redistribute it and/or modify it under the terms of the GNU General |
| 7 |
* Public License (GPL); either version 2, or (at your option) any |
| 8 |
* later version. |
| 9 |
*/ |
| 10 |
|
| 11 |
#ifndef _KPROBES_C_ |
| 12 |
#define _KPROBES_C_ |
| 13 |
|
| 14 |
// Warn of misconfigured kernels |
| 15 |
#if !defined(CONFIG_KPROBES) |
| 16 |
#error "Need CONFIG_KPROBES!" |
| 17 |
#endif |
| 18 |
|
| 19 |
// PR26074: if kallsyms_on_each_symbol() is not exported we try to get it via relocations. |
| 20 |
// Need to define the runtime.h variable here to check if it's defined from kprobes.c. |
| 21 |
// (This code would go in a header if it were used anywhere else.) |
| 22 |
// |
| 23 |
// PR11514: kallsyms_on_each_symbol() should not be used on PPC64. |
| 24 |
#ifndef CONFIG_PPC64 |
| 25 |
#define STAPCONF_KALLSYMS_ON_EACH_SYMBOL |
| 26 |
#if !defined(STAPCONF_KALLSYMS_ON_EACH_SYMBOL_EXPORTED) |
| 27 |
extern void *_stp_kallsyms_on_each_symbol; |
| 28 |
#endif |
| 29 |
#endif |
| 30 |
|
| 31 |
// No export check and gates. This one seems simply like a non-export. |
| 32 |
extern void *_stp_module_kallsyms_on_each_symbol; |
| 33 |
|
| 34 |
#if defined(STAPCONF_KALLSYMS_ON_EACH_SYMBOL) && defined(STAPCONF_KALLSYMS_ON_EACH_SYMBOL_EXPORTED) |
| 35 |
#define USE_KALLSYMS_ON_EACH_SYMBOL (1) |
| 36 |
#elif defined(STAPCONF_KALLSYMS_ON_EACH_SYMBOL) |
| 37 |
// XXX May not be available; if available, is passed through relocation: |
| 38 |
#define USE_KALLSYMS_ON_EACH_SYMBOL (_stp_kallsyms_on_each_symbol != NULL) |
| 39 |
#else |
| 40 |
#define USE_KALLSYMS_ON_EACH_SYMBOL (0) |
| 41 |
#endif |
| 42 |
|
| 43 |
#include <linux/kprobes.h> |
| 44 |
#include <linux/module.h> |
| 45 |
|
| 46 |
#ifdef DEBUG_KPROBES |
| 47 |
#define dbug_stapkp(args...) do { \ |
| 48 |
_stp_dbug(__FUNCTION__, __LINE__, args); \ |
| 49 |
} while (0) |
| 50 |
#define dbug_stapkp_cond(cond, args...) do { \ |
| 51 |
if (cond) \ |
| 52 |
dbug_stapkp(args); \ |
| 53 |
} while (0) |
| 54 |
#else |
| 55 |
#define dbug_stapkp(args...) do { } while (0) |
| 56 |
#define dbug_stapkp_cond(cond, args...) do { } while (0) |
| 57 |
#endif |
| 58 |
|
| 59 |
#ifndef KRETACTIVE |
| 60 |
#define KRETACTIVE (max(15, 6 * (int)num_possible_cpus())) |
| 61 |
#endif |
| 62 |
|
| 63 |
// This shouldn't happen, but check as a precaution. If we're on kver >= 2.6.30, |
| 64 |
// then we must also have STP_ON_THE_FLY_TIMER_ENABLE (which is turned on for |
| 65 |
// kver >= 2.6.17, see translate_pass()). This indicates that the background |
| 66 |
// timer is available and thus that kprobes can be armed/disarmed on-the-fly. |
| 67 |
#if LINUX_VERSION_CODE >= KERNEL_VERSION(2,6,30) \ |
| 68 |
&& !defined(STP_ON_THE_FLY_TIMER_ENABLE) |
| 69 |
#error "STP_ON_THE_FLY_TIMER_ENABLE undefined" |
| 70 |
#endif |
| 71 |
|
| 72 |
// NB: this struct is set up by the stapkp_prepare_* functions prior to |
| 73 |
// registering and zero'ed out again after each unregister |
| 74 |
struct stap_kprobe { |
| 75 |
union { struct kprobe kp; struct kretprobe krp; } u; |
| 76 |
#ifdef __ia64__ |
| 77 |
// PR6028: We register a second dummy probe at the same address so that the |
| 78 |
// kernel uses aggr_kprobe. This is needed ensure that the bspcache is always |
| 79 |
// valid. |
| 80 |
struct kprobe dummy; |
| 81 |
#endif |
| 82 |
}; |
| 83 |
|
| 84 |
|
| 85 |
struct stap_kprobe_probe { |
| 86 |
const unsigned return_p:1; |
| 87 |
const unsigned maxactive_p:1; |
| 88 |
const unsigned optional_p:1; |
| 89 |
unsigned registered_p:1; |
| 90 |
const unsigned short maxactive_val; |
| 91 |
|
| 92 |
// data saved in the kretprobe_instance packet |
| 93 |
const unsigned short saved_longs; |
| 94 |
const unsigned short saved_strings; |
| 95 |
|
| 96 |
// These macros declare the module and section strings as either const char[] |
| 97 |
// or const char * const. Their actual types are determined at translate-time |
| 98 |
// in dwarf_derived_probe_group::emit_module_decls(). |
| 99 |
STAP_KPROBE_PROBE_STR_module; |
| 100 |
STAP_KPROBE_PROBE_STR_section; |
| 101 |
|
| 102 |
// For the majority of dwarf-based kprobes, we'll use address-based |
| 103 |
// probing. But, for dwarf-based kprobes in modules, we need to |
| 104 |
// switch to using symbol_name+offset (on kernels that support |
| 105 |
// symbol_name+offset probing). |
| 106 |
const unsigned long address; |
| 107 |
|
| 108 |
// Note we can't really check for STAPCONF_KPROBE_SYMBOL_NAME here, |
| 109 |
// since that complicates the init logic too much. |
| 110 |
const char *symbol_name; |
| 111 |
unsigned int offset; |
| 112 |
|
| 113 |
const struct stap_probe * const probe; |
| 114 |
const struct stap_probe * const entry_probe; |
| 115 |
struct stap_kprobe * const kprobe; |
| 116 |
}; |
| 117 |
|
| 118 |
|
| 119 |
// Forward declare the main entry functions (stap-generated) |
| 120 |
static int |
| 121 |
enter_kprobe_probe(struct kprobe *inst, |
| 122 |
struct pt_regs *regs); |
| 123 |
static int |
| 124 |
enter_kretprobe_common(struct kretprobe_instance *inst, |
| 125 |
struct pt_regs *regs, int entry); |
| 126 |
|
| 127 |
// Helper entry functions for kretprobes |
| 128 |
static int |
| 129 |
enter_kretprobe_probe(struct kretprobe_instance *inst, |
| 130 |
struct pt_regs *regs) |
| 131 |
{ |
| 132 |
return enter_kretprobe_common(inst, regs, 0); |
| 133 |
} |
| 134 |
|
| 135 |
static int |
| 136 |
enter_kretprobe_entry_probe(struct kretprobe_instance *inst, |
| 137 |
struct pt_regs *regs) |
| 138 |
{ |
| 139 |
return enter_kretprobe_common(inst, regs, 1); |
| 140 |
} |
| 141 |
|
| 142 |
|
| 143 |
static unsigned long |
| 144 |
stapkp_relocate_addr(struct stap_kprobe_probe *skp) |
| 145 |
{ |
| 146 |
return _stp_kmodule_relocate(skp->module, skp->section, skp->address); |
| 147 |
} |
| 148 |
|
| 149 |
|
| 150 |
static int |
| 151 |
stapkp_prepare_kprobe(struct stap_kprobe_probe *skp) |
| 152 |
{ |
| 153 |
struct kprobe *kp = &skp->kprobe->u.kp; |
| 154 |
unsigned long addr = 0; |
| 155 |
|
| 156 |
// PR28557 Try a pass resolving the address now with the currently |
| 157 |
// known module/section addresses within our own stap-symbols tables. |
| 158 |
addr = stapkp_relocate_addr(skp); |
| 159 |
if (addr != 0) { |
| 160 |
kp->addr = (void*) addr; |
| 161 |
} |
| 162 |
|
| 163 |
// fall back to kallsyms-based or kernel kprobes-delegated symbolic |
| 164 |
// registration |
| 165 |
else if (skp->symbol_name) { |
| 166 |
// If we're doing symbolic name + offset probing (that gets |
| 167 |
// converted to an address), it doesn't really matter if the |
| 168 |
// symbol is in a module and the module isn't loaded right |
| 169 |
// now. The registration will fail, but will get tried again |
| 170 |
// when the module is loaded. |
| 171 |
if (USE_KALLSYMS_ON_EACH_SYMBOL && kp->addr == 0) |
| 172 |
return 4; |
| 173 |
else if (!USE_KALLSYMS_ON_EACH_SYMBOL) { |
| 174 |
// If we don't have kallsyms_on_each_symbol(), we'll use |
| 175 |
// symbol_name+offset probing and let |
| 176 |
// register_kprobe()/register_kretprobe() call |
| 177 |
// kallsyms_lookup_name() for us. However, on kernels < 3.11, |
| 178 |
// module_kallsyms_lookup_name() (called from |
| 179 |
// kallsyms_lookup_name()) has a bug where it modifies its |
| 180 |
// argument. So, for those kernels we'll workaround the bug by |
| 181 |
// duplicating the string (so we go from read-only memory in the |
| 182 |
// initialized struct data to read-write allocated memory). The |
| 183 |
// memory gets freed when the probe is unregistered. |
| 184 |
// |
| 185 |
// This bug was fixed in kernel 3.11+ by the following commit: |
| 186 |
// |
| 187 |
// commit 4f6de4d51f4a3ab06a85e91e708cc89a513ef30c |
| 188 |
// Author: Mathias Krause <[email protected]> |
| 189 |
// Date: Tue Jul 2 15:35:11 2013 +0930 |
| 190 |
// |
| 191 |
// module: don't modify argument of module_kallsyms_lookup_name() |
| 192 |
#if LINUX_VERSION_CODE < KERNEL_VERSION(3,11,0) |
| 193 |
if (kp->symbol_name == NULL) |
| 194 |
kp->symbol_name = kstrdup(skp->symbol_name, STP_ALLOC_FLAGS); |
| 195 |
#else |
| 196 |
kp->symbol_name = (typeof(kp->symbol_name))skp->symbol_name; |
| 197 |
#endif |
| 198 |
kp->offset = skp->offset; |
| 199 |
} |
| 200 |
} |
| 201 |
|
| 202 |
// NB: success paths need to come here, to set the pre_handler |
| 203 |
kp->pre_handler = &enter_kprobe_probe; |
| 204 |
|
| 205 |
#ifdef __ia64__ // PR6028 |
| 206 |
skp->kprobe->dummy.addr = kp->addr; |
| 207 |
skp->kprobe->dummy.pre_handler = NULL; |
| 208 |
skp->kprobe->dummy.symbol_name = kp->symbol_name; |
| 209 |
#endif |
| 210 |
|
| 211 |
#if LINUX_VERSION_CODE >= KERNEL_VERSION(2,6,30) |
| 212 |
if (!skp->probe->cond_enabled) { |
| 213 |
kp->flags |= KPROBE_FLAG_DISABLED; |
| 214 |
dbug_otf("registering as disabled (kprobe) pidx %zu\n", |
| 215 |
skp->probe->index); |
| 216 |
} |
| 217 |
#endif |
| 218 |
|
| 219 |
return 0; |
| 220 |
} |
| 221 |
|
| 222 |
|
| 223 |
static int |
| 224 |
stapkp_arch_register_kprobe(struct stap_kprobe_probe *skp) |
| 225 |
{ |
| 226 |
int ret = 0; |
| 227 |
struct kprobe *kp = &skp->kprobe->u.kp; |
| 228 |
|
| 229 |
#ifndef __ia64__ |
| 230 |
ret = register_kprobe(kp); |
| 231 |
if (ret == 0) { |
| 232 |
if (skp->symbol_name) |
| 233 |
dbug_stapkp("+kprobe %s+%u\n", kp->symbol_name, kp->offset); |
| 234 |
else |
| 235 |
dbug_stapkp("+kprobe %p\n", kp->addr); |
| 236 |
} |
| 237 |
#else // PR6028 |
| 238 |
ret = register_kprobe(&skp->kprobe->dummy); |
| 239 |
if (ret == 0) { |
| 240 |
ret = register_kprobe(kp); |
| 241 |
if (ret != 0) |
| 242 |
unregister_kprobe(&skp->kprobe->dummy); |
| 243 |
} |
| 244 |
dbug_stapkp_cond(ret == 0, "+kprobe %p\n", skp->kprobe->dummy.addr); |
| 245 |
dbug_stapkp_cond(ret == 0, "+kprobe %p\n", kp->addr); |
| 246 |
#endif |
| 247 |
|
| 248 |
skp->registered_p = (ret ? 0 : 1); |
| 249 |
|
| 250 |
return ret; |
| 251 |
} |
| 252 |
|
| 253 |
|
| 254 |
static int |
| 255 |
stapkp_register_kprobe(struct stap_kprobe_probe *skp) |
| 256 |
{ |
| 257 |
int ret = stapkp_prepare_kprobe(skp); |
| 258 |
if (ret == 0) |
| 259 |
ret = stapkp_arch_register_kprobe(skp); |
| 260 |
|
| 261 |
dbug_stapkp("stapkp_register_kprobe module %s addr 0x%lx rc %d\n", |
| 262 |
skp->module, (unsigned long)skp->address, |
| 263 |
ret); |
| 264 |
|
| 265 |
return ret; |
| 266 |
} |
| 267 |
|
| 268 |
|
| 269 |
static int |
| 270 |
stapkp_prepare_kretprobe(struct stap_kprobe_probe *skp) |
| 271 |
{ |
| 272 |
struct kretprobe *krp = &skp->kprobe->u.krp; |
| 273 |
unsigned long addr = 0; |
| 274 |
|
| 275 |
// PR28557 Try a pass resolving the address now with the currently |
| 276 |
// known module/section addresses within our own stap-symbols tables. |
| 277 |
addr = stapkp_relocate_addr(skp); |
| 278 |
if (addr != 0) { |
| 279 |
krp->kp.addr = (void*) addr; |
| 280 |
} |
| 281 |
// fall back to kallsyms-based or kernel kprobes-delegated symbolic |
| 282 |
// registration |
| 283 |
else if (skp->symbol_name) { |
| 284 |
if (USE_KALLSYMS_ON_EACH_SYMBOL && krp->kp.addr == 0) |
| 285 |
return 4; |
| 286 |
else if (!USE_KALLSYMS_ON_EACH_SYMBOL) { |
| 287 |
#if LINUX_VERSION_CODE < KERNEL_VERSION(3,11,0) |
| 288 |
if (krp->kp.symbol_name == NULL) |
| 289 |
krp->kp.symbol_name = kstrdup(skp->symbol_name, STP_ALLOC_FLAGS); |
| 290 |
#else |
| 291 |
krp->kp.symbol_name = (typeof(krp->kp.symbol_name))skp->symbol_name; |
| 292 |
#endif |
| 293 |
krp->kp.offset = skp->offset; |
| 294 |
} |
| 295 |
} |
| 296 |
|
| 297 |
if (skp->maxactive_p) |
| 298 |
krp->maxactive = skp->maxactive_val; |
| 299 |
else |
| 300 |
krp->maxactive = KRETACTIVE; |
| 301 |
|
| 302 |
krp->handler = &enter_kretprobe_probe; |
| 303 |
|
| 304 |
#if LINUX_VERSION_CODE >= KERNEL_VERSION(2,6,25) |
| 305 |
if (skp->entry_probe) { |
| 306 |
krp->entry_handler = &enter_kretprobe_entry_probe; |
| 307 |
krp->data_size = skp->saved_longs * sizeof(int64_t) + |
| 308 |
skp->saved_strings * MAXSTRINGLEN; |
| 309 |
} |
| 310 |
#endif |
| 311 |
|
| 312 |
#ifdef __ia64__ // PR6028 |
| 313 |
skp->kprobe->dummy.addr = krp->kp.addr; |
| 314 |
skp->kprobe->dummy.pre_handler = NULL; |
| 315 |
skp->kprobe->dummy.symbol_name = krp->kp.symbol_name; |
| 316 |
#endif |
| 317 |
|
| 318 |
#if LINUX_VERSION_CODE >= KERNEL_VERSION(2,6,30) |
| 319 |
if (!skp->probe->cond_enabled) { |
| 320 |
krp->kp.flags |= KPROBE_FLAG_DISABLED; |
| 321 |
dbug_otf("registering as disabled (kretprobe) pidx %zu\n", |
| 322 |
skp->probe->index); |
| 323 |
} |
| 324 |
#endif |
| 325 |
|
| 326 |
return 0; |
| 327 |
} |
| 328 |
|
| 329 |
|
| 330 |
static int |
| 331 |
stapkp_arch_register_kretprobe(struct stap_kprobe_probe *skp) |
| 332 |
{ |
| 333 |
int ret = 0; |
| 334 |
struct kretprobe *krp = &skp->kprobe->u.krp; |
| 335 |
|
| 336 |
#ifndef __ia64__ |
| 337 |
ret = register_kretprobe(krp); |
| 338 |
dbug_stapkp_cond(ret == 0, "+kretprobe %p\n", krp->kp.addr); |
| 339 |
#else // PR6028 |
| 340 |
ret = register_kprobe(&skp->kprobe->dummy); |
| 341 |
if (ret == 0) { |
| 342 |
ret = register_kretprobe(krp); |
| 343 |
if (ret != 0) |
| 344 |
unregister_kprobe(&skp->kprobe->dummy); |
| 345 |
} |
| 346 |
dbug_stapkp_cond(ret == 0, "+kprobe %p\n", skp->kprobe->dummy.addr); |
| 347 |
dbug_stapkp_cond(ret == 0, "+kretprobe %p\n", krp->kp.addr); |
| 348 |
#endif |
| 349 |
|
| 350 |
skp->registered_p = (ret ? 0 : 1); |
| 351 |
|
| 352 |
return ret; |
| 353 |
} |
| 354 |
|
| 355 |
|
| 356 |
static int |
| 357 |
stapkp_register_kretprobe(struct stap_kprobe_probe *skp) |
| 358 |
{ |
| 359 |
int ret = stapkp_prepare_kretprobe(skp); |
| 360 |
if (ret == 0) |
| 361 |
ret = stapkp_arch_register_kretprobe(skp); |
| 362 |
return ret; |
| 363 |
} |
| 364 |
|
| 365 |
|
| 366 |
static int |
| 367 |
stapkp_register_probe(struct stap_kprobe_probe *skp) |
| 368 |
{ |
| 369 |
dbug_stapkp("stapkp_register_probe module %s addr 0x%lx reg %d\n", |
| 370 |
skp->module, (unsigned long)skp->address, |
| 371 |
skp->registered_p); |
| 372 |
|
| 373 |
if (skp->registered_p) |
| 374 |
return 0; |
| 375 |
|
| 376 |
return skp->return_p ? stapkp_register_kretprobe(skp) |
| 377 |
: stapkp_register_kprobe(skp); |
| 378 |
} |
| 379 |
|
| 380 |
|
| 381 |
static void |
| 382 |
stapkp_add_missed(struct stap_kprobe_probe *skp) |
| 383 |
{ |
| 384 |
if (skp->return_p) { |
| 385 |
|
| 386 |
struct kretprobe *krp = &skp->kprobe->u.krp; |
| 387 |
|
| 388 |
atomic_add(krp->nmissed, skipped_count()); |
| 389 |
#ifdef STP_TIMING |
| 390 |
if (krp->nmissed) |
| 391 |
_stp_warn ("Skipped due to missed kretprobe/1 on '%s': %d\n", |
| 392 |
skp->probe->pp, krp->nmissed); |
| 393 |
#endif |
| 394 |
|
| 395 |
atomic_add(krp->kp.nmissed, skipped_count()); |
| 396 |
#ifdef STP_TIMING |
| 397 |
if (krp->kp.nmissed) |
| 398 |
_stp_warn ("Skipped due to missed kretprobe/2 on '%s': %lu\n", |
| 399 |
skp->probe->pp, krp->kp.nmissed); |
| 400 |
#endif |
| 401 |
|
| 402 |
} else { |
| 403 |
|
| 404 |
struct kprobe *kp = &skp->kprobe->u.kp; |
| 405 |
|
| 406 |
atomic_add (kp->nmissed, skipped_count()); |
| 407 |
#ifdef STP_TIMING |
| 408 |
if (kp->nmissed) |
| 409 |
_stp_warn ("Skipped due to missed kprobe on '%s': %lu\n", |
| 410 |
skp->probe->pp, kp->nmissed); |
| 411 |
#endif |
| 412 |
} |
| 413 |
} |
| 414 |
|
| 415 |
|
| 416 |
static void |
| 417 |
stapkp_unregister_probe(struct stap_kprobe_probe *skp) |
| 418 |
{ |
| 419 |
struct stap_kprobe *sk = skp->kprobe; |
| 420 |
|
| 421 |
if (!skp->registered_p) |
| 422 |
return; |
| 423 |
|
| 424 |
if (skp->return_p) { |
| 425 |
unregister_kretprobe (&sk->u.krp); |
| 426 |
if (skp->symbol_name) |
| 427 |
dbug_stapkp("-kretprobe %s:%d\n", sk->u.krp.kp.symbol_name, |
| 428 |
sk->u.krp.kp.offset); |
| 429 |
else |
| 430 |
dbug_stapkp("-kretprobe %p\n", sk->u.krp.kp.addr); |
| 431 |
} else { |
| 432 |
unregister_kprobe (&sk->u.kp); |
| 433 |
if (skp->symbol_name) |
| 434 |
dbug_stapkp("-kprobe %s:%u\n", sk->u.kp.symbol_name, |
| 435 |
sk->u.kp.offset); |
| 436 |
else |
| 437 |
dbug_stapkp("-kprobe %p\n", sk->u.kp.addr); |
| 438 |
} |
| 439 |
|
| 440 |
#if defined(__ia64__) |
| 441 |
unregister_kprobe (&sk->dummy); |
| 442 |
dbug_stapkp("-kprobe %p\n", sk->dummy.addr); |
| 443 |
#endif |
| 444 |
|
| 445 |
skp->registered_p = 0; |
| 446 |
|
| 447 |
stapkp_add_missed(skp); |
| 448 |
|
| 449 |
#if LINUX_VERSION_CODE < KERNEL_VERSION(3,11,0) |
| 450 |
if (skp->symbol_name != NULL) { |
| 451 |
if (skp->return_p) { |
| 452 |
if (sk->u.krp.kp.symbol_name != NULL) |
| 453 |
kfree(sk->u.krp.kp.symbol_name); |
| 454 |
} |
| 455 |
else { |
| 456 |
if (sk->u.kp.symbol_name != NULL) |
| 457 |
kfree(sk->u.kp.symbol_name); |
| 458 |
} |
| 459 |
} |
| 460 |
#endif |
| 461 |
|
| 462 |
// PR16861: kprobes may have left some things in the k[ret]probe struct. |
| 463 |
// Let's reset it to be sure it's safe for re-use. |
| 464 |
memset(sk, 0, sizeof(struct stap_kprobe)); |
| 465 |
} |
| 466 |
|
| 467 |
|
| 468 |
#if defined(STAPCONF_UNREGISTER_KPROBES) |
| 469 |
|
| 470 |
// The actual size is set later on in |
| 471 |
// generic_kprobe_derived_probe_group::emit_module_decls(). |
| 472 |
static void * stap_unreg_kprobes[]; |
| 473 |
|
| 474 |
enum collect_type { |
| 475 |
COLLECT_KPROBES, |
| 476 |
#if defined(__ia64__) |
| 477 |
COLLECT_DUMMYS, |
| 478 |
#endif |
| 479 |
COLLECT_KRETPROBES |
| 480 |
}; |
| 481 |
|
| 482 |
static size_t |
| 483 |
stapkp_collect_registered_probes(struct stap_kprobe_probe *probes, |
| 484 |
size_t nprobes, enum collect_type type) |
| 485 |
{ |
| 486 |
size_t i, j; |
| 487 |
|
| 488 |
j = 0; |
| 489 |
for (i = 0; i < nprobes; i++) { |
| 490 |
|
| 491 |
struct stap_kprobe_probe *skp = &probes[i]; |
| 492 |
struct stap_kprobe *sk = skp->kprobe; |
| 493 |
|
| 494 |
if (!skp->registered_p) |
| 495 |
continue; |
| 496 |
|
| 497 |
if (type == COLLECT_KPROBES && !skp->return_p) |
| 498 |
stap_unreg_kprobes[j++] = &sk->u.kp; |
| 499 |
else if (type == COLLECT_KRETPROBES && skp->return_p) |
| 500 |
stap_unreg_kprobes[j++] = &sk->u.krp; |
| 501 |
#if defined(__ia64__) |
| 502 |
else if (type == COLLECT_DUMMYS) |
| 503 |
stap_unreg_kprobes[j++] = &sk->dummy; |
| 504 |
#endif |
| 505 |
} |
| 506 |
|
| 507 |
return j; |
| 508 |
} |
| 509 |
|
| 510 |
static void |
| 511 |
stapkp_batch_unregister_probes(struct stap_kprobe_probe *probes, |
| 512 |
size_t nprobes) |
| 513 |
{ |
| 514 |
size_t i, n; |
| 515 |
|
| 516 |
n = stapkp_collect_registered_probes(probes, |
| 517 |
nprobes, COLLECT_KPROBES); |
| 518 |
unregister_kprobes((struct kprobe **)stap_unreg_kprobes, n); |
| 519 |
dbug_stapkp_cond(n > 0, "-kprobe * %zd\n", n); |
| 520 |
|
| 521 |
n = stapkp_collect_registered_probes(probes, |
| 522 |
nprobes, COLLECT_KRETPROBES); |
| 523 |
unregister_kretprobes((struct kretprobe **)stap_unreg_kprobes, n); |
| 524 |
dbug_stapkp_cond(n > 0, "-kretprobe * %zd\n", n); |
| 525 |
|
| 526 |
#ifdef __ia64__ |
| 527 |
n = stapkp_collect_registered_probes(probes, |
| 528 |
nprobes, COLLECT_DUMMYS); |
| 529 |
unregister_kprobes((struct kprobe **)stap_unreg_kprobes, n); |
| 530 |
dbug_stapkp_cond(n > 0, "-kprobe * %zd\n", n); |
| 531 |
#endif |
| 532 |
|
| 533 |
// Now for all of those we just unregistered, we need to update registered_p |
| 534 |
// and account for (and possibly report) missed hits. |
| 535 |
for (i = 0; i < nprobes; i++) { |
| 536 |
|
| 537 |
struct stap_kprobe_probe *skp = &probes[i]; |
| 538 |
|
| 539 |
if (!skp->registered_p) |
| 540 |
continue; |
| 541 |
|
| 542 |
skp->registered_p = 0; |
| 543 |
|
| 544 |
stapkp_add_missed(skp); |
| 545 |
|
| 546 |
#if LINUX_VERSION_CODE < KERNEL_VERSION(3,11,0) |
| 547 |
if (skp->symbol_name != NULL) { |
| 548 |
if (skp->return_p) { |
| 549 |
if (skp->kprobe->u.krp.kp.symbol_name != NULL) |
| 550 |
kfree(skp->kprobe->u.krp.kp.symbol_name); |
| 551 |
} |
| 552 |
else { |
| 553 |
if (skp->kprobe->u.kp.symbol_name != NULL) |
| 554 |
kfree(skp->kprobe->u.kp.symbol_name); |
| 555 |
} |
| 556 |
} |
| 557 |
#endif |
| 558 |
|
| 559 |
// PR16861: kprobes may have left some things in the k[ret]probe struct. |
| 560 |
// Let's reset it to be sure it's safe for re-use. |
| 561 |
memset(skp->kprobe, 0, sizeof(struct stap_kprobe)); |
| 562 |
} |
| 563 |
} |
| 564 |
|
| 565 |
#endif /* STAPCONF_UNREGISTER_KPROBES */ |
| 566 |
|
| 567 |
|
| 568 |
static void |
| 569 |
stapkp_unregister_probes(struct stap_kprobe_probe *probes, |
| 570 |
size_t nprobes) |
| 571 |
{ |
| 572 |
#if defined(STAPCONF_UNREGISTER_KPROBES) |
| 573 |
|
| 574 |
// Unregister using batch mode |
| 575 |
stapkp_batch_unregister_probes(probes, nprobes); |
| 576 |
|
| 577 |
#else |
| 578 |
|
| 579 |
// We'll have to unregister them one by one |
| 580 |
size_t i; |
| 581 |
for (i = 0; i < nprobes; i++) { |
| 582 |
|
| 583 |
struct stap_kprobe_probe *skp = &probes[i]; |
| 584 |
|
| 585 |
if (!skp->registered_p) |
| 586 |
continue; |
| 587 |
|
| 588 |
stapkp_unregister_probe(skp); |
| 589 |
} |
| 590 |
|
| 591 |
#endif |
| 592 |
} |
| 593 |
|
| 594 |
|
| 595 |
#if LINUX_VERSION_CODE >= KERNEL_VERSION(2,6,30) |
| 596 |
|
| 597 |
static int |
| 598 |
stapkp_enabled(struct stap_kprobe_probe *skp) |
| 599 |
{ |
| 600 |
if (!skp->registered_p) |
| 601 |
return 0; |
| 602 |
|
| 603 |
return skp->return_p ? !kprobe_disabled(&skp->kprobe->u.krp.kp) |
| 604 |
: !kprobe_disabled(&skp->kprobe->u.kp); |
| 605 |
} |
| 606 |
|
| 607 |
|
| 608 |
static int |
| 609 |
stapkp_should_enable_probe(struct stap_kprobe_probe *skp) |
| 610 |
{ |
| 611 |
return skp->registered_p |
| 612 |
&& !stapkp_enabled(skp) |
| 613 |
&& skp->probe->cond_enabled; |
| 614 |
} |
| 615 |
|
| 616 |
|
| 617 |
static int |
| 618 |
stapkp_enable_probe(struct stap_kprobe_probe *skp) |
| 619 |
{ |
| 620 |
int ret = 0; |
| 621 |
|
| 622 |
dbug_otf("enabling (k%sprobe) pidx %zu\n", |
| 623 |
skp->return_p ? "ret" : "", skp->probe->index); |
| 624 |
|
| 625 |
ret = skp->return_p ? enable_kretprobe(&skp->kprobe->u.krp) |
| 626 |
: enable_kprobe(&skp->kprobe->u.kp); |
| 627 |
|
| 628 |
if (ret != 0) { |
| 629 |
stapkp_unregister_probe(skp); |
| 630 |
dbug_otf("failed to enable (k%sprobe) pidx %zu (rc %d)\n", |
| 631 |
skp->return_p ? "ret" : "", skp->probe->index, ret); |
| 632 |
} |
| 633 |
|
| 634 |
return ret; |
| 635 |
} |
| 636 |
|
| 637 |
|
| 638 |
static int |
| 639 |
stapkp_should_disable_probe(struct stap_kprobe_probe *skp) |
| 640 |
{ |
| 641 |
return skp->registered_p |
| 642 |
&& stapkp_enabled(skp) |
| 643 |
&& !skp->probe->cond_enabled; |
| 644 |
} |
| 645 |
|
| 646 |
|
| 647 |
static int |
| 648 |
stapkp_disable_probe(struct stap_kprobe_probe *skp) |
| 649 |
{ |
| 650 |
int ret = 0; |
| 651 |
|
| 652 |
dbug_otf("disabling (k%sprobe) pidx %zu\n", |
| 653 |
skp->return_p ? "ret" : "", skp->probe->index); |
| 654 |
|
| 655 |
ret = skp->return_p ? disable_kretprobe(&skp->kprobe->u.krp) |
| 656 |
: disable_kprobe(&skp->kprobe->u.kp); |
| 657 |
|
| 658 |
if (ret != 0) { |
| 659 |
stapkp_unregister_probe(skp); |
| 660 |
dbug_otf("failed to disable (k%sprobe) pidx %zu (rc %d)\n", |
| 661 |
skp->return_p ? "ret" : "", skp->probe->index, ret); |
| 662 |
} |
| 663 |
|
| 664 |
return ret; |
| 665 |
} |
| 666 |
|
| 667 |
|
| 668 |
static int |
| 669 |
stapkp_refresh_probe(struct stap_kprobe_probe *skp) |
| 670 |
{ |
| 671 |
if (stapkp_should_enable_probe(skp)) |
| 672 |
return stapkp_enable_probe(skp); |
| 673 |
if (stapkp_should_disable_probe(skp)) |
| 674 |
return stapkp_disable_probe(skp); |
| 675 |
return 0; |
| 676 |
} |
| 677 |
|
| 678 |
#endif /* LINUX_VERSION_CODE >= 2.6.30 */ |
| 679 |
|
| 680 |
|
| 681 |
struct stapkp_symbol_data { |
| 682 |
struct stap_kprobe_probe *probes; |
| 683 |
size_t nprobes; /* number of probes in "probes" */ |
| 684 |
size_t probe_max; /* number of probes to process */ |
| 685 |
const char *modname; |
| 686 |
}; |
| 687 |
|
| 688 |
|
| 689 |
static int |
| 690 |
__stapkp_symbol_callback(void *data, const char *name, |
| 691 |
struct module *mod, unsigned long addr) |
| 692 |
{ |
| 693 |
struct stapkp_symbol_data *sd = data; |
| 694 |
size_t i; |
| 695 |
|
| 696 |
if ((mod && sd->modname && strcmp(mod->name, sd->modname) != 0) |
| 697 |
|| (!mod && sd->modname)) |
| 698 |
return 0; |
| 699 |
|
| 700 |
for (i = 0; i < sd->nprobes; i++) { |
| 701 |
struct stap_kprobe_probe *skp = &sd->probes[i]; |
| 702 |
int update_addr = 0; |
| 703 |
|
| 704 |
if (! skp->symbol_name) |
| 705 |
continue; |
| 706 |
|
| 707 |
// If (1) We're probing a module symbol and we're in that module |
| 708 |
// and the names match; or (2) we're probing a symbol in the |
| 709 |
// kernel and the names match, then update the k[ret]probe |
| 710 |
// address. |
| 711 |
if (mod && skp->module && strcmp(mod->name, skp->module) == 0) { |
| 712 |
char *colon = strchr(skp->symbol_name, ':'); |
| 713 |
|
| 714 |
if (colon != NULL && strcmp(name, colon+1) == 0) |
| 715 |
update_addr = 1; |
| 716 |
} |
| 717 |
else if (!mod && (skp->module == NULL || skp->module[0] == '\0') |
| 718 |
&& strcmp(name, skp->symbol_name) == 0) |
| 719 |
update_addr = 1; |
| 720 |
if (update_addr) { |
| 721 |
|
| 722 |
if (skp->return_p) |
| 723 |
skp->kprobe->u.krp.kp.addr = (void *)(addr + skp->offset); |
| 724 |
else |
| 725 |
skp->kprobe->u.kp.addr = (void *)(addr + skp->offset); |
| 726 |
// Note that we could have more than 1 probe at the same |
| 727 |
// symbol (with the same or differing offsets), so we can't |
| 728 |
// return here. |
| 729 |
// |
| 730 |
// But we can quit if we've processed all the needed probes. |
| 731 |
--sd->probe_max; |
| 732 |
if (sd->probe_max == 0) |
| 733 |
return -1; |
| 734 |
} |
| 735 |
} |
| 736 |
return 0; |
| 737 |
} |
| 738 |
|
| 739 |
static int |
| 740 |
#if defined(STAPCONF_KALLSYMS_6_4) |
| 741 |
stapkp_symbol_callback(void *data, const char *name, |
| 742 |
unsigned long addr) |
| 743 |
{ |
| 744 |
struct module *mod = NULL; |
| 745 |
#else |
| 746 |
stapkp_symbol_callback(void *data, const char *name, |
| 747 |
struct module *mod, unsigned long addr) |
| 748 |
{ |
| 749 |
#endif |
| 750 |
return __stapkp_symbol_callback(data, name, mod, addr); |
| 751 |
} |
| 752 |
|
| 753 |
static int |
| 754 |
stapkp_init(struct stap_kprobe_probe *probes, |
| 755 |
size_t nprobes) |
| 756 |
{ |
| 757 |
size_t i; |
| 758 |
|
| 759 |
if (USE_KALLSYMS_ON_EACH_SYMBOL) { |
| 760 |
// If we have any symbol_name+offset probes, we need to try to |
| 761 |
// convert those into address-based probes. |
| 762 |
size_t probe_max = 0; |
| 763 |
for (i = 0; i < nprobes; i++) { |
| 764 |
struct stap_kprobe_probe *skp = &probes[i]; |
| 765 |
|
| 766 |
if (! skp->symbol_name) |
| 767 |
continue; |
| 768 |
++probe_max; |
| 769 |
} |
| 770 |
if (probe_max > 0) { |
| 771 |
// Here we're going to try to convert any symbol_name+offset |
| 772 |
// probes into address probes. |
| 773 |
struct stapkp_symbol_data sd; |
| 774 |
dbug_stapkp("looking up %lu probes\n", probe_max); |
| 775 |
sd.probes = probes; |
| 776 |
sd.nprobes = nprobes; |
| 777 |
sd.probe_max = probe_max; |
| 778 |
sd.modname = NULL; |
| 779 |
#ifdef STAPCONF_MODULE_MUTEX |
| 780 |
mutex_lock(&module_mutex); |
| 781 |
#endif |
| 782 |
kallsyms_on_each_symbol(stapkp_symbol_callback, &sd); |
| 783 |
#if defined(STAPCONF_KALLSYMS_6_3) || defined(STAPCONF_KALLSYMS_6_4) |
| 784 |
module_kallsyms_on_each_symbol(sd.modname, stapkp_symbol_callback, &sd); |
| 785 |
#elif LINUX_VERSION_CODE >= KERNEL_VERSION(5,12,0) |
| 786 |
module_kallsyms_on_each_symbol(stapkp_symbol_callback, &sd); |
| 787 |
#endif |
| 788 |
#ifdef STAPCONF_MODULE_MUTEX |
| 789 |
mutex_unlock(&module_mutex); |
| 790 |
#endif |
| 791 |
dbug_stapkp("found %lu probes\n", sd.probe_max); |
| 792 |
} |
| 793 |
} |
| 794 |
|
| 795 |
for (i = 0; i < nprobes; i++) { |
| 796 |
struct stap_kprobe_probe *skp = &probes[i]; |
| 797 |
int rc = 0; |
| 798 |
|
| 799 |
rc = stapkp_register_probe(skp); |
| 800 |
if (rc == 1) // failed to relocate addr? |
| 801 |
continue; // don't fuss about it, module probably not loaded |
| 802 |
|
| 803 |
// PR32695 gcc might statically determine the strcmp will nevery match |
| 804 |
#pragma GCC diagnostic ignored "-Wstring-compare" |
| 805 |
// NB: We keep going even if a probe failed to register (PR6749). We only |
| 806 |
// warn about it if it wasn't optional and isn't in a module. |
| 807 |
if (rc && !skp->optional_p |
| 808 |
&& ((skp->module == NULL) || skp->module[0] == '\0' |
| 809 |
|| strcmp(skp->module, "kernel") == 0)) { |
| 810 |
if (skp->symbol_name) |
| 811 |
_stp_warn("probe %s (%s+%u) registration error [man warning::pass5] (rc %d)", |
| 812 |
skp->probe->pp, skp->symbol_name, skp->offset, rc); |
| 813 |
else |
| 814 |
_stp_warn("probe %s (address 0x%lx) registration error [man warning::pass5] (rc %d)", |
| 815 |
skp->probe->pp, stapkp_relocate_addr(skp), rc); |
| 816 |
} |
| 817 |
} |
| 818 |
|
| 819 |
return 0; |
| 820 |
} |
| 821 |
|
| 822 |
|
| 823 |
/* stapkp_refresh is called for two reasons: either a kprobe needs to be |
| 824 |
* enabled/disabled (modname is NULL), or a module has been loaded/unloaded and |
| 825 |
* kprobes need to be registered/unregistered (modname is !NULL). */ |
| 826 |
static void |
| 827 |
stapkp_refresh(const char *modname, |
| 828 |
struct stap_kprobe_probe *probes, |
| 829 |
size_t nprobes) |
| 830 |
{ |
| 831 |
size_t i; |
| 832 |
|
| 833 |
dbug_stapkp("refresh %lu probes with module %s\n", nprobes, modname ?: "?"); |
| 834 |
|
| 835 |
if (USE_KALLSYMS_ON_EACH_SYMBOL) { |
| 836 |
if (modname) { |
| 837 |
size_t probe_max = 0; |
| 838 |
for (i = 0; i < nprobes; i++) { |
| 839 |
struct stap_kprobe_probe *skp = &probes[i]; |
| 840 |
|
| 841 |
// If this probe is in the same module that is being |
| 842 |
// loaded/unloaded and the probe is symbol_name+offset based |
| 843 |
// and it isn't registered (so the module must be loaded), |
| 844 |
// try to convert all probes in the same module to |
| 845 |
// address-based probes. |
| 846 |
if (skp->module && strcmp(modname, skp->module) == 0 |
| 847 |
&& skp->symbol_name && skp->registered_p == 0) |
| 848 |
++probe_max; |
| 849 |
} |
| 850 |
if (probe_max > 0) { |
| 851 |
struct stapkp_symbol_data sd; |
| 852 |
sd.probes = probes; |
| 853 |
sd.nprobes = nprobes; |
| 854 |
sd.probe_max = probe_max; |
| 855 |
sd.modname = modname; |
| 856 |
#ifdef STAPCONF_MODULE_MUTEX |
| 857 |
mutex_lock(&module_mutex); |
| 858 |
#endif |
| 859 |
kallsyms_on_each_symbol(stapkp_symbol_callback, &sd); |
| 860 |
#if defined(STAPCONF_KALLSYMS_6_3) || defined(STAPCONF_KALLSYMS_6_4) |
| 861 |
module_kallsyms_on_each_symbol(sd.modname, stapkp_symbol_callback, &sd); |
| 862 |
#elif LINUX_VERSION_CODE >= KERNEL_VERSION(5,12,0) |
| 863 |
module_kallsyms_on_each_symbol(stapkp_symbol_callback, &sd); |
| 864 |
#endif |
| 865 |
#ifdef STAPCONF_MODULE_MUTEX |
| 866 |
mutex_unlock(&module_mutex); |
| 867 |
#endif |
| 868 |
} |
| 869 |
} |
| 870 |
} |
| 871 |
|
| 872 |
for (i = 0; i < nprobes; i++) { |
| 873 |
|
| 874 |
struct stap_kprobe_probe *skp = &probes[i]; |
| 875 |
|
| 876 |
// was this probe's target module loaded/unloaded |
| 877 |
if (modname && skp->module |
| 878 |
&& strcmp(modname, skp->module) == 0) { |
| 879 |
int rc; |
| 880 |
unsigned long addr = (! skp->symbol_name |
| 881 |
? stapkp_relocate_addr(skp) : 0); |
| 882 |
|
| 883 |
// module being loaded? |
| 884 |
if (skp->registered_p == 0 && (addr != 0 || skp->symbol_name)) |
| 885 |
stapkp_register_probe(skp); |
| 886 |
// module/section being unloaded? |
| 887 |
else if (skp->registered_p == 1 && addr == 0) |
| 888 |
stapkp_unregister_probe(skp); |
| 889 |
|
| 890 |
} |
| 891 |
#if LINUX_VERSION_CODE >= KERNEL_VERSION(2,6,30) |
| 892 |
else if (stapkp_should_enable_probe(skp) |
| 893 |
|| stapkp_should_disable_probe(skp)) { |
| 894 |
stapkp_refresh_probe(skp); |
| 895 |
} |
| 896 |
#endif |
| 897 |
} |
| 898 |
} |
| 899 |
|
| 900 |
|
| 901 |
static void |
| 902 |
stapkp_exit(struct stap_kprobe_probe *probes, |
| 903 |
size_t nprobes) |
| 904 |
{ |
| 905 |
stapkp_unregister_probes(probes, nprobes); |
| 906 |
} |
| 907 |
|
| 908 |
|
| 909 |
#endif /* _KPROBES_C_ */ |