| 1 |
/* -*- linux-c -*- |
| 2 |
* Common functions for using inode-based uprobes |
| 3 |
* Copyright (C) 2011-2020 Red Hat Inc. |
| 4 |
* |
| 5 |
* This file is part of systemtap, and is free software. You can |
| 6 |
* redistribute it and/or modify it under the terms of the GNU General |
| 7 |
* Public License (GPL); either version 2, or (at your option) any |
| 8 |
* later version. |
| 9 |
*/ |
| 10 |
|
| 11 |
#ifndef _UPROBES_INODE_C_ |
| 12 |
#define _UPROBES_INODE_C_ |
| 13 |
|
| 14 |
#include <linux/fs.h> |
| 15 |
#include <linux/list.h> |
| 16 |
#include <linux/namei.h> |
| 17 |
#include <linux/mutex.h> |
| 18 |
#include <linux/spinlock.h> |
| 19 |
#include <linux/uprobes.h> |
| 20 |
#include <linux/fs_struct.h> |
| 21 |
|
| 22 |
/* STAPIU: SystemTap Inode Uprobes */ |
| 23 |
|
| 24 |
|
| 25 |
// PR13489, inodes-uprobes export kludge |
| 26 |
#if !defined(CONFIG_UPROBES) |
| 27 |
#error "not to be built without CONFIG_UPROBES" |
| 28 |
#endif |
| 29 |
|
| 30 |
#if !defined(STAPCONF_UPROBE_REGISTER_EXPORTED) |
| 31 |
// First get the right typeof(name) that's found in uprobes.h |
| 32 |
#if defined(STAPCONF_OLD_INODE_UPROBES) |
| 33 |
typedef typeof(®ister_uprobe) uprobe_register_fn; |
| 34 |
#else |
| 35 |
typedef typeof(&uprobe_register) uprobe_register_fn; |
| 36 |
#endif |
| 37 |
// Then define the typecasted call via function pointer |
| 38 |
#define uprobe_register(a,b,c) \ |
| 39 |
ibt_wrapper(int, (* (uprobe_register_fn)kallsyms_uprobe_register)((a), (b), (c))) |
| 40 |
#elif defined(STAPCONF_OLD_INODE_UPROBES) |
| 41 |
// In this case, just need to map the new name to the old |
| 42 |
#define uprobe_register register_uprobe |
| 43 |
#endif |
| 44 |
|
| 45 |
#if !defined(STAPCONF_UPROBE_UNREGISTER_EXPORTED) |
| 46 |
// First get the right typeof(name) that's found in uprobes.h |
| 47 |
#if defined(STAPCONF_OLD_INODE_UPROBES) |
| 48 |
typedef typeof(&unregister_uprobe) uprobe_unregister_fn; |
| 49 |
#else |
| 50 |
#if defined(STAPCONF_PR32194_UPROBE_REGISTER_UNREGISTER) |
| 51 |
typedef typeof(&uprobe_unregister_nosync) uprobe_unregister_fn; |
| 52 |
#else |
| 53 |
typedef typeof(&uprobe_unregister) uprobe_unregister_fn; |
| 54 |
#endif |
| 55 |
#endif |
| 56 |
// Then define the typecasted call via function pointer |
| 57 |
#define uprobe_unregister(a,b,c) \ |
| 58 |
void_ibt_wrapper((* (uprobe_unregister_fn)kallsyms_uprobe_unregister)((a),(b),(c))) |
| 59 |
#elif defined(STAPCONF_OLD_INODE_UPROBES) |
| 60 |
// In this case, just need to map the new name to the old |
| 61 |
#define uprobe_unregister unregister_uprobe |
| 62 |
#endif |
| 63 |
|
| 64 |
|
| 65 |
// uprobes started setting REG_IP itself starting in kernel commit 74e59dfc. |
| 66 |
// There's no direct indicator of this, but commit da1816b1 in the same patch |
| 67 |
// series defines UPROBE_HANDLER_MASK, so that's a decent trigger for us. |
| 68 |
#ifndef UPROBE_HANDLER_MASK |
| 69 |
#define STAPIU_NEEDS_REG_IP 1 |
| 70 |
#if !defined(STAPCONF_UPROBE_GET_SWBP_ADDR_EXPORTED) |
| 71 |
// First typedef from the original decl, then #define it as a typecasted call. |
| 72 |
typedef typeof(&uprobe_get_swbp_addr) uprobe_get_swbp_addr_fn; |
| 73 |
#define uprobe_get_swbp_addr(a) \ |
| 74 |
ibt_wrapper(unsigned long, (* (uprobe_get_swbp_addr_fn)kallsyms_uprobe_get_swbp_addr)((a))) |
| 75 |
#endif |
| 76 |
#endif |
| 77 |
|
| 78 |
|
| 79 |
/* A uprobe attached to a particular inode on behalf of a particular |
| 80 |
consumer. NB: such a uprobe instance affects all processes that |
| 81 |
map that inode, so it is not tagged or associated with a |
| 82 |
process. This object is owned by a stapiu_consumer. */ |
| 83 |
struct stapiu_instance { |
| 84 |
struct list_head instance_list; // to find other instances e.g. during shutdown |
| 85 |
|
| 86 |
struct uprobe_consumer kconsumer; // the kernel-side struct for uprobe callbacks etc. |
| 87 |
struct inode *inode; // refcounted |
| 88 |
unsigned registered_p:1; // whether the this kconsumer is registered (= armed, live) |
| 89 |
|
| 90 |
struct stapiu_consumer *sconsumer; // whose instance are we |
| 91 |
}; |
| 92 |
|
| 93 |
|
| 94 |
/* A snippet to record the per-process vm where a particular |
| 95 |
executable/solib was mapped. Used for sdt semaphore setting, and |
| 96 |
for identifying processes of our interest (vs. disinterest) for |
| 97 |
uprobe hits. This object is owned by a stapiu_consumer. We use |
| 98 |
the same inode* as the stapiu_instance, and have the same lifespan, |
| 99 |
so don't bother separately refcount it. |
| 100 |
*/ |
| 101 |
struct stapiu_process { |
| 102 |
struct list_head process_list; // to find other processes |
| 103 |
|
| 104 |
struct inode *inode; // the inode* for solib or executable |
| 105 |
unsigned long relocation; // the mmap'ed .text address |
| 106 |
unsigned long base; // the address to apply sdt offsets against |
| 107 |
pid_t tgid; // pid |
| 108 |
}; |
| 109 |
|
| 110 |
|
| 111 |
/* A consumer is a declaration of a family of uprobes we want to |
| 112 |
place, on one or more IDENTICAL files specified by name or buildid. |
| 113 |
When a matching binaries are found, new stapiu_instances are |
| 114 |
created for it. Note that uprobes are file-based, not |
| 115 |
process-based. The files are identical enough that we store only |
| 116 |
one uprobe address, sdt-semaphore offset etc., for the whole lot. |
| 117 |
|
| 118 |
But we also need some per-process data saved: for sys/sdt.h |
| 119 |
semaphores, on older kernels that don't have semaphore management |
| 120 |
capabilities natively, we need to track all the processes where a |
| 121 |
given binary is mapped, and their base addresses. That way we can |
| 122 |
incrementally compute the semaphore address across the several |
| 123 |
mmap(2) operations involved in shlib loading. We also track |
| 124 |
processes because that's how we tell apart a uprobe hit from an |
| 125 |
interesting process subtree (like due to stap -c ...) from a hit |
| 126 |
on some other process (which we want to ignore) --- but only in |
| 127 |
_stp_target-set mode. |
| 128 |
*/ |
| 129 |
struct stapiu_consumer { |
| 130 |
// NB: task_finder looks for PROCESSES only |
| 131 |
// info to identify the matching exec: pathname/buildid/pid |
| 132 |
struct stap_task_finder_target finder; |
| 133 |
// ... so if we want to probe SHARED LIBRARIES, we need the |
| 134 |
// task_finder to track many processes (not filter on procname / |
| 135 |
// build_id), for which we receive mmap callbacks, then we filter |
| 136 |
// the mmap callbacks against the following fields. |
| 137 |
const char *solib_pathname; |
| 138 |
unsigned long solib_build_id_vaddr; |
| 139 |
const char *solib_build_id; |
| 140 |
int solib_build_id_len; |
| 141 |
|
| 142 |
// The key by which we can match the _stp_module[] element |
| 143 |
const char *module_name; |
| 144 |
|
| 145 |
struct mutex consumer_lock; |
| 146 |
// a lock to protect lists etc. from iteration/modification; NB: not |
| 147 |
// held during uprobe hits; NB: must be a "sleeping lock" not a |
| 148 |
// "spinning lock" |
| 149 |
|
| 150 |
// what kind of uprobe and where to put it |
| 151 |
const unsigned return_p:1; |
| 152 |
loff_t offset; /* the probe offset within the inode */ |
| 153 |
loff_t sdt_sem_offset; /* the semaphore offset from process->base */ |
| 154 |
// those result in this: |
| 155 |
struct list_head instance_list_head; // the resulting uprobe instances for this consumer |
| 156 |
|
| 157 |
struct list_head process_list_head; // the processes for this consumer |
| 158 |
spinlock_t process_list_lock; // protect list; used briefly from even atomic contexts |
| 159 |
|
| 160 |
// List of perf counters used by each probe |
| 161 |
// This list is an index into struct stap_perf_probe, |
| 162 |
long perf_counters_dim; |
| 163 |
long *perf_counters; |
| 164 |
void (*perf_read_handler)(long *values); |
| 165 |
|
| 166 |
const struct stap_probe * const probe; // the script-level probe handler metadata: pp(), probe handler function |
| 167 |
#if defined(STAPCONF_PR32194_UPROBE_REGISTER_UNREGISTER) |
| 168 |
struct uprobe * uprobe; |
| 169 |
#endif |
| 170 |
}; |
| 171 |
|
| 172 |
|
| 173 |
|
| 174 |
|
| 175 |
/* The stap-generated probe handler for all inode-uprobes. */ |
| 176 |
static int |
| 177 |
stapiu_probe_handler (struct stapiu_consumer *c, struct pt_regs *regs); |
| 178 |
|
| 179 |
static inline struct inode * |
| 180 |
stap_get_real_inode(struct dentry *dentry) |
| 181 |
{ |
| 182 |
if (unlikely(dentry == NULL)) |
| 183 |
return NULL; |
| 184 |
|
| 185 |
return |
| 186 |
#ifdef STAPCONF_D_REAL_INODE |
| 187 |
d_real_inode(dentry); |
| 188 |
#else |
| 189 |
dentry->d_inode; |
| 190 |
#endif |
| 191 |
} |
| 192 |
|
| 193 |
static int |
| 194 |
stapiu_change_plus(struct stapiu_consumer* c, struct task_struct *task, |
| 195 |
unsigned long relocation, unsigned long length, |
| 196 |
unsigned long offset, unsigned long vm_flags, |
| 197 |
struct inode *inode); |
| 198 |
|
| 199 |
#ifdef STAPCONF_UPROBES_CB_DATA |
| 200 |
static int |
| 201 |
stapiu_probe_prehandler (struct uprobe_consumer *inst, |
| 202 |
struct pt_regs *regs, |
| 203 |
__u64 *data __attribute__((unused))) |
| 204 |
#else |
| 205 |
static int |
| 206 |
stapiu_probe_prehandler (struct uprobe_consumer *inst, |
| 207 |
struct pt_regs *regs) |
| 208 |
|
| 209 |
#endif |
| 210 |
{ |
| 211 |
int ret; |
| 212 |
struct stapiu_instance *instance = |
| 213 |
container_of(inst, struct stapiu_instance, kconsumer); |
| 214 |
struct stapiu_consumer *c = instance->sconsumer; |
| 215 |
|
| 216 |
if (_stp_target) // need we filter by pid at all? |
| 217 |
{ |
| 218 |
struct stapiu_process *p, *process = NULL; |
| 219 |
unsigned long flags; |
| 220 |
|
| 221 |
// First find the related process, set by stapiu_change_plus. |
| 222 |
// NB: This is a linear search performed for every probe hit! |
| 223 |
// This could be an algorithmic problem if the list gets large, |
| 224 |
// but we'll wait until this is demonstratedly a hotspot before |
| 225 |
// optimizing. NB: on rhel7 sometimes we're invoked from atomic |
| 226 |
// context, so must be careful to use the spinlock, not the |
| 227 |
// mutex. |
| 228 |
spin_lock_irqsave(&c->process_list_lock, flags); |
| 229 |
list_for_each_entry(p, &c->process_list_head, process_list) { |
| 230 |
if (p->tgid == current->tgid) { |
| 231 |
process = p; |
| 232 |
break; |
| 233 |
} |
| 234 |
} |
| 235 |
spin_unlock_irqrestore(&c->process_list_lock, flags); |
| 236 |
if (!process) { |
| 237 |
/* We know that we're in -c/-x mode, but this process is not |
| 238 |
in the process hierarchy, so the uprobe should be ignored |
| 239 |
and future hits prevented. PR15278 |
| 240 |
*/ |
| 241 |
#ifdef UPROBE_HANDLER_REMOVE |
| 242 |
/* Once we're past the starting phase, we can be sure that any |
| 243 |
* processes which are executing code in a mapping have already |
| 244 |
* been through task_finder. So if it's not in our list of |
| 245 |
* target->processes, it can safely get removed. */ |
| 246 |
if (stap_task_finder_complete()) |
| 247 |
return UPROBE_HANDLER_REMOVE; |
| 248 |
#endif |
| 249 |
return 0; |
| 250 |
} |
| 251 |
} |
| 252 |
|
| 253 |
#ifdef STAPIU_NEEDS_REG_IP |
| 254 |
/* Make it look like the IP is set as it would in the actual user task |
| 255 |
* before calling the real probe handler. */ |
| 256 |
{ |
| 257 |
unsigned long saved_ip = REG_IP(regs); |
| 258 |
SET_REG_IP(regs, uprobe_get_swbp_addr(regs)); |
| 259 |
#endif |
| 260 |
|
| 261 |
ret = stapiu_probe_handler(c, regs); |
| 262 |
|
| 263 |
#ifdef STAPIU_NEEDS_REG_IP |
| 264 |
/* Reset IP regs on return, so we don't confuse uprobes. */ |
| 265 |
SET_REG_IP(regs, saved_ip); |
| 266 |
} |
| 267 |
#endif |
| 268 |
|
| 269 |
return ret; |
| 270 |
} |
| 271 |
|
| 272 |
#ifdef STAPCONF_UPROBES_CB_DATA |
| 273 |
static int |
| 274 |
stapiu_retprobe_prehandler (struct uprobe_consumer *inst, |
| 275 |
unsigned long func __attribute__((unused)), |
| 276 |
struct pt_regs *regs, |
| 277 |
__u64 *data) |
| 278 |
{ |
| 279 |
return stapiu_probe_prehandler(inst, regs, data); |
| 280 |
} |
| 281 |
#else |
| 282 |
static int |
| 283 |
stapiu_retprobe_prehandler (struct uprobe_consumer *inst, |
| 284 |
unsigned long func __attribute__((unused)), |
| 285 |
struct pt_regs *regs) |
| 286 |
{ |
| 287 |
return stapiu_probe_prehandler(inst, regs); |
| 288 |
} |
| 289 |
#endif |
| 290 |
|
| 291 |
|
| 292 |
|
| 293 |
static int |
| 294 |
stapiu_register (struct stapiu_instance* inst, struct stapiu_consumer* c) |
| 295 |
{ |
| 296 |
int ret = 0; |
| 297 |
|
| 298 |
dbug_uprobes("registering (u%sprobe) at inode-offset " |
| 299 |
"%lu:%p pidx %zu target filename:%s buildid:%s\n", |
| 300 |
c->return_p ? "ret" : "", |
| 301 |
(unsigned long) inst->inode->i_ino, |
| 302 |
(void*) (uintptr_t) c->offset, |
| 303 |
c->probe->index, |
| 304 |
((char*)c->finder.procname ?: ((char*)c->solib_pathname ?: "")), |
| 305 |
((char*)c->finder.build_id ?: ((char*)c->solib_build_id ?: ""))); |
| 306 |
|
| 307 |
if (!c->return_p) { |
| 308 |
inst->kconsumer.handler = stapiu_probe_prehandler; |
| 309 |
} else { |
| 310 |
#if defined(STAPCONF_INODE_URETPROBES) |
| 311 |
inst->kconsumer.ret_handler = stapiu_retprobe_prehandler; |
| 312 |
#else |
| 313 |
ret = EINVAL; |
| 314 |
#endif |
| 315 |
} |
| 316 |
if (ret == 0) |
| 317 |
#if defined(STAPCONF_PR32194_UPROBE_REGISTER_UNREGISTER) |
| 318 |
c->uprobe = uprobe_register (inst->inode, c->offset, 0, &inst->kconsumer); |
| 319 |
|
| 320 |
if (c->uprobe == NULL) |
| 321 |
#else |
| 322 |
ret = uprobe_register (inst->inode, c->offset, &inst->kconsumer); |
| 323 |
|
| 324 |
if (ret) |
| 325 |
#endif |
| 326 |
_stp_warn("probe %s at inode-offset %lu:%p " |
| 327 |
"registration error [man warning::pass5] (rc %d)", |
| 328 |
c->probe->pp, |
| 329 |
(unsigned long) inst->inode->i_ino, |
| 330 |
(void*) (uintptr_t) c->offset, |
| 331 |
ret); |
| 332 |
|
| 333 |
inst->registered_p = (ret ? 0 : 1); |
| 334 |
return ret; |
| 335 |
} |
| 336 |
|
| 337 |
|
| 338 |
static void |
| 339 |
stapiu_unregister (struct stapiu_instance* inst, struct stapiu_consumer* c) |
| 340 |
{ |
| 341 |
dbug_uprobes("unregistering (u%sprobe) at inode-offset " |
| 342 |
"%lu:%p pidx %zu\n", |
| 343 |
c->return_p ? "ret" : "", |
| 344 |
(unsigned long) inst->inode->i_ino, |
| 345 |
(void*) (uintptr_t) c->offset, |
| 346 |
c->probe->index); |
| 347 |
|
| 348 |
#if defined(STAPCONF_PR32194_UPROBE_REGISTER_UNREGISTER) |
| 349 |
(void) uprobe_unregister_nosync (c->uprobe, &inst->kconsumer); |
| 350 |
(void) uprobe_unregister_sync(); |
| 351 |
#else |
| 352 |
(void) uprobe_unregister (inst->inode, c->offset, &inst->kconsumer); |
| 353 |
#endif |
| 354 |
inst->registered_p = 0; |
| 355 |
} |
| 356 |
|
| 357 |
|
| 358 |
|
| 359 |
/* Read-modify-write a semaphore in an arbitrary task, usually +/- 1. */ |
| 360 |
static int |
| 361 |
stapiu_write_task_semaphore(struct task_struct* task, |
| 362 |
unsigned long addr, short delta) |
| 363 |
{ |
| 364 |
int count, rc = 0; |
| 365 |
unsigned short sdt_semaphore = 0; /* NB: fixed size */ |
| 366 |
/* XXX: need to analyze possibility of race condition */ |
| 367 |
count = __access_process_vm_noflush(task, addr, |
| 368 |
&sdt_semaphore, sizeof(sdt_semaphore), 0); |
| 369 |
if (count != sizeof(sdt_semaphore)) |
| 370 |
rc = 1; |
| 371 |
else { |
| 372 |
sdt_semaphore += delta; |
| 373 |
count = __access_process_vm_noflush(task, addr, |
| 374 |
&sdt_semaphore, sizeof(sdt_semaphore), 1); |
| 375 |
rc = (count == sizeof(sdt_semaphore)) ? 0 : 1; |
| 376 |
} |
| 377 |
return rc; |
| 378 |
} |
| 379 |
|
| 380 |
|
| 381 |
static void |
| 382 |
stapiu_decrement_process_semaphores(struct stapiu_process *p, |
| 383 |
struct stapiu_consumer *c) |
| 384 |
{ |
| 385 |
struct task_struct *task; |
| 386 |
rcu_read_lock(); |
| 387 |
|
| 388 |
/* We'd like to call find_task_by_pid_ns() here, but it isn't |
| 389 |
* exported. So, we call what it calls... */ |
| 390 |
task = pid_task(find_pid_ns(p->tgid, &init_pid_ns), PIDTYPE_PID); |
| 391 |
|
| 392 |
/* The task may have exited while we weren't watching. */ |
| 393 |
if (task) { |
| 394 |
/* Holding the rcu read lock makes us atomic, and we |
| 395 |
* can't write userspace memory while atomic (which |
| 396 |
* could pagefault). So, instead we lock the task |
| 397 |
* structure, then release the rcu read lock. */ |
| 398 |
get_task_struct(task); |
| 399 |
rcu_read_unlock(); |
| 400 |
|
| 401 |
if (c->sdt_sem_offset) { |
| 402 |
unsigned long addr = p->base + c->sdt_sem_offset; |
| 403 |
stapiu_write_task_semaphore(task, addr, -1); |
| 404 |
} |
| 405 |
put_task_struct(task); |
| 406 |
} else { |
| 407 |
rcu_read_unlock(); |
| 408 |
} |
| 409 |
} |
| 410 |
|
| 411 |
|
| 412 |
/* As part of shutdown, we need to decrement the semaphores in every task we've |
| 413 |
* been attached to. */ |
| 414 |
static void |
| 415 |
stapiu_decrement_semaphores(struct stapiu_consumer *consumers, size_t nconsumers) |
| 416 |
{ |
| 417 |
size_t i; |
| 418 |
/* NB: no process_list_lock use needed as the task_finder engine is |
| 419 |
* already stopped by now, so no one else will mess with us. We need |
| 420 |
* to be sleepable for access_process_vm. */ |
| 421 |
for (i = 0; i < nconsumers; ++i) { |
| 422 |
struct stapiu_consumer *c = &consumers[i]; |
| 423 |
struct stapiu_process *p; |
| 424 |
int has_semaphores = 0; |
| 425 |
|
| 426 |
if (! c->sdt_sem_offset) |
| 427 |
continue; |
| 428 |
|
| 429 |
list_for_each_entry(p, &c->process_list_head, process_list) |
| 430 |
stapiu_decrement_process_semaphores(p, c); |
| 431 |
} |
| 432 |
} |
| 433 |
|
| 434 |
|
| 435 |
/* Unregister all uprobe consumers of each target inode. */ |
| 436 |
static void |
| 437 |
stapiu_consumer_unreg(struct stapiu_consumer *c) |
| 438 |
{ |
| 439 |
struct stapiu_instance *inst, *in2; |
| 440 |
struct stapiu_process *p, *tmp; |
| 441 |
|
| 442 |
// no need for locking protection; by the time this cleanup |
| 443 |
// is triggered, no further list modifying ops can also go |
| 444 |
list_for_each_entry_safe(inst, in2, &c->instance_list_head, instance_list) { |
| 445 |
if (inst->registered_p) |
| 446 |
stapiu_unregister(inst, c); |
| 447 |
if (inst->inode) |
| 448 |
iput(inst->inode); |
| 449 |
list_del(&inst->instance_list); |
| 450 |
_stp_kfree(inst); |
| 451 |
} |
| 452 |
|
| 453 |
// NB: it's hypothetically possible for the same process to show up |
| 454 |
// multiple times in the list. Don't break after the first. |
| 455 |
list_for_each_entry_safe(p, tmp, &c->process_list_head, process_list) { |
| 456 |
list_del(&p->process_list); |
| 457 |
// no refcount used for the inode field |
| 458 |
_stp_kfree (p); |
| 459 |
} |
| 460 |
} |
| 461 |
|
| 462 |
|
| 463 |
/* Register/unregister a target's uprobe consumers if their associated probe |
| 464 |
* handlers have their conditions enabled/disabled. */ |
| 465 |
static void |
| 466 |
stapiu_consumer_refresh(struct stapiu_consumer *c) |
| 467 |
{ |
| 468 |
struct stapiu_instance *inst; |
| 469 |
|
| 470 |
mutex_lock(& c->consumer_lock); |
| 471 |
|
| 472 |
list_for_each_entry(inst, &c->instance_list_head, instance_list) { |
| 473 |
if (inst->registered_p && !c->probe->cond_enabled) |
| 474 |
stapiu_unregister(inst, c); |
| 475 |
else if (!inst->registered_p && c->probe->cond_enabled) |
| 476 |
stapiu_register(inst, c); |
| 477 |
} |
| 478 |
|
| 479 |
mutex_unlock(& c->consumer_lock); |
| 480 |
} |
| 481 |
|
| 482 |
|
| 483 |
/* Cleanup every target. */ |
| 484 |
static void |
| 485 |
stapiu_exit(struct stapiu_consumer *consumers, size_t nconsumers) |
| 486 |
{ |
| 487 |
size_t i; |
| 488 |
stapiu_decrement_semaphores(consumers, nconsumers); |
| 489 |
for (i = 0; i < nconsumers; ++i) { |
| 490 |
struct stapiu_consumer *c = &consumers[i]; |
| 491 |
// protect against conceivable stapiu_refresh() at same time |
| 492 |
mutex_lock(& c->consumer_lock); |
| 493 |
stapiu_consumer_unreg(c); |
| 494 |
mutex_unlock(& c->consumer_lock); |
| 495 |
stap_cleanup_task_finder_target(&c->finder); |
| 496 |
/* NB: task_finder needs no unregister. */ |
| 497 |
} |
| 498 |
} |
| 499 |
|
| 500 |
|
| 501 |
/* Initialize every consumer. */ |
| 502 |
static int |
| 503 |
stapiu_init(struct stapiu_consumer *consumers, size_t nconsumers) |
| 504 |
{ |
| 505 |
int ret = 0; |
| 506 |
size_t i; |
| 507 |
bool mnt_ns_switched = false; |
| 508 |
|
| 509 |
might_sleep(); |
| 510 |
|
| 511 |
for (i = 0; i < nconsumers; ++i) { |
| 512 |
struct stapiu_consumer *c = &consumers[i]; |
| 513 |
INIT_LIST_HEAD(&c->instance_list_head); |
| 514 |
INIT_LIST_HEAD(&c->process_list_head); |
| 515 |
mutex_init(&c->consumer_lock); |
| 516 |
spin_lock_init(&c->process_list_lock); |
| 517 |
|
| 518 |
if (strcmp(c->finder.purpose, "inode-uprobes") == 0) { |
| 519 |
/* PR31014: Uprobes registered in task finder would block the |
| 520 |
* target processes for long time; so, we register them early |
| 521 |
* here upon module loading. */ |
| 522 |
|
| 523 |
struct path path; |
| 524 |
struct inode *inode = NULL; |
| 525 |
|
| 526 |
const char *pathname = c->finder.procname; |
| 527 |
if (pathname == NULL) |
| 528 |
pathname = c->solib_pathname; |
| 529 |
|
| 530 |
if (pathname == NULL) { |
| 531 |
if (likely(c->solib_build_id_len > 0)) { |
| 532 |
// this is a buildid-based uprobe; we'll postpone the uprobe |
| 533 |
// registration to the task finder callbacks. |
| 534 |
continue; |
| 535 |
} |
| 536 |
|
| 537 |
ret = EINVAL; |
| 538 |
_stp_error("Couldn't find proc name or so lib path for the inode uprobe\n"); |
| 539 |
break; |
| 540 |
} |
| 541 |
|
| 542 |
if (!mnt_ns_switched) { |
| 543 |
ret = stap_switch_to_target_mnt_ns_if_needed(&mnt_ns_switched); |
| 544 |
if (unlikely(ret != 0)) |
| 545 |
break; |
| 546 |
} |
| 547 |
|
| 548 |
/* NB Alas. we cannot check build id here since we do not have loaded |
| 549 |
* memory pages to work with at this early stage. Fortunately, one |
| 550 |
* advantage of the early uprobe registration is that we can ensure |
| 551 |
* it is the file path in stap's own mount namespace and build id |
| 552 |
* checks do not matter too much anyway. (Executables of the same |
| 553 |
* file paths in other mount namespaces won't hit, as desired.) */ |
| 554 |
ret = kern_path(pathname, LOOKUP_FOLLOW, &path); |
| 555 |
if (unlikely(ret != 0)) { |
| 556 |
_stp_error("Couldn't resolve target program file path '%s': %d\n", |
| 557 |
pathname, ret); |
| 558 |
break; |
| 559 |
} |
| 560 |
|
| 561 |
inode = stap_get_real_inode(path.dentry); |
| 562 |
if (likely(inode != NULL)) { |
| 563 |
ret = stapiu_change_plus(c, NULL, 0, TASK_SIZE, 0, 0, inode); |
| 564 |
path_put(&path); |
| 565 |
if (unlikely(ret != 0)) |
| 566 |
break; |
| 567 |
} else { |
| 568 |
path_put(&path); |
| 569 |
} |
| 570 |
|
| 571 |
/* NB uprobe_register() calls are expensive (8+ ms) and blocking; |
| 572 |
* make sure we yield in between the calls. */ |
| 573 |
cond_resched(); |
| 574 |
} |
| 575 |
} |
| 576 |
|
| 577 |
if (mnt_ns_switched) { |
| 578 |
int rc; |
| 579 |
|
| 580 |
rc = stap_switch_to_orig_mnt_ns_if_needed(); |
| 581 |
if (unlikely(rc != 0)) { |
| 582 |
int j, n; |
| 583 |
|
| 584 |
n = ret != 0 ? i + 1 : nconsumers; |
| 585 |
for (j = 0; j < n; ++j) { |
| 586 |
struct stapiu_consumer *c = &consumers[j]; |
| 587 |
// protect against conceivable stapiu_refresh() at same time |
| 588 |
mutex_lock(& c->consumer_lock); |
| 589 |
stapiu_consumer_unreg(c); |
| 590 |
mutex_unlock(& c->consumer_lock); |
| 591 |
} |
| 592 |
|
| 593 |
return rc; |
| 594 |
} |
| 595 |
} |
| 596 |
|
| 597 |
if (unlikely(ret != 0)) { |
| 598 |
for ( ;; ) { |
| 599 |
struct stapiu_consumer *c = &consumers[i]; |
| 600 |
// protect against conceivable stapiu_refresh() at same time |
| 601 |
mutex_lock(& c->consumer_lock); |
| 602 |
stapiu_consumer_unreg(c); |
| 603 |
mutex_unlock(& c->consumer_lock); |
| 604 |
if (i == 0) |
| 605 |
break; |
| 606 |
i--; |
| 607 |
} |
| 608 |
return ret; |
| 609 |
} |
| 610 |
|
| 611 |
for (i = 0; i < nconsumers; ++i) { |
| 612 |
struct stapiu_consumer *c = &consumers[i]; |
| 613 |
dbug_uprobes("registering task-finder for procname:%s buildid:%s\n", |
| 614 |
((char*)c->finder.procname ?: ""), |
| 615 |
((char*)c->finder.build_id ?: "")); |
| 616 |
|
| 617 |
ret = stap_register_task_finder_target(&c->finder); |
| 618 |
if (ret != 0) { |
| 619 |
_stp_error("Couldn't register task finder target for file '%s': %d\n", |
| 620 |
c->finder.procname, ret); |
| 621 |
break; |
| 622 |
} |
| 623 |
} |
| 624 |
|
| 625 |
if (unlikely(ret != 0)) { |
| 626 |
int j; |
| 627 |
for (j = 0; j < nconsumers; ++j) { |
| 628 |
struct stapiu_consumer *c = &consumers[j]; |
| 629 |
// protect against conceivable stapiu_refresh() at same time |
| 630 |
mutex_lock(& c->consumer_lock); |
| 631 |
stapiu_consumer_unreg(c); |
| 632 |
mutex_unlock(& c->consumer_lock); |
| 633 |
} |
| 634 |
for ( ;; ) { |
| 635 |
struct stapiu_consumer *c = &consumers[i]; |
| 636 |
stap_cleanup_task_finder_target(&c->finder); |
| 637 |
if (i == 0) |
| 638 |
break; |
| 639 |
i--; |
| 640 |
} |
| 641 |
return ret; |
| 642 |
} |
| 643 |
|
| 644 |
return 0; |
| 645 |
} |
| 646 |
|
| 647 |
|
| 648 |
/* Refresh the entire inode-uprobes subsystem. */ |
| 649 |
static void |
| 650 |
stapiu_refresh(struct stapiu_consumer *consumers, size_t nconsumers) |
| 651 |
{ |
| 652 |
size_t i; |
| 653 |
|
| 654 |
for (i = 0; i < nconsumers; ++i) { |
| 655 |
struct stapiu_consumer *c = &consumers[i]; |
| 656 |
stapiu_consumer_refresh(c); |
| 657 |
} |
| 658 |
} |
| 659 |
|
| 660 |
|
| 661 |
/* Task-finder found a process with a target that we're interested in. |
| 662 |
Time to create a stapiu_instance for this inode/consumer combination. */ |
| 663 |
static int |
| 664 |
stapiu_change_plus(struct stapiu_consumer* c, struct task_struct *task, |
| 665 |
unsigned long relocation, unsigned long length, |
| 666 |
unsigned long offset, unsigned long vm_flags, |
| 667 |
struct inode *inode) |
| 668 |
{ |
| 669 |
int rc = 0; |
| 670 |
struct stapiu_instance *i; |
| 671 |
struct stapiu_instance *inst = NULL; |
| 672 |
struct stapiu_process *p; |
| 673 |
int j; |
| 674 |
unsigned long flags; |
| 675 |
|
| 676 |
if (! inode) { |
| 677 |
rc = -EINVAL; |
| 678 |
goto out; |
| 679 |
} |
| 680 |
|
| 681 |
if (task != NULL) { |
| 682 |
/* Do the buildid check. NB: on F29+, offset may not equal |
| 683 |
0 for LOADable "R E" segments, because the read-only .note.* |
| 684 |
stuff may have been loaded earlier, separately. PR23890. */ |
| 685 |
// NB: this is not really necessary for buildid-based probes, |
| 686 |
// which had this verified already. |
| 687 |
rc = _stp_usermodule_check(task, c->module_name, |
| 688 |
relocation - offset); |
| 689 |
if (rc) |
| 690 |
goto out; |
| 691 |
} |
| 692 |
|
| 693 |
dbug_uprobes("notified for inode-offset arrival u%sprobe " |
| 694 |
"%lu:%p pidx %zu target procname:%s buildid:%s\n", |
| 695 |
c->return_p ? "ret" : "", |
| 696 |
(unsigned long) inode->i_ino, |
| 697 |
(void*) (uintptr_t) c->offset, |
| 698 |
c->probe->index, |
| 699 |
((char*)c->finder.procname ?: ((char*)c->solib_pathname ?: "")), |
| 700 |
((char*)c->finder.build_id ?: ((char*)c->solib_build_id ?: ""))); |
| 701 |
|
| 702 |
/* Check the buildid of the target (if we haven't already). We |
| 703 |
* lock the target so we don't have concurrency issues. */ |
| 704 |
mutex_lock(&c->consumer_lock); |
| 705 |
|
| 706 |
// Check if we already have an instance for this inode. This is normal: |
| 707 |
// if a different process maps the same solib, or forks into the same |
| 708 |
// executable. In this case, we must not re-register the same uprobe. |
| 709 |
list_for_each_entry(i, &c->instance_list_head, instance_list) { |
| 710 |
if (i->inode == inode) { |
| 711 |
inst = i; |
| 712 |
break; |
| 713 |
} |
| 714 |
} |
| 715 |
|
| 716 |
if (!inst) { // new instance; need new uprobe etc. |
| 717 |
if (task != NULL) { |
| 718 |
if (c->solib_build_id_len == 0) { |
| 719 |
// instance should have been initialized early in stapiu_init; if not, |
| 720 |
// this must be a mismatch. And we do nothing here. |
| 721 |
goto out1; |
| 722 |
} |
| 723 |
/* At this point, we know it is a buildid-based uprobe; |
| 724 |
* go ahead to register a new uprobe. */ |
| 725 |
} |
| 726 |
|
| 727 |
inst = _stp_kzalloc(sizeof(struct stapiu_instance)); |
| 728 |
if (! inst) { |
| 729 |
rc = -ENOMEM; |
| 730 |
goto out1; |
| 731 |
} |
| 732 |
|
| 733 |
inst->sconsumer = c; // back link essential; that's how we go from uprobe *handler callback |
| 734 |
|
| 735 |
/* Grab the inode first (to prevent TOCTTOU problems). */ |
| 736 |
inst->inode = igrab(inode); |
| 737 |
if (!inst->inode) { |
| 738 |
rc = -EINVAL; |
| 739 |
goto out2; |
| 740 |
} |
| 741 |
|
| 742 |
// Add the inode/instance to the list |
| 743 |
list_add(&inst->instance_list, &c->instance_list_head); |
| 744 |
|
| 745 |
// Register the actual uprobe if cond_enabled already |
| 746 |
if (c->probe->cond_enabled) |
| 747 |
(void) stapiu_register(inst, c); |
| 748 |
} |
| 749 |
|
| 750 |
// ... but we may have to do per-process work anyway: perfctr |
| 751 |
// initialization and sdt.h semaphore manipulation! |
| 752 |
|
| 753 |
// Perform perfctr registration if required |
| 754 |
if (task != NULL) { |
| 755 |
for (j=0; j < c->perf_counters_dim; j++) { |
| 756 |
if ((c->perf_counters)[j] > -1) |
| 757 |
(void) _stp_perf_read_init ((c->perf_counters)[j], task); |
| 758 |
} |
| 759 |
} |
| 760 |
|
| 761 |
// NB: process_list[] already extended up in stapiu_mmap_found(). |
| 762 |
|
| 763 |
rc = 0; |
| 764 |
goto out1; |
| 765 |
|
| 766 |
out2: |
| 767 |
_stp_kfree(inst); |
| 768 |
out1: |
| 769 |
mutex_unlock(&c->consumer_lock); |
| 770 |
out: |
| 771 |
return rc; |
| 772 |
} |
| 773 |
|
| 774 |
|
| 775 |
/* Task-finder found a writable mapping in our interested target. |
| 776 |
* Increment the semaphore now. */ |
| 777 |
static int |
| 778 |
stapiu_change_semaphore_plus(struct stapiu_consumer* c, struct task_struct *task, |
| 779 |
unsigned long relocation, struct inode* inode) |
| 780 |
{ |
| 781 |
int rc = 0; |
| 782 |
struct stapiu_process *p; |
| 783 |
int any_found; |
| 784 |
unsigned long flags; |
| 785 |
|
| 786 |
if (! c->sdt_sem_offset) // nothing to do |
| 787 |
return 0; |
| 788 |
|
| 789 |
dbug_uprobes("considering semaphore (u%sprobe) pid %ld inode 0x%lx" |
| 790 |
"pidx %zu\n", |
| 791 |
c->return_p ? "ret" : "", |
| 792 |
(long) task->tgid, |
| 793 |
(unsigned long) inode, |
| 794 |
c->probe->index); |
| 795 |
|
| 796 |
// NB: we mustn't hold a lock while changing the task memory, |
| 797 |
// but we need a lock to protect the process_list from concurrent |
| 798 |
// add/delete. So hold a spinlock during iteration until the first |
| 799 |
// hit, then unlock & process. NB: We could in principle have multiple |
| 800 |
// instances of the same process in the list (e.g., if the process |
| 801 |
// somehow maps in the same solib multiple times). We can't easily |
| 802 |
// both iterate this list (in a spinlock-protected safe way), and |
| 803 |
// relax the spinlock enough to do a safe stapiu_write_task_semaphore() |
| 804 |
// call within the loop. So we will hit only the first copy in our list. |
| 805 |
any_found = 0; |
| 806 |
spin_lock_irqsave(&c->process_list_lock, flags); |
| 807 |
/* Look through all the consumer's processes and increment semaphores. */ |
| 808 |
list_for_each_entry(p, &c->process_list_head, process_list) { |
| 809 |
unsigned long addr = p->base + c->sdt_sem_offset; |
| 810 |
int rc2; |
| 811 |
if (p->tgid != task->tgid) continue; // skip other processes in the list |
| 812 |
if (p->inode != inode) continue; // skip other inodes |
| 813 |
|
| 814 |
// unlock list and process write for this entry |
| 815 |
spin_unlock_irqrestore(&c->process_list_lock, flags); |
| 816 |
any_found=1; |
| 817 |
|
| 818 |
dbug_uprobes("incrementing semaphore (u%sprobe) pid %ld " |
| 819 |
"pidx %zu address 0x%lx\n", |
| 820 |
c->return_p ? "ret" : "", |
| 821 |
(long) task->tgid, |
| 822 |
c->probe->index, |
| 823 |
(unsigned long) addr); |
| 824 |
|
| 825 |
rc2 = stapiu_write_task_semaphore(task, addr, +1); |
| 826 |
if (!rc) |
| 827 |
rc = rc2; |
| 828 |
break; // exit list_for_each loop |
| 829 |
} |
| 830 |
if (! any_found) |
| 831 |
spin_unlock_irqrestore(&c->process_list_lock, flags); |
| 832 |
else |
| 833 |
{}; // already unlocked |
| 834 |
|
| 835 |
return rc; |
| 836 |
} |
| 837 |
|
| 838 |
|
| 839 |
/* Task-finder found a mapping that's now going away. We don't need to worry |
| 840 |
* about the semaphores, so we can just release the process slot. */ |
| 841 |
static int |
| 842 |
stapiu_change_minus(struct stapiu_consumer* c, struct task_struct *task, |
| 843 |
unsigned long addr, unsigned long length) |
| 844 |
{ |
| 845 |
// We don't need do anything really. |
| 846 |
// A process going away means: |
| 847 |
// - its uprobes will no longer fire: no problem, the uprobe inode |
| 848 |
// is shared and persistent |
| 849 |
// - its sdt semaphores (if any) will be nonzero: no problem, the |
| 850 |
// process is dying anyway |
| 851 |
// - the stapiu_consumer's process_list linked list will have a record |
| 852 |
// of the dead process: well, not great, it'll be cleaned up eventually, |
| 853 |
// and cleaning it up NOW is tricky - we could use the process_list_lock |
| 854 |
// to protect the list (as done in stapiu_change_semaphore_plus), |
| 855 |
// but not our sleepy mutex: |
| 856 |
// |
| 857 |
// [ 1955.410237] ? stapiu_change_minus+0x38/0xf0 [stap_54a723c01c50d972590a5c901516849_15522] |
| 858 |
// [ 1955.411583] __mutex_lock+0x35/0x820 |
| 859 |
// [ 1955.416858] ? _raw_spin_unlock+0x1f/0x30 |
| 860 |
// [ 1955.419649] ? utrace_control+0xbe/0x2d0 [stap_54a723c01c50d972590a5c901516849_15522] |
| 861 |
// [ 1955.421702] stapiu_change_minus+0x38/0xf0 [stap_54a723c01c50d972590a5c901516849_15522] |
| 862 |
// [ 1955.425147] ? __stp_utrace_task_finder_target_exec+0x74/0xc0 [stap_54a723c01c50d972590a5c901516849_15522] |
| 863 |
// [ 1955.429773] ? utrace_report_exec+0xdb/0x140 [stap_54a723c01c50d972590a5c901516849_15522] |
| 864 |
// [ 1955.431398] ? __do_execve_file+0xa05/0xb30 |
| 865 |
// [ 1955.432923] ? do_execve+0x27/0x30 |
| 866 |
// [ 1955.436334] ? __x64_sys_execve+0x27/0x30 |
| 867 |
// [ 1955.437700] ? do_syscall_64+0x5c/0xa0 |
| 868 |
|
| 869 |
// But as an optimization - to avoid having them build up indefinitely, |
| 870 |
// and make semaphore operations go slowly, we will nuke matching entries anyway. |
| 871 |
unsigned long flags; |
| 872 |
struct stapiu_process *p, *tmp; |
| 873 |
unsigned nmatch=0; |
| 874 |
|
| 875 |
spin_lock_irqsave(&c->process_list_lock, flags); |
| 876 |
list_for_each_entry_safe(p, tmp, &c->process_list_head, process_list) { |
| 877 |
// we nuke by matching semaphore address (where ..._semaphore_plus wrote) |
| 878 |
// against the address range being unmapped |
| 879 |
unsigned long semaddr = p->base + c->sdt_sem_offset; |
| 880 |
if (p->tgid != task->tgid) // skip other processes in the list |
| 881 |
continue; |
| 882 |
if (semaddr >= addr && semaddr < addr + length) { |
| 883 |
list_del(&p->process_list); |
| 884 |
_stp_kfree (p); |
| 885 |
nmatch ++; |
| 886 |
} |
| 887 |
} |
| 888 |
spin_unlock_irqrestore(&c->process_list_lock, flags); |
| 889 |
|
| 890 |
if (nmatch > 0) |
| 891 |
dbug_uprobes("notified for inode-offset departure u%sprobe " |
| 892 |
"pidx %zu matches:%u procname:%s buildid:%s\n", |
| 893 |
c->return_p ? "ret" : "", |
| 894 |
c->probe->index, |
| 895 |
nmatch, |
| 896 |
((char*)c->finder.procname ?: ((char*)c->solib_pathname ?: "")), |
| 897 |
((char*)c->finder.build_id ?: ((char*)c->solib_build_id ?: ""))); |
| 898 |
|
| 899 |
return 0; |
| 900 |
} |
| 901 |
|
| 902 |
|
| 903 |
static struct inode * |
| 904 |
stapiu_get_task_inode(struct task_struct *task) |
| 905 |
{ |
| 906 |
struct mm_struct *mm; |
| 907 |
struct file* vm_file; |
| 908 |
struct inode *inode = NULL; |
| 909 |
|
| 910 |
// Grab the inode associated with the task. |
| 911 |
// |
| 912 |
// Note we're not calling get_task_mm()/mmput() here. Since |
| 913 |
// we're in the the context of task, the mm should stick |
| 914 |
// around without locking it (and mmput() can sleep). |
| 915 |
mm = task->mm; |
| 916 |
if (! mm) { |
| 917 |
/* If the thread doesn't have a mm_struct, it is |
| 918 |
* a kernel thread which we need to skip. */ |
| 919 |
return NULL; |
| 920 |
} |
| 921 |
|
| 922 |
vm_file = stap_find_exe_file(mm); |
| 923 |
if (vm_file) { |
| 924 |
if (vm_file->f_path.dentry) { |
| 925 |
// PR31024: we must get the real inode since for overlay fs, we |
| 926 |
// might only get the proxying inode instead. |
| 927 |
inode = stap_get_real_inode(vm_file->f_path.dentry); |
| 928 |
} |
| 929 |
fput(vm_file); |
| 930 |
} |
| 931 |
return inode; |
| 932 |
} |
| 933 |
|
| 934 |
|
| 935 |
/* The task_finder_callback we use for ET_EXEC targets. */ |
| 936 |
static int |
| 937 |
stapiu_process_found(struct stap_task_finder_target *tf_target, |
| 938 |
struct task_struct *task, int register_p, int process_p) |
| 939 |
{ |
| 940 |
struct stapiu_consumer *c = container_of(tf_target, struct stapiu_consumer, finder); |
| 941 |
|
| 942 |
if (!process_p) |
| 943 |
return 0; /* ignore threads */ |
| 944 |
|
| 945 |
dbug_uprobes("process_found pid=%ld f.p=%s f.b=%s c.p=%s c.b=%s\n", |
| 946 |
(long)task->tgid, |
| 947 |
((char*)c->finder.procname ?: ""), |
| 948 |
((char*)c->finder.build_id ?: ""), |
| 949 |
((char*)c->solib_pathname ?: ""), |
| 950 |
((char*)c->solib_build_id ?: "")); |
| 951 |
|
| 952 |
/* ET_EXEC events are like shlib events, but with 0 relocation bases */ |
| 953 |
if (register_p) { |
| 954 |
int rc = -EINVAL; |
| 955 |
struct inode *inode = stapiu_get_task_inode(task); |
| 956 |
|
| 957 |
if (inode) { |
| 958 |
// Add a stapiu_process record to the consumer, so that |
| 959 |
// the semaphore increment logic will accept this task. |
| 960 |
struct stapiu_process* p; |
| 961 |
unsigned long flags; |
| 962 |
p = _stp_kzalloc(sizeof(struct stapiu_process)); |
| 963 |
if (p) { |
| 964 |
p->tgid = task->tgid; |
| 965 |
p->relocation = 0; |
| 966 |
p->inode = inode; |
| 967 |
p->base = 0; |
| 968 |
spin_lock_irqsave (&c->process_list_lock, flags); |
| 969 |
list_add(&p->process_list, &c->process_list_head); |
| 970 |
spin_unlock_irqrestore (&c->process_list_lock, flags); |
| 971 |
} else { |
| 972 |
_stp_warn("out of memory tracking executable in process %ld\n", |
| 973 |
(long) task->tgid); |
| 974 |
} |
| 975 |
|
| 976 |
rc = stapiu_change_plus(c, task, 0, TASK_SIZE, 0, 0, inode); |
| 977 |
|
| 978 |
stapiu_change_semaphore_plus(c, task, 0, inode); |
| 979 |
} |
| 980 |
return rc; |
| 981 |
} else |
| 982 |
return stapiu_change_minus(c, task, 0, TASK_SIZE); |
| 983 |
} |
| 984 |
|
| 985 |
|
| 986 |
bool |
| 987 |
__verify_build_id (struct task_struct *tsk, unsigned long addr, |
| 988 |
unsigned const char *build_id, int build_id_len); |
| 989 |
// defined in task_finder2.c |
| 990 |
|
| 991 |
|
| 992 |
|
| 993 |
/* The task_finder_mmap_callback. These callbacks are NOT |
| 994 |
pre-filtered for buildid or pathname matches (because task_finder |
| 995 |
deals with TASKS only), so we get to do that here. */ |
| 996 |
static int |
| 997 |
stapiu_mmap_found(struct stap_task_finder_target *tf_target, |
| 998 |
struct task_struct *task, |
| 999 |
char *path, struct dentry *dentry, |
| 1000 |
unsigned long addr, unsigned long length, |
| 1001 |
unsigned long offset, unsigned long vm_flags) |
| 1002 |
{ |
| 1003 |
struct stapiu_consumer *c = |
| 1004 |
container_of(tf_target, struct stapiu_consumer, finder); |
| 1005 |
int rc = 0; |
| 1006 |
struct stapiu_process* p; |
| 1007 |
int known_mapping_p; |
| 1008 |
unsigned long flags; |
| 1009 |
struct inode *real_inode = stap_get_real_inode(dentry); |
| 1010 |
|
| 1011 |
/* |
| 1012 |
We need to verify that this file/mmap corresponds to the given stapiu_consumer. |
| 1013 |
One could compare (inode) file name, but that won't work with buildid-based |
| 1014 |
uprobes. For those, one cannot just |
| 1015 |
|
| 1016 |
__verify_build_id(... addr - offset + c->solib_build_id_vaddr ...) |
| 1017 |
|
| 1018 |
because dlopen()ing a shared library involves multiple mmaps, including |
| 1019 |
some at repeating/offset addresses. See glibc _dl_map_segments() in various |
| 1020 |
versions. So by the fourth call (!) on modern glibc's, we get a VM_WRITE-able |
| 1021 |
data segment mapped, but that's at a load/mapping address that is offset by a |
| 1022 |
page from the base (file offset=0) mapping. |
| 1023 |
|
| 1024 |
e.g. on Fedora 32 / glibc 2.31, with testsuite/libsdt_buildid.so: |
| 1025 |
|
| 1026 |
Program Headers: |
| 1027 |
Type Offset VirtAddr PhysAddr FileSiz MemSiz Flg Align |
| 1028 |
LOAD 0x000000 0x0000000000000000 0x0000000000000000 0x0004b8 0x0004b8 R 0x1000 |
| 1029 |
LOAD 0x001000 0x0000000000001000 0x0000000000001000 0x000161 0x000161 R E 0x1000 |
| 1030 |
LOAD 0x002000 0x0000000000002000 0x0000000000002000 0x0000cc 0x0000cc R 0x1000 |
| 1031 |
LOAD 0x002df8 0x0000000000003df8 0x0000000000003df8 0x000232 0x000238 RW 0x1000 |
| 1032 |
DYNAMIC 0x002e10 0x0000000000003e10 0x0000000000003e10 0x0001d0 0x0001d0 RW 0x8 |
| 1033 |
|
| 1034 |
strace: |
| 1035 |
openat(AT_FDCWD, ".../libsdt_buildid.so", O_RDONLY|O_CLOEXEC) = 3 |
| 1036 |
mmap(NULL, 16432, PROT_READ, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x148c764ac000 |
| 1037 |
mmap(0x148c764ad000, 4096, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x1000) = 0x148c764ad000 |
| 1038 |
mmap(0x148c764ae000, 4096, PROT_READ, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x2000) = 0x148c764ae000 |
| 1039 |
mmap(0x148c764af000, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x2000) = 0x148c764af000 |
| 1040 |
|
| 1041 |
Note how the virtual mapping for the fourth mmap (also) maps file-offset 0x2000 at |
| 1042 |
vm offset 0x3000. |
| 1043 |
|
| 1044 |
So what we do is rely on the name/buildid validation tests being run |
| 1045 |
-earlier- in the dlopen/mmap sequence to validate near-future |
| 1046 |
mmap()s. We search the c->process_list[] for a mapping that already |
| 1047 |
overlaps the new range, and if so, consider it validated ... whether |
| 1048 |
for the solib_pathname or the solib_build_id case. |
| 1049 |
|
| 1050 |
This is complicated for startup-time traversal of processes/mmaps, |
| 1051 |
where it seems sometimes we get notifications out of temporal sequence. |
| 1052 |
*/ |
| 1053 |
|
| 1054 |
known_mapping_p = 0; |
| 1055 |
spin_lock_irqsave(&c->process_list_lock, flags); |
| 1056 |
list_for_each_entry(p, &c->process_list_head, process_list) { |
| 1057 |
if (p->tgid != task->tgid) continue; |
| 1058 |
if (p->inode != real_inode) continue; |
| 1059 |
known_mapping_p = 1; |
| 1060 |
break; |
| 1061 |
} |
| 1062 |
spin_unlock_irqrestore(&c->process_list_lock, flags); |
| 1063 |
|
| 1064 |
|
| 1065 |
// Check if this mapping (solib) is of interest: whether we expect |
| 1066 |
// it by buildid or name. |
| 1067 |
|
| 1068 |
if (! known_mapping_p) { |
| 1069 |
/* The file path or build-id must match. The build-id address |
| 1070 |
* is calculated using start address of this vma, the file |
| 1071 |
* offset of the vma start address and the file offset of |
| 1072 |
* the build-id. */ |
| 1073 |
if (c->solib_pathname && path && strcmp (path, c->solib_pathname)) |
| 1074 |
return 0; |
| 1075 |
if (c->solib_build_id_len > 0 && !__verify_build_id(task, |
| 1076 |
addr - offset + c->solib_build_id_vaddr, |
| 1077 |
c->solib_build_id, |
| 1078 |
c->solib_build_id_len)) |
| 1079 |
return 0; |
| 1080 |
} |
| 1081 |
|
| 1082 |
// If we made it this far, we have an interesting solib. |
| 1083 |
|
| 1084 |
dbug_uprobes("mmap_found pid=%ld path=%s addr=0x%lx length=%lu offset=%lu flags=0x%lx known=%d\n", |
| 1085 |
(long) task->tgid, path, addr, length, offset, vm_flags, known_mapping_p); |
| 1086 |
|
| 1087 |
if (! known_mapping_p) { |
| 1088 |
// OK, let's add it. The first mapping should be a VM_READ mapping |
| 1089 |
// of the entire solib file, which will also serve as the apprx. |
| 1090 |
// outer bounds of the repeatedly-mapped segments. |
| 1091 |
|
| 1092 |
#if 0 |
| 1093 |
// Consider an assumption about the dlopen/mmap sequence |
| 1094 |
// If it comes out of sequence, we could get length/base wrong in the stored |
| 1095 |
// stapiu_process, which could lead us to miscalculate semaphore addresses. |
| 1096 |
// |
| 1097 |
// However, this has been observed on task-finder initial-enumeration case, |
| 1098 |
// (sdt_misc.exp, where a solib test is already running when stap starts). |
| 1099 |
if (offset != 0) |
| 1100 |
return 0; |
| 1101 |
#endif |
| 1102 |
|
| 1103 |
// Associate this consumer with this process. If we encounter |
| 1104 |
// resource problems here, we don't really have to undo the uprobe |
| 1105 |
// registrations etc. already in effect. It may break correct |
| 1106 |
// tracking of process hierarchy in -c/-x operation, but too bad. |
| 1107 |
p = _stp_kzalloc(sizeof(struct stapiu_process)); |
| 1108 |
if (p) { |
| 1109 |
p->tgid = task->tgid; |
| 1110 |
p->relocation = addr; |
| 1111 |
p->inode = real_inode; |
| 1112 |
p->base = addr-offset; // ... in case caught this during the second mmap |
| 1113 |
spin_lock_irqsave (&c->process_list_lock, flags); |
| 1114 |
list_add(&p->process_list, &c->process_list_head); |
| 1115 |
spin_unlock_irqrestore (&c->process_list_lock, flags); |
| 1116 |
} else |
| 1117 |
_stp_warn("out of memory tracking solib %s in process %ld\n", |
| 1118 |
path, (long) task->tgid); |
| 1119 |
} |
| 1120 |
|
| 1121 |
/* Check non-writable, executable sections for probes. */ |
| 1122 |
if ((vm_flags & VM_EXEC) && !(vm_flags & VM_WRITE)) |
| 1123 |
rc = stapiu_change_plus(c, task, addr, length, |
| 1124 |
offset, vm_flags, real_inode); |
| 1125 |
|
| 1126 |
/* Check writeable sections for semaphores. |
| 1127 |
* NB: They may have also been executable for the check above, |
| 1128 |
* if we're running a kernel that lacks noexec mappings. |
| 1129 |
* So long as there's no error (rc == 0), we need to look |
| 1130 |
* for semaphores too. |
| 1131 |
*/ |
| 1132 |
|
| 1133 |
if ((rc == 0) && (vm_flags & VM_WRITE)) |
| 1134 |
rc = stapiu_change_semaphore_plus(c, task, addr, real_inode); |
| 1135 |
|
| 1136 |
return rc; |
| 1137 |
} |
| 1138 |
|
| 1139 |
|
| 1140 |
/* The task_finder_munmap_callback */ |
| 1141 |
static int |
| 1142 |
stapiu_munmap_found(struct stap_task_finder_target *tf_target, |
| 1143 |
struct task_struct *task, |
| 1144 |
unsigned long addr, unsigned long length) |
| 1145 |
{ |
| 1146 |
struct stapiu_consumer *c = |
| 1147 |
container_of(tf_target, struct stapiu_consumer, finder); |
| 1148 |
|
| 1149 |
return stapiu_change_minus(c, task, addr, length); |
| 1150 |
} |
| 1151 |
|
| 1152 |
|
| 1153 |
/* The task_finder_callback we use for ET_DYN targets. |
| 1154 |
* This just forces an unmap of everything as the process exits. (PR11151) |
| 1155 |
*/ |
| 1156 |
static int |
| 1157 |
stapiu_process_munmap(struct stap_task_finder_target *tf_target, |
| 1158 |
struct task_struct *task, |
| 1159 |
int register_p, int process_p) |
| 1160 |
{ |
| 1161 |
struct stapiu_consumer *c = |
| 1162 |
container_of(tf_target, struct stapiu_consumer, finder); |
| 1163 |
|
| 1164 |
if (!process_p) |
| 1165 |
return 0; /* ignore threads */ |
| 1166 |
|
| 1167 |
/* Covering 0->TASK_SIZE means "unmap everything" */ |
| 1168 |
if (!register_p) |
| 1169 |
return stapiu_change_minus(c, task, 0, TASK_SIZE); |
| 1170 |
return 0; |
| 1171 |
} |
| 1172 |
|
| 1173 |
|
| 1174 |
#endif /* _UPROBES_INODE_C_ */ |