Debian Codegraph @

runtime/linux/uprobes-inode.c

raw
1 /* -*- linux-c -*-
2 * Common functions for using inode-based uprobes
3 * Copyright (C) 2011-2020 Red Hat Inc.
4 *
5 * This file is part of systemtap, and is free software. You can
6 * redistribute it and/or modify it under the terms of the GNU General
7 * Public License (GPL); either version 2, or (at your option) any
8 * later version.
9 */
10
11 #ifndef _UPROBES_INODE_C_
12 #define _UPROBES_INODE_C_
13
14 #include <linux/fs.h>
15 #include <linux/list.h>
16 #include <linux/namei.h>
17 #include <linux/mutex.h>
18 #include <linux/spinlock.h>
19 #include <linux/uprobes.h>
20 #include <linux/fs_struct.h>
21
22 /* STAPIU: SystemTap Inode Uprobes */
23
24
25 // PR13489, inodes-uprobes export kludge
26 #if !defined(CONFIG_UPROBES)
27 #error "not to be built without CONFIG_UPROBES"
28 #endif
29
30 #if !defined(STAPCONF_UPROBE_REGISTER_EXPORTED)
31 // First get the right typeof(name) that's found in uprobes.h
32 #if defined(STAPCONF_OLD_INODE_UPROBES)
33 typedef typeof(&register_uprobe) uprobe_register_fn;
34 #else
35 typedef typeof(&uprobe_register) uprobe_register_fn;
36 #endif
37 // Then define the typecasted call via function pointer
38 #define uprobe_register(a,b,c) \
39 ibt_wrapper(int, (* (uprobe_register_fn)kallsyms_uprobe_register)((a), (b), (c)))
40 #elif defined(STAPCONF_OLD_INODE_UPROBES)
41 // In this case, just need to map the new name to the old
42 #define uprobe_register register_uprobe
43 #endif
44
45 #if !defined(STAPCONF_UPROBE_UNREGISTER_EXPORTED)
46 // First get the right typeof(name) that's found in uprobes.h
47 #if defined(STAPCONF_OLD_INODE_UPROBES)
48 typedef typeof(&unregister_uprobe) uprobe_unregister_fn;
49 #else
50 #if defined(STAPCONF_PR32194_UPROBE_REGISTER_UNREGISTER)
51 typedef typeof(&uprobe_unregister_nosync) uprobe_unregister_fn;
52 #else
53 typedef typeof(&uprobe_unregister) uprobe_unregister_fn;
54 #endif
55 #endif
56 // Then define the typecasted call via function pointer
57 #define uprobe_unregister(a,b,c) \
58 void_ibt_wrapper((* (uprobe_unregister_fn)kallsyms_uprobe_unregister)((a),(b),(c)))
59 #elif defined(STAPCONF_OLD_INODE_UPROBES)
60 // In this case, just need to map the new name to the old
61 #define uprobe_unregister unregister_uprobe
62 #endif
63
64
65 // uprobes started setting REG_IP itself starting in kernel commit 74e59dfc.
66 // There's no direct indicator of this, but commit da1816b1 in the same patch
67 // series defines UPROBE_HANDLER_MASK, so that's a decent trigger for us.
68 #ifndef UPROBE_HANDLER_MASK
69 #define STAPIU_NEEDS_REG_IP 1
70 #if !defined(STAPCONF_UPROBE_GET_SWBP_ADDR_EXPORTED)
71 // First typedef from the original decl, then #define it as a typecasted call.
72 typedef typeof(&uprobe_get_swbp_addr) uprobe_get_swbp_addr_fn;
73 #define uprobe_get_swbp_addr(a) \
74 ibt_wrapper(unsigned long, (* (uprobe_get_swbp_addr_fn)kallsyms_uprobe_get_swbp_addr)((a)))
75 #endif
76 #endif
77
78
79 /* A uprobe attached to a particular inode on behalf of a particular
80 consumer. NB: such a uprobe instance affects all processes that
81 map that inode, so it is not tagged or associated with a
82 process. This object is owned by a stapiu_consumer. */
83 struct stapiu_instance {
84 struct list_head instance_list; // to find other instances e.g. during shutdown
85
86 struct uprobe_consumer kconsumer; // the kernel-side struct for uprobe callbacks etc.
87 struct inode *inode; // refcounted
88 unsigned registered_p:1; // whether the this kconsumer is registered (= armed, live)
89
90 struct stapiu_consumer *sconsumer; // whose instance are we
91 };
92
93
94 /* A snippet to record the per-process vm where a particular
95 executable/solib was mapped. Used for sdt semaphore setting, and
96 for identifying processes of our interest (vs. disinterest) for
97 uprobe hits. This object is owned by a stapiu_consumer. We use
98 the same inode* as the stapiu_instance, and have the same lifespan,
99 so don't bother separately refcount it.
100 */
101 struct stapiu_process {
102 struct list_head process_list; // to find other processes
103
104 struct inode *inode; // the inode* for solib or executable
105 unsigned long relocation; // the mmap'ed .text address
106 unsigned long base; // the address to apply sdt offsets against
107 pid_t tgid; // pid
108 };
109
110
111 /* A consumer is a declaration of a family of uprobes we want to
112 place, on one or more IDENTICAL files specified by name or buildid.
113 When a matching binaries are found, new stapiu_instances are
114 created for it. Note that uprobes are file-based, not
115 process-based. The files are identical enough that we store only
116 one uprobe address, sdt-semaphore offset etc., for the whole lot.
117
118 But we also need some per-process data saved: for sys/sdt.h
119 semaphores, on older kernels that don't have semaphore management
120 capabilities natively, we need to track all the processes where a
121 given binary is mapped, and their base addresses. That way we can
122 incrementally compute the semaphore address across the several
123 mmap(2) operations involved in shlib loading. We also track
124 processes because that's how we tell apart a uprobe hit from an
125 interesting process subtree (like due to stap -c ...) from a hit
126 on some other process (which we want to ignore) --- but only in
127 _stp_target-set mode.
128 */
129 struct stapiu_consumer {
130 // NB: task_finder looks for PROCESSES only
131 // info to identify the matching exec: pathname/buildid/pid
132 struct stap_task_finder_target finder;
133 // ... so if we want to probe SHARED LIBRARIES, we need the
134 // task_finder to track many processes (not filter on procname /
135 // build_id), for which we receive mmap callbacks, then we filter
136 // the mmap callbacks against the following fields.
137 const char *solib_pathname;
138 unsigned long solib_build_id_vaddr;
139 const char *solib_build_id;
140 int solib_build_id_len;
141
142 // The key by which we can match the _stp_module[] element
143 const char *module_name;
144
145 struct mutex consumer_lock;
146 // a lock to protect lists etc. from iteration/modification; NB: not
147 // held during uprobe hits; NB: must be a "sleeping lock" not a
148 // "spinning lock"
149
150 // what kind of uprobe and where to put it
151 const unsigned return_p:1;
152 loff_t offset; /* the probe offset within the inode */
153 loff_t sdt_sem_offset; /* the semaphore offset from process->base */
154 // those result in this:
155 struct list_head instance_list_head; // the resulting uprobe instances for this consumer
156
157 struct list_head process_list_head; // the processes for this consumer
158 spinlock_t process_list_lock; // protect list; used briefly from even atomic contexts
159
160 // List of perf counters used by each probe
161 // This list is an index into struct stap_perf_probe,
162 long perf_counters_dim;
163 long *perf_counters;
164 void (*perf_read_handler)(long *values);
165
166 const struct stap_probe * const probe; // the script-level probe handler metadata: pp(), probe handler function
167 #if defined(STAPCONF_PR32194_UPROBE_REGISTER_UNREGISTER)
168 struct uprobe * uprobe;
169 #endif
170 };
171
172
173
174
175 /* The stap-generated probe handler for all inode-uprobes. */
176 static int
177 stapiu_probe_handler (struct stapiu_consumer *c, struct pt_regs *regs);
178
179 static inline struct inode *
180 stap_get_real_inode(struct dentry *dentry)
181 {
182 if (unlikely(dentry == NULL))
183 return NULL;
184
185 return
186 #ifdef STAPCONF_D_REAL_INODE
187 d_real_inode(dentry);
188 #else
189 dentry->d_inode;
190 #endif
191 }
192
193 static int
194 stapiu_change_plus(struct stapiu_consumer* c, struct task_struct *task,
195 unsigned long relocation, unsigned long length,
196 unsigned long offset, unsigned long vm_flags,
197 struct inode *inode);
198
199 #ifdef STAPCONF_UPROBES_CB_DATA
200 static int
201 stapiu_probe_prehandler (struct uprobe_consumer *inst,
202 struct pt_regs *regs,
203 __u64 *data __attribute__((unused)))
204 #else
205 static int
206 stapiu_probe_prehandler (struct uprobe_consumer *inst,
207 struct pt_regs *regs)
208
209 #endif
210 {
211 int ret;
212 struct stapiu_instance *instance =
213 container_of(inst, struct stapiu_instance, kconsumer);
214 struct stapiu_consumer *c = instance->sconsumer;
215
216 if (_stp_target) // need we filter by pid at all?
217 {
218 struct stapiu_process *p, *process = NULL;
219 unsigned long flags;
220
221 // First find the related process, set by stapiu_change_plus.
222 // NB: This is a linear search performed for every probe hit!
223 // This could be an algorithmic problem if the list gets large,
224 // but we'll wait until this is demonstratedly a hotspot before
225 // optimizing. NB: on rhel7 sometimes we're invoked from atomic
226 // context, so must be careful to use the spinlock, not the
227 // mutex.
228 spin_lock_irqsave(&c->process_list_lock, flags);
229 list_for_each_entry(p, &c->process_list_head, process_list) {
230 if (p->tgid == current->tgid) {
231 process = p;
232 break;
233 }
234 }
235 spin_unlock_irqrestore(&c->process_list_lock, flags);
236 if (!process) {
237 /* We know that we're in -c/-x mode, but this process is not
238 in the process hierarchy, so the uprobe should be ignored
239 and future hits prevented. PR15278
240 */
241 #ifdef UPROBE_HANDLER_REMOVE
242 /* Once we're past the starting phase, we can be sure that any
243 * processes which are executing code in a mapping have already
244 * been through task_finder. So if it's not in our list of
245 * target->processes, it can safely get removed. */
246 if (stap_task_finder_complete())
247 return UPROBE_HANDLER_REMOVE;
248 #endif
249 return 0;
250 }
251 }
252
253 #ifdef STAPIU_NEEDS_REG_IP
254 /* Make it look like the IP is set as it would in the actual user task
255 * before calling the real probe handler. */
256 {
257 unsigned long saved_ip = REG_IP(regs);
258 SET_REG_IP(regs, uprobe_get_swbp_addr(regs));
259 #endif
260
261 ret = stapiu_probe_handler(c, regs);
262
263 #ifdef STAPIU_NEEDS_REG_IP
264 /* Reset IP regs on return, so we don't confuse uprobes. */
265 SET_REG_IP(regs, saved_ip);
266 }
267 #endif
268
269 return ret;
270 }
271
272 #ifdef STAPCONF_UPROBES_CB_DATA
273 static int
274 stapiu_retprobe_prehandler (struct uprobe_consumer *inst,
275 unsigned long func __attribute__((unused)),
276 struct pt_regs *regs,
277 __u64 *data)
278 {
279 return stapiu_probe_prehandler(inst, regs, data);
280 }
281 #else
282 static int
283 stapiu_retprobe_prehandler (struct uprobe_consumer *inst,
284 unsigned long func __attribute__((unused)),
285 struct pt_regs *regs)
286 {
287 return stapiu_probe_prehandler(inst, regs);
288 }
289 #endif
290
291
292
293 static int
294 stapiu_register (struct stapiu_instance* inst, struct stapiu_consumer* c)
295 {
296 int ret = 0;
297
298 dbug_uprobes("registering (u%sprobe) at inode-offset "
299 "%lu:%p pidx %zu target filename:%s buildid:%s\n",
300 c->return_p ? "ret" : "",
301 (unsigned long) inst->inode->i_ino,
302 (void*) (uintptr_t) c->offset,
303 c->probe->index,
304 ((char*)c->finder.procname ?: ((char*)c->solib_pathname ?: "")),
305 ((char*)c->finder.build_id ?: ((char*)c->solib_build_id ?: "")));
306
307 if (!c->return_p) {
308 inst->kconsumer.handler = stapiu_probe_prehandler;
309 } else {
310 #if defined(STAPCONF_INODE_URETPROBES)
311 inst->kconsumer.ret_handler = stapiu_retprobe_prehandler;
312 #else
313 ret = EINVAL;
314 #endif
315 }
316 if (ret == 0)
317 #if defined(STAPCONF_PR32194_UPROBE_REGISTER_UNREGISTER)
318 c->uprobe = uprobe_register (inst->inode, c->offset, 0, &inst->kconsumer);
319
320 if (c->uprobe == NULL)
321 #else
322 ret = uprobe_register (inst->inode, c->offset, &inst->kconsumer);
323
324 if (ret)
325 #endif
326 _stp_warn("probe %s at inode-offset %lu:%p "
327 "registration error [man warning::pass5] (rc %d)",
328 c->probe->pp,
329 (unsigned long) inst->inode->i_ino,
330 (void*) (uintptr_t) c->offset,
331 ret);
332
333 inst->registered_p = (ret ? 0 : 1);
334 return ret;
335 }
336
337
338 static void
339 stapiu_unregister (struct stapiu_instance* inst, struct stapiu_consumer* c)
340 {
341 dbug_uprobes("unregistering (u%sprobe) at inode-offset "
342 "%lu:%p pidx %zu\n",
343 c->return_p ? "ret" : "",
344 (unsigned long) inst->inode->i_ino,
345 (void*) (uintptr_t) c->offset,
346 c->probe->index);
347
348 #if defined(STAPCONF_PR32194_UPROBE_REGISTER_UNREGISTER)
349 (void) uprobe_unregister_nosync (c->uprobe, &inst->kconsumer);
350 (void) uprobe_unregister_sync();
351 #else
352 (void) uprobe_unregister (inst->inode, c->offset, &inst->kconsumer);
353 #endif
354 inst->registered_p = 0;
355 }
356
357
358
359 /* Read-modify-write a semaphore in an arbitrary task, usually +/- 1. */
360 static int
361 stapiu_write_task_semaphore(struct task_struct* task,
362 unsigned long addr, short delta)
363 {
364 int count, rc = 0;
365 unsigned short sdt_semaphore = 0; /* NB: fixed size */
366 /* XXX: need to analyze possibility of race condition */
367 count = __access_process_vm_noflush(task, addr,
368 &sdt_semaphore, sizeof(sdt_semaphore), 0);
369 if (count != sizeof(sdt_semaphore))
370 rc = 1;
371 else {
372 sdt_semaphore += delta;
373 count = __access_process_vm_noflush(task, addr,
374 &sdt_semaphore, sizeof(sdt_semaphore), 1);
375 rc = (count == sizeof(sdt_semaphore)) ? 0 : 1;
376 }
377 return rc;
378 }
379
380
381 static void
382 stapiu_decrement_process_semaphores(struct stapiu_process *p,
383 struct stapiu_consumer *c)
384 {
385 struct task_struct *task;
386 rcu_read_lock();
387
388 /* We'd like to call find_task_by_pid_ns() here, but it isn't
389 * exported. So, we call what it calls... */
390 task = pid_task(find_pid_ns(p->tgid, &init_pid_ns), PIDTYPE_PID);
391
392 /* The task may have exited while we weren't watching. */
393 if (task) {
394 /* Holding the rcu read lock makes us atomic, and we
395 * can't write userspace memory while atomic (which
396 * could pagefault). So, instead we lock the task
397 * structure, then release the rcu read lock. */
398 get_task_struct(task);
399 rcu_read_unlock();
400
401 if (c->sdt_sem_offset) {
402 unsigned long addr = p->base + c->sdt_sem_offset;
403 stapiu_write_task_semaphore(task, addr, -1);
404 }
405 put_task_struct(task);
406 } else {
407 rcu_read_unlock();
408 }
409 }
410
411
412 /* As part of shutdown, we need to decrement the semaphores in every task we've
413 * been attached to. */
414 static void
415 stapiu_decrement_semaphores(struct stapiu_consumer *consumers, size_t nconsumers)
416 {
417 size_t i;
418 /* NB: no process_list_lock use needed as the task_finder engine is
419 * already stopped by now, so no one else will mess with us. We need
420 * to be sleepable for access_process_vm. */
421 for (i = 0; i < nconsumers; ++i) {
422 struct stapiu_consumer *c = &consumers[i];
423 struct stapiu_process *p;
424 int has_semaphores = 0;
425
426 if (! c->sdt_sem_offset)
427 continue;
428
429 list_for_each_entry(p, &c->process_list_head, process_list)
430 stapiu_decrement_process_semaphores(p, c);
431 }
432 }
433
434
435 /* Unregister all uprobe consumers of each target inode. */
436 static void
437 stapiu_consumer_unreg(struct stapiu_consumer *c)
438 {
439 struct stapiu_instance *inst, *in2;
440 struct stapiu_process *p, *tmp;
441
442 // no need for locking protection; by the time this cleanup
443 // is triggered, no further list modifying ops can also go
444 list_for_each_entry_safe(inst, in2, &c->instance_list_head, instance_list) {
445 if (inst->registered_p)
446 stapiu_unregister(inst, c);
447 if (inst->inode)
448 iput(inst->inode);
449 list_del(&inst->instance_list);
450 _stp_kfree(inst);
451 }
452
453 // NB: it's hypothetically possible for the same process to show up
454 // multiple times in the list. Don't break after the first.
455 list_for_each_entry_safe(p, tmp, &c->process_list_head, process_list) {
456 list_del(&p->process_list);
457 // no refcount used for the inode field
458 _stp_kfree (p);
459 }
460 }
461
462
463 /* Register/unregister a target's uprobe consumers if their associated probe
464 * handlers have their conditions enabled/disabled. */
465 static void
466 stapiu_consumer_refresh(struct stapiu_consumer *c)
467 {
468 struct stapiu_instance *inst;
469
470 mutex_lock(& c->consumer_lock);
471
472 list_for_each_entry(inst, &c->instance_list_head, instance_list) {
473 if (inst->registered_p && !c->probe->cond_enabled)
474 stapiu_unregister(inst, c);
475 else if (!inst->registered_p && c->probe->cond_enabled)
476 stapiu_register(inst, c);
477 }
478
479 mutex_unlock(& c->consumer_lock);
480 }
481
482
483 /* Cleanup every target. */
484 static void
485 stapiu_exit(struct stapiu_consumer *consumers, size_t nconsumers)
486 {
487 size_t i;
488 stapiu_decrement_semaphores(consumers, nconsumers);
489 for (i = 0; i < nconsumers; ++i) {
490 struct stapiu_consumer *c = &consumers[i];
491 // protect against conceivable stapiu_refresh() at same time
492 mutex_lock(& c->consumer_lock);
493 stapiu_consumer_unreg(c);
494 mutex_unlock(& c->consumer_lock);
495 stap_cleanup_task_finder_target(&c->finder);
496 /* NB: task_finder needs no unregister. */
497 }
498 }
499
500
501 /* Initialize every consumer. */
502 static int
503 stapiu_init(struct stapiu_consumer *consumers, size_t nconsumers)
504 {
505 int ret = 0;
506 size_t i;
507 bool mnt_ns_switched = false;
508
509 might_sleep();
510
511 for (i = 0; i < nconsumers; ++i) {
512 struct stapiu_consumer *c = &consumers[i];
513 INIT_LIST_HEAD(&c->instance_list_head);
514 INIT_LIST_HEAD(&c->process_list_head);
515 mutex_init(&c->consumer_lock);
516 spin_lock_init(&c->process_list_lock);
517
518 if (strcmp(c->finder.purpose, "inode-uprobes") == 0) {
519 /* PR31014: Uprobes registered in task finder would block the
520 * target processes for long time; so, we register them early
521 * here upon module loading. */
522
523 struct path path;
524 struct inode *inode = NULL;
525
526 const char *pathname = c->finder.procname;
527 if (pathname == NULL)
528 pathname = c->solib_pathname;
529
530 if (pathname == NULL) {
531 if (likely(c->solib_build_id_len > 0)) {
532 // this is a buildid-based uprobe; we'll postpone the uprobe
533 // registration to the task finder callbacks.
534 continue;
535 }
536
537 ret = EINVAL;
538 _stp_error("Couldn't find proc name or so lib path for the inode uprobe\n");
539 break;
540 }
541
542 if (!mnt_ns_switched) {
543 ret = stap_switch_to_target_mnt_ns_if_needed(&mnt_ns_switched);
544 if (unlikely(ret != 0))
545 break;
546 }
547
548 /* NB Alas. we cannot check build id here since we do not have loaded
549 * memory pages to work with at this early stage. Fortunately, one
550 * advantage of the early uprobe registration is that we can ensure
551 * it is the file path in stap's own mount namespace and build id
552 * checks do not matter too much anyway. (Executables of the same
553 * file paths in other mount namespaces won't hit, as desired.) */
554 ret = kern_path(pathname, LOOKUP_FOLLOW, &path);
555 if (unlikely(ret != 0)) {
556 _stp_error("Couldn't resolve target program file path '%s': %d\n",
557 pathname, ret);
558 break;
559 }
560
561 inode = stap_get_real_inode(path.dentry);
562 if (likely(inode != NULL)) {
563 ret = stapiu_change_plus(c, NULL, 0, TASK_SIZE, 0, 0, inode);
564 path_put(&path);
565 if (unlikely(ret != 0))
566 break;
567 } else {
568 path_put(&path);
569 }
570
571 /* NB uprobe_register() calls are expensive (8+ ms) and blocking;
572 * make sure we yield in between the calls. */
573 cond_resched();
574 }
575 }
576
577 if (mnt_ns_switched) {
578 int rc;
579
580 rc = stap_switch_to_orig_mnt_ns_if_needed();
581 if (unlikely(rc != 0)) {
582 int j, n;
583
584 n = ret != 0 ? i + 1 : nconsumers;
585 for (j = 0; j < n; ++j) {
586 struct stapiu_consumer *c = &consumers[j];
587 // protect against conceivable stapiu_refresh() at same time
588 mutex_lock(& c->consumer_lock);
589 stapiu_consumer_unreg(c);
590 mutex_unlock(& c->consumer_lock);
591 }
592
593 return rc;
594 }
595 }
596
597 if (unlikely(ret != 0)) {
598 for ( ;; ) {
599 struct stapiu_consumer *c = &consumers[i];
600 // protect against conceivable stapiu_refresh() at same time
601 mutex_lock(& c->consumer_lock);
602 stapiu_consumer_unreg(c);
603 mutex_unlock(& c->consumer_lock);
604 if (i == 0)
605 break;
606 i--;
607 }
608 return ret;
609 }
610
611 for (i = 0; i < nconsumers; ++i) {
612 struct stapiu_consumer *c = &consumers[i];
613 dbug_uprobes("registering task-finder for procname:%s buildid:%s\n",
614 ((char*)c->finder.procname ?: ""),
615 ((char*)c->finder.build_id ?: ""));
616
617 ret = stap_register_task_finder_target(&c->finder);
618 if (ret != 0) {
619 _stp_error("Couldn't register task finder target for file '%s': %d\n",
620 c->finder.procname, ret);
621 break;
622 }
623 }
624
625 if (unlikely(ret != 0)) {
626 int j;
627 for (j = 0; j < nconsumers; ++j) {
628 struct stapiu_consumer *c = &consumers[j];
629 // protect against conceivable stapiu_refresh() at same time
630 mutex_lock(& c->consumer_lock);
631 stapiu_consumer_unreg(c);
632 mutex_unlock(& c->consumer_lock);
633 }
634 for ( ;; ) {
635 struct stapiu_consumer *c = &consumers[i];
636 stap_cleanup_task_finder_target(&c->finder);
637 if (i == 0)
638 break;
639 i--;
640 }
641 return ret;
642 }
643
644 return 0;
645 }
646
647
648 /* Refresh the entire inode-uprobes subsystem. */
649 static void
650 stapiu_refresh(struct stapiu_consumer *consumers, size_t nconsumers)
651 {
652 size_t i;
653
654 for (i = 0; i < nconsumers; ++i) {
655 struct stapiu_consumer *c = &consumers[i];
656 stapiu_consumer_refresh(c);
657 }
658 }
659
660
661 /* Task-finder found a process with a target that we're interested in.
662 Time to create a stapiu_instance for this inode/consumer combination. */
663 static int
664 stapiu_change_plus(struct stapiu_consumer* c, struct task_struct *task,
665 unsigned long relocation, unsigned long length,
666 unsigned long offset, unsigned long vm_flags,
667 struct inode *inode)
668 {
669 int rc = 0;
670 struct stapiu_instance *i;
671 struct stapiu_instance *inst = NULL;
672 struct stapiu_process *p;
673 int j;
674 unsigned long flags;
675
676 if (! inode) {
677 rc = -EINVAL;
678 goto out;
679 }
680
681 if (task != NULL) {
682 /* Do the buildid check. NB: on F29+, offset may not equal
683 0 for LOADable "R E" segments, because the read-only .note.*
684 stuff may have been loaded earlier, separately. PR23890. */
685 // NB: this is not really necessary for buildid-based probes,
686 // which had this verified already.
687 rc = _stp_usermodule_check(task, c->module_name,
688 relocation - offset);
689 if (rc)
690 goto out;
691 }
692
693 dbug_uprobes("notified for inode-offset arrival u%sprobe "
694 "%lu:%p pidx %zu target procname:%s buildid:%s\n",
695 c->return_p ? "ret" : "",
696 (unsigned long) inode->i_ino,
697 (void*) (uintptr_t) c->offset,
698 c->probe->index,
699 ((char*)c->finder.procname ?: ((char*)c->solib_pathname ?: "")),
700 ((char*)c->finder.build_id ?: ((char*)c->solib_build_id ?: "")));
701
702 /* Check the buildid of the target (if we haven't already). We
703 * lock the target so we don't have concurrency issues. */
704 mutex_lock(&c->consumer_lock);
705
706 // Check if we already have an instance for this inode. This is normal:
707 // if a different process maps the same solib, or forks into the same
708 // executable. In this case, we must not re-register the same uprobe.
709 list_for_each_entry(i, &c->instance_list_head, instance_list) {
710 if (i->inode == inode) {
711 inst = i;
712 break;
713 }
714 }
715
716 if (!inst) { // new instance; need new uprobe etc.
717 if (task != NULL) {
718 if (c->solib_build_id_len == 0) {
719 // instance should have been initialized early in stapiu_init; if not,
720 // this must be a mismatch. And we do nothing here.
721 goto out1;
722 }
723 /* At this point, we know it is a buildid-based uprobe;
724 * go ahead to register a new uprobe. */
725 }
726
727 inst = _stp_kzalloc(sizeof(struct stapiu_instance));
728 if (! inst) {
729 rc = -ENOMEM;
730 goto out1;
731 }
732
733 inst->sconsumer = c; // back link essential; that's how we go from uprobe *handler callback
734
735 /* Grab the inode first (to prevent TOCTTOU problems). */
736 inst->inode = igrab(inode);
737 if (!inst->inode) {
738 rc = -EINVAL;
739 goto out2;
740 }
741
742 // Add the inode/instance to the list
743 list_add(&inst->instance_list, &c->instance_list_head);
744
745 // Register the actual uprobe if cond_enabled already
746 if (c->probe->cond_enabled)
747 (void) stapiu_register(inst, c);
748 }
749
750 // ... but we may have to do per-process work anyway: perfctr
751 // initialization and sdt.h semaphore manipulation!
752
753 // Perform perfctr registration if required
754 if (task != NULL) {
755 for (j=0; j < c->perf_counters_dim; j++) {
756 if ((c->perf_counters)[j] > -1)
757 (void) _stp_perf_read_init ((c->perf_counters)[j], task);
758 }
759 }
760
761 // NB: process_list[] already extended up in stapiu_mmap_found().
762
763 rc = 0;
764 goto out1;
765
766 out2:
767 _stp_kfree(inst);
768 out1:
769 mutex_unlock(&c->consumer_lock);
770 out:
771 return rc;
772 }
773
774
775 /* Task-finder found a writable mapping in our interested target.
776 * Increment the semaphore now. */
777 static int
778 stapiu_change_semaphore_plus(struct stapiu_consumer* c, struct task_struct *task,
779 unsigned long relocation, struct inode* inode)
780 {
781 int rc = 0;
782 struct stapiu_process *p;
783 int any_found;
784 unsigned long flags;
785
786 if (! c->sdt_sem_offset) // nothing to do
787 return 0;
788
789 dbug_uprobes("considering semaphore (u%sprobe) pid %ld inode 0x%lx"
790 "pidx %zu\n",
791 c->return_p ? "ret" : "",
792 (long) task->tgid,
793 (unsigned long) inode,
794 c->probe->index);
795
796 // NB: we mustn't hold a lock while changing the task memory,
797 // but we need a lock to protect the process_list from concurrent
798 // add/delete. So hold a spinlock during iteration until the first
799 // hit, then unlock & process. NB: We could in principle have multiple
800 // instances of the same process in the list (e.g., if the process
801 // somehow maps in the same solib multiple times). We can't easily
802 // both iterate this list (in a spinlock-protected safe way), and
803 // relax the spinlock enough to do a safe stapiu_write_task_semaphore()
804 // call within the loop. So we will hit only the first copy in our list.
805 any_found = 0;
806 spin_lock_irqsave(&c->process_list_lock, flags);
807 /* Look through all the consumer's processes and increment semaphores. */
808 list_for_each_entry(p, &c->process_list_head, process_list) {
809 unsigned long addr = p->base + c->sdt_sem_offset;
810 int rc2;
811 if (p->tgid != task->tgid) continue; // skip other processes in the list
812 if (p->inode != inode) continue; // skip other inodes
813
814 // unlock list and process write for this entry
815 spin_unlock_irqrestore(&c->process_list_lock, flags);
816 any_found=1;
817
818 dbug_uprobes("incrementing semaphore (u%sprobe) pid %ld "
819 "pidx %zu address 0x%lx\n",
820 c->return_p ? "ret" : "",
821 (long) task->tgid,
822 c->probe->index,
823 (unsigned long) addr);
824
825 rc2 = stapiu_write_task_semaphore(task, addr, +1);
826 if (!rc)
827 rc = rc2;
828 break; // exit list_for_each loop
829 }
830 if (! any_found)
831 spin_unlock_irqrestore(&c->process_list_lock, flags);
832 else
833 {}; // already unlocked
834
835 return rc;
836 }
837
838
839 /* Task-finder found a mapping that's now going away. We don't need to worry
840 * about the semaphores, so we can just release the process slot. */
841 static int
842 stapiu_change_minus(struct stapiu_consumer* c, struct task_struct *task,
843 unsigned long addr, unsigned long length)
844 {
845 // We don't need do anything really.
846 // A process going away means:
847 // - its uprobes will no longer fire: no problem, the uprobe inode
848 // is shared and persistent
849 // - its sdt semaphores (if any) will be nonzero: no problem, the
850 // process is dying anyway
851 // - the stapiu_consumer's process_list linked list will have a record
852 // of the dead process: well, not great, it'll be cleaned up eventually,
853 // and cleaning it up NOW is tricky - we could use the process_list_lock
854 // to protect the list (as done in stapiu_change_semaphore_plus),
855 // but not our sleepy mutex:
856 //
857 // [ 1955.410237] ? stapiu_change_minus+0x38/0xf0 [stap_54a723c01c50d972590a5c901516849_15522]
858 // [ 1955.411583] __mutex_lock+0x35/0x820
859 // [ 1955.416858] ? _raw_spin_unlock+0x1f/0x30
860 // [ 1955.419649] ? utrace_control+0xbe/0x2d0 [stap_54a723c01c50d972590a5c901516849_15522]
861 // [ 1955.421702] stapiu_change_minus+0x38/0xf0 [stap_54a723c01c50d972590a5c901516849_15522]
862 // [ 1955.425147] ? __stp_utrace_task_finder_target_exec+0x74/0xc0 [stap_54a723c01c50d972590a5c901516849_15522]
863 // [ 1955.429773] ? utrace_report_exec+0xdb/0x140 [stap_54a723c01c50d972590a5c901516849_15522]
864 // [ 1955.431398] ? __do_execve_file+0xa05/0xb30
865 // [ 1955.432923] ? do_execve+0x27/0x30
866 // [ 1955.436334] ? __x64_sys_execve+0x27/0x30
867 // [ 1955.437700] ? do_syscall_64+0x5c/0xa0
868
869 // But as an optimization - to avoid having them build up indefinitely,
870 // and make semaphore operations go slowly, we will nuke matching entries anyway.
871 unsigned long flags;
872 struct stapiu_process *p, *tmp;
873 unsigned nmatch=0;
874
875 spin_lock_irqsave(&c->process_list_lock, flags);
876 list_for_each_entry_safe(p, tmp, &c->process_list_head, process_list) {
877 // we nuke by matching semaphore address (where ..._semaphore_plus wrote)
878 // against the address range being unmapped
879 unsigned long semaddr = p->base + c->sdt_sem_offset;
880 if (p->tgid != task->tgid) // skip other processes in the list
881 continue;
882 if (semaddr >= addr && semaddr < addr + length) {
883 list_del(&p->process_list);
884 _stp_kfree (p);
885 nmatch ++;
886 }
887 }
888 spin_unlock_irqrestore(&c->process_list_lock, flags);
889
890 if (nmatch > 0)
891 dbug_uprobes("notified for inode-offset departure u%sprobe "
892 "pidx %zu matches:%u procname:%s buildid:%s\n",
893 c->return_p ? "ret" : "",
894 c->probe->index,
895 nmatch,
896 ((char*)c->finder.procname ?: ((char*)c->solib_pathname ?: "")),
897 ((char*)c->finder.build_id ?: ((char*)c->solib_build_id ?: "")));
898
899 return 0;
900 }
901
902
903 static struct inode *
904 stapiu_get_task_inode(struct task_struct *task)
905 {
906 struct mm_struct *mm;
907 struct file* vm_file;
908 struct inode *inode = NULL;
909
910 // Grab the inode associated with the task.
911 //
912 // Note we're not calling get_task_mm()/mmput() here. Since
913 // we're in the the context of task, the mm should stick
914 // around without locking it (and mmput() can sleep).
915 mm = task->mm;
916 if (! mm) {
917 /* If the thread doesn't have a mm_struct, it is
918 * a kernel thread which we need to skip. */
919 return NULL;
920 }
921
922 vm_file = stap_find_exe_file(mm);
923 if (vm_file) {
924 if (vm_file->f_path.dentry) {
925 // PR31024: we must get the real inode since for overlay fs, we
926 // might only get the proxying inode instead.
927 inode = stap_get_real_inode(vm_file->f_path.dentry);
928 }
929 fput(vm_file);
930 }
931 return inode;
932 }
933
934
935 /* The task_finder_callback we use for ET_EXEC targets. */
936 static int
937 stapiu_process_found(struct stap_task_finder_target *tf_target,
938 struct task_struct *task, int register_p, int process_p)
939 {
940 struct stapiu_consumer *c = container_of(tf_target, struct stapiu_consumer, finder);
941
942 if (!process_p)
943 return 0; /* ignore threads */
944
945 dbug_uprobes("process_found pid=%ld f.p=%s f.b=%s c.p=%s c.b=%s\n",
946 (long)task->tgid,
947 ((char*)c->finder.procname ?: ""),
948 ((char*)c->finder.build_id ?: ""),
949 ((char*)c->solib_pathname ?: ""),
950 ((char*)c->solib_build_id ?: ""));
951
952 /* ET_EXEC events are like shlib events, but with 0 relocation bases */
953 if (register_p) {
954 int rc = -EINVAL;
955 struct inode *inode = stapiu_get_task_inode(task);
956
957 if (inode) {
958 // Add a stapiu_process record to the consumer, so that
959 // the semaphore increment logic will accept this task.
960 struct stapiu_process* p;
961 unsigned long flags;
962 p = _stp_kzalloc(sizeof(struct stapiu_process));
963 if (p) {
964 p->tgid = task->tgid;
965 p->relocation = 0;
966 p->inode = inode;
967 p->base = 0;
968 spin_lock_irqsave (&c->process_list_lock, flags);
969 list_add(&p->process_list, &c->process_list_head);
970 spin_unlock_irqrestore (&c->process_list_lock, flags);
971 } else {
972 _stp_warn("out of memory tracking executable in process %ld\n",
973 (long) task->tgid);
974 }
975
976 rc = stapiu_change_plus(c, task, 0, TASK_SIZE, 0, 0, inode);
977
978 stapiu_change_semaphore_plus(c, task, 0, inode);
979 }
980 return rc;
981 } else
982 return stapiu_change_minus(c, task, 0, TASK_SIZE);
983 }
984
985
986 bool
987 __verify_build_id (struct task_struct *tsk, unsigned long addr,
988 unsigned const char *build_id, int build_id_len);
989 // defined in task_finder2.c
990
991
992
993 /* The task_finder_mmap_callback. These callbacks are NOT
994 pre-filtered for buildid or pathname matches (because task_finder
995 deals with TASKS only), so we get to do that here. */
996 static int
997 stapiu_mmap_found(struct stap_task_finder_target *tf_target,
998 struct task_struct *task,
999 char *path, struct dentry *dentry,
1000 unsigned long addr, unsigned long length,
1001 unsigned long offset, unsigned long vm_flags)
1002 {
1003 struct stapiu_consumer *c =
1004 container_of(tf_target, struct stapiu_consumer, finder);
1005 int rc = 0;
1006 struct stapiu_process* p;
1007 int known_mapping_p;
1008 unsigned long flags;
1009 struct inode *real_inode = stap_get_real_inode(dentry);
1010
1011 /*
1012 We need to verify that this file/mmap corresponds to the given stapiu_consumer.
1013 One could compare (inode) file name, but that won't work with buildid-based
1014 uprobes. For those, one cannot just
1015
1016 __verify_build_id(... addr - offset + c->solib_build_id_vaddr ...)
1017
1018 because dlopen()ing a shared library involves multiple mmaps, including
1019 some at repeating/offset addresses. See glibc _dl_map_segments() in various
1020 versions. So by the fourth call (!) on modern glibc's, we get a VM_WRITE-able
1021 data segment mapped, but that's at a load/mapping address that is offset by a
1022 page from the base (file offset=0) mapping.
1023
1024 e.g. on Fedora 32 / glibc 2.31, with testsuite/libsdt_buildid.so:
1025
1026 Program Headers:
1027 Type Offset VirtAddr PhysAddr FileSiz MemSiz Flg Align
1028 LOAD 0x000000 0x0000000000000000 0x0000000000000000 0x0004b8 0x0004b8 R 0x1000
1029 LOAD 0x001000 0x0000000000001000 0x0000000000001000 0x000161 0x000161 R E 0x1000
1030 LOAD 0x002000 0x0000000000002000 0x0000000000002000 0x0000cc 0x0000cc R 0x1000
1031 LOAD 0x002df8 0x0000000000003df8 0x0000000000003df8 0x000232 0x000238 RW 0x1000
1032 DYNAMIC 0x002e10 0x0000000000003e10 0x0000000000003e10 0x0001d0 0x0001d0 RW 0x8
1033
1034 strace:
1035 openat(AT_FDCWD, ".../libsdt_buildid.so", O_RDONLY|O_CLOEXEC) = 3
1036 mmap(NULL, 16432, PROT_READ, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x148c764ac000
1037 mmap(0x148c764ad000, 4096, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x1000) = 0x148c764ad000
1038 mmap(0x148c764ae000, 4096, PROT_READ, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x2000) = 0x148c764ae000
1039 mmap(0x148c764af000, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x2000) = 0x148c764af000
1040
1041 Note how the virtual mapping for the fourth mmap (also) maps file-offset 0x2000 at
1042 vm offset 0x3000.
1043
1044 So what we do is rely on the name/buildid validation tests being run
1045 -earlier- in the dlopen/mmap sequence to validate near-future
1046 mmap()s. We search the c->process_list[] for a mapping that already
1047 overlaps the new range, and if so, consider it validated ... whether
1048 for the solib_pathname or the solib_build_id case.
1049
1050 This is complicated for startup-time traversal of processes/mmaps,
1051 where it seems sometimes we get notifications out of temporal sequence.
1052 */
1053
1054 known_mapping_p = 0;
1055 spin_lock_irqsave(&c->process_list_lock, flags);
1056 list_for_each_entry(p, &c->process_list_head, process_list) {
1057 if (p->tgid != task->tgid) continue;
1058 if (p->inode != real_inode) continue;
1059 known_mapping_p = 1;
1060 break;
1061 }
1062 spin_unlock_irqrestore(&c->process_list_lock, flags);
1063
1064
1065 // Check if this mapping (solib) is of interest: whether we expect
1066 // it by buildid or name.
1067
1068 if (! known_mapping_p) {
1069 /* The file path or build-id must match. The build-id address
1070 * is calculated using start address of this vma, the file
1071 * offset of the vma start address and the file offset of
1072 * the build-id. */
1073 if (c->solib_pathname && path && strcmp (path, c->solib_pathname))
1074 return 0;
1075 if (c->solib_build_id_len > 0 && !__verify_build_id(task,
1076 addr - offset + c->solib_build_id_vaddr,
1077 c->solib_build_id,
1078 c->solib_build_id_len))
1079 return 0;
1080 }
1081
1082 // If we made it this far, we have an interesting solib.
1083
1084 dbug_uprobes("mmap_found pid=%ld path=%s addr=0x%lx length=%lu offset=%lu flags=0x%lx known=%d\n",
1085 (long) task->tgid, path, addr, length, offset, vm_flags, known_mapping_p);
1086
1087 if (! known_mapping_p) {
1088 // OK, let's add it. The first mapping should be a VM_READ mapping
1089 // of the entire solib file, which will also serve as the apprx.
1090 // outer bounds of the repeatedly-mapped segments.
1091
1092 #if 0
1093 // Consider an assumption about the dlopen/mmap sequence
1094 // If it comes out of sequence, we could get length/base wrong in the stored
1095 // stapiu_process, which could lead us to miscalculate semaphore addresses.
1096 //
1097 // However, this has been observed on task-finder initial-enumeration case,
1098 // (sdt_misc.exp, where a solib test is already running when stap starts).
1099 if (offset != 0)
1100 return 0;
1101 #endif
1102
1103 // Associate this consumer with this process. If we encounter
1104 // resource problems here, we don't really have to undo the uprobe
1105 // registrations etc. already in effect. It may break correct
1106 // tracking of process hierarchy in -c/-x operation, but too bad.
1107 p = _stp_kzalloc(sizeof(struct stapiu_process));
1108 if (p) {
1109 p->tgid = task->tgid;
1110 p->relocation = addr;
1111 p->inode = real_inode;
1112 p->base = addr-offset; // ... in case caught this during the second mmap
1113 spin_lock_irqsave (&c->process_list_lock, flags);
1114 list_add(&p->process_list, &c->process_list_head);
1115 spin_unlock_irqrestore (&c->process_list_lock, flags);
1116 } else
1117 _stp_warn("out of memory tracking solib %s in process %ld\n",
1118 path, (long) task->tgid);
1119 }
1120
1121 /* Check non-writable, executable sections for probes. */
1122 if ((vm_flags & VM_EXEC) && !(vm_flags & VM_WRITE))
1123 rc = stapiu_change_plus(c, task, addr, length,
1124 offset, vm_flags, real_inode);
1125
1126 /* Check writeable sections for semaphores.
1127 * NB: They may have also been executable for the check above,
1128 * if we're running a kernel that lacks noexec mappings.
1129 * So long as there's no error (rc == 0), we need to look
1130 * for semaphores too.
1131 */
1132
1133 if ((rc == 0) && (vm_flags & VM_WRITE))
1134 rc = stapiu_change_semaphore_plus(c, task, addr, real_inode);
1135
1136 return rc;
1137 }
1138
1139
1140 /* The task_finder_munmap_callback */
1141 static int
1142 stapiu_munmap_found(struct stap_task_finder_target *tf_target,
1143 struct task_struct *task,
1144 unsigned long addr, unsigned long length)
1145 {
1146 struct stapiu_consumer *c =
1147 container_of(tf_target, struct stapiu_consumer, finder);
1148
1149 return stapiu_change_minus(c, task, addr, length);
1150 }
1151
1152
1153 /* The task_finder_callback we use for ET_DYN targets.
1154 * This just forces an unmap of everything as the process exits. (PR11151)
1155 */
1156 static int
1157 stapiu_process_munmap(struct stap_task_finder_target *tf_target,
1158 struct task_struct *task,
1159 int register_p, int process_p)
1160 {
1161 struct stapiu_consumer *c =
1162 container_of(tf_target, struct stapiu_consumer, finder);
1163
1164 if (!process_p)
1165 return 0; /* ignore threads */
1166
1167 /* Covering 0->TASK_SIZE means "unmap everything" */
1168 if (!register_p)
1169 return stapiu_change_minus(c, task, 0, TASK_SIZE);
1170 return 0;
1171 }
1172
1173
1174 #endif /* _UPROBES_INODE_C_ */