Debian Codegraph @

runtime/linux/task_finder2.c

raw
1 #ifndef TASK_FINDER2_C
2 #define TASK_FINDER2_C
3
4 #include "stp_utrace.c"
5
6 #include <linux/list.h>
7 #include <linux/binfmts.h>
8 #include <linux/mount.h>
9 #include "stap_mmap_lock.h"
10 #ifndef STAPCONF_TASK_UID
11 #include <linux/cred.h>
12 #endif
13 #include "../uidgid_compatibility.h"
14 #include "syscall.h"
15 #include "task_finder_map.c"
16 #include "task_finder_vma.c"
17 #include "linux/stap_fs.h"
18
19 #ifndef VMA_ITERATOR
20 #define VMA_ITERATOR(name, mm, addr) \
21 struct mm_struct *name = mm
22 #define for_each_vma(vmi, vma) \
23 for (vma = vmi->mmap; vma; vma = vma->vm_next)
24 #endif
25
26 static LIST_HEAD(__stp_task_finder_list);
27
28 struct stap_task_finder_target;
29
30 #define __STP_TF_UNITIALIZED 0
31 #define __STP_TF_STARTING 1
32 #define __STP_TF_RUNNING 2
33 #define __STP_TF_STOPPING 3
34 #define __STP_TF_STOPPED 4
35 static atomic_t __stp_task_finder_state = ATOMIC_INIT(__STP_TF_UNITIALIZED);
36 static atomic_t __stp_task_finder_complete = ATOMIC_INIT(0);
37 static atomic_t __stp_inuse_count = ATOMIC_INIT (0);
38
39 #define __stp_tf_handler_start() (atomic_inc(&__stp_inuse_count))
40 #define __stp_tf_handler_end() (atomic_dec(&__stp_inuse_count))
41
42 #ifdef DEBUG_TASK_FINDER
43 static atomic_t __stp_attach_count = ATOMIC_INIT (0);
44
45 #define debug_task_finder_attach() (atomic_inc(&__stp_attach_count))
46 #define debug_task_finder_detach() (atomic_dec(&__stp_attach_count))
47 #define debug_task_finder_report() \
48 dbug_task(1, "attach count: %d, inuse count: %d\n", \
49 atomic_read(&__stp_attach_count), \
50 atomic_read(&__stp_inuse_count))
51 #else
52 #define debug_task_finder_attach() /* empty */
53 #define debug_task_finder_detach() /* empty */
54 #define debug_task_finder_report() /* empty */
55 #endif /* !DEBUG_TASK_FINDER */
56
57 typedef int (*stap_task_finder_callback)(struct stap_task_finder_target *tgt,
58 struct task_struct *tsk,
59 int register_p,
60 int process_p);
61
62 typedef int
63 (*stap_task_finder_mmap_callback)(struct stap_task_finder_target *tgt,
64 struct task_struct *tsk,
65 char *path,
66 struct dentry *dentry,
67 unsigned long addr,
68 unsigned long length,
69 unsigned long offset,
70 unsigned long vm_flags);
71 typedef int
72 (*stap_task_finder_munmap_callback)(struct stap_task_finder_target *tgt,
73 struct task_struct *tsk,
74 unsigned long addr,
75 unsigned long length);
76
77 typedef int
78 (*stap_task_finder_mprotect_callback)(struct stap_task_finder_target *tgt,
79 struct task_struct *tsk,
80 unsigned long addr,
81 unsigned long length,
82 int prot);
83
84 struct stap_task_finder_target {
85 /* private: */
86 struct list_head list; /* __stp_task_finder_list linkage */
87 struct list_head callback_list_head;
88 struct list_head callback_list;
89 struct utrace_engine_ops ops;
90 size_t pathlen;
91 unsigned engine_attached:1;
92 unsigned mmap_events:1;
93 unsigned munmap_events:1;
94 unsigned mprotect_events:1;
95 unsigned new_procname_p:1; /* set to true when a new procname
96 buffer is dynamically allocated
97 (like in
98 stap_register_task_finder_target) */
99
100 /* public: */
101 pid_t pid;
102 int build_id_len;
103 uint64_t build_id_vaddr;
104 unsigned char *build_id;
105 const char *procname;
106 const char *purpose;
107 stap_task_finder_callback callback;
108 stap_task_finder_mmap_callback mmap_callback;
109 stap_task_finder_munmap_callback munmap_callback;
110 stap_task_finder_mprotect_callback mprotect_callback;
111 };
112
113 static LIST_HEAD(__stp_tf_task_work_list);
114 static STP_DEFINE_SPINLOCK(__stp_tf_task_work_list_lock);
115 struct __stp_tf_task_work {
116 struct list_head list;
117 struct task_struct *task;
118 void *data;
119 struct task_work work;
120 task_work_func_t func;
121 };
122
123 static void
124 __stp_tf_task_work_free(struct task_work *work)
125 {
126 struct __stp_tf_task_work *tf_work =
127 container_of(work, typeof(*tf_work), work);
128
129 _stp_kfree(tf_work);
130 }
131
132 static void
133 __stp_tf_task_worker_fn(struct task_work *work)
134 {
135 /* Go through all the queued task workers and dequeue them in order */
136 while (1) {
137 struct __stp_tf_task_work *entry, *tf_work = NULL;
138 unsigned long flags;
139
140 /* Search for task workers for this task */
141 stp_spin_lock_irqsave(&__stp_tf_task_work_list_lock, flags);
142 list_for_each_entry(entry, &__stp_tf_task_work_list, list) {
143 if (entry->task == current) {
144 tf_work = entry;
145 list_del(&tf_work->list);
146 break;
147 }
148 }
149 stp_spin_unlock_irqrestore(&__stp_tf_task_work_list_lock, flags);
150
151 /* No more workers to execute for this task */
152 if (!tf_work)
153 break;
154
155 /* Run the task worker outside the spin lock so it can sleep */
156 tf_work->func(&tf_work->work);
157 }
158
159 /*
160 * Free the tf_work associated with this task work. It's guaranteed to
161 * not be on the task work list anymore because we drained all of the
162 * workers for this task.
163 */
164 __stp_tf_task_work_free(work);
165 stp_task_work_func_done();
166 }
167
168 static void
169 __stp_tf_init_task_work(struct task_work *work, task_work_func_t func)
170 {
171 struct __stp_tf_task_work *tf_work =
172 container_of(work, typeof(*tf_work), work);
173
174 tf_work->func = func;
175 stp_init_task_work(work, __stp_tf_task_worker_fn);
176 }
177
178 static int
179 __stp_tf_task_work_add(struct task_struct *task, struct task_work *work)
180 {
181 struct __stp_tf_task_work *tf_work =
182 container_of(work, typeof(*tf_work), work);
183 unsigned long flags;
184 int rc;
185
186 /* Associate this tf_work with the target task it'll run within */
187 tf_work->task = task;
188
189 /*
190 * Queue the task worker onto the list. Once the tf_work is added to the
191 * list, the pointer is only safe while the list lock is held. So we
192 * need to add the task work while we're still holding the list lock.
193 */
194 stp_spin_lock_irqsave(&__stp_tf_task_work_list_lock, flags);
195 list_add_tail(&tf_work->list, &__stp_tf_task_work_list);
196 rc = stp_task_work_add(task, work);
197 if (rc)
198 list_del(&tf_work->list);
199 stp_spin_unlock_irqrestore(&__stp_tf_task_work_list_lock, flags);
200
201 return rc;
202 }
203
204 /*
205 * Allocate a 'struct task_work' for use. Internally keeps track of
206 * allocated structs for use when shutting down.
207 *
208 * Returns NULL in the case of a memory allocation failure.
209 *
210 * Note that it remembers the current task, so if we need to allocate
211 * a 'struct task_work' for a task that isn't current, we'll need a
212 * __stp_tf_alloc_task_work_for_task(task) variant.
213 */
214 static struct task_work *
215 __stp_tf_alloc_task_work(void *data)
216 {
217 struct __stp_tf_task_work *tf_work;
218 unsigned long flags;
219
220 tf_work = _stp_kmalloc(sizeof(*tf_work));
221 if (tf_work == NULL) {
222 _stp_error("Unable to allocate space for task_work");
223 return NULL;
224 }
225
226 tf_work->data = data;
227
228 return &tf_work->work;
229 }
230
231 /*
232 * Cancel (and free) all outstanding task work requests.
233 */
234 static void __stp_tf_cancel_all_task_work(void)
235 {
236 struct __stp_tf_task_work *node, *tmp;
237 unsigned long flags;
238
239 // Cancel all remaining requests.
240 stp_spin_lock_irqsave(&__stp_tf_task_work_list_lock, flags);
241 restart:
242 list_for_each_entry_safe(node, tmp, &__stp_tf_task_work_list, list) {
243 struct __stp_tf_task_work *tf_work;
244 struct task_work *work;
245
246 work = stp_task_work_cancel(node->task, node->work.func);
247 if (!work)
248 continue;
249
250 /*
251 * There can be multiple queued task workers with the same
252 * worker func, so there's no guarantee that tf_work == node.
253 * Therefore, we can only free what stp_task_work_cancel() just
254 * gave us; freeing 'node' would be unsafe.
255 */
256 tf_work = container_of(work, typeof(*tf_work), work);
257 list_del(&tf_work->list);
258 _stp_kfree(tf_work);
259
260 /*
261 * If the tf_work we just freed was the next node in the list,
262 * then we need to restart the list iteration because
263 * list_for_each_entry_safe() can't cope with the next node
264 * being freed. We still need to use list_for_each_entry_safe()
265 * because we need to get through one successful pass through
266 * the entire list, since it's not guaranteed that this list
267 * will be empty when this function exits, as there can still be
268 * active task workers running, which is fine since the
269 * stp_task_work API will wait for all task workers to finish
270 * before allowing the module to unload.
271 */
272 if (tf_work == tmp)
273 goto restart;
274 }
275 stp_spin_unlock_irqrestore(&__stp_tf_task_work_list_lock, flags);
276 }
277
278 static u32
279 __stp_utrace_task_finder_target_exec(u32 action,
280 struct utrace_engine *engine,
281 const struct linux_binfmt *fmt,
282 const struct linux_binprm *bprm,
283 struct pt_regs *regs);
284
285 static u32
286 __stp_utrace_task_finder_target_death(struct utrace_engine *engine,
287 bool group_dead, int signal);
288
289 static u32
290 __stp_utrace_task_finder_target_quiesce(u32 action,
291 struct utrace_engine *engine,
292 unsigned long event);
293
294 static u32
295 __stp_utrace_task_finder_target_syscall_entry(u32 action,
296 struct utrace_engine *engine,
297 struct pt_regs *regs);
298
299 static u32
300 __stp_utrace_task_finder_target_syscall_exit(u32 action,
301 struct utrace_engine *engine,
302 struct pt_regs *regs);
303
304 static void
305 __stp_call_mmap_callbacks_for_task(struct stap_task_finder_target *tgt,
306 struct task_struct *tsk);
307
308 static void
309 stap_cleanup_task_finder_target(struct stap_task_finder_target *tgt)
310 {
311 if (tgt->new_procname_p) {
312 _stp_kfree((void *) tgt->procname);
313 tgt->procname = "<freed>";
314 tgt->new_procname_p = 0;
315 }
316 }
317
318 static int
319 stap_register_task_finder_target(struct stap_task_finder_target *new_tgt)
320 {
321 // Since this __stp_task_finder_list is (currently) only
322 // written to in one big setup operation before the task
323 // finder process is started, we don't need to lock it.
324 struct list_head *node;
325 struct stap_task_finder_target *tgt = NULL;
326 int found_node = 0;
327
328 if (atomic_read(&__stp_task_finder_state) != __STP_TF_UNITIALIZED) {
329 _stp_error("task_finder already started, no new targets allowed");
330 return EBUSY;
331 }
332
333 if (new_tgt == NULL)
334 return EFAULT;
335
336 if (new_tgt->procname != NULL) {
337 int ret;
338 char *p;
339 size_t len;
340 const char *pathname = new_tgt->procname;
341
342 p = stap_real_path(pathname, &len);
343 if (p != NULL) {
344 if (unlikely(IS_ERR(p)))
345 return -PTR_ERR(p);
346
347 /* stap_real_path() allocates a new string buffer in
348 * p. */
349
350 if (unlikely(new_tgt->new_procname_p))
351 _stp_kfree((void *) new_tgt->procname);
352 else
353 new_tgt->new_procname_p = 1;
354
355 /* NB: this buffer will be freed by
356 * stap_cleanup_task_finder_target() */
357 new_tgt->procname = (const char *) p;
358 new_tgt->pathlen = len;
359
360 } else {
361 new_tgt->pathlen = strlen(pathname);
362 }
363
364 } else
365 new_tgt->pathlen = 0;
366
367 // Make sure everything is initialized properly.
368 new_tgt->engine_attached = 0;
369 new_tgt->mmap_events = 0;
370 new_tgt->munmap_events = 0;
371 new_tgt->mprotect_events = 0;
372 memset(&new_tgt->ops, 0, sizeof(new_tgt->ops));
373 new_tgt->ops.report_exec = &__stp_utrace_task_finder_target_exec;
374 new_tgt->ops.report_death = &__stp_utrace_task_finder_target_death;
375 new_tgt->ops.report_quiesce = &__stp_utrace_task_finder_target_quiesce;
376 new_tgt->ops.report_syscall_entry = \
377 &__stp_utrace_task_finder_target_syscall_entry;
378 new_tgt->ops.report_syscall_exit = \
379 &__stp_utrace_task_finder_target_syscall_exit;
380
381 // Search the list for an existing entry for procname/pid/build-id.
382 list_for_each(node, &__stp_task_finder_list) {
383 tgt = list_entry(node, struct stap_task_finder_target, list);
384 if (tgt == new_tgt) {
385 _stp_error("target already registered");
386 return EINVAL;
387 }
388 if (tgt != NULL
389 /* procname-based target */
390 && ((new_tgt->pathlen > 0
391 && tgt->pathlen == new_tgt->pathlen
392 && strcmp(tgt->procname, new_tgt->procname) == 0)
393 /* pid-based target (a specific pid or all
394 * pids) */
395 || (new_tgt->pathlen == 0 && tgt->pathlen == 0
396 && tgt->pid == new_tgt->pid)
397 /* buildid-based target */
398 || (new_tgt->build_id_len > 0
399 && new_tgt->build_id_len == tgt->build_id_len
400 && strcmp(new_tgt->build_id, tgt->build_id) == 0))) {
401 found_node = 1;
402 break;
403 }
404 }
405
406 // If we didn't find a matching existing entry, add the new
407 // target to the task list.
408 if (! found_node) {
409 INIT_LIST_HEAD(&new_tgt->callback_list_head);
410 list_add_tail(&new_tgt->list, &__stp_task_finder_list);
411 tgt = new_tgt;
412 }
413
414 // Add this target to the callback list for this task.
415 list_add_tail(&new_tgt->callback_list, &tgt->callback_list_head);
416
417 // If the new target has any m* callbacks, remember this.
418 if (new_tgt->mmap_callback != NULL)
419 tgt->mmap_events = 1;
420 if (new_tgt->munmap_callback != NULL)
421 tgt->munmap_events = 1;
422 if (new_tgt->mprotect_callback != NULL)
423 tgt->mprotect_events = 1;
424 return 0;
425 }
426
427 static int
428 stap_utrace_detach(struct task_struct *tsk,
429 const struct utrace_engine_ops *ops)
430 {
431 struct utrace_engine *engine;
432 struct mm_struct *mm;
433 int rc = 0;
434
435 // Ignore invalid tasks.
436 if (tsk == NULL || tsk->pid <= 0)
437 return 0;
438
439 #ifdef PF_KTHREAD
440 // Ignore kernel threads. On systems without PF_KTHREAD,
441 // we're ok, since kernel threads won't be matched by the
442 // utrace_attach_task() call below.
443 if (tsk->flags & PF_KTHREAD)
444 return 0;
445 #endif
446
447 // Notice we're not calling get_task_mm() here. Normally we
448 // avoid tasks with no mm, because those are kernel threads.
449 // So, why is this function different? When a thread is in
450 // the process of dying, its mm gets freed. Then, later the
451 // thread gets in the dying state and the thread's DEATH event
452 // handler gets called (if any).
453 //
454 // If a thread is in this "mortally wounded" state - no mm
455 // but not dead - and at that moment this function is called,
456 // we'd miss detaching from it if we were checking to see if
457 // it had an mm.
458
459 engine = utrace_attach_task(tsk, UTRACE_ATTACH_MATCH_OPS, ops, 0);
460 if (IS_ERR(engine)) {
461 rc = -PTR_ERR(engine);
462 if (rc != ENOENT) {
463 _stp_error("utrace_attach_task returned error %d on pid %d",
464 rc, tsk->pid);
465 }
466 else {
467 rc = 0;
468 }
469 }
470 else if (unlikely(engine == NULL)) {
471 _stp_error("utrace_attach returned NULL on pid %d",
472 (int)tsk->pid);
473 rc = EFAULT;
474 }
475 else {
476 rc = utrace_control(tsk, engine, UTRACE_DETACH);
477 switch (rc) {
478 case 0: /* success */
479 debug_task_finder_detach();
480 break;
481 case -ESRCH: /* REAP callback already begun */
482 case -EALREADY: /* DEATH callback already begun */
483 rc = 0; /* ignore these errors */
484 break;
485 case -EINPROGRESS:
486 do {
487 rc = utrace_barrier(tsk, engine);
488 } while (rc == -ERESTARTSYS);
489 if (rc == 0 || rc == -ESRCH || rc == -EALREADY) {
490 rc = 0;
491 debug_task_finder_detach();
492 } else {
493 rc = -rc;
494 _stp_error("utrace_barrier returned error %d on pid %d", rc, tsk->pid);
495 }
496 break;
497 default:
498 rc = -rc;
499 _stp_error("utrace_control returned error %d on pid %d",
500 rc, tsk->pid);
501 break;
502 }
503 utrace_engine_put(engine);
504 }
505 return rc;
506 }
507
508 static char *
509 __stp_get_mm_path(struct mm_struct *mm, char *buf, int buflen)
510 {
511 struct file *vm_file = stap_find_exe_file(mm);
512 char *rc = NULL;
513
514 if (vm_file) {
515 #ifdef STAPCONF_DPATH_PATH
516 rc = d_path(&(vm_file->f_path), buf, buflen);
517 #else
518 rc = d_path(vm_file->f_dentry, vm_file->f_vfsmnt,
519 buf, buflen);
520 #endif
521 fput(vm_file);
522 }
523 else {
524 *buf = '\0';
525 rc = ERR_PTR(-ENOENT);
526 }
527 return rc;
528 }
529
530 /*
531 * All user threads get an engine with __STP_TASK_FINDER_EVENTS events
532 * attached to it so the task_finder layer can monitor new thread
533 * creation/death.
534 */
535 #define __STP_TASK_FINDER_EVENTS (UTRACE_EVENT(CLONE) \
536 | UTRACE_EVENT(EXEC) \
537 | UTRACE_EVENT(DEATH))
538
539 /*
540 * __STP_TASK_BASE_EVENTS: base events for stap_task_finder_target's
541 * without map callback's
542 *
543 * __STP_TASK_VM_BASE_EVENTS: base events for
544 * stap_task_finder_target's with map callback's
545 */
546 #define __STP_TASK_BASE_EVENTS (UTRACE_EVENT(DEATH)|UTRACE_EVENT(EXEC))
547
548 #define __STP_TASK_VM_BASE_EVENTS (__STP_TASK_BASE_EVENTS \
549 | UTRACE_EVENT(SYSCALL_ENTRY)\
550 | UTRACE_EVENT(SYSCALL_EXIT))
551
552 /*
553 * All "interesting" threads get an engine with
554 * __STP_ATTACHED_TASK_EVENTS events attached to it. After the thread
555 * quiesces, we reset the events to __STP_ATTACHED_TASK_BASE_EVENTS
556 * events.
557 */
558 #define __STP_ATTACHED_TASK_EVENTS (UTRACE_EVENT(DEATH) \
559 | UTRACE_EVENT(QUIESCE))
560
561 #define __STP_ATTACHED_TASK_BASE_EVENTS(tgt) \
562 (((tgt)->mmap_events || (tgt)->munmap_events \
563 || (tgt)->mprotect_events) \
564 ? __STP_TASK_VM_BASE_EVENTS : __STP_TASK_BASE_EVENTS)
565
566 static int
567 __stp_utrace_attach(struct task_struct *tsk,
568 const struct utrace_engine_ops *ops, void *data,
569 unsigned long event_flags,
570 enum utrace_resume_action action)
571 {
572 struct utrace_engine *engine;
573 int rc = 0;
574
575 // Ignore invalid tasks.
576 if (tsk == NULL || tsk->pid <= 0)
577 return EPERM;
578
579 #ifdef PF_KTHREAD
580 // Ignore kernel threads
581 if (tsk->flags & PF_KTHREAD)
582 return EPERM;
583 #endif
584
585 // Ignore threads with no mm (which are either kernel threads
586 // or "mortally wounded" threads).
587 //
588 // Note we're not calling get_task_mm()/mmput() here. Since
589 // we're in the the context of that task, the mm should stick
590 // around without locking it (and mmput() can sleep).
591 if (! tsk->mm)
592 return EPERM;
593
594 engine = utrace_attach_task(tsk, UTRACE_ATTACH_CREATE, ops, data);
595 if (IS_ERR(engine)) {
596 int error = -PTR_ERR(engine);
597 if (error != ESRCH && error != ENOENT) {
598 _stp_error("utrace_attach returned error %d on pid %d",
599 error, (int)tsk->pid);
600 rc = error;
601 }
602 }
603 else if (unlikely(engine == NULL)) {
604 _stp_error("utrace_attach returned NULL on pid %d",
605 (int)tsk->pid);
606 rc = EFAULT;
607 }
608 else {
609 rc = utrace_set_events(tsk, engine, event_flags);
610 dbug_task(2, "utrace_set_events(%lu) returned %d", event_flags,
611 rc);
612 if (rc == -EINPROGRESS) {
613 /*
614 * It's running our callback, so we have to
615 * synchronize. We can't keep rcu_read_lock,
616 * so the task pointer might die. But it's
617 * safe to call utrace_barrier() even with a
618 * stale task pointer, if we have an engine
619 * ref.
620 */
621 do {
622 rc = utrace_barrier(tsk, engine);
623 } while (rc == -ERESTARTSYS);
624 if (rc != 0 && rc != -ESRCH && rc != -EALREADY)
625 _stp_error("utrace_barrier returned error %d on pid %d",
626 rc, (int)tsk->pid);
627 }
628 if (rc == 0) {
629 debug_task_finder_attach();
630
631 if (action != UTRACE_RESUME) {
632 rc = utrace_control(tsk, engine, action);
633 dbug_task(2, "utrace_control(%d) returned %d", action, rc);
634 /* If utrace_control() returns
635 * EINPROGRESS when we're trying to
636 * stop/interrupt, that means the task
637 * hasn't stopped quite yet, but will
638 * soon. Ignore this error. */
639 if (rc != 0 && rc != -EINPROGRESS) {
640 _stp_error("utrace_control returned error %d on pid %d",
641 rc, (int)tsk->pid);
642 }
643 rc = 0;
644 }
645 }
646 else if (rc != -ESRCH && rc != -EALREADY)
647 _stp_error("utrace_set_events2 returned error %d on pid %d",
648 rc, (int)tsk->pid);
649 utrace_engine_put(engine);
650 }
651 return rc;
652 }
653
654 static int
655 stap_utrace_attach(struct task_struct *tsk,
656 const struct utrace_engine_ops *ops, void *data,
657 unsigned long event_flags)
658 {
659 return __stp_utrace_attach(tsk, ops, data, event_flags, UTRACE_RESUME);
660 }
661
662 static inline void
663 __stp_call_callbacks(struct stap_task_finder_target *tgt,
664 struct task_struct *tsk, int register_p, int process_p)
665 {
666 struct list_head *cb_node;
667 int rc;
668
669 dbug_task(2, "entering tgt=%p tsk=%p pid=%d", tgt, tsk, tsk ? tsk->tgid : -1);
670
671 if (tgt == NULL || tsk == NULL)
672 return;
673
674 list_for_each(cb_node, &tgt->callback_list_head) {
675 struct stap_task_finder_target *cb_tgt;
676
677 cb_tgt = list_entry(cb_node, struct stap_task_finder_target,
678 callback_list);
679 if (cb_tgt == NULL || cb_tgt->callback == NULL)
680 continue;
681
682 rc = cb_tgt->callback(cb_tgt, tsk, register_p, process_p);
683
684 dbug_task(1, "tgt %s callback returned %d (proc=%s pid=%d, "
685 "pathlen=%d, engine-attached=%d)",
686 (cb_tgt->purpose?:""), rc, cb_tgt->procname,
687 cb_tgt->pid, (int) cb_tgt->pathlen,
688 cb_tgt->engine_attached);
689
690 if (rc != 0) {
691 _stp_warn("task_finder %s%scallback for task %d failed: %d",
692 (cb_tgt->purpose?:""), (cb_tgt->purpose?" ":""),
693 (int)tsk->pid, rc);
694 }
695 }
696 }
697
698 bool
699 __verify_build_id(struct task_struct *tsk, unsigned long addr,
700 unsigned const char *build_id, int build_id_len);
701
702 bool
703 __verify_build_id(struct task_struct *tsk, unsigned long addr,
704 unsigned const char *build_id, int build_id_len)
705 {
706 #define MAX_HEXSTR_LEN 64
707 int i;
708 unsigned char tsk_build_id[MAX_HEXSTR_LEN + 1];
709
710 if (build_id_len > MAX_HEXSTR_LEN)
711 return false;
712
713 for (i = 0; i < build_id_len / 2; i++) {
714 int rc;
715 unsigned char b;
716
717 if ((rc = __access_process_vm_noflush(tsk, addr + i, &b, 1, 0)) != 1)
718 return false;
719
720 tsk_build_id[i * 2] = "0123456789abcdef"[b >> 4];
721 tsk_build_id[i * 2 + 1] = "0123456789abcdef"[b & 0xf];
722 }
723 tsk_build_id[build_id_len] = '\0';
724
725 if (strcmp(build_id, tsk_build_id)) {
726 dbug_task(2, "target build-id not matched: [%s] @ 0x%lx != [%s]\n",
727 build_id, addr, tsk_build_id);
728 return false;
729 }
730
731 return true;
732 }
733
734 static void
735 __stp_call_mmap_callbacks(struct stap_task_finder_target *tgt,
736 struct task_struct *tsk, char *path,
737 struct dentry *dentry,
738 unsigned long addr, unsigned long length,
739 unsigned long offset, unsigned long vm_flags)
740 {
741 struct list_head *cb_node;
742 int rc;
743
744 if (tgt == NULL || tsk == NULL)
745 return;
746
747 dbug_task_vma(1,
748 "pid %d, a/l/o/p/path 0x%lx 0x%lx 0x%lx %c%c%c%c %s\n",
749 tsk->pid, addr, length, offset,
750 vm_flags & VM_READ ? 'r' : '-',
751 vm_flags & VM_WRITE ? 'w' : '-',
752 vm_flags & VM_EXEC ? 'x' : '-',
753 vm_flags & VM_MAYSHARE ? 's' : 'p',
754 path);
755 list_for_each(cb_node, &tgt->callback_list_head) {
756 struct stap_task_finder_target *cb_tgt;
757
758 cb_tgt = list_entry(cb_node, struct stap_task_finder_target,
759 callback_list);
760 if (cb_tgt == NULL || cb_tgt->mmap_callback == NULL)
761 continue;
762
763 rc = cb_tgt->mmap_callback(cb_tgt, tsk, path, dentry,
764 addr, length, offset, vm_flags);
765 if (rc != 0) {
766 _stp_warn("task_finder mmap %s%scallback for task %d failed: %d",
767 (cb_tgt->purpose?:""), (cb_tgt->purpose?" ":""),
768 (int)tsk->pid, rc);
769 }
770 }
771 }
772
773
774 static struct vm_area_struct *
775 __stp_find_file_based_vma(struct mm_struct *mm, unsigned long addr)
776 {
777 struct vm_area_struct *vma = find_vma(mm, addr);
778
779 // I'm not positive why the checking for vm_start > addr is
780 // necessary, but it seems to be (sometimes find_vma() returns
781 // a vma that addr doesn't belong to).
782 if (vma && (vma->vm_file == NULL || vma->vm_start > addr))
783 vma = NULL;
784 return vma;
785 }
786
787
788 static void
789 __stp_call_mmap_callbacks_with_addr(struct stap_task_finder_target *tgt,
790 struct task_struct *tsk,
791 unsigned long addr)
792 {
793 struct mm_struct *mm;
794 struct vm_area_struct *vma;
795 char *mmpath_buf = NULL;
796 char *mmpath = NULL;
797 struct dentry *dentry = NULL;
798 unsigned long length = 0;
799 unsigned long offset = 0;
800 unsigned long vm_flags = 0;
801
802 // __stp_call_mmap_callbacks_with_addr() is only called when
803 // tsk is current, so there isn't any danger of mm going
804 // away. So, we don't need to call get_task_mm()/mmput()
805 // (which avoids the possibility of sleeping).
806 mm = tsk->mm;
807 if (! mm)
808 return;
809
810 // The down_read() function can sleep, so we'll call
811 // down_read_trylock() instead, which can fail.
812 if (! mmap_read_trylock(mm))
813 return;
814 vma = __stp_find_file_based_vma(mm, addr);
815 if (vma) {
816 // Cache information we need from the vma
817 addr = vma->vm_start;
818 length = vma->vm_end - vma->vm_start;
819 offset = (vma->vm_pgoff << PAGE_SHIFT);
820 vm_flags = vma->vm_flags;
821 #ifdef STAPCONF_DPATH_PATH
822 dentry = vma->vm_file->f_path.dentry;
823 #else
824 dentry = vma->vm_file->f_dentry;
825 #endif
826
827 // Allocate space for a path
828 mmpath_buf = _stp_kmalloc(PATH_MAX);
829 if (mmpath_buf == NULL) {
830 mmap_read_unlock(mm);
831 _stp_error("Unable to allocate space for path");
832 return;
833 }
834 else {
835 // Grab the path associated with this vma.
836 #ifdef STAPCONF_DPATH_PATH
837 mmpath = d_path(&(vma->vm_file->f_path), mmpath_buf,
838 PATH_MAX);
839 #else
840 mmpath = d_path(vma->vm_file->f_dentry,
841 vma->vm_file->f_vfsmnt, mmpath_buf,
842 PATH_MAX);
843 #endif
844 if (mmpath == NULL || IS_ERR(mmpath)) {
845 long err = ((mmpath == NULL) ? 0
846 : -PTR_ERR(mmpath));
847 _stp_error("Unable to get path (error %ld) for pid %d",
848 err, (int)tsk->pid);
849 mmpath = NULL;
850 }
851 }
852 }
853
854 // At this point, we're done with the vma (assuming we found
855 // one). We can't hold the 'mmap_sem' semaphore while making
856 // callbacks.
857 mmap_read_unlock(mm);
858
859 if (mmpath)
860 __stp_call_mmap_callbacks(tgt, tsk, mmpath, dentry, addr,
861 length, offset, vm_flags);
862
863 // Cleanup.
864 if (mmpath_buf)
865 _stp_kfree(mmpath_buf);
866 return;
867 }
868
869
870 static inline void
871 __stp_call_munmap_callbacks(struct stap_task_finder_target *tgt,
872 struct task_struct *tsk, unsigned long addr,
873 unsigned long length)
874 {
875 struct list_head *cb_node;
876 int rc;
877
878 if (tgt == NULL || tsk == NULL)
879 return;
880
881 list_for_each(cb_node, &tgt->callback_list_head) {
882 struct stap_task_finder_target *cb_tgt;
883
884 cb_tgt = list_entry(cb_node, struct stap_task_finder_target,
885 callback_list);
886 if (cb_tgt == NULL || cb_tgt->munmap_callback == NULL)
887 continue;
888
889 rc = cb_tgt->munmap_callback(cb_tgt, tsk, addr, length);
890 if (rc != 0) {
891 _stp_warn("task_finder munmap %s%scallback for task %d failed: %d",
892 (cb_tgt->purpose?:""), (cb_tgt->purpose?" ":""),
893 (int)tsk->pid, rc);
894 }
895 }
896 }
897
898 static inline void
899 __stp_call_mprotect_callbacks(struct stap_task_finder_target *tgt,
900 struct task_struct *tsk, unsigned long addr,
901 unsigned long length, int prot)
902 {
903 struct list_head *cb_node;
904 int rc;
905
906 if (tgt == NULL || tsk == NULL)
907 return;
908
909 list_for_each(cb_node, &tgt->callback_list_head) {
910 struct stap_task_finder_target *cb_tgt;
911
912 cb_tgt = list_entry(cb_node, struct stap_task_finder_target,
913 callback_list);
914 if (cb_tgt == NULL || cb_tgt->mprotect_callback == NULL)
915 continue;
916
917 rc = cb_tgt->mprotect_callback(cb_tgt, tsk, addr, length,
918 prot);
919 if (rc != 0) {
920 _stp_warn("task_finder mprotect %s%scallback for task %d failed: %d",
921 (cb_tgt->purpose?:""), (cb_tgt->purpose?" ":""),
922 (int)tsk->pid, rc);
923 }
924 }
925 }
926
927 static inline void
928 __stp_utrace_attach_match_filename(struct task_struct *tsk,
929 const char * const filename,
930 int process_p)
931 {
932 size_t filelen;
933 struct list_head *tgt_node;
934 struct stap_task_finder_target *tgt;
935 uid_t tsk_euid;
936
937 #ifdef STAPCONF_TASK_UID
938 tsk_euid = tsk->euid;
939 #else
940 #if defined(CONFIG_USER_NS) || (LINUX_VERSION_CODE >= KERNEL_VERSION(3,14,0))
941 tsk_euid = from_kuid_munged(current_user_ns(), task_euid(tsk));
942 #else
943 tsk_euid = task_euid(tsk);
944 #endif
945 #endif
946 filelen = strlen(filename);
947 list_for_each(tgt_node, &__stp_task_finder_list) {
948 int rc;
949
950 tgt = list_entry(tgt_node, struct stap_task_finder_target,
951 list);
952 // If we've got a matching procname or a matching build-id
953 // or we're probing all threads, we've got a match. We've
954 // got to keep matching since a single thread could match a
955 // procname/build-id and match an "all thread" probe.
956 if (tgt == NULL)
957 continue;
958 /* buildid-based target ... gets checked in __stp_tf_quiesce_worker */
959 /* procname-based target */
960 else if (tgt->pathlen > 0
961 && (tgt->pathlen != filelen
962 || strcmp(tgt->procname, filename) != 0))
963 {
964 dbug_task(2, "target path NOT matched: [%s] != [%s]",
965 tgt->procname, filename);
966 continue;
967 }
968 /* Ignore pid-based target, they were handled at startup. */
969 else if (tgt->pid != 0)
970 continue;
971 /* Notice that "pid == 0" (which means to probe all
972 * threads) falls through. */
973
974 #if ! STP_PRIVILEGE_CONTAINS (STP_PRIVILEGE, STP_PR_STAPDEV) && \
975 ! STP_PRIVILEGE_CONTAINS (STP_PRIVILEGE, STP_PR_STAPSYS)
976 /* Make sure unprivileged users only probe their own threads. */
977 if (_stp_uid != tsk_euid) {
978 if (tgt->pid != 0) {
979 _stp_warn("Process %d does not belong to unprivileged user %d",
980 tsk->pid, _stp_uid);
981 }
982 continue;
983 }
984 #endif
985
986
987 // Set up events we need for attached tasks. We won't
988 // actually call the callbacks here - we'll call them
989 // when the thread gets quiesced.
990 rc = __stp_utrace_attach(tsk, &tgt->ops, tgt,
991 __STP_ATTACHED_TASK_EVENTS,
992 UTRACE_INTERRUPT);
993 if (rc != 0 && rc != EPERM)
994 break;
995 tgt->engine_attached = 1;
996 }
997 }
998
999 // This function handles the details of getting a task's associated
1000 // procname, and calling __stp_utrace_attach_match_filename() to
1001 // attach to it if we find the procname "interesting". So, what's the
1002 // difference between path_tsk and match_tsk? Normally they are the
1003 // same, except in one case. In an UTRACE_EVENT(EXEC), we need to
1004 // detach engines from the newly exec'ed process (since its path has
1005 // changed). In this case, we have to match the path of the parent
1006 // (path_tsk) against the child (match_tsk).
1007
1008 static void
1009 __stp_utrace_attach_match_tsk(struct task_struct *path_tsk,
1010 struct task_struct *match_tsk, int process_p)
1011 {
1012 struct mm_struct *mm;
1013 char *mmpath_buf;
1014 char *mmpath;
1015
1016 #if 0
1017 printk(KERN_ERR "%s:%d entry\n", __FUNCTION__, __LINE__);
1018 #endif
1019 if (path_tsk == NULL || path_tsk->pid <= 0
1020 || match_tsk == NULL || match_tsk->pid <= 0
1021 || (_stp_target && match_tsk != path_tsk))
1022 return;
1023
1024 // Grab the path associated with the path_tsk.
1025 //
1026 // Note we're not calling get_task_mm()/mmput() here. Since
1027 // we're in the the context of path_task, the mm should stick
1028 // around without locking it (and mmput() can sleep).
1029 mm = path_tsk->mm;
1030 if (! mm) {
1031 /* If the thread doesn't have a mm_struct, it is
1032 * a kernel thread which we need to skip. */
1033 return;
1034 }
1035
1036 // Allocate space for a path
1037 mmpath_buf = _stp_kmalloc(PATH_MAX);
1038 if (mmpath_buf == NULL) {
1039 _stp_error("Unable to allocate space for path");
1040 return;
1041 }
1042
1043 // Grab the path associated with the new task
1044 mmpath = __stp_get_mm_path(mm, mmpath_buf, PATH_MAX);
1045 if (mmpath == NULL || IS_ERR(mmpath)) {
1046 int rc = -PTR_ERR(mmpath);
1047 if (rc != ENOENT)
1048 _stp_error("Unable to get path (error %d) for pid %d",
1049 rc, (int)path_tsk->pid);
1050 }
1051 else {
1052 #if 0
1053 _stp_dbug(__FUNCTION__, __LINE__,
1054 "calling __stp_utrace_attach_match_filename(%p, %s, %d, %d)\n",
1055 match_tsk, mmpath, register_p, process_p);
1056 #endif
1057 __stp_utrace_attach_match_filename(match_tsk, mmpath,
1058 process_p);
1059 }
1060
1061 _stp_kfree(mmpath_buf);
1062 return;
1063 }
1064
1065 static void
1066 __stp_tf_clone_worker(struct task_work *work)
1067 {
1068 struct __stp_tf_task_work *tf_work = \
1069 container_of(work, struct __stp_tf_task_work, work);
1070 struct utrace_engine_ops *ops = \
1071 (struct utrace_engine_ops *)tf_work->data;
1072 struct task_struct *parent = tf_work->task;
1073 int rc;
1074
1075 might_sleep();
1076 if (atomic_read(&__stp_task_finder_state) != __STP_TF_RUNNING
1077 || current->flags & PF_EXITING)
1078 return;
1079
1080 __stp_tf_handler_start();
1081
1082 // On clone, attach to the child, but we might need to sleep...
1083 rc = __stp_utrace_attach(current, ops, 0,
1084 __STP_TASK_FINDER_EVENTS, UTRACE_RESUME);
1085 if (rc == 0 || rc == EPERM) {
1086 // Assume that if the thread is a thread group leader,
1087 // it is a process.
1088 __stp_utrace_attach_match_tsk(parent, current,
1089 (current->pid == current->tgid));
1090 }
1091
1092 __stp_tf_handler_end();
1093 }
1094
1095
1096 static u32
1097 __stp_utrace_task_finder_report_clone(u32 action,
1098 struct utrace_engine *engine,
1099 unsigned long clone_flags,
1100 struct task_struct *child)
1101 {
1102 int rc;
1103 struct task_work *work;
1104
1105 if (atomic_read(&__stp_task_finder_state) != __STP_TF_RUNNING) {
1106 debug_task_finder_detach();
1107 return UTRACE_DETACH;
1108 }
1109
1110 __stp_tf_handler_start();
1111
1112 // We can't sleep in tracepoint handlers.
1113 // __stp_utrace_attach() might need to call utrace_barrier(),
1114 // which can sleep when task != current. So, arrange for the
1115 // child task to truly stop.
1116 work = __stp_tf_alloc_task_work((void *)(engine->ops));
1117 if (work == NULL) {
1118 _stp_error("Unable to allocate space for task_work");
1119 __stp_tf_handler_end();
1120 return UTRACE_RESUME;
1121 }
1122 __stp_tf_init_task_work(work, &__stp_tf_clone_worker);
1123 rc = __stp_tf_task_work_add(child, work);
1124 if (rc) {
1125 __stp_tf_task_work_free(work);
1126 // stp_task_work_add() returns -ESRCH if the task has already
1127 // passed exit_task_work(). Just ignore this error.
1128 if (rc != -ESRCH)
1129 printk(KERN_ERR "%s:%d - stp_task_work_add() returned %d\n",
1130 __FUNCTION__, __LINE__, rc);
1131 }
1132
1133 __stp_tf_handler_end();
1134 return UTRACE_RESUME;
1135 }
1136
1137 static u32
1138 __stp_utrace_task_finder_report_exec(u32 action,
1139 struct utrace_engine *engine,
1140 const struct linux_binfmt *fmt,
1141 const struct linux_binprm *bprm,
1142 struct pt_regs *regs)
1143 {
1144 size_t filelen;
1145 struct list_head *tgt_node;
1146 struct stap_task_finder_target *tgt;
1147 int found_node = 0;
1148
1149 if (atomic_read(&__stp_task_finder_state) != __STP_TF_RUNNING) {
1150 debug_task_finder_detach();
1151 return UTRACE_DETACH;
1152 }
1153
1154 __stp_tf_handler_start();
1155
1156 // If the original task was "interesting",
1157 // __stp_utrace_task_finder_target_exec() will handle calling
1158 // callbacks.
1159
1160 // We assume that all exec's are exec'ing a new process. Note
1161 // that we don't use bprm->filename, since that path can be
1162 // relative.
1163 __stp_utrace_attach_match_tsk(current, current, 1);
1164
1165 __stp_tf_handler_end();
1166 return UTRACE_RESUME;
1167 }
1168
1169 static u32
1170 stap_utrace_task_finder_report_death(struct utrace_engine *engine,
1171 bool group_dead, int signal)
1172 {
1173 debug_task_finder_detach();
1174 return UTRACE_DETACH;
1175 }
1176
1177 static u32
1178 __stp_utrace_task_finder_target_exec(u32 action,
1179 struct utrace_engine *engine,
1180 const struct linux_binfmt *fmt,
1181 const struct linux_binprm *bprm,
1182 struct pt_regs *regs)
1183 {
1184 struct task_struct *tsk = current;
1185 struct stap_task_finder_target *tgt = engine->data;
1186 int rc;
1187
1188 if (atomic_read(&__stp_task_finder_state) != __STP_TF_RUNNING) {
1189 debug_task_finder_detach();
1190 return UTRACE_DETACH;
1191 }
1192
1193 __stp_tf_handler_start();
1194
1195 // We'll hardcode this as a process end. If a thread
1196 // calls exec() (which it isn't supposed to), the kernel
1197 // "promotes" it to being a process. Call the callbacks.
1198 if (tgt != NULL && tsk != NULL) {
1199 __stp_call_callbacks(tgt, tsk, 0, 1);
1200 }
1201
1202 // Note that we don't want to set engine_attached to 0 here -
1203 // only when *all* threads using this engine have been
1204 // detached.
1205
1206 // Let __stp_utrace_task_finder_report_exec() call
1207 // __stp_utrace_attach_match_tsk() to figure out if the
1208 // exec'ed program is "interesting".
1209
1210 __stp_tf_handler_end();
1211 debug_task_finder_detach();
1212 return UTRACE_DETACH;
1213 }
1214
1215 static u32
1216 __stp_utrace_task_finder_target_death(struct utrace_engine *engine,
1217 bool group_dead, int signal)
1218 {
1219 struct task_struct *tsk = current;
1220 struct stap_task_finder_target *tgt = engine->data;
1221
1222 if (atomic_read(&__stp_task_finder_state) != __STP_TF_RUNNING) {
1223 debug_task_finder_detach();
1224 return UTRACE_DETACH;
1225 }
1226
1227 __stp_tf_handler_start();
1228 // The first implementation of this added a
1229 // UTRACE_EVENT(DEATH) handler to
1230 // __stp_utrace_task_finder_ops. However, dead threads don't
1231 // have a mm_struct, so we can't find the exe's path. So, we
1232 // don't know which callback(s) to call.
1233 //
1234 // So, now when an "interesting" thread is found, we add a
1235 // separate UTRACE_EVENT(DEATH) handler for each attached
1236 // handler.
1237 if (tgt != NULL && tsk != NULL) {
1238 __stp_call_callbacks(tgt, tsk, 0,
1239 ((tsk->signal == NULL)
1240 || (atomic_read(&tsk->signal->live) == 0)));
1241 }
1242
1243 __stp_tf_handler_end();
1244 debug_task_finder_detach();
1245 return UTRACE_DETACH;
1246 }
1247
1248 static void
1249 __stp_call_mmap_callbacks_for_task(struct stap_task_finder_target *tgt,
1250 struct task_struct *tsk)
1251 {
1252 struct mm_struct *mm;
1253 char *mmpath_buf;
1254 char *mmpath;
1255 struct vm_area_struct *vma;
1256 int file_based_vmas = 0;
1257 struct vma_cache_t {
1258 #ifdef STAPCONF_DPATH_PATH
1259 struct path f_path;
1260 #else
1261 struct vfsmount *f_vfsmnt;
1262 #endif
1263 struct dentry *dentry;
1264 unsigned long addr;
1265 unsigned long length;
1266 unsigned long offset;
1267 unsigned long vm_flags;
1268 };
1269 struct vma_cache_t *vma_cache = NULL;
1270 struct vma_cache_t *vma_cache_p;
1271
1272 // Call the mmap_callback for every vma associated with
1273 // a file.
1274 //
1275 // Note we're not calling get_task_mm()/mmput() here. Since
1276 // we're in the the context of that task, the mm should stick
1277 // around without locking it (and mmput() can sleep).
1278 mm = tsk->mm;
1279 if (! mm)
1280 return;
1281
1282 // Allocate space for a path
1283 mmpath_buf = _stp_kmalloc(PATH_MAX);
1284 if (mmpath_buf == NULL) {
1285 _stp_error("Unable to allocate space for path");
1286 return;
1287 }
1288
1289 // The down_read() function can sleep, so we'll call
1290 // down_read_trylock() instead, which can fail.
1291 if (! mmap_read_trylock(mm)) {
1292 _stp_kfree(mmpath_buf);
1293 return;
1294 }
1295
1296 {
1297 // Need surrounding {} because of VMA_ITERATOR variable
1298 // First find the number of file-based vmas.
1299 VMA_ITERATOR(vmi, mm, 0);
1300 for_each_vma(vmi, vma) {
1301 if (vma->vm_file)
1302 file_based_vmas++;
1303 }
1304 }
1305
1306 // Now allocate an array to cache vma information in.
1307 if (file_based_vmas > 0)
1308 vma_cache = _stp_vzalloc(sizeof(struct vma_cache_t)
1309 * file_based_vmas);
1310 if (vma_cache != NULL) {
1311 // Loop through the vmas again, and cache needed information.
1312 VMA_ITERATOR(vmi, mm, 0);
1313 vma_cache_p = vma_cache;
1314 for_each_vma(vmi,vma) {
1315 if (vma->vm_file) {
1316 #ifdef STAPCONF_DPATH_PATH
1317 // Notice we're increasing the reference
1318 // count for 'f_path'. This way it won't
1319 // get deleted from out under us.
1320 vma_cache_p->f_path = vma->vm_file->f_path;
1321 path_get(&vma_cache_p->f_path);
1322 vma_cache_p->dentry = vma->vm_file->f_path.dentry;
1323 #else
1324 // Notice we're increasing the reference
1325 // count for 'dentry' and 'f_vfsmnt'.
1326 // This way they won't get deleted from
1327 // out under us.
1328 vma_cache_p->dentry = vma->vm_file->f_dentry;
1329 dget(vma_cache_p->dentry);
1330 vma_cache_p->f_vfsmnt = vma->vm_file->f_vfsmnt;
1331 mntget(vma_cache_p->f_vfsmnt);
1332 vma_cache_p->dentry = vma->vm_file->f_dentry;
1333 #endif
1334 vma_cache_p->addr = vma->vm_start;
1335 vma_cache_p->length = vma->vm_end - vma->vm_start;
1336 vma_cache_p->offset = (vma->vm_pgoff << PAGE_SHIFT);
1337 vma_cache_p->vm_flags = vma->vm_flags;
1338 vma_cache_p++;
1339 }
1340 }
1341 }
1342
1343 // At this point, we're done with the vmas (assuming we found
1344 // any). We can't hold the 'mmap_sem' semaphore while making
1345 // callbacks.
1346 mmap_read_unlock(mm);
1347
1348 if (vma_cache) {
1349 int i;
1350
1351 // Loop over our cached information and make callbacks
1352 // based on it.
1353 vma_cache_p = vma_cache;
1354 for (i = 0; i < file_based_vmas; i++) {
1355 #ifdef STAPCONF_DPATH_PATH
1356 mmpath = d_path(&vma_cache_p->f_path, mmpath_buf,
1357 PATH_MAX);
1358 path_put(&vma_cache_p->f_path);
1359 #else
1360 mmpath = d_path(vma_cache_p->dentry,
1361 vma_cache_p->f_vfsmnt, mmpath_buf,
1362 PATH_MAX);
1363 dput(vma_cache_p->dentry);
1364 mntput(vma_cache_p->f_vfsmnt);
1365 #endif
1366 if (mmpath == NULL || IS_ERR(mmpath)) {
1367 long err = ((mmpath == NULL) ? 0
1368 : -PTR_ERR(mmpath));
1369 _stp_error("Unable to get path (error %ld) for pid %d",
1370 err, (int)tsk->pid);
1371 }
1372 else {
1373 __stp_call_mmap_callbacks(tgt, tsk, mmpath,
1374 vma_cache_p->dentry,
1375 vma_cache_p->addr,
1376 vma_cache_p->length,
1377 vma_cache_p->offset,
1378 vma_cache_p->vm_flags);
1379 }
1380 vma_cache_p++;
1381 }
1382 _stp_vfree(vma_cache);
1383 }
1384
1385 _stp_kfree(mmpath_buf);
1386 }
1387
1388 static void
1389 __stp_tf_quiesce_worker(struct task_work *work)
1390 {
1391 struct __stp_tf_task_work *tf_work = \
1392 container_of(work, struct __stp_tf_task_work, work);
1393 struct stap_task_finder_target *tgt = \
1394 (struct stap_task_finder_target *)tf_work->data;
1395
1396 might_sleep();
1397 if (atomic_read(&__stp_task_finder_state) != __STP_TF_RUNNING
1398 || current->flags & PF_EXITING)
1399 return;
1400
1401 /* If we had a build-id based executable probe (so we have a
1402 * tgt->build_id) set, we could not check it back in
1403 * __stp_utrace_attach_* because we can't do sleepy
1404 * access_process_vm() calls from there. BUt now that we're
1405 * in process context, quiesced, finally we can check. If we
1406 * were build-id based, and the build-id does not match, then
1407 * we UTRACE_DETACH from this process and skip the callbacks.
1408 *
1409 * XXX: For processes that do match, we redo this check every
1410 * time this callbacks is encountered somehow. That's
1411 * probably unnecessary.
1412 */
1413 if (tgt->build_id_len > 0) {
1414 int ok = __verify_build_id(current,
1415 tgt->build_id_vaddr,
1416 tgt->build_id,
1417 tgt->build_id_len);
1418
1419 dbug_task(2, "verified buildid-target process pid=%ld ok=%d\n",
1420 (long) current->tgid, ok);
1421 if (!ok) {
1422 // stap_utrace_detach (current, & tgt->ops);
1423 return;
1424 }
1425 }
1426
1427 __stp_tf_handler_start();
1428
1429 /* NB make sure we run mmap callbacks before other callbacks
1430 * like 'probe process.begin' handlers so that the vma tracker
1431 * is already initialized in the latter contexts */
1432
1433 /* If this is just a thread other than the thread group
1434 * leader, don't bother inform map callback clients about its
1435 * memory map, since they will simply duplicate each other. */
1436 if (tgt->mmap_events == 1 && current->tgid == current->pid) {
1437 __stp_call_mmap_callbacks_for_task(tgt, current);
1438 }
1439
1440 /* Call the callbacks. Assume that if the thread is a
1441 * thread group leader, it is a process. */
1442 __stp_call_callbacks(tgt, current, 1, (current->pid == current->tgid));
1443
1444 __stp_tf_handler_end();
1445 }
1446
1447 static u32
1448 __stp_utrace_task_finder_target_quiesce(u32 action,
1449 struct utrace_engine *engine,
1450 unsigned long event)
1451 {
1452 struct task_struct *tsk = current;
1453 struct stap_task_finder_target *tgt = engine->data;
1454 struct task_work *work;
1455 int rc;
1456
1457 if (atomic_read(&__stp_task_finder_state) != __STP_TF_RUNNING) {
1458 debug_task_finder_detach();
1459 return UTRACE_DETACH;
1460 }
1461
1462 if (tgt == NULL || tsk == NULL) {
1463 debug_task_finder_detach();
1464 return UTRACE_DETACH;
1465 }
1466
1467 __stp_tf_handler_start();
1468
1469 // Turn off quiesce handling
1470 rc = utrace_set_events(tsk, engine,
1471 __STP_ATTACHED_TASK_BASE_EVENTS(tgt));
1472
1473 if (rc == -EINPROGRESS) {
1474 /*
1475 * It's running our callback, so we have to
1476 * synchronize. We can't keep rcu_read_lock,
1477 * so the task pointer might die. But it's
1478 * safe to call utrace_barrier() even with
1479 * a stale task pointer, if we have an engine ref.
1480 */
1481 do {
1482 rc = utrace_barrier(tsk, engine);
1483 } while (rc == -ERESTARTSYS);
1484 if (rc == 0)
1485 rc = utrace_set_events(tsk, engine,
1486 __STP_ATTACHED_TASK_BASE_EVENTS(tgt));
1487 else if (rc != -ESRCH && rc != -EALREADY)
1488 _stp_error("utrace_barrier returned error %d on pid %d",
1489 rc, (int)tsk->pid);
1490 }
1491 if (rc != 0)
1492 _stp_error("utrace_set_events returned error %d on pid %d",
1493 rc, (int)tsk->pid);
1494
1495 /*
1496 * We could be in atomic context in a syscall tracepoint, which runs
1497 * in an RCU read-side critical section. We can't detect this with
1498 * in_atomic() on non-PREEMPT kernels (i.e., CONFIG_PREEMPT=n) so we
1499 * must always use a task worker here because there's no way to tell if
1500 * sleeping is okay.
1501 */
1502 work = __stp_tf_alloc_task_work(tgt);
1503 if (work == NULL) {
1504 __stp_tf_handler_end();
1505 _stp_error("Unable to allocate space for task_work");
1506 return UTRACE_RESUME;
1507 }
1508 __stp_tf_init_task_work(work, &__stp_tf_quiesce_worker);
1509
1510 rc = __stp_tf_task_work_add(tsk, work);
1511 if (rc) {
1512 __stp_tf_task_work_free(work);
1513 /* stp_task_work_add() returns -ESRCH if the task has
1514 * already passed exit_task_work(). Just ignore this
1515 * error. */
1516 if (rc != -ESRCH)
1517 printk(KERN_ERR "%s:%d - stp_task_work_add() returned %d\n",
1518 __FUNCTION__, __LINE__, rc);
1519 }
1520
1521 __stp_tf_handler_end();
1522 return UTRACE_RESUME;
1523 }
1524
1525
1526 /* FIXME: in the brave new world, we'll use target individual
1527 * syscalls, instead of tracing all syscalls for the map stuff.
1528 * However, process.syscall will still need to target all syscalls. */
1529 static u32
1530 __stp_utrace_task_finder_target_syscall_entry(u32 action,
1531 struct utrace_engine *engine,
1532 struct pt_regs *regs)
1533 {
1534 struct task_struct *tsk = current;
1535 struct stap_task_finder_target *tgt = engine->data;
1536 long syscall_no;
1537 unsigned long args[8] = { 0L }; /* large enough for syscall_get_arguments() target */
1538 int rc;
1539 int is_mmap_or_mmap2 = 0;
1540 int is_mprotect = 0;
1541 int is_munmap = 0;
1542
1543 if (atomic_read(&__stp_task_finder_state) != __STP_TF_RUNNING) {
1544 debug_task_finder_detach();
1545 return UTRACE_DETACH;
1546 }
1547
1548 if (unlikely(tgt == NULL))
1549 return UTRACE_RESUME;
1550
1551 // See if syscall is one we're interested in. On x86_64, this
1552 // is a potentially expensive operation (since we have to
1553 // check and see if it is a 32-bit task). So, cache the
1554 // results.
1555 //
1556 // FIXME: do we need to handle mremap()?
1557 syscall_no = _stp_syscall_get_nr(tsk, regs);
1558 is_mmap_or_mmap2 = (syscall_no == MMAP_SYSCALL_NO(tsk)
1559 || syscall_no == MMAP2_SYSCALL_NO(tsk) ? 1 : 0);
1560 if (!is_mmap_or_mmap2) {
1561 is_mprotect = (syscall_no == MPROTECT_SYSCALL_NO(tsk) ? 1 : 0);
1562 if (!is_mprotect) {
1563 is_munmap = (syscall_no == MUNMAP_SYSCALL_NO(tsk)
1564 ? 1 : 0);
1565 }
1566 }
1567 if (!is_mmap_or_mmap2 && !is_mprotect && !is_munmap)
1568 return UTRACE_RESUME;
1569
1570 // The syscall is one we're interested in, but do we have a
1571 // handler for it?
1572 if ((is_mmap_or_mmap2 && tgt->mmap_events == 0)
1573 || (is_mprotect && tgt->mprotect_events == 0)
1574 || (is_munmap && tgt->munmap_events == 0))
1575 return UTRACE_RESUME;
1576
1577 // Save the needed arguments. Note that for mmap, we really
1578 // just need the return value, so there is no need to save
1579 // any arguments.
1580 __stp_tf_handler_start();
1581 if (is_munmap) {
1582 // We need 2 arguments for munmap()
1583 _stp_syscall_get_arguments(tsk, regs, 0, 2, args);
1584 }
1585 else if (is_mprotect) {
1586 // We need 3 arguments for mprotect()
1587 _stp_syscall_get_arguments(tsk, regs, 0, 3, args);
1588 }
1589
1590 // Remember the syscall information
1591 rc = __stp_tf_add_map(tsk, syscall_no, args[0], args[1], args[2]);
1592 if (rc != 0)
1593 _stp_error("__stp_tf_add_map returned error %d on pid %d",
1594 rc, tsk->pid);
1595 __stp_tf_handler_end();
1596 return UTRACE_RESUME;
1597 }
1598
1599 static void
1600 __stp_tf_mmap_worker(struct task_work *work)
1601 {
1602 struct __stp_tf_task_work *tf_work = \
1603 container_of(work, struct __stp_tf_task_work, work);
1604 struct stap_task_finder_target *tgt = \
1605 (struct stap_task_finder_target *)tf_work->data;
1606 struct __stp_tf_map_entry *entry;
1607
1608 might_sleep();
1609
1610 // See if we can find saved syscall info.
1611 entry = __stp_tf_get_map_entry(current);
1612 if (entry == NULL)
1613 return;
1614
1615 if (atomic_read(&__stp_task_finder_state) != __STP_TF_RUNNING
1616 || current->flags & PF_EXITING) {
1617 __stp_tf_remove_map_entry(entry);
1618 return;
1619 }
1620
1621 __stp_tf_handler_start();
1622
1623 if (entry->syscall_no == MUNMAP_SYSCALL_NO(current)) {
1624 // Call the callbacks
1625 __stp_call_munmap_callbacks(tgt, current, entry->arg0,
1626 entry->arg1);
1627 }
1628 else if (entry->syscall_no == MMAP_SYSCALL_NO(current)
1629 || entry->syscall_no == MMAP2_SYSCALL_NO(current)) {
1630 // Call the callbacks. Note that arg0 is really the
1631 // return value of mmap()/mmap2().
1632 __stp_call_mmap_callbacks_with_addr(tgt, current, entry->arg0);
1633 }
1634 else { // mprotect
1635 // Call the callbacks
1636 __stp_call_mprotect_callbacks(tgt, current, entry->arg0,
1637 entry->arg1, entry->arg2);
1638 }
1639 __stp_tf_remove_map_entry(entry);
1640
1641 __stp_tf_handler_end();
1642 }
1643
1644 static u32
1645 __stp_utrace_task_finder_target_syscall_exit(u32 action,
1646 struct utrace_engine *engine,
1647 struct pt_regs *regs)
1648 {
1649 struct task_struct *tsk = current;
1650 struct stap_task_finder_target *tgt = engine->data;
1651 unsigned long rv;
1652 struct __stp_tf_map_entry *entry;
1653 struct task_work *work;
1654 int rc;
1655
1656 if (atomic_read(&__stp_task_finder_state) != __STP_TF_RUNNING) {
1657 debug_task_finder_detach();
1658 return UTRACE_DETACH;
1659 }
1660
1661 if (tgt == NULL)
1662 return UTRACE_RESUME;
1663
1664 // See if we can find saved syscall info. If we can, it must
1665 // be one of the syscalls we are interested in (and we must
1666 // have callbacks to call for it).
1667 entry = __stp_tf_get_map_entry(tsk);
1668 if (entry == NULL)
1669 return UTRACE_RESUME;
1670
1671 // Get return value
1672 __stp_tf_handler_start();
1673 rv = syscall_get_return_value(tsk, regs);
1674
1675 dbug_task_vma(1,
1676 "tsk %d found %s(0x%lx), returned 0x%lx\n",
1677 tsk->pid,
1678 ((entry->syscall_no == MMAP_SYSCALL_NO(tsk)) ? "mmap"
1679 : ((entry->syscall_no == MMAP2_SYSCALL_NO(tsk)) ? "mmap2"
1680 : ((entry->syscall_no == MPROTECT_SYSCALL_NO(tsk))
1681 ? "mprotect"
1682 : ((entry->syscall_no == MUNMAP_SYSCALL_NO(tsk))
1683 ? "munmap"
1684 : "UNKNOWN")))),
1685 entry->arg0, rv);
1686
1687 /* If this is mmap()/mmap2(), we need to remember the
1688 * return value. We'll use entry->arg0, since
1689 * mmap()/mmap2() doesn't use that info. */
1690 if (entry->syscall_no == MMAP_SYSCALL_NO(tsk)
1691 || entry->syscall_no == MMAP2_SYSCALL_NO(tsk)) {
1692 entry->arg0 = rv;
1693 }
1694
1695 /*
1696 * We could be in atomic context in a syscall tracepoint, which runs
1697 * in an RCU read-side critical section. We can't detect this with
1698 * in_atomic() on non-PREEMPT kernels (i.e., CONFIG_PREEMPT=n) so we
1699 * must always use a task worker here because there's no way to tell if
1700 * sleeping is okay.
1701 */
1702 work = __stp_tf_alloc_task_work(tgt);
1703 if (work == NULL) {
1704 _stp_error("Unable to allocate space for task_work");
1705 __stp_tf_remove_map_entry(entry);
1706 __stp_tf_handler_end();
1707 return UTRACE_RESUME;
1708 }
1709 __stp_tf_init_task_work(work, &__stp_tf_mmap_worker);
1710 rc = __stp_tf_task_work_add(tsk, work);
1711 if (rc) {
1712 __stp_tf_task_work_free(work);
1713 /* stp_task_work_add() returns -ESRCH if the task has
1714 * already passed exit_task_work(). Just ignore this
1715 * error. */
1716 if (rc != -ESRCH)
1717 printk(KERN_ERR "%s:%d - stp_task_work_add() returned %d\n",
1718 __FUNCTION__, __LINE__, rc);
1719 }
1720
1721 __stp_tf_handler_end();
1722 return UTRACE_RESUME;
1723 }
1724
1725 static struct utrace_engine_ops __stp_utrace_task_finder_ops = {
1726 .report_clone = __stp_utrace_task_finder_report_clone,
1727 .report_exec = __stp_utrace_task_finder_report_exec,
1728 .report_death = stap_utrace_task_finder_report_death,
1729 };
1730
1731 static void
1732 stap_stop_task_finder(void);
1733
1734 static int
1735 stap_start_task_finder(void)
1736 {
1737 int rc = 0;
1738 struct task_struct *grp, *tsk;
1739 char *mmpath_buf;
1740 uid_t tsk_euid;
1741
1742 if (atomic_inc_return(&__stp_task_finder_state) != __STP_TF_STARTING) {
1743 atomic_dec(&__stp_task_finder_state);
1744 _stp_error("task_finder already started");
1745 return EBUSY;
1746 }
1747
1748 mmpath_buf = _stp_kmalloc(PATH_MAX);
1749 if (mmpath_buf == NULL) {
1750 _stp_error("Unable to allocate space for path");
1751 return ENOMEM;
1752 }
1753
1754 rc = utrace_init();
1755 if (rc) { /* PR14781, handle utrace alloc failure. */
1756 _stp_kfree(mmpath_buf);
1757 _stp_error("Failed to initialize utrace hooks");
1758 return ENOMEM; /* XXX: or some other one. */
1759 }
1760
1761 __stp_tf_map_initialize();
1762
1763 atomic_set(&__stp_task_finder_state, __STP_TF_RUNNING);
1764
1765 rcu_read_lock();
1766 for_each_process_thread(grp, tsk) {
1767 struct mm_struct *mm;
1768 char *mmpath;
1769 size_t mmpathlen;
1770 struct list_head *tgt_node;
1771
1772 /* If in stap -c/-x mode, skip over other processes. */
1773 if (_stp_target && tsk->tgid != _stp_target)
1774 continue;
1775
1776 rc = __stp_utrace_attach(tsk, &__stp_utrace_task_finder_ops, 0,
1777 __STP_TASK_FINDER_EVENTS,
1778 UTRACE_RESUME);
1779
1780 dbug_task(2, "__stp_utrace_attach() for pid %d returned %d",
1781 tsk->tgid, rc);
1782
1783 if (rc == EPERM) {
1784 /* Ignore EPERM errors, which mean this wasn't
1785 * a thread we can attach to. */
1786 rc = 0;
1787 continue;
1788 }
1789 else if (rc != 0) {
1790 /* If we get a real error, quit. */
1791 goto stf_err;
1792 }
1793
1794 // Grab the path associated with this task.
1795 //
1796 // Note we aren't calling get_task_mm()/mmput() here.
1797 // Instead we're calling task_lock()/task_unlock().
1798 // We really only need to lock the mm, but mmput() can
1799 // sleep so we can't call it. Also note that
1800 // __stp_get_mm_path() grabs the mmap semaphore, which
1801 // should also keep us safe.
1802 task_lock(tsk);
1803 if (! tsk->mm) {
1804 /* If the thread doesn't have a mm_struct, it
1805 * is a kernel thread which we need to
1806 * skip. */
1807 task_unlock(tsk);
1808 continue;
1809 }
1810 mmpath = __stp_get_mm_path(tsk->mm, mmpath_buf, PATH_MAX);
1811 task_unlock(tsk);
1812 if (mmpath == NULL || IS_ERR(mmpath)) {
1813 rc = PTR_ERR(mmpath);
1814 /* If this was our target then it's a fatal error */
1815 if (!_stp_target && rc == -ENOENT) {
1816 _stp_warn("Unable to get path (error %d) for pid %d",
1817 rc, (int)tsk->pid);
1818 rc = 0; /* ignore ENOENT */
1819 continue;
1820 }
1821 else {
1822 _stp_error("Unable to get path (error %d) for pid %d",
1823 rc, (int)tsk->pid);
1824 goto stf_err;
1825 }
1826 }
1827
1828 /* Check the thread's exe's path/pid/build-id against our list. */
1829 #ifdef STAPCONF_TASK_UID
1830 tsk_euid = tsk->euid;
1831 #else
1832 #if defined(CONFIG_USER_NS) || (LINUX_VERSION_CODE >= KERNEL_VERSION(3,14,0))
1833 tsk_euid = from_kuid_munged(current_user_ns(), task_euid(tsk));
1834 #else
1835 tsk_euid = task_euid(tsk);
1836 #endif
1837 #endif
1838 mmpathlen = strlen(mmpath);
1839 list_for_each(tgt_node, &__stp_task_finder_list) {
1840 struct stap_task_finder_target *tgt;
1841
1842 tgt = list_entry(tgt_node,
1843 struct stap_task_finder_target, list);
1844 if (tgt == NULL)
1845 continue;
1846 /* buildid-based target ... gets checked in __stp_tf_quiesce_worker */
1847 /* procname-based target */
1848 else if (tgt->build_id == 0 && tgt->pathlen > 0
1849 && (tgt->pathlen != mmpathlen
1850 || strcmp(tgt->procname, mmpath) != 0))
1851 {
1852 dbug_task(2, "target path not matched: [%s] != [%s]",
1853 tgt->procname, mmpath);
1854 continue;
1855 }
1856 /* pid-based target */
1857 else if (tgt->pid != 0 && tgt->pid != tsk->pid)
1858 continue;
1859 /* Notice that "pid == 0" (which means to
1860 * probe all threads) falls through. */
1861
1862 #if ! STP_PRIVILEGE_CONTAINS (STP_PRIVILEGE, STP_PR_STAPDEV) && \
1863 ! STP_PRIVILEGE_CONTAINS (STP_PRIVILEGE, STP_PR_STAPSYS)
1864 /* Make sure unprivileged users only probe their own threads. */
1865 if (_stp_uid != tsk_euid) {
1866 if (tgt->pid != 0 || _stp_target) {
1867 _stp_warn("Process %d does not belong to unprivileged user %d",
1868 tsk->pid, _stp_uid);
1869 }
1870 continue;
1871 }
1872 #endif
1873
1874 // Set up events we need for attached tasks.
1875 rc = __stp_utrace_attach(tsk, &tgt->ops, tgt,
1876 __STP_ATTACHED_TASK_EVENTS,
1877 UTRACE_INTERRUPT);
1878 dbug_task(2, "__stp_utrace_attach() for %d returned %d", tsk->tgid,
1879 rc);
1880
1881 if (rc != 0 && rc != EPERM)
1882 goto stf_err;
1883 rc = 0; /* ignore EPERM */
1884 tgt->engine_attached = 1;
1885 }
1886 }
1887 stf_err:
1888 rcu_read_unlock();
1889 _stp_kfree(mmpath_buf);
1890 debug_task_finder_report(); // report at end for utrace engine counting
1891 /*
1892 * We need to do our own cleanup if something failed. A full task finder
1893 * stop is needed because utrace is guaranteed to be live by this point.
1894 */
1895 if (rc)
1896 stap_stop_task_finder();
1897 return rc;
1898 }
1899
1900
1901 static void
1902 stap_task_finder_post_init(void)
1903 {
1904 struct task_struct *grp, *tsk;
1905
1906 if (atomic_read(&__stp_task_finder_state) != __STP_TF_RUNNING) {
1907 _stp_error("task_finder not running?");
1908 return;
1909 }
1910
1911 dbug_task(2, "entry.");
1912 rcu_read_lock();
1913 for_each_process_thread(grp, tsk) {
1914 struct list_head *tgt_node;
1915
1916 if (atomic_read(&__stp_task_finder_state) != __STP_TF_RUNNING) {
1917 dbug_task(2, "exiting early...");
1918 break;
1919 }
1920
1921 /* If in stap -c/-x mode, skip over other processes. */
1922 if (_stp_target && tsk->tgid != _stp_target)
1923 continue;
1924
1925 /* Only "poke" thread group leaders. */
1926 if (tsk->tgid != tsk->pid)
1927 continue;
1928
1929 /* See if we need to "poke" this thread. */
1930 list_for_each(tgt_node, &__stp_task_finder_list) {
1931 struct stap_task_finder_target *tgt;
1932 struct utrace_engine *engine;
1933
1934 tgt = list_entry(tgt_node,
1935 struct stap_task_finder_target, list);
1936 if (tgt == NULL || !tgt->engine_attached)
1937 continue;
1938
1939 // If we found an "interesting" task earlier,
1940 // stop it.
1941 engine = utrace_attach_task(tsk,
1942 UTRACE_ATTACH_MATCH_OPS,
1943 &tgt->ops, tgt);
1944 if (engine != NULL && !IS_ERR(engine)) {
1945 /* We found a target task. Stop it. */
1946 int rc = utrace_control(tsk, engine,
1947 UTRACE_INTERRUPT);
1948 /* If utrace_control() returns
1949 * EINPROGRESS when we're
1950 * trying to stop/interrupt,
1951 * that means the task hasn't
1952 * stopped quite yet, but will
1953 * soon. Ignore this
1954 * error. */
1955 if (rc != 0 && rc != -EINPROGRESS) {
1956 _stp_error("utrace_control returned error %d on pid %d",
1957 rc, (int)tsk->pid);
1958 }
1959
1960 dbug_task(2, "utrace_control(UTRACE_INTERRUPT) for pid %d "
1961 "returned %d (%d)", tsk->pid, rc, -EINPROGRESS);
1962
1963 utrace_engine_put(engine);
1964
1965 /* Since we only need to interrupt
1966 * the task once, not once per
1967 * engine, get out of this loop. */
1968 break;
1969 }
1970 }
1971 }
1972 rcu_read_unlock();
1973 atomic_set(&__stp_task_finder_complete, 1);
1974 dbug_task(2, "exit.");
1975 return;
1976 }
1977
1978
1979 /* Indicates whether task_finder has complete coverage of all processes.
1980 * e.g. uprobes prefilter can be sure it's safe to REMOVE from an unknown process.
1981 */
1982 static inline int
1983 stap_task_finder_complete(void)
1984 {
1985 return atomic_read(&__stp_task_finder_complete) != 0;
1986 }
1987
1988
1989 static void
1990 stap_stop_task_finder(void)
1991 {
1992 #ifdef DEBUG_TASK_FINDER
1993 int i = 0;
1994
1995 printk(KERN_ERR "%s:%d - entry\n", __FUNCTION__, __LINE__);
1996 #endif
1997 if (atomic_read(&__stp_task_finder_state) == __STP_TF_UNITIALIZED)
1998 return;
1999
2000 atomic_set(&__stp_task_finder_state, __STP_TF_STOPPING);
2001 debug_task_finder_report();
2002
2003 // The utrace_shutdown() function detaches and cleans up
2004 // everything for us - we don't have to go through each
2005 // engine. This also means that the attach_count could end up
2006 // > 0 (since we don't got through each engine individually).
2007 utrace_shutdown();
2008
2009 debug_task_finder_report();
2010 atomic_set(&__stp_task_finder_state, __STP_TF_STOPPED);
2011
2012 /* Now that all the engines are detached, make sure
2013 * all the callbacks are finished. If they aren't, we'll
2014 * crash the kernel when the module is removed. */
2015 while (atomic_read(&__stp_inuse_count) != 0) {
2016 schedule();
2017 #ifdef DEBUG_TASK_FINDER
2018 i++;
2019 #endif
2020 }
2021 #ifdef DEBUG_TASK_FINDER
2022 if (i > 0)
2023 printk(KERN_ERR "it took %d polling loops to quit.\n", i);
2024 #endif
2025 debug_task_finder_report();
2026
2027 /* Make sure all pending task_work requests are canceled. */
2028 __stp_tf_cancel_all_task_work();
2029
2030 /* Call utrace_exit(), which also calls stp_task_work_exit()
2031 * to wait on any running task_work items. */
2032 utrace_exit();
2033
2034 #ifdef DEBUG_TASK_FINDER
2035 printk(KERN_ERR "%s:%d - exit\n", __FUNCTION__, __LINE__);
2036 #endif
2037 }
2038
2039 #endif /* TASK_FINDER2_C */