| 1 |
#ifndef TASK_FINDER2_C |
| 2 |
#define TASK_FINDER2_C |
| 3 |
|
| 4 |
#include "stp_utrace.c" |
| 5 |
|
| 6 |
#include <linux/list.h> |
| 7 |
#include <linux/binfmts.h> |
| 8 |
#include <linux/mount.h> |
| 9 |
#include "stap_mmap_lock.h" |
| 10 |
#ifndef STAPCONF_TASK_UID |
| 11 |
#include <linux/cred.h> |
| 12 |
#endif |
| 13 |
#include "../uidgid_compatibility.h" |
| 14 |
#include "syscall.h" |
| 15 |
#include "task_finder_map.c" |
| 16 |
#include "task_finder_vma.c" |
| 17 |
#include "linux/stap_fs.h" |
| 18 |
|
| 19 |
#ifndef VMA_ITERATOR |
| 20 |
#define VMA_ITERATOR(name, mm, addr) \ |
| 21 |
struct mm_struct *name = mm |
| 22 |
#define for_each_vma(vmi, vma) \ |
| 23 |
for (vma = vmi->mmap; vma; vma = vma->vm_next) |
| 24 |
#endif |
| 25 |
|
| 26 |
static LIST_HEAD(__stp_task_finder_list); |
| 27 |
|
| 28 |
struct stap_task_finder_target; |
| 29 |
|
| 30 |
#define __STP_TF_UNITIALIZED 0 |
| 31 |
#define __STP_TF_STARTING 1 |
| 32 |
#define __STP_TF_RUNNING 2 |
| 33 |
#define __STP_TF_STOPPING 3 |
| 34 |
#define __STP_TF_STOPPED 4 |
| 35 |
static atomic_t __stp_task_finder_state = ATOMIC_INIT(__STP_TF_UNITIALIZED); |
| 36 |
static atomic_t __stp_task_finder_complete = ATOMIC_INIT(0); |
| 37 |
static atomic_t __stp_inuse_count = ATOMIC_INIT (0); |
| 38 |
|
| 39 |
#define __stp_tf_handler_start() (atomic_inc(&__stp_inuse_count)) |
| 40 |
#define __stp_tf_handler_end() (atomic_dec(&__stp_inuse_count)) |
| 41 |
|
| 42 |
#ifdef DEBUG_TASK_FINDER |
| 43 |
static atomic_t __stp_attach_count = ATOMIC_INIT (0); |
| 44 |
|
| 45 |
#define debug_task_finder_attach() (atomic_inc(&__stp_attach_count)) |
| 46 |
#define debug_task_finder_detach() (atomic_dec(&__stp_attach_count)) |
| 47 |
#define debug_task_finder_report() \ |
| 48 |
dbug_task(1, "attach count: %d, inuse count: %d\n", \ |
| 49 |
atomic_read(&__stp_attach_count), \ |
| 50 |
atomic_read(&__stp_inuse_count)) |
| 51 |
#else |
| 52 |
#define debug_task_finder_attach() /* empty */ |
| 53 |
#define debug_task_finder_detach() /* empty */ |
| 54 |
#define debug_task_finder_report() /* empty */ |
| 55 |
#endif /* !DEBUG_TASK_FINDER */ |
| 56 |
|
| 57 |
typedef int (*stap_task_finder_callback)(struct stap_task_finder_target *tgt, |
| 58 |
struct task_struct *tsk, |
| 59 |
int register_p, |
| 60 |
int process_p); |
| 61 |
|
| 62 |
typedef int |
| 63 |
(*stap_task_finder_mmap_callback)(struct stap_task_finder_target *tgt, |
| 64 |
struct task_struct *tsk, |
| 65 |
char *path, |
| 66 |
struct dentry *dentry, |
| 67 |
unsigned long addr, |
| 68 |
unsigned long length, |
| 69 |
unsigned long offset, |
| 70 |
unsigned long vm_flags); |
| 71 |
typedef int |
| 72 |
(*stap_task_finder_munmap_callback)(struct stap_task_finder_target *tgt, |
| 73 |
struct task_struct *tsk, |
| 74 |
unsigned long addr, |
| 75 |
unsigned long length); |
| 76 |
|
| 77 |
typedef int |
| 78 |
(*stap_task_finder_mprotect_callback)(struct stap_task_finder_target *tgt, |
| 79 |
struct task_struct *tsk, |
| 80 |
unsigned long addr, |
| 81 |
unsigned long length, |
| 82 |
int prot); |
| 83 |
|
| 84 |
struct stap_task_finder_target { |
| 85 |
/* private: */ |
| 86 |
struct list_head list; /* __stp_task_finder_list linkage */ |
| 87 |
struct list_head callback_list_head; |
| 88 |
struct list_head callback_list; |
| 89 |
struct utrace_engine_ops ops; |
| 90 |
size_t pathlen; |
| 91 |
unsigned engine_attached:1; |
| 92 |
unsigned mmap_events:1; |
| 93 |
unsigned munmap_events:1; |
| 94 |
unsigned mprotect_events:1; |
| 95 |
unsigned new_procname_p:1; /* set to true when a new procname |
| 96 |
buffer is dynamically allocated |
| 97 |
(like in |
| 98 |
stap_register_task_finder_target) */ |
| 99 |
|
| 100 |
/* public: */ |
| 101 |
pid_t pid; |
| 102 |
int build_id_len; |
| 103 |
uint64_t build_id_vaddr; |
| 104 |
unsigned char *build_id; |
| 105 |
const char *procname; |
| 106 |
const char *purpose; |
| 107 |
stap_task_finder_callback callback; |
| 108 |
stap_task_finder_mmap_callback mmap_callback; |
| 109 |
stap_task_finder_munmap_callback munmap_callback; |
| 110 |
stap_task_finder_mprotect_callback mprotect_callback; |
| 111 |
}; |
| 112 |
|
| 113 |
static LIST_HEAD(__stp_tf_task_work_list); |
| 114 |
static STP_DEFINE_SPINLOCK(__stp_tf_task_work_list_lock); |
| 115 |
struct __stp_tf_task_work { |
| 116 |
struct list_head list; |
| 117 |
struct task_struct *task; |
| 118 |
void *data; |
| 119 |
struct task_work work; |
| 120 |
task_work_func_t func; |
| 121 |
}; |
| 122 |
|
| 123 |
static void |
| 124 |
__stp_tf_task_work_free(struct task_work *work) |
| 125 |
{ |
| 126 |
struct __stp_tf_task_work *tf_work = |
| 127 |
container_of(work, typeof(*tf_work), work); |
| 128 |
|
| 129 |
_stp_kfree(tf_work); |
| 130 |
} |
| 131 |
|
| 132 |
static void |
| 133 |
__stp_tf_task_worker_fn(struct task_work *work) |
| 134 |
{ |
| 135 |
/* Go through all the queued task workers and dequeue them in order */ |
| 136 |
while (1) { |
| 137 |
struct __stp_tf_task_work *entry, *tf_work = NULL; |
| 138 |
unsigned long flags; |
| 139 |
|
| 140 |
/* Search for task workers for this task */ |
| 141 |
stp_spin_lock_irqsave(&__stp_tf_task_work_list_lock, flags); |
| 142 |
list_for_each_entry(entry, &__stp_tf_task_work_list, list) { |
| 143 |
if (entry->task == current) { |
| 144 |
tf_work = entry; |
| 145 |
list_del(&tf_work->list); |
| 146 |
break; |
| 147 |
} |
| 148 |
} |
| 149 |
stp_spin_unlock_irqrestore(&__stp_tf_task_work_list_lock, flags); |
| 150 |
|
| 151 |
/* No more workers to execute for this task */ |
| 152 |
if (!tf_work) |
| 153 |
break; |
| 154 |
|
| 155 |
/* Run the task worker outside the spin lock so it can sleep */ |
| 156 |
tf_work->func(&tf_work->work); |
| 157 |
} |
| 158 |
|
| 159 |
/* |
| 160 |
* Free the tf_work associated with this task work. It's guaranteed to |
| 161 |
* not be on the task work list anymore because we drained all of the |
| 162 |
* workers for this task. |
| 163 |
*/ |
| 164 |
__stp_tf_task_work_free(work); |
| 165 |
stp_task_work_func_done(); |
| 166 |
} |
| 167 |
|
| 168 |
static void |
| 169 |
__stp_tf_init_task_work(struct task_work *work, task_work_func_t func) |
| 170 |
{ |
| 171 |
struct __stp_tf_task_work *tf_work = |
| 172 |
container_of(work, typeof(*tf_work), work); |
| 173 |
|
| 174 |
tf_work->func = func; |
| 175 |
stp_init_task_work(work, __stp_tf_task_worker_fn); |
| 176 |
} |
| 177 |
|
| 178 |
static int |
| 179 |
__stp_tf_task_work_add(struct task_struct *task, struct task_work *work) |
| 180 |
{ |
| 181 |
struct __stp_tf_task_work *tf_work = |
| 182 |
container_of(work, typeof(*tf_work), work); |
| 183 |
unsigned long flags; |
| 184 |
int rc; |
| 185 |
|
| 186 |
/* Associate this tf_work with the target task it'll run within */ |
| 187 |
tf_work->task = task; |
| 188 |
|
| 189 |
/* |
| 190 |
* Queue the task worker onto the list. Once the tf_work is added to the |
| 191 |
* list, the pointer is only safe while the list lock is held. So we |
| 192 |
* need to add the task work while we're still holding the list lock. |
| 193 |
*/ |
| 194 |
stp_spin_lock_irqsave(&__stp_tf_task_work_list_lock, flags); |
| 195 |
list_add_tail(&tf_work->list, &__stp_tf_task_work_list); |
| 196 |
rc = stp_task_work_add(task, work); |
| 197 |
if (rc) |
| 198 |
list_del(&tf_work->list); |
| 199 |
stp_spin_unlock_irqrestore(&__stp_tf_task_work_list_lock, flags); |
| 200 |
|
| 201 |
return rc; |
| 202 |
} |
| 203 |
|
| 204 |
/* |
| 205 |
* Allocate a 'struct task_work' for use. Internally keeps track of |
| 206 |
* allocated structs for use when shutting down. |
| 207 |
* |
| 208 |
* Returns NULL in the case of a memory allocation failure. |
| 209 |
* |
| 210 |
* Note that it remembers the current task, so if we need to allocate |
| 211 |
* a 'struct task_work' for a task that isn't current, we'll need a |
| 212 |
* __stp_tf_alloc_task_work_for_task(task) variant. |
| 213 |
*/ |
| 214 |
static struct task_work * |
| 215 |
__stp_tf_alloc_task_work(void *data) |
| 216 |
{ |
| 217 |
struct __stp_tf_task_work *tf_work; |
| 218 |
unsigned long flags; |
| 219 |
|
| 220 |
tf_work = _stp_kmalloc(sizeof(*tf_work)); |
| 221 |
if (tf_work == NULL) { |
| 222 |
_stp_error("Unable to allocate space for task_work"); |
| 223 |
return NULL; |
| 224 |
} |
| 225 |
|
| 226 |
tf_work->data = data; |
| 227 |
|
| 228 |
return &tf_work->work; |
| 229 |
} |
| 230 |
|
| 231 |
/* |
| 232 |
* Cancel (and free) all outstanding task work requests. |
| 233 |
*/ |
| 234 |
static void __stp_tf_cancel_all_task_work(void) |
| 235 |
{ |
| 236 |
struct __stp_tf_task_work *node, *tmp; |
| 237 |
unsigned long flags; |
| 238 |
|
| 239 |
// Cancel all remaining requests. |
| 240 |
stp_spin_lock_irqsave(&__stp_tf_task_work_list_lock, flags); |
| 241 |
restart: |
| 242 |
list_for_each_entry_safe(node, tmp, &__stp_tf_task_work_list, list) { |
| 243 |
struct __stp_tf_task_work *tf_work; |
| 244 |
struct task_work *work; |
| 245 |
|
| 246 |
work = stp_task_work_cancel(node->task, node->work.func); |
| 247 |
if (!work) |
| 248 |
continue; |
| 249 |
|
| 250 |
/* |
| 251 |
* There can be multiple queued task workers with the same |
| 252 |
* worker func, so there's no guarantee that tf_work == node. |
| 253 |
* Therefore, we can only free what stp_task_work_cancel() just |
| 254 |
* gave us; freeing 'node' would be unsafe. |
| 255 |
*/ |
| 256 |
tf_work = container_of(work, typeof(*tf_work), work); |
| 257 |
list_del(&tf_work->list); |
| 258 |
_stp_kfree(tf_work); |
| 259 |
|
| 260 |
/* |
| 261 |
* If the tf_work we just freed was the next node in the list, |
| 262 |
* then we need to restart the list iteration because |
| 263 |
* list_for_each_entry_safe() can't cope with the next node |
| 264 |
* being freed. We still need to use list_for_each_entry_safe() |
| 265 |
* because we need to get through one successful pass through |
| 266 |
* the entire list, since it's not guaranteed that this list |
| 267 |
* will be empty when this function exits, as there can still be |
| 268 |
* active task workers running, which is fine since the |
| 269 |
* stp_task_work API will wait for all task workers to finish |
| 270 |
* before allowing the module to unload. |
| 271 |
*/ |
| 272 |
if (tf_work == tmp) |
| 273 |
goto restart; |
| 274 |
} |
| 275 |
stp_spin_unlock_irqrestore(&__stp_tf_task_work_list_lock, flags); |
| 276 |
} |
| 277 |
|
| 278 |
static u32 |
| 279 |
__stp_utrace_task_finder_target_exec(u32 action, |
| 280 |
struct utrace_engine *engine, |
| 281 |
const struct linux_binfmt *fmt, |
| 282 |
const struct linux_binprm *bprm, |
| 283 |
struct pt_regs *regs); |
| 284 |
|
| 285 |
static u32 |
| 286 |
__stp_utrace_task_finder_target_death(struct utrace_engine *engine, |
| 287 |
bool group_dead, int signal); |
| 288 |
|
| 289 |
static u32 |
| 290 |
__stp_utrace_task_finder_target_quiesce(u32 action, |
| 291 |
struct utrace_engine *engine, |
| 292 |
unsigned long event); |
| 293 |
|
| 294 |
static u32 |
| 295 |
__stp_utrace_task_finder_target_syscall_entry(u32 action, |
| 296 |
struct utrace_engine *engine, |
| 297 |
struct pt_regs *regs); |
| 298 |
|
| 299 |
static u32 |
| 300 |
__stp_utrace_task_finder_target_syscall_exit(u32 action, |
| 301 |
struct utrace_engine *engine, |
| 302 |
struct pt_regs *regs); |
| 303 |
|
| 304 |
static void |
| 305 |
__stp_call_mmap_callbacks_for_task(struct stap_task_finder_target *tgt, |
| 306 |
struct task_struct *tsk); |
| 307 |
|
| 308 |
static void |
| 309 |
stap_cleanup_task_finder_target(struct stap_task_finder_target *tgt) |
| 310 |
{ |
| 311 |
if (tgt->new_procname_p) { |
| 312 |
_stp_kfree((void *) tgt->procname); |
| 313 |
tgt->procname = "<freed>"; |
| 314 |
tgt->new_procname_p = 0; |
| 315 |
} |
| 316 |
} |
| 317 |
|
| 318 |
static int |
| 319 |
stap_register_task_finder_target(struct stap_task_finder_target *new_tgt) |
| 320 |
{ |
| 321 |
// Since this __stp_task_finder_list is (currently) only |
| 322 |
// written to in one big setup operation before the task |
| 323 |
// finder process is started, we don't need to lock it. |
| 324 |
struct list_head *node; |
| 325 |
struct stap_task_finder_target *tgt = NULL; |
| 326 |
int found_node = 0; |
| 327 |
|
| 328 |
if (atomic_read(&__stp_task_finder_state) != __STP_TF_UNITIALIZED) { |
| 329 |
_stp_error("task_finder already started, no new targets allowed"); |
| 330 |
return EBUSY; |
| 331 |
} |
| 332 |
|
| 333 |
if (new_tgt == NULL) |
| 334 |
return EFAULT; |
| 335 |
|
| 336 |
if (new_tgt->procname != NULL) { |
| 337 |
int ret; |
| 338 |
char *p; |
| 339 |
size_t len; |
| 340 |
const char *pathname = new_tgt->procname; |
| 341 |
|
| 342 |
p = stap_real_path(pathname, &len); |
| 343 |
if (p != NULL) { |
| 344 |
if (unlikely(IS_ERR(p))) |
| 345 |
return -PTR_ERR(p); |
| 346 |
|
| 347 |
/* stap_real_path() allocates a new string buffer in |
| 348 |
* p. */ |
| 349 |
|
| 350 |
if (unlikely(new_tgt->new_procname_p)) |
| 351 |
_stp_kfree((void *) new_tgt->procname); |
| 352 |
else |
| 353 |
new_tgt->new_procname_p = 1; |
| 354 |
|
| 355 |
/* NB: this buffer will be freed by |
| 356 |
* stap_cleanup_task_finder_target() */ |
| 357 |
new_tgt->procname = (const char *) p; |
| 358 |
new_tgt->pathlen = len; |
| 359 |
|
| 360 |
} else { |
| 361 |
new_tgt->pathlen = strlen(pathname); |
| 362 |
} |
| 363 |
|
| 364 |
} else |
| 365 |
new_tgt->pathlen = 0; |
| 366 |
|
| 367 |
// Make sure everything is initialized properly. |
| 368 |
new_tgt->engine_attached = 0; |
| 369 |
new_tgt->mmap_events = 0; |
| 370 |
new_tgt->munmap_events = 0; |
| 371 |
new_tgt->mprotect_events = 0; |
| 372 |
memset(&new_tgt->ops, 0, sizeof(new_tgt->ops)); |
| 373 |
new_tgt->ops.report_exec = &__stp_utrace_task_finder_target_exec; |
| 374 |
new_tgt->ops.report_death = &__stp_utrace_task_finder_target_death; |
| 375 |
new_tgt->ops.report_quiesce = &__stp_utrace_task_finder_target_quiesce; |
| 376 |
new_tgt->ops.report_syscall_entry = \ |
| 377 |
&__stp_utrace_task_finder_target_syscall_entry; |
| 378 |
new_tgt->ops.report_syscall_exit = \ |
| 379 |
&__stp_utrace_task_finder_target_syscall_exit; |
| 380 |
|
| 381 |
// Search the list for an existing entry for procname/pid/build-id. |
| 382 |
list_for_each(node, &__stp_task_finder_list) { |
| 383 |
tgt = list_entry(node, struct stap_task_finder_target, list); |
| 384 |
if (tgt == new_tgt) { |
| 385 |
_stp_error("target already registered"); |
| 386 |
return EINVAL; |
| 387 |
} |
| 388 |
if (tgt != NULL |
| 389 |
/* procname-based target */ |
| 390 |
&& ((new_tgt->pathlen > 0 |
| 391 |
&& tgt->pathlen == new_tgt->pathlen |
| 392 |
&& strcmp(tgt->procname, new_tgt->procname) == 0) |
| 393 |
/* pid-based target (a specific pid or all |
| 394 |
* pids) */ |
| 395 |
|| (new_tgt->pathlen == 0 && tgt->pathlen == 0 |
| 396 |
&& tgt->pid == new_tgt->pid) |
| 397 |
/* buildid-based target */ |
| 398 |
|| (new_tgt->build_id_len > 0 |
| 399 |
&& new_tgt->build_id_len == tgt->build_id_len |
| 400 |
&& strcmp(new_tgt->build_id, tgt->build_id) == 0))) { |
| 401 |
found_node = 1; |
| 402 |
break; |
| 403 |
} |
| 404 |
} |
| 405 |
|
| 406 |
// If we didn't find a matching existing entry, add the new |
| 407 |
// target to the task list. |
| 408 |
if (! found_node) { |
| 409 |
INIT_LIST_HEAD(&new_tgt->callback_list_head); |
| 410 |
list_add_tail(&new_tgt->list, &__stp_task_finder_list); |
| 411 |
tgt = new_tgt; |
| 412 |
} |
| 413 |
|
| 414 |
// Add this target to the callback list for this task. |
| 415 |
list_add_tail(&new_tgt->callback_list, &tgt->callback_list_head); |
| 416 |
|
| 417 |
// If the new target has any m* callbacks, remember this. |
| 418 |
if (new_tgt->mmap_callback != NULL) |
| 419 |
tgt->mmap_events = 1; |
| 420 |
if (new_tgt->munmap_callback != NULL) |
| 421 |
tgt->munmap_events = 1; |
| 422 |
if (new_tgt->mprotect_callback != NULL) |
| 423 |
tgt->mprotect_events = 1; |
| 424 |
return 0; |
| 425 |
} |
| 426 |
|
| 427 |
static int |
| 428 |
stap_utrace_detach(struct task_struct *tsk, |
| 429 |
const struct utrace_engine_ops *ops) |
| 430 |
{ |
| 431 |
struct utrace_engine *engine; |
| 432 |
struct mm_struct *mm; |
| 433 |
int rc = 0; |
| 434 |
|
| 435 |
// Ignore invalid tasks. |
| 436 |
if (tsk == NULL || tsk->pid <= 0) |
| 437 |
return 0; |
| 438 |
|
| 439 |
#ifdef PF_KTHREAD |
| 440 |
// Ignore kernel threads. On systems without PF_KTHREAD, |
| 441 |
// we're ok, since kernel threads won't be matched by the |
| 442 |
// utrace_attach_task() call below. |
| 443 |
if (tsk->flags & PF_KTHREAD) |
| 444 |
return 0; |
| 445 |
#endif |
| 446 |
|
| 447 |
// Notice we're not calling get_task_mm() here. Normally we |
| 448 |
// avoid tasks with no mm, because those are kernel threads. |
| 449 |
// So, why is this function different? When a thread is in |
| 450 |
// the process of dying, its mm gets freed. Then, later the |
| 451 |
// thread gets in the dying state and the thread's DEATH event |
| 452 |
// handler gets called (if any). |
| 453 |
// |
| 454 |
// If a thread is in this "mortally wounded" state - no mm |
| 455 |
// but not dead - and at that moment this function is called, |
| 456 |
// we'd miss detaching from it if we were checking to see if |
| 457 |
// it had an mm. |
| 458 |
|
| 459 |
engine = utrace_attach_task(tsk, UTRACE_ATTACH_MATCH_OPS, ops, 0); |
| 460 |
if (IS_ERR(engine)) { |
| 461 |
rc = -PTR_ERR(engine); |
| 462 |
if (rc != ENOENT) { |
| 463 |
_stp_error("utrace_attach_task returned error %d on pid %d", |
| 464 |
rc, tsk->pid); |
| 465 |
} |
| 466 |
else { |
| 467 |
rc = 0; |
| 468 |
} |
| 469 |
} |
| 470 |
else if (unlikely(engine == NULL)) { |
| 471 |
_stp_error("utrace_attach returned NULL on pid %d", |
| 472 |
(int)tsk->pid); |
| 473 |
rc = EFAULT; |
| 474 |
} |
| 475 |
else { |
| 476 |
rc = utrace_control(tsk, engine, UTRACE_DETACH); |
| 477 |
switch (rc) { |
| 478 |
case 0: /* success */ |
| 479 |
debug_task_finder_detach(); |
| 480 |
break; |
| 481 |
case -ESRCH: /* REAP callback already begun */ |
| 482 |
case -EALREADY: /* DEATH callback already begun */ |
| 483 |
rc = 0; /* ignore these errors */ |
| 484 |
break; |
| 485 |
case -EINPROGRESS: |
| 486 |
do { |
| 487 |
rc = utrace_barrier(tsk, engine); |
| 488 |
} while (rc == -ERESTARTSYS); |
| 489 |
if (rc == 0 || rc == -ESRCH || rc == -EALREADY) { |
| 490 |
rc = 0; |
| 491 |
debug_task_finder_detach(); |
| 492 |
} else { |
| 493 |
rc = -rc; |
| 494 |
_stp_error("utrace_barrier returned error %d on pid %d", rc, tsk->pid); |
| 495 |
} |
| 496 |
break; |
| 497 |
default: |
| 498 |
rc = -rc; |
| 499 |
_stp_error("utrace_control returned error %d on pid %d", |
| 500 |
rc, tsk->pid); |
| 501 |
break; |
| 502 |
} |
| 503 |
utrace_engine_put(engine); |
| 504 |
} |
| 505 |
return rc; |
| 506 |
} |
| 507 |
|
| 508 |
static char * |
| 509 |
__stp_get_mm_path(struct mm_struct *mm, char *buf, int buflen) |
| 510 |
{ |
| 511 |
struct file *vm_file = stap_find_exe_file(mm); |
| 512 |
char *rc = NULL; |
| 513 |
|
| 514 |
if (vm_file) { |
| 515 |
#ifdef STAPCONF_DPATH_PATH |
| 516 |
rc = d_path(&(vm_file->f_path), buf, buflen); |
| 517 |
#else |
| 518 |
rc = d_path(vm_file->f_dentry, vm_file->f_vfsmnt, |
| 519 |
buf, buflen); |
| 520 |
#endif |
| 521 |
fput(vm_file); |
| 522 |
} |
| 523 |
else { |
| 524 |
*buf = '\0'; |
| 525 |
rc = ERR_PTR(-ENOENT); |
| 526 |
} |
| 527 |
return rc; |
| 528 |
} |
| 529 |
|
| 530 |
/* |
| 531 |
* All user threads get an engine with __STP_TASK_FINDER_EVENTS events |
| 532 |
* attached to it so the task_finder layer can monitor new thread |
| 533 |
* creation/death. |
| 534 |
*/ |
| 535 |
#define __STP_TASK_FINDER_EVENTS (UTRACE_EVENT(CLONE) \ |
| 536 |
| UTRACE_EVENT(EXEC) \ |
| 537 |
| UTRACE_EVENT(DEATH)) |
| 538 |
|
| 539 |
/* |
| 540 |
* __STP_TASK_BASE_EVENTS: base events for stap_task_finder_target's |
| 541 |
* without map callback's |
| 542 |
* |
| 543 |
* __STP_TASK_VM_BASE_EVENTS: base events for |
| 544 |
* stap_task_finder_target's with map callback's |
| 545 |
*/ |
| 546 |
#define __STP_TASK_BASE_EVENTS (UTRACE_EVENT(DEATH)|UTRACE_EVENT(EXEC)) |
| 547 |
|
| 548 |
#define __STP_TASK_VM_BASE_EVENTS (__STP_TASK_BASE_EVENTS \ |
| 549 |
| UTRACE_EVENT(SYSCALL_ENTRY)\ |
| 550 |
| UTRACE_EVENT(SYSCALL_EXIT)) |
| 551 |
|
| 552 |
/* |
| 553 |
* All "interesting" threads get an engine with |
| 554 |
* __STP_ATTACHED_TASK_EVENTS events attached to it. After the thread |
| 555 |
* quiesces, we reset the events to __STP_ATTACHED_TASK_BASE_EVENTS |
| 556 |
* events. |
| 557 |
*/ |
| 558 |
#define __STP_ATTACHED_TASK_EVENTS (UTRACE_EVENT(DEATH) \ |
| 559 |
| UTRACE_EVENT(QUIESCE)) |
| 560 |
|
| 561 |
#define __STP_ATTACHED_TASK_BASE_EVENTS(tgt) \ |
| 562 |
(((tgt)->mmap_events || (tgt)->munmap_events \ |
| 563 |
|| (tgt)->mprotect_events) \ |
| 564 |
? __STP_TASK_VM_BASE_EVENTS : __STP_TASK_BASE_EVENTS) |
| 565 |
|
| 566 |
static int |
| 567 |
__stp_utrace_attach(struct task_struct *tsk, |
| 568 |
const struct utrace_engine_ops *ops, void *data, |
| 569 |
unsigned long event_flags, |
| 570 |
enum utrace_resume_action action) |
| 571 |
{ |
| 572 |
struct utrace_engine *engine; |
| 573 |
int rc = 0; |
| 574 |
|
| 575 |
// Ignore invalid tasks. |
| 576 |
if (tsk == NULL || tsk->pid <= 0) |
| 577 |
return EPERM; |
| 578 |
|
| 579 |
#ifdef PF_KTHREAD |
| 580 |
// Ignore kernel threads |
| 581 |
if (tsk->flags & PF_KTHREAD) |
| 582 |
return EPERM; |
| 583 |
#endif |
| 584 |
|
| 585 |
// Ignore threads with no mm (which are either kernel threads |
| 586 |
// or "mortally wounded" threads). |
| 587 |
// |
| 588 |
// Note we're not calling get_task_mm()/mmput() here. Since |
| 589 |
// we're in the the context of that task, the mm should stick |
| 590 |
// around without locking it (and mmput() can sleep). |
| 591 |
if (! tsk->mm) |
| 592 |
return EPERM; |
| 593 |
|
| 594 |
engine = utrace_attach_task(tsk, UTRACE_ATTACH_CREATE, ops, data); |
| 595 |
if (IS_ERR(engine)) { |
| 596 |
int error = -PTR_ERR(engine); |
| 597 |
if (error != ESRCH && error != ENOENT) { |
| 598 |
_stp_error("utrace_attach returned error %d on pid %d", |
| 599 |
error, (int)tsk->pid); |
| 600 |
rc = error; |
| 601 |
} |
| 602 |
} |
| 603 |
else if (unlikely(engine == NULL)) { |
| 604 |
_stp_error("utrace_attach returned NULL on pid %d", |
| 605 |
(int)tsk->pid); |
| 606 |
rc = EFAULT; |
| 607 |
} |
| 608 |
else { |
| 609 |
rc = utrace_set_events(tsk, engine, event_flags); |
| 610 |
dbug_task(2, "utrace_set_events(%lu) returned %d", event_flags, |
| 611 |
rc); |
| 612 |
if (rc == -EINPROGRESS) { |
| 613 |
/* |
| 614 |
* It's running our callback, so we have to |
| 615 |
* synchronize. We can't keep rcu_read_lock, |
| 616 |
* so the task pointer might die. But it's |
| 617 |
* safe to call utrace_barrier() even with a |
| 618 |
* stale task pointer, if we have an engine |
| 619 |
* ref. |
| 620 |
*/ |
| 621 |
do { |
| 622 |
rc = utrace_barrier(tsk, engine); |
| 623 |
} while (rc == -ERESTARTSYS); |
| 624 |
if (rc != 0 && rc != -ESRCH && rc != -EALREADY) |
| 625 |
_stp_error("utrace_barrier returned error %d on pid %d", |
| 626 |
rc, (int)tsk->pid); |
| 627 |
} |
| 628 |
if (rc == 0) { |
| 629 |
debug_task_finder_attach(); |
| 630 |
|
| 631 |
if (action != UTRACE_RESUME) { |
| 632 |
rc = utrace_control(tsk, engine, action); |
| 633 |
dbug_task(2, "utrace_control(%d) returned %d", action, rc); |
| 634 |
/* If utrace_control() returns |
| 635 |
* EINPROGRESS when we're trying to |
| 636 |
* stop/interrupt, that means the task |
| 637 |
* hasn't stopped quite yet, but will |
| 638 |
* soon. Ignore this error. */ |
| 639 |
if (rc != 0 && rc != -EINPROGRESS) { |
| 640 |
_stp_error("utrace_control returned error %d on pid %d", |
| 641 |
rc, (int)tsk->pid); |
| 642 |
} |
| 643 |
rc = 0; |
| 644 |
} |
| 645 |
} |
| 646 |
else if (rc != -ESRCH && rc != -EALREADY) |
| 647 |
_stp_error("utrace_set_events2 returned error %d on pid %d", |
| 648 |
rc, (int)tsk->pid); |
| 649 |
utrace_engine_put(engine); |
| 650 |
} |
| 651 |
return rc; |
| 652 |
} |
| 653 |
|
| 654 |
static int |
| 655 |
stap_utrace_attach(struct task_struct *tsk, |
| 656 |
const struct utrace_engine_ops *ops, void *data, |
| 657 |
unsigned long event_flags) |
| 658 |
{ |
| 659 |
return __stp_utrace_attach(tsk, ops, data, event_flags, UTRACE_RESUME); |
| 660 |
} |
| 661 |
|
| 662 |
static inline void |
| 663 |
__stp_call_callbacks(struct stap_task_finder_target *tgt, |
| 664 |
struct task_struct *tsk, int register_p, int process_p) |
| 665 |
{ |
| 666 |
struct list_head *cb_node; |
| 667 |
int rc; |
| 668 |
|
| 669 |
dbug_task(2, "entering tgt=%p tsk=%p pid=%d", tgt, tsk, tsk ? tsk->tgid : -1); |
| 670 |
|
| 671 |
if (tgt == NULL || tsk == NULL) |
| 672 |
return; |
| 673 |
|
| 674 |
list_for_each(cb_node, &tgt->callback_list_head) { |
| 675 |
struct stap_task_finder_target *cb_tgt; |
| 676 |
|
| 677 |
cb_tgt = list_entry(cb_node, struct stap_task_finder_target, |
| 678 |
callback_list); |
| 679 |
if (cb_tgt == NULL || cb_tgt->callback == NULL) |
| 680 |
continue; |
| 681 |
|
| 682 |
rc = cb_tgt->callback(cb_tgt, tsk, register_p, process_p); |
| 683 |
|
| 684 |
dbug_task(1, "tgt %s callback returned %d (proc=%s pid=%d, " |
| 685 |
"pathlen=%d, engine-attached=%d)", |
| 686 |
(cb_tgt->purpose?:""), rc, cb_tgt->procname, |
| 687 |
cb_tgt->pid, (int) cb_tgt->pathlen, |
| 688 |
cb_tgt->engine_attached); |
| 689 |
|
| 690 |
if (rc != 0) { |
| 691 |
_stp_warn("task_finder %s%scallback for task %d failed: %d", |
| 692 |
(cb_tgt->purpose?:""), (cb_tgt->purpose?" ":""), |
| 693 |
(int)tsk->pid, rc); |
| 694 |
} |
| 695 |
} |
| 696 |
} |
| 697 |
|
| 698 |
bool |
| 699 |
__verify_build_id(struct task_struct *tsk, unsigned long addr, |
| 700 |
unsigned const char *build_id, int build_id_len); |
| 701 |
|
| 702 |
bool |
| 703 |
__verify_build_id(struct task_struct *tsk, unsigned long addr, |
| 704 |
unsigned const char *build_id, int build_id_len) |
| 705 |
{ |
| 706 |
#define MAX_HEXSTR_LEN 64 |
| 707 |
int i; |
| 708 |
unsigned char tsk_build_id[MAX_HEXSTR_LEN + 1]; |
| 709 |
|
| 710 |
if (build_id_len > MAX_HEXSTR_LEN) |
| 711 |
return false; |
| 712 |
|
| 713 |
for (i = 0; i < build_id_len / 2; i++) { |
| 714 |
int rc; |
| 715 |
unsigned char b; |
| 716 |
|
| 717 |
if ((rc = __access_process_vm_noflush(tsk, addr + i, &b, 1, 0)) != 1) |
| 718 |
return false; |
| 719 |
|
| 720 |
tsk_build_id[i * 2] = "0123456789abcdef"[b >> 4]; |
| 721 |
tsk_build_id[i * 2 + 1] = "0123456789abcdef"[b & 0xf]; |
| 722 |
} |
| 723 |
tsk_build_id[build_id_len] = '\0'; |
| 724 |
|
| 725 |
if (strcmp(build_id, tsk_build_id)) { |
| 726 |
dbug_task(2, "target build-id not matched: [%s] @ 0x%lx != [%s]\n", |
| 727 |
build_id, addr, tsk_build_id); |
| 728 |
return false; |
| 729 |
} |
| 730 |
|
| 731 |
return true; |
| 732 |
} |
| 733 |
|
| 734 |
static void |
| 735 |
__stp_call_mmap_callbacks(struct stap_task_finder_target *tgt, |
| 736 |
struct task_struct *tsk, char *path, |
| 737 |
struct dentry *dentry, |
| 738 |
unsigned long addr, unsigned long length, |
| 739 |
unsigned long offset, unsigned long vm_flags) |
| 740 |
{ |
| 741 |
struct list_head *cb_node; |
| 742 |
int rc; |
| 743 |
|
| 744 |
if (tgt == NULL || tsk == NULL) |
| 745 |
return; |
| 746 |
|
| 747 |
dbug_task_vma(1, |
| 748 |
"pid %d, a/l/o/p/path 0x%lx 0x%lx 0x%lx %c%c%c%c %s\n", |
| 749 |
tsk->pid, addr, length, offset, |
| 750 |
vm_flags & VM_READ ? 'r' : '-', |
| 751 |
vm_flags & VM_WRITE ? 'w' : '-', |
| 752 |
vm_flags & VM_EXEC ? 'x' : '-', |
| 753 |
vm_flags & VM_MAYSHARE ? 's' : 'p', |
| 754 |
path); |
| 755 |
list_for_each(cb_node, &tgt->callback_list_head) { |
| 756 |
struct stap_task_finder_target *cb_tgt; |
| 757 |
|
| 758 |
cb_tgt = list_entry(cb_node, struct stap_task_finder_target, |
| 759 |
callback_list); |
| 760 |
if (cb_tgt == NULL || cb_tgt->mmap_callback == NULL) |
| 761 |
continue; |
| 762 |
|
| 763 |
rc = cb_tgt->mmap_callback(cb_tgt, tsk, path, dentry, |
| 764 |
addr, length, offset, vm_flags); |
| 765 |
if (rc != 0) { |
| 766 |
_stp_warn("task_finder mmap %s%scallback for task %d failed: %d", |
| 767 |
(cb_tgt->purpose?:""), (cb_tgt->purpose?" ":""), |
| 768 |
(int)tsk->pid, rc); |
| 769 |
} |
| 770 |
} |
| 771 |
} |
| 772 |
|
| 773 |
|
| 774 |
static struct vm_area_struct * |
| 775 |
__stp_find_file_based_vma(struct mm_struct *mm, unsigned long addr) |
| 776 |
{ |
| 777 |
struct vm_area_struct *vma = find_vma(mm, addr); |
| 778 |
|
| 779 |
// I'm not positive why the checking for vm_start > addr is |
| 780 |
// necessary, but it seems to be (sometimes find_vma() returns |
| 781 |
// a vma that addr doesn't belong to). |
| 782 |
if (vma && (vma->vm_file == NULL || vma->vm_start > addr)) |
| 783 |
vma = NULL; |
| 784 |
return vma; |
| 785 |
} |
| 786 |
|
| 787 |
|
| 788 |
static void |
| 789 |
__stp_call_mmap_callbacks_with_addr(struct stap_task_finder_target *tgt, |
| 790 |
struct task_struct *tsk, |
| 791 |
unsigned long addr) |
| 792 |
{ |
| 793 |
struct mm_struct *mm; |
| 794 |
struct vm_area_struct *vma; |
| 795 |
char *mmpath_buf = NULL; |
| 796 |
char *mmpath = NULL; |
| 797 |
struct dentry *dentry = NULL; |
| 798 |
unsigned long length = 0; |
| 799 |
unsigned long offset = 0; |
| 800 |
unsigned long vm_flags = 0; |
| 801 |
|
| 802 |
// __stp_call_mmap_callbacks_with_addr() is only called when |
| 803 |
// tsk is current, so there isn't any danger of mm going |
| 804 |
// away. So, we don't need to call get_task_mm()/mmput() |
| 805 |
// (which avoids the possibility of sleeping). |
| 806 |
mm = tsk->mm; |
| 807 |
if (! mm) |
| 808 |
return; |
| 809 |
|
| 810 |
// The down_read() function can sleep, so we'll call |
| 811 |
// down_read_trylock() instead, which can fail. |
| 812 |
if (! mmap_read_trylock(mm)) |
| 813 |
return; |
| 814 |
vma = __stp_find_file_based_vma(mm, addr); |
| 815 |
if (vma) { |
| 816 |
// Cache information we need from the vma |
| 817 |
addr = vma->vm_start; |
| 818 |
length = vma->vm_end - vma->vm_start; |
| 819 |
offset = (vma->vm_pgoff << PAGE_SHIFT); |
| 820 |
vm_flags = vma->vm_flags; |
| 821 |
#ifdef STAPCONF_DPATH_PATH |
| 822 |
dentry = vma->vm_file->f_path.dentry; |
| 823 |
#else |
| 824 |
dentry = vma->vm_file->f_dentry; |
| 825 |
#endif |
| 826 |
|
| 827 |
// Allocate space for a path |
| 828 |
mmpath_buf = _stp_kmalloc(PATH_MAX); |
| 829 |
if (mmpath_buf == NULL) { |
| 830 |
mmap_read_unlock(mm); |
| 831 |
_stp_error("Unable to allocate space for path"); |
| 832 |
return; |
| 833 |
} |
| 834 |
else { |
| 835 |
// Grab the path associated with this vma. |
| 836 |
#ifdef STAPCONF_DPATH_PATH |
| 837 |
mmpath = d_path(&(vma->vm_file->f_path), mmpath_buf, |
| 838 |
PATH_MAX); |
| 839 |
#else |
| 840 |
mmpath = d_path(vma->vm_file->f_dentry, |
| 841 |
vma->vm_file->f_vfsmnt, mmpath_buf, |
| 842 |
PATH_MAX); |
| 843 |
#endif |
| 844 |
if (mmpath == NULL || IS_ERR(mmpath)) { |
| 845 |
long err = ((mmpath == NULL) ? 0 |
| 846 |
: -PTR_ERR(mmpath)); |
| 847 |
_stp_error("Unable to get path (error %ld) for pid %d", |
| 848 |
err, (int)tsk->pid); |
| 849 |
mmpath = NULL; |
| 850 |
} |
| 851 |
} |
| 852 |
} |
| 853 |
|
| 854 |
// At this point, we're done with the vma (assuming we found |
| 855 |
// one). We can't hold the 'mmap_sem' semaphore while making |
| 856 |
// callbacks. |
| 857 |
mmap_read_unlock(mm); |
| 858 |
|
| 859 |
if (mmpath) |
| 860 |
__stp_call_mmap_callbacks(tgt, tsk, mmpath, dentry, addr, |
| 861 |
length, offset, vm_flags); |
| 862 |
|
| 863 |
// Cleanup. |
| 864 |
if (mmpath_buf) |
| 865 |
_stp_kfree(mmpath_buf); |
| 866 |
return; |
| 867 |
} |
| 868 |
|
| 869 |
|
| 870 |
static inline void |
| 871 |
__stp_call_munmap_callbacks(struct stap_task_finder_target *tgt, |
| 872 |
struct task_struct *tsk, unsigned long addr, |
| 873 |
unsigned long length) |
| 874 |
{ |
| 875 |
struct list_head *cb_node; |
| 876 |
int rc; |
| 877 |
|
| 878 |
if (tgt == NULL || tsk == NULL) |
| 879 |
return; |
| 880 |
|
| 881 |
list_for_each(cb_node, &tgt->callback_list_head) { |
| 882 |
struct stap_task_finder_target *cb_tgt; |
| 883 |
|
| 884 |
cb_tgt = list_entry(cb_node, struct stap_task_finder_target, |
| 885 |
callback_list); |
| 886 |
if (cb_tgt == NULL || cb_tgt->munmap_callback == NULL) |
| 887 |
continue; |
| 888 |
|
| 889 |
rc = cb_tgt->munmap_callback(cb_tgt, tsk, addr, length); |
| 890 |
if (rc != 0) { |
| 891 |
_stp_warn("task_finder munmap %s%scallback for task %d failed: %d", |
| 892 |
(cb_tgt->purpose?:""), (cb_tgt->purpose?" ":""), |
| 893 |
(int)tsk->pid, rc); |
| 894 |
} |
| 895 |
} |
| 896 |
} |
| 897 |
|
| 898 |
static inline void |
| 899 |
__stp_call_mprotect_callbacks(struct stap_task_finder_target *tgt, |
| 900 |
struct task_struct *tsk, unsigned long addr, |
| 901 |
unsigned long length, int prot) |
| 902 |
{ |
| 903 |
struct list_head *cb_node; |
| 904 |
int rc; |
| 905 |
|
| 906 |
if (tgt == NULL || tsk == NULL) |
| 907 |
return; |
| 908 |
|
| 909 |
list_for_each(cb_node, &tgt->callback_list_head) { |
| 910 |
struct stap_task_finder_target *cb_tgt; |
| 911 |
|
| 912 |
cb_tgt = list_entry(cb_node, struct stap_task_finder_target, |
| 913 |
callback_list); |
| 914 |
if (cb_tgt == NULL || cb_tgt->mprotect_callback == NULL) |
| 915 |
continue; |
| 916 |
|
| 917 |
rc = cb_tgt->mprotect_callback(cb_tgt, tsk, addr, length, |
| 918 |
prot); |
| 919 |
if (rc != 0) { |
| 920 |
_stp_warn("task_finder mprotect %s%scallback for task %d failed: %d", |
| 921 |
(cb_tgt->purpose?:""), (cb_tgt->purpose?" ":""), |
| 922 |
(int)tsk->pid, rc); |
| 923 |
} |
| 924 |
} |
| 925 |
} |
| 926 |
|
| 927 |
static inline void |
| 928 |
__stp_utrace_attach_match_filename(struct task_struct *tsk, |
| 929 |
const char * const filename, |
| 930 |
int process_p) |
| 931 |
{ |
| 932 |
size_t filelen; |
| 933 |
struct list_head *tgt_node; |
| 934 |
struct stap_task_finder_target *tgt; |
| 935 |
uid_t tsk_euid; |
| 936 |
|
| 937 |
#ifdef STAPCONF_TASK_UID |
| 938 |
tsk_euid = tsk->euid; |
| 939 |
#else |
| 940 |
#if defined(CONFIG_USER_NS) || (LINUX_VERSION_CODE >= KERNEL_VERSION(3,14,0)) |
| 941 |
tsk_euid = from_kuid_munged(current_user_ns(), task_euid(tsk)); |
| 942 |
#else |
| 943 |
tsk_euid = task_euid(tsk); |
| 944 |
#endif |
| 945 |
#endif |
| 946 |
filelen = strlen(filename); |
| 947 |
list_for_each(tgt_node, &__stp_task_finder_list) { |
| 948 |
int rc; |
| 949 |
|
| 950 |
tgt = list_entry(tgt_node, struct stap_task_finder_target, |
| 951 |
list); |
| 952 |
// If we've got a matching procname or a matching build-id |
| 953 |
// or we're probing all threads, we've got a match. We've |
| 954 |
// got to keep matching since a single thread could match a |
| 955 |
// procname/build-id and match an "all thread" probe. |
| 956 |
if (tgt == NULL) |
| 957 |
continue; |
| 958 |
/* buildid-based target ... gets checked in __stp_tf_quiesce_worker */ |
| 959 |
/* procname-based target */ |
| 960 |
else if (tgt->pathlen > 0 |
| 961 |
&& (tgt->pathlen != filelen |
| 962 |
|| strcmp(tgt->procname, filename) != 0)) |
| 963 |
{ |
| 964 |
dbug_task(2, "target path NOT matched: [%s] != [%s]", |
| 965 |
tgt->procname, filename); |
| 966 |
continue; |
| 967 |
} |
| 968 |
/* Ignore pid-based target, they were handled at startup. */ |
| 969 |
else if (tgt->pid != 0) |
| 970 |
continue; |
| 971 |
/* Notice that "pid == 0" (which means to probe all |
| 972 |
* threads) falls through. */ |
| 973 |
|
| 974 |
#if ! STP_PRIVILEGE_CONTAINS (STP_PRIVILEGE, STP_PR_STAPDEV) && \ |
| 975 |
! STP_PRIVILEGE_CONTAINS (STP_PRIVILEGE, STP_PR_STAPSYS) |
| 976 |
/* Make sure unprivileged users only probe their own threads. */ |
| 977 |
if (_stp_uid != tsk_euid) { |
| 978 |
if (tgt->pid != 0) { |
| 979 |
_stp_warn("Process %d does not belong to unprivileged user %d", |
| 980 |
tsk->pid, _stp_uid); |
| 981 |
} |
| 982 |
continue; |
| 983 |
} |
| 984 |
#endif |
| 985 |
|
| 986 |
|
| 987 |
// Set up events we need for attached tasks. We won't |
| 988 |
// actually call the callbacks here - we'll call them |
| 989 |
// when the thread gets quiesced. |
| 990 |
rc = __stp_utrace_attach(tsk, &tgt->ops, tgt, |
| 991 |
__STP_ATTACHED_TASK_EVENTS, |
| 992 |
UTRACE_INTERRUPT); |
| 993 |
if (rc != 0 && rc != EPERM) |
| 994 |
break; |
| 995 |
tgt->engine_attached = 1; |
| 996 |
} |
| 997 |
} |
| 998 |
|
| 999 |
// This function handles the details of getting a task's associated |
| 1000 |
// procname, and calling __stp_utrace_attach_match_filename() to |
| 1001 |
// attach to it if we find the procname "interesting". So, what's the |
| 1002 |
// difference between path_tsk and match_tsk? Normally they are the |
| 1003 |
// same, except in one case. In an UTRACE_EVENT(EXEC), we need to |
| 1004 |
// detach engines from the newly exec'ed process (since its path has |
| 1005 |
// changed). In this case, we have to match the path of the parent |
| 1006 |
// (path_tsk) against the child (match_tsk). |
| 1007 |
|
| 1008 |
static void |
| 1009 |
__stp_utrace_attach_match_tsk(struct task_struct *path_tsk, |
| 1010 |
struct task_struct *match_tsk, int process_p) |
| 1011 |
{ |
| 1012 |
struct mm_struct *mm; |
| 1013 |
char *mmpath_buf; |
| 1014 |
char *mmpath; |
| 1015 |
|
| 1016 |
#if 0 |
| 1017 |
printk(KERN_ERR "%s:%d entry\n", __FUNCTION__, __LINE__); |
| 1018 |
#endif |
| 1019 |
if (path_tsk == NULL || path_tsk->pid <= 0 |
| 1020 |
|| match_tsk == NULL || match_tsk->pid <= 0 |
| 1021 |
|| (_stp_target && match_tsk != path_tsk)) |
| 1022 |
return; |
| 1023 |
|
| 1024 |
// Grab the path associated with the path_tsk. |
| 1025 |
// |
| 1026 |
// Note we're not calling get_task_mm()/mmput() here. Since |
| 1027 |
// we're in the the context of path_task, the mm should stick |
| 1028 |
// around without locking it (and mmput() can sleep). |
| 1029 |
mm = path_tsk->mm; |
| 1030 |
if (! mm) { |
| 1031 |
/* If the thread doesn't have a mm_struct, it is |
| 1032 |
* a kernel thread which we need to skip. */ |
| 1033 |
return; |
| 1034 |
} |
| 1035 |
|
| 1036 |
// Allocate space for a path |
| 1037 |
mmpath_buf = _stp_kmalloc(PATH_MAX); |
| 1038 |
if (mmpath_buf == NULL) { |
| 1039 |
_stp_error("Unable to allocate space for path"); |
| 1040 |
return; |
| 1041 |
} |
| 1042 |
|
| 1043 |
// Grab the path associated with the new task |
| 1044 |
mmpath = __stp_get_mm_path(mm, mmpath_buf, PATH_MAX); |
| 1045 |
if (mmpath == NULL || IS_ERR(mmpath)) { |
| 1046 |
int rc = -PTR_ERR(mmpath); |
| 1047 |
if (rc != ENOENT) |
| 1048 |
_stp_error("Unable to get path (error %d) for pid %d", |
| 1049 |
rc, (int)path_tsk->pid); |
| 1050 |
} |
| 1051 |
else { |
| 1052 |
#if 0 |
| 1053 |
_stp_dbug(__FUNCTION__, __LINE__, |
| 1054 |
"calling __stp_utrace_attach_match_filename(%p, %s, %d, %d)\n", |
| 1055 |
match_tsk, mmpath, register_p, process_p); |
| 1056 |
#endif |
| 1057 |
__stp_utrace_attach_match_filename(match_tsk, mmpath, |
| 1058 |
process_p); |
| 1059 |
} |
| 1060 |
|
| 1061 |
_stp_kfree(mmpath_buf); |
| 1062 |
return; |
| 1063 |
} |
| 1064 |
|
| 1065 |
static void |
| 1066 |
__stp_tf_clone_worker(struct task_work *work) |
| 1067 |
{ |
| 1068 |
struct __stp_tf_task_work *tf_work = \ |
| 1069 |
container_of(work, struct __stp_tf_task_work, work); |
| 1070 |
struct utrace_engine_ops *ops = \ |
| 1071 |
(struct utrace_engine_ops *)tf_work->data; |
| 1072 |
struct task_struct *parent = tf_work->task; |
| 1073 |
int rc; |
| 1074 |
|
| 1075 |
might_sleep(); |
| 1076 |
if (atomic_read(&__stp_task_finder_state) != __STP_TF_RUNNING |
| 1077 |
|| current->flags & PF_EXITING) |
| 1078 |
return; |
| 1079 |
|
| 1080 |
__stp_tf_handler_start(); |
| 1081 |
|
| 1082 |
// On clone, attach to the child, but we might need to sleep... |
| 1083 |
rc = __stp_utrace_attach(current, ops, 0, |
| 1084 |
__STP_TASK_FINDER_EVENTS, UTRACE_RESUME); |
| 1085 |
if (rc == 0 || rc == EPERM) { |
| 1086 |
// Assume that if the thread is a thread group leader, |
| 1087 |
// it is a process. |
| 1088 |
__stp_utrace_attach_match_tsk(parent, current, |
| 1089 |
(current->pid == current->tgid)); |
| 1090 |
} |
| 1091 |
|
| 1092 |
__stp_tf_handler_end(); |
| 1093 |
} |
| 1094 |
|
| 1095 |
|
| 1096 |
static u32 |
| 1097 |
__stp_utrace_task_finder_report_clone(u32 action, |
| 1098 |
struct utrace_engine *engine, |
| 1099 |
unsigned long clone_flags, |
| 1100 |
struct task_struct *child) |
| 1101 |
{ |
| 1102 |
int rc; |
| 1103 |
struct task_work *work; |
| 1104 |
|
| 1105 |
if (atomic_read(&__stp_task_finder_state) != __STP_TF_RUNNING) { |
| 1106 |
debug_task_finder_detach(); |
| 1107 |
return UTRACE_DETACH; |
| 1108 |
} |
| 1109 |
|
| 1110 |
__stp_tf_handler_start(); |
| 1111 |
|
| 1112 |
// We can't sleep in tracepoint handlers. |
| 1113 |
// __stp_utrace_attach() might need to call utrace_barrier(), |
| 1114 |
// which can sleep when task != current. So, arrange for the |
| 1115 |
// child task to truly stop. |
| 1116 |
work = __stp_tf_alloc_task_work((void *)(engine->ops)); |
| 1117 |
if (work == NULL) { |
| 1118 |
_stp_error("Unable to allocate space for task_work"); |
| 1119 |
__stp_tf_handler_end(); |
| 1120 |
return UTRACE_RESUME; |
| 1121 |
} |
| 1122 |
__stp_tf_init_task_work(work, &__stp_tf_clone_worker); |
| 1123 |
rc = __stp_tf_task_work_add(child, work); |
| 1124 |
if (rc) { |
| 1125 |
__stp_tf_task_work_free(work); |
| 1126 |
// stp_task_work_add() returns -ESRCH if the task has already |
| 1127 |
// passed exit_task_work(). Just ignore this error. |
| 1128 |
if (rc != -ESRCH) |
| 1129 |
printk(KERN_ERR "%s:%d - stp_task_work_add() returned %d\n", |
| 1130 |
__FUNCTION__, __LINE__, rc); |
| 1131 |
} |
| 1132 |
|
| 1133 |
__stp_tf_handler_end(); |
| 1134 |
return UTRACE_RESUME; |
| 1135 |
} |
| 1136 |
|
| 1137 |
static u32 |
| 1138 |
__stp_utrace_task_finder_report_exec(u32 action, |
| 1139 |
struct utrace_engine *engine, |
| 1140 |
const struct linux_binfmt *fmt, |
| 1141 |
const struct linux_binprm *bprm, |
| 1142 |
struct pt_regs *regs) |
| 1143 |
{ |
| 1144 |
size_t filelen; |
| 1145 |
struct list_head *tgt_node; |
| 1146 |
struct stap_task_finder_target *tgt; |
| 1147 |
int found_node = 0; |
| 1148 |
|
| 1149 |
if (atomic_read(&__stp_task_finder_state) != __STP_TF_RUNNING) { |
| 1150 |
debug_task_finder_detach(); |
| 1151 |
return UTRACE_DETACH; |
| 1152 |
} |
| 1153 |
|
| 1154 |
__stp_tf_handler_start(); |
| 1155 |
|
| 1156 |
// If the original task was "interesting", |
| 1157 |
// __stp_utrace_task_finder_target_exec() will handle calling |
| 1158 |
// callbacks. |
| 1159 |
|
| 1160 |
// We assume that all exec's are exec'ing a new process. Note |
| 1161 |
// that we don't use bprm->filename, since that path can be |
| 1162 |
// relative. |
| 1163 |
__stp_utrace_attach_match_tsk(current, current, 1); |
| 1164 |
|
| 1165 |
__stp_tf_handler_end(); |
| 1166 |
return UTRACE_RESUME; |
| 1167 |
} |
| 1168 |
|
| 1169 |
static u32 |
| 1170 |
stap_utrace_task_finder_report_death(struct utrace_engine *engine, |
| 1171 |
bool group_dead, int signal) |
| 1172 |
{ |
| 1173 |
debug_task_finder_detach(); |
| 1174 |
return UTRACE_DETACH; |
| 1175 |
} |
| 1176 |
|
| 1177 |
static u32 |
| 1178 |
__stp_utrace_task_finder_target_exec(u32 action, |
| 1179 |
struct utrace_engine *engine, |
| 1180 |
const struct linux_binfmt *fmt, |
| 1181 |
const struct linux_binprm *bprm, |
| 1182 |
struct pt_regs *regs) |
| 1183 |
{ |
| 1184 |
struct task_struct *tsk = current; |
| 1185 |
struct stap_task_finder_target *tgt = engine->data; |
| 1186 |
int rc; |
| 1187 |
|
| 1188 |
if (atomic_read(&__stp_task_finder_state) != __STP_TF_RUNNING) { |
| 1189 |
debug_task_finder_detach(); |
| 1190 |
return UTRACE_DETACH; |
| 1191 |
} |
| 1192 |
|
| 1193 |
__stp_tf_handler_start(); |
| 1194 |
|
| 1195 |
// We'll hardcode this as a process end. If a thread |
| 1196 |
// calls exec() (which it isn't supposed to), the kernel |
| 1197 |
// "promotes" it to being a process. Call the callbacks. |
| 1198 |
if (tgt != NULL && tsk != NULL) { |
| 1199 |
__stp_call_callbacks(tgt, tsk, 0, 1); |
| 1200 |
} |
| 1201 |
|
| 1202 |
// Note that we don't want to set engine_attached to 0 here - |
| 1203 |
// only when *all* threads using this engine have been |
| 1204 |
// detached. |
| 1205 |
|
| 1206 |
// Let __stp_utrace_task_finder_report_exec() call |
| 1207 |
// __stp_utrace_attach_match_tsk() to figure out if the |
| 1208 |
// exec'ed program is "interesting". |
| 1209 |
|
| 1210 |
__stp_tf_handler_end(); |
| 1211 |
debug_task_finder_detach(); |
| 1212 |
return UTRACE_DETACH; |
| 1213 |
} |
| 1214 |
|
| 1215 |
static u32 |
| 1216 |
__stp_utrace_task_finder_target_death(struct utrace_engine *engine, |
| 1217 |
bool group_dead, int signal) |
| 1218 |
{ |
| 1219 |
struct task_struct *tsk = current; |
| 1220 |
struct stap_task_finder_target *tgt = engine->data; |
| 1221 |
|
| 1222 |
if (atomic_read(&__stp_task_finder_state) != __STP_TF_RUNNING) { |
| 1223 |
debug_task_finder_detach(); |
| 1224 |
return UTRACE_DETACH; |
| 1225 |
} |
| 1226 |
|
| 1227 |
__stp_tf_handler_start(); |
| 1228 |
// The first implementation of this added a |
| 1229 |
// UTRACE_EVENT(DEATH) handler to |
| 1230 |
// __stp_utrace_task_finder_ops. However, dead threads don't |
| 1231 |
// have a mm_struct, so we can't find the exe's path. So, we |
| 1232 |
// don't know which callback(s) to call. |
| 1233 |
// |
| 1234 |
// So, now when an "interesting" thread is found, we add a |
| 1235 |
// separate UTRACE_EVENT(DEATH) handler for each attached |
| 1236 |
// handler. |
| 1237 |
if (tgt != NULL && tsk != NULL) { |
| 1238 |
__stp_call_callbacks(tgt, tsk, 0, |
| 1239 |
((tsk->signal == NULL) |
| 1240 |
|| (atomic_read(&tsk->signal->live) == 0))); |
| 1241 |
} |
| 1242 |
|
| 1243 |
__stp_tf_handler_end(); |
| 1244 |
debug_task_finder_detach(); |
| 1245 |
return UTRACE_DETACH; |
| 1246 |
} |
| 1247 |
|
| 1248 |
static void |
| 1249 |
__stp_call_mmap_callbacks_for_task(struct stap_task_finder_target *tgt, |
| 1250 |
struct task_struct *tsk) |
| 1251 |
{ |
| 1252 |
struct mm_struct *mm; |
| 1253 |
char *mmpath_buf; |
| 1254 |
char *mmpath; |
| 1255 |
struct vm_area_struct *vma; |
| 1256 |
int file_based_vmas = 0; |
| 1257 |
struct vma_cache_t { |
| 1258 |
#ifdef STAPCONF_DPATH_PATH |
| 1259 |
struct path f_path; |
| 1260 |
#else |
| 1261 |
struct vfsmount *f_vfsmnt; |
| 1262 |
#endif |
| 1263 |
struct dentry *dentry; |
| 1264 |
unsigned long addr; |
| 1265 |
unsigned long length; |
| 1266 |
unsigned long offset; |
| 1267 |
unsigned long vm_flags; |
| 1268 |
}; |
| 1269 |
struct vma_cache_t *vma_cache = NULL; |
| 1270 |
struct vma_cache_t *vma_cache_p; |
| 1271 |
|
| 1272 |
// Call the mmap_callback for every vma associated with |
| 1273 |
// a file. |
| 1274 |
// |
| 1275 |
// Note we're not calling get_task_mm()/mmput() here. Since |
| 1276 |
// we're in the the context of that task, the mm should stick |
| 1277 |
// around without locking it (and mmput() can sleep). |
| 1278 |
mm = tsk->mm; |
| 1279 |
if (! mm) |
| 1280 |
return; |
| 1281 |
|
| 1282 |
// Allocate space for a path |
| 1283 |
mmpath_buf = _stp_kmalloc(PATH_MAX); |
| 1284 |
if (mmpath_buf == NULL) { |
| 1285 |
_stp_error("Unable to allocate space for path"); |
| 1286 |
return; |
| 1287 |
} |
| 1288 |
|
| 1289 |
// The down_read() function can sleep, so we'll call |
| 1290 |
// down_read_trylock() instead, which can fail. |
| 1291 |
if (! mmap_read_trylock(mm)) { |
| 1292 |
_stp_kfree(mmpath_buf); |
| 1293 |
return; |
| 1294 |
} |
| 1295 |
|
| 1296 |
{ |
| 1297 |
// Need surrounding {} because of VMA_ITERATOR variable |
| 1298 |
// First find the number of file-based vmas. |
| 1299 |
VMA_ITERATOR(vmi, mm, 0); |
| 1300 |
for_each_vma(vmi, vma) { |
| 1301 |
if (vma->vm_file) |
| 1302 |
file_based_vmas++; |
| 1303 |
} |
| 1304 |
} |
| 1305 |
|
| 1306 |
// Now allocate an array to cache vma information in. |
| 1307 |
if (file_based_vmas > 0) |
| 1308 |
vma_cache = _stp_vzalloc(sizeof(struct vma_cache_t) |
| 1309 |
* file_based_vmas); |
| 1310 |
if (vma_cache != NULL) { |
| 1311 |
// Loop through the vmas again, and cache needed information. |
| 1312 |
VMA_ITERATOR(vmi, mm, 0); |
| 1313 |
vma_cache_p = vma_cache; |
| 1314 |
for_each_vma(vmi,vma) { |
| 1315 |
if (vma->vm_file) { |
| 1316 |
#ifdef STAPCONF_DPATH_PATH |
| 1317 |
// Notice we're increasing the reference |
| 1318 |
// count for 'f_path'. This way it won't |
| 1319 |
// get deleted from out under us. |
| 1320 |
vma_cache_p->f_path = vma->vm_file->f_path; |
| 1321 |
path_get(&vma_cache_p->f_path); |
| 1322 |
vma_cache_p->dentry = vma->vm_file->f_path.dentry; |
| 1323 |
#else |
| 1324 |
// Notice we're increasing the reference |
| 1325 |
// count for 'dentry' and 'f_vfsmnt'. |
| 1326 |
// This way they won't get deleted from |
| 1327 |
// out under us. |
| 1328 |
vma_cache_p->dentry = vma->vm_file->f_dentry; |
| 1329 |
dget(vma_cache_p->dentry); |
| 1330 |
vma_cache_p->f_vfsmnt = vma->vm_file->f_vfsmnt; |
| 1331 |
mntget(vma_cache_p->f_vfsmnt); |
| 1332 |
vma_cache_p->dentry = vma->vm_file->f_dentry; |
| 1333 |
#endif |
| 1334 |
vma_cache_p->addr = vma->vm_start; |
| 1335 |
vma_cache_p->length = vma->vm_end - vma->vm_start; |
| 1336 |
vma_cache_p->offset = (vma->vm_pgoff << PAGE_SHIFT); |
| 1337 |
vma_cache_p->vm_flags = vma->vm_flags; |
| 1338 |
vma_cache_p++; |
| 1339 |
} |
| 1340 |
} |
| 1341 |
} |
| 1342 |
|
| 1343 |
// At this point, we're done with the vmas (assuming we found |
| 1344 |
// any). We can't hold the 'mmap_sem' semaphore while making |
| 1345 |
// callbacks. |
| 1346 |
mmap_read_unlock(mm); |
| 1347 |
|
| 1348 |
if (vma_cache) { |
| 1349 |
int i; |
| 1350 |
|
| 1351 |
// Loop over our cached information and make callbacks |
| 1352 |
// based on it. |
| 1353 |
vma_cache_p = vma_cache; |
| 1354 |
for (i = 0; i < file_based_vmas; i++) { |
| 1355 |
#ifdef STAPCONF_DPATH_PATH |
| 1356 |
mmpath = d_path(&vma_cache_p->f_path, mmpath_buf, |
| 1357 |
PATH_MAX); |
| 1358 |
path_put(&vma_cache_p->f_path); |
| 1359 |
#else |
| 1360 |
mmpath = d_path(vma_cache_p->dentry, |
| 1361 |
vma_cache_p->f_vfsmnt, mmpath_buf, |
| 1362 |
PATH_MAX); |
| 1363 |
dput(vma_cache_p->dentry); |
| 1364 |
mntput(vma_cache_p->f_vfsmnt); |
| 1365 |
#endif |
| 1366 |
if (mmpath == NULL || IS_ERR(mmpath)) { |
| 1367 |
long err = ((mmpath == NULL) ? 0 |
| 1368 |
: -PTR_ERR(mmpath)); |
| 1369 |
_stp_error("Unable to get path (error %ld) for pid %d", |
| 1370 |
err, (int)tsk->pid); |
| 1371 |
} |
| 1372 |
else { |
| 1373 |
__stp_call_mmap_callbacks(tgt, tsk, mmpath, |
| 1374 |
vma_cache_p->dentry, |
| 1375 |
vma_cache_p->addr, |
| 1376 |
vma_cache_p->length, |
| 1377 |
vma_cache_p->offset, |
| 1378 |
vma_cache_p->vm_flags); |
| 1379 |
} |
| 1380 |
vma_cache_p++; |
| 1381 |
} |
| 1382 |
_stp_vfree(vma_cache); |
| 1383 |
} |
| 1384 |
|
| 1385 |
_stp_kfree(mmpath_buf); |
| 1386 |
} |
| 1387 |
|
| 1388 |
static void |
| 1389 |
__stp_tf_quiesce_worker(struct task_work *work) |
| 1390 |
{ |
| 1391 |
struct __stp_tf_task_work *tf_work = \ |
| 1392 |
container_of(work, struct __stp_tf_task_work, work); |
| 1393 |
struct stap_task_finder_target *tgt = \ |
| 1394 |
(struct stap_task_finder_target *)tf_work->data; |
| 1395 |
|
| 1396 |
might_sleep(); |
| 1397 |
if (atomic_read(&__stp_task_finder_state) != __STP_TF_RUNNING |
| 1398 |
|| current->flags & PF_EXITING) |
| 1399 |
return; |
| 1400 |
|
| 1401 |
/* If we had a build-id based executable probe (so we have a |
| 1402 |
* tgt->build_id) set, we could not check it back in |
| 1403 |
* __stp_utrace_attach_* because we can't do sleepy |
| 1404 |
* access_process_vm() calls from there. BUt now that we're |
| 1405 |
* in process context, quiesced, finally we can check. If we |
| 1406 |
* were build-id based, and the build-id does not match, then |
| 1407 |
* we UTRACE_DETACH from this process and skip the callbacks. |
| 1408 |
* |
| 1409 |
* XXX: For processes that do match, we redo this check every |
| 1410 |
* time this callbacks is encountered somehow. That's |
| 1411 |
* probably unnecessary. |
| 1412 |
*/ |
| 1413 |
if (tgt->build_id_len > 0) { |
| 1414 |
int ok = __verify_build_id(current, |
| 1415 |
tgt->build_id_vaddr, |
| 1416 |
tgt->build_id, |
| 1417 |
tgt->build_id_len); |
| 1418 |
|
| 1419 |
dbug_task(2, "verified buildid-target process pid=%ld ok=%d\n", |
| 1420 |
(long) current->tgid, ok); |
| 1421 |
if (!ok) { |
| 1422 |
// stap_utrace_detach (current, & tgt->ops); |
| 1423 |
return; |
| 1424 |
} |
| 1425 |
} |
| 1426 |
|
| 1427 |
__stp_tf_handler_start(); |
| 1428 |
|
| 1429 |
/* NB make sure we run mmap callbacks before other callbacks |
| 1430 |
* like 'probe process.begin' handlers so that the vma tracker |
| 1431 |
* is already initialized in the latter contexts */ |
| 1432 |
|
| 1433 |
/* If this is just a thread other than the thread group |
| 1434 |
* leader, don't bother inform map callback clients about its |
| 1435 |
* memory map, since they will simply duplicate each other. */ |
| 1436 |
if (tgt->mmap_events == 1 && current->tgid == current->pid) { |
| 1437 |
__stp_call_mmap_callbacks_for_task(tgt, current); |
| 1438 |
} |
| 1439 |
|
| 1440 |
/* Call the callbacks. Assume that if the thread is a |
| 1441 |
* thread group leader, it is a process. */ |
| 1442 |
__stp_call_callbacks(tgt, current, 1, (current->pid == current->tgid)); |
| 1443 |
|
| 1444 |
__stp_tf_handler_end(); |
| 1445 |
} |
| 1446 |
|
| 1447 |
static u32 |
| 1448 |
__stp_utrace_task_finder_target_quiesce(u32 action, |
| 1449 |
struct utrace_engine *engine, |
| 1450 |
unsigned long event) |
| 1451 |
{ |
| 1452 |
struct task_struct *tsk = current; |
| 1453 |
struct stap_task_finder_target *tgt = engine->data; |
| 1454 |
struct task_work *work; |
| 1455 |
int rc; |
| 1456 |
|
| 1457 |
if (atomic_read(&__stp_task_finder_state) != __STP_TF_RUNNING) { |
| 1458 |
debug_task_finder_detach(); |
| 1459 |
return UTRACE_DETACH; |
| 1460 |
} |
| 1461 |
|
| 1462 |
if (tgt == NULL || tsk == NULL) { |
| 1463 |
debug_task_finder_detach(); |
| 1464 |
return UTRACE_DETACH; |
| 1465 |
} |
| 1466 |
|
| 1467 |
__stp_tf_handler_start(); |
| 1468 |
|
| 1469 |
// Turn off quiesce handling |
| 1470 |
rc = utrace_set_events(tsk, engine, |
| 1471 |
__STP_ATTACHED_TASK_BASE_EVENTS(tgt)); |
| 1472 |
|
| 1473 |
if (rc == -EINPROGRESS) { |
| 1474 |
/* |
| 1475 |
* It's running our callback, so we have to |
| 1476 |
* synchronize. We can't keep rcu_read_lock, |
| 1477 |
* so the task pointer might die. But it's |
| 1478 |
* safe to call utrace_barrier() even with |
| 1479 |
* a stale task pointer, if we have an engine ref. |
| 1480 |
*/ |
| 1481 |
do { |
| 1482 |
rc = utrace_barrier(tsk, engine); |
| 1483 |
} while (rc == -ERESTARTSYS); |
| 1484 |
if (rc == 0) |
| 1485 |
rc = utrace_set_events(tsk, engine, |
| 1486 |
__STP_ATTACHED_TASK_BASE_EVENTS(tgt)); |
| 1487 |
else if (rc != -ESRCH && rc != -EALREADY) |
| 1488 |
_stp_error("utrace_barrier returned error %d on pid %d", |
| 1489 |
rc, (int)tsk->pid); |
| 1490 |
} |
| 1491 |
if (rc != 0) |
| 1492 |
_stp_error("utrace_set_events returned error %d on pid %d", |
| 1493 |
rc, (int)tsk->pid); |
| 1494 |
|
| 1495 |
/* |
| 1496 |
* We could be in atomic context in a syscall tracepoint, which runs |
| 1497 |
* in an RCU read-side critical section. We can't detect this with |
| 1498 |
* in_atomic() on non-PREEMPT kernels (i.e., CONFIG_PREEMPT=n) so we |
| 1499 |
* must always use a task worker here because there's no way to tell if |
| 1500 |
* sleeping is okay. |
| 1501 |
*/ |
| 1502 |
work = __stp_tf_alloc_task_work(tgt); |
| 1503 |
if (work == NULL) { |
| 1504 |
__stp_tf_handler_end(); |
| 1505 |
_stp_error("Unable to allocate space for task_work"); |
| 1506 |
return UTRACE_RESUME; |
| 1507 |
} |
| 1508 |
__stp_tf_init_task_work(work, &__stp_tf_quiesce_worker); |
| 1509 |
|
| 1510 |
rc = __stp_tf_task_work_add(tsk, work); |
| 1511 |
if (rc) { |
| 1512 |
__stp_tf_task_work_free(work); |
| 1513 |
/* stp_task_work_add() returns -ESRCH if the task has |
| 1514 |
* already passed exit_task_work(). Just ignore this |
| 1515 |
* error. */ |
| 1516 |
if (rc != -ESRCH) |
| 1517 |
printk(KERN_ERR "%s:%d - stp_task_work_add() returned %d\n", |
| 1518 |
__FUNCTION__, __LINE__, rc); |
| 1519 |
} |
| 1520 |
|
| 1521 |
__stp_tf_handler_end(); |
| 1522 |
return UTRACE_RESUME; |
| 1523 |
} |
| 1524 |
|
| 1525 |
|
| 1526 |
/* FIXME: in the brave new world, we'll use target individual |
| 1527 |
* syscalls, instead of tracing all syscalls for the map stuff. |
| 1528 |
* However, process.syscall will still need to target all syscalls. */ |
| 1529 |
static u32 |
| 1530 |
__stp_utrace_task_finder_target_syscall_entry(u32 action, |
| 1531 |
struct utrace_engine *engine, |
| 1532 |
struct pt_regs *regs) |
| 1533 |
{ |
| 1534 |
struct task_struct *tsk = current; |
| 1535 |
struct stap_task_finder_target *tgt = engine->data; |
| 1536 |
long syscall_no; |
| 1537 |
unsigned long args[8] = { 0L }; /* large enough for syscall_get_arguments() target */ |
| 1538 |
int rc; |
| 1539 |
int is_mmap_or_mmap2 = 0; |
| 1540 |
int is_mprotect = 0; |
| 1541 |
int is_munmap = 0; |
| 1542 |
|
| 1543 |
if (atomic_read(&__stp_task_finder_state) != __STP_TF_RUNNING) { |
| 1544 |
debug_task_finder_detach(); |
| 1545 |
return UTRACE_DETACH; |
| 1546 |
} |
| 1547 |
|
| 1548 |
if (unlikely(tgt == NULL)) |
| 1549 |
return UTRACE_RESUME; |
| 1550 |
|
| 1551 |
// See if syscall is one we're interested in. On x86_64, this |
| 1552 |
// is a potentially expensive operation (since we have to |
| 1553 |
// check and see if it is a 32-bit task). So, cache the |
| 1554 |
// results. |
| 1555 |
// |
| 1556 |
// FIXME: do we need to handle mremap()? |
| 1557 |
syscall_no = _stp_syscall_get_nr(tsk, regs); |
| 1558 |
is_mmap_or_mmap2 = (syscall_no == MMAP_SYSCALL_NO(tsk) |
| 1559 |
|| syscall_no == MMAP2_SYSCALL_NO(tsk) ? 1 : 0); |
| 1560 |
if (!is_mmap_or_mmap2) { |
| 1561 |
is_mprotect = (syscall_no == MPROTECT_SYSCALL_NO(tsk) ? 1 : 0); |
| 1562 |
if (!is_mprotect) { |
| 1563 |
is_munmap = (syscall_no == MUNMAP_SYSCALL_NO(tsk) |
| 1564 |
? 1 : 0); |
| 1565 |
} |
| 1566 |
} |
| 1567 |
if (!is_mmap_or_mmap2 && !is_mprotect && !is_munmap) |
| 1568 |
return UTRACE_RESUME; |
| 1569 |
|
| 1570 |
// The syscall is one we're interested in, but do we have a |
| 1571 |
// handler for it? |
| 1572 |
if ((is_mmap_or_mmap2 && tgt->mmap_events == 0) |
| 1573 |
|| (is_mprotect && tgt->mprotect_events == 0) |
| 1574 |
|| (is_munmap && tgt->munmap_events == 0)) |
| 1575 |
return UTRACE_RESUME; |
| 1576 |
|
| 1577 |
// Save the needed arguments. Note that for mmap, we really |
| 1578 |
// just need the return value, so there is no need to save |
| 1579 |
// any arguments. |
| 1580 |
__stp_tf_handler_start(); |
| 1581 |
if (is_munmap) { |
| 1582 |
// We need 2 arguments for munmap() |
| 1583 |
_stp_syscall_get_arguments(tsk, regs, 0, 2, args); |
| 1584 |
} |
| 1585 |
else if (is_mprotect) { |
| 1586 |
// We need 3 arguments for mprotect() |
| 1587 |
_stp_syscall_get_arguments(tsk, regs, 0, 3, args); |
| 1588 |
} |
| 1589 |
|
| 1590 |
// Remember the syscall information |
| 1591 |
rc = __stp_tf_add_map(tsk, syscall_no, args[0], args[1], args[2]); |
| 1592 |
if (rc != 0) |
| 1593 |
_stp_error("__stp_tf_add_map returned error %d on pid %d", |
| 1594 |
rc, tsk->pid); |
| 1595 |
__stp_tf_handler_end(); |
| 1596 |
return UTRACE_RESUME; |
| 1597 |
} |
| 1598 |
|
| 1599 |
static void |
| 1600 |
__stp_tf_mmap_worker(struct task_work *work) |
| 1601 |
{ |
| 1602 |
struct __stp_tf_task_work *tf_work = \ |
| 1603 |
container_of(work, struct __stp_tf_task_work, work); |
| 1604 |
struct stap_task_finder_target *tgt = \ |
| 1605 |
(struct stap_task_finder_target *)tf_work->data; |
| 1606 |
struct __stp_tf_map_entry *entry; |
| 1607 |
|
| 1608 |
might_sleep(); |
| 1609 |
|
| 1610 |
// See if we can find saved syscall info. |
| 1611 |
entry = __stp_tf_get_map_entry(current); |
| 1612 |
if (entry == NULL) |
| 1613 |
return; |
| 1614 |
|
| 1615 |
if (atomic_read(&__stp_task_finder_state) != __STP_TF_RUNNING |
| 1616 |
|| current->flags & PF_EXITING) { |
| 1617 |
__stp_tf_remove_map_entry(entry); |
| 1618 |
return; |
| 1619 |
} |
| 1620 |
|
| 1621 |
__stp_tf_handler_start(); |
| 1622 |
|
| 1623 |
if (entry->syscall_no == MUNMAP_SYSCALL_NO(current)) { |
| 1624 |
// Call the callbacks |
| 1625 |
__stp_call_munmap_callbacks(tgt, current, entry->arg0, |
| 1626 |
entry->arg1); |
| 1627 |
} |
| 1628 |
else if (entry->syscall_no == MMAP_SYSCALL_NO(current) |
| 1629 |
|| entry->syscall_no == MMAP2_SYSCALL_NO(current)) { |
| 1630 |
// Call the callbacks. Note that arg0 is really the |
| 1631 |
// return value of mmap()/mmap2(). |
| 1632 |
__stp_call_mmap_callbacks_with_addr(tgt, current, entry->arg0); |
| 1633 |
} |
| 1634 |
else { // mprotect |
| 1635 |
// Call the callbacks |
| 1636 |
__stp_call_mprotect_callbacks(tgt, current, entry->arg0, |
| 1637 |
entry->arg1, entry->arg2); |
| 1638 |
} |
| 1639 |
__stp_tf_remove_map_entry(entry); |
| 1640 |
|
| 1641 |
__stp_tf_handler_end(); |
| 1642 |
} |
| 1643 |
|
| 1644 |
static u32 |
| 1645 |
__stp_utrace_task_finder_target_syscall_exit(u32 action, |
| 1646 |
struct utrace_engine *engine, |
| 1647 |
struct pt_regs *regs) |
| 1648 |
{ |
| 1649 |
struct task_struct *tsk = current; |
| 1650 |
struct stap_task_finder_target *tgt = engine->data; |
| 1651 |
unsigned long rv; |
| 1652 |
struct __stp_tf_map_entry *entry; |
| 1653 |
struct task_work *work; |
| 1654 |
int rc; |
| 1655 |
|
| 1656 |
if (atomic_read(&__stp_task_finder_state) != __STP_TF_RUNNING) { |
| 1657 |
debug_task_finder_detach(); |
| 1658 |
return UTRACE_DETACH; |
| 1659 |
} |
| 1660 |
|
| 1661 |
if (tgt == NULL) |
| 1662 |
return UTRACE_RESUME; |
| 1663 |
|
| 1664 |
// See if we can find saved syscall info. If we can, it must |
| 1665 |
// be one of the syscalls we are interested in (and we must |
| 1666 |
// have callbacks to call for it). |
| 1667 |
entry = __stp_tf_get_map_entry(tsk); |
| 1668 |
if (entry == NULL) |
| 1669 |
return UTRACE_RESUME; |
| 1670 |
|
| 1671 |
// Get return value |
| 1672 |
__stp_tf_handler_start(); |
| 1673 |
rv = syscall_get_return_value(tsk, regs); |
| 1674 |
|
| 1675 |
dbug_task_vma(1, |
| 1676 |
"tsk %d found %s(0x%lx), returned 0x%lx\n", |
| 1677 |
tsk->pid, |
| 1678 |
((entry->syscall_no == MMAP_SYSCALL_NO(tsk)) ? "mmap" |
| 1679 |
: ((entry->syscall_no == MMAP2_SYSCALL_NO(tsk)) ? "mmap2" |
| 1680 |
: ((entry->syscall_no == MPROTECT_SYSCALL_NO(tsk)) |
| 1681 |
? "mprotect" |
| 1682 |
: ((entry->syscall_no == MUNMAP_SYSCALL_NO(tsk)) |
| 1683 |
? "munmap" |
| 1684 |
: "UNKNOWN")))), |
| 1685 |
entry->arg0, rv); |
| 1686 |
|
| 1687 |
/* If this is mmap()/mmap2(), we need to remember the |
| 1688 |
* return value. We'll use entry->arg0, since |
| 1689 |
* mmap()/mmap2() doesn't use that info. */ |
| 1690 |
if (entry->syscall_no == MMAP_SYSCALL_NO(tsk) |
| 1691 |
|| entry->syscall_no == MMAP2_SYSCALL_NO(tsk)) { |
| 1692 |
entry->arg0 = rv; |
| 1693 |
} |
| 1694 |
|
| 1695 |
/* |
| 1696 |
* We could be in atomic context in a syscall tracepoint, which runs |
| 1697 |
* in an RCU read-side critical section. We can't detect this with |
| 1698 |
* in_atomic() on non-PREEMPT kernels (i.e., CONFIG_PREEMPT=n) so we |
| 1699 |
* must always use a task worker here because there's no way to tell if |
| 1700 |
* sleeping is okay. |
| 1701 |
*/ |
| 1702 |
work = __stp_tf_alloc_task_work(tgt); |
| 1703 |
if (work == NULL) { |
| 1704 |
_stp_error("Unable to allocate space for task_work"); |
| 1705 |
__stp_tf_remove_map_entry(entry); |
| 1706 |
__stp_tf_handler_end(); |
| 1707 |
return UTRACE_RESUME; |
| 1708 |
} |
| 1709 |
__stp_tf_init_task_work(work, &__stp_tf_mmap_worker); |
| 1710 |
rc = __stp_tf_task_work_add(tsk, work); |
| 1711 |
if (rc) { |
| 1712 |
__stp_tf_task_work_free(work); |
| 1713 |
/* stp_task_work_add() returns -ESRCH if the task has |
| 1714 |
* already passed exit_task_work(). Just ignore this |
| 1715 |
* error. */ |
| 1716 |
if (rc != -ESRCH) |
| 1717 |
printk(KERN_ERR "%s:%d - stp_task_work_add() returned %d\n", |
| 1718 |
__FUNCTION__, __LINE__, rc); |
| 1719 |
} |
| 1720 |
|
| 1721 |
__stp_tf_handler_end(); |
| 1722 |
return UTRACE_RESUME; |
| 1723 |
} |
| 1724 |
|
| 1725 |
static struct utrace_engine_ops __stp_utrace_task_finder_ops = { |
| 1726 |
.report_clone = __stp_utrace_task_finder_report_clone, |
| 1727 |
.report_exec = __stp_utrace_task_finder_report_exec, |
| 1728 |
.report_death = stap_utrace_task_finder_report_death, |
| 1729 |
}; |
| 1730 |
|
| 1731 |
static void |
| 1732 |
stap_stop_task_finder(void); |
| 1733 |
|
| 1734 |
static int |
| 1735 |
stap_start_task_finder(void) |
| 1736 |
{ |
| 1737 |
int rc = 0; |
| 1738 |
struct task_struct *grp, *tsk; |
| 1739 |
char *mmpath_buf; |
| 1740 |
uid_t tsk_euid; |
| 1741 |
|
| 1742 |
if (atomic_inc_return(&__stp_task_finder_state) != __STP_TF_STARTING) { |
| 1743 |
atomic_dec(&__stp_task_finder_state); |
| 1744 |
_stp_error("task_finder already started"); |
| 1745 |
return EBUSY; |
| 1746 |
} |
| 1747 |
|
| 1748 |
mmpath_buf = _stp_kmalloc(PATH_MAX); |
| 1749 |
if (mmpath_buf == NULL) { |
| 1750 |
_stp_error("Unable to allocate space for path"); |
| 1751 |
return ENOMEM; |
| 1752 |
} |
| 1753 |
|
| 1754 |
rc = utrace_init(); |
| 1755 |
if (rc) { /* PR14781, handle utrace alloc failure. */ |
| 1756 |
_stp_kfree(mmpath_buf); |
| 1757 |
_stp_error("Failed to initialize utrace hooks"); |
| 1758 |
return ENOMEM; /* XXX: or some other one. */ |
| 1759 |
} |
| 1760 |
|
| 1761 |
__stp_tf_map_initialize(); |
| 1762 |
|
| 1763 |
atomic_set(&__stp_task_finder_state, __STP_TF_RUNNING); |
| 1764 |
|
| 1765 |
rcu_read_lock(); |
| 1766 |
for_each_process_thread(grp, tsk) { |
| 1767 |
struct mm_struct *mm; |
| 1768 |
char *mmpath; |
| 1769 |
size_t mmpathlen; |
| 1770 |
struct list_head *tgt_node; |
| 1771 |
|
| 1772 |
/* If in stap -c/-x mode, skip over other processes. */ |
| 1773 |
if (_stp_target && tsk->tgid != _stp_target) |
| 1774 |
continue; |
| 1775 |
|
| 1776 |
rc = __stp_utrace_attach(tsk, &__stp_utrace_task_finder_ops, 0, |
| 1777 |
__STP_TASK_FINDER_EVENTS, |
| 1778 |
UTRACE_RESUME); |
| 1779 |
|
| 1780 |
dbug_task(2, "__stp_utrace_attach() for pid %d returned %d", |
| 1781 |
tsk->tgid, rc); |
| 1782 |
|
| 1783 |
if (rc == EPERM) { |
| 1784 |
/* Ignore EPERM errors, which mean this wasn't |
| 1785 |
* a thread we can attach to. */ |
| 1786 |
rc = 0; |
| 1787 |
continue; |
| 1788 |
} |
| 1789 |
else if (rc != 0) { |
| 1790 |
/* If we get a real error, quit. */ |
| 1791 |
goto stf_err; |
| 1792 |
} |
| 1793 |
|
| 1794 |
// Grab the path associated with this task. |
| 1795 |
// |
| 1796 |
// Note we aren't calling get_task_mm()/mmput() here. |
| 1797 |
// Instead we're calling task_lock()/task_unlock(). |
| 1798 |
// We really only need to lock the mm, but mmput() can |
| 1799 |
// sleep so we can't call it. Also note that |
| 1800 |
// __stp_get_mm_path() grabs the mmap semaphore, which |
| 1801 |
// should also keep us safe. |
| 1802 |
task_lock(tsk); |
| 1803 |
if (! tsk->mm) { |
| 1804 |
/* If the thread doesn't have a mm_struct, it |
| 1805 |
* is a kernel thread which we need to |
| 1806 |
* skip. */ |
| 1807 |
task_unlock(tsk); |
| 1808 |
continue; |
| 1809 |
} |
| 1810 |
mmpath = __stp_get_mm_path(tsk->mm, mmpath_buf, PATH_MAX); |
| 1811 |
task_unlock(tsk); |
| 1812 |
if (mmpath == NULL || IS_ERR(mmpath)) { |
| 1813 |
rc = PTR_ERR(mmpath); |
| 1814 |
/* If this was our target then it's a fatal error */ |
| 1815 |
if (!_stp_target && rc == -ENOENT) { |
| 1816 |
_stp_warn("Unable to get path (error %d) for pid %d", |
| 1817 |
rc, (int)tsk->pid); |
| 1818 |
rc = 0; /* ignore ENOENT */ |
| 1819 |
continue; |
| 1820 |
} |
| 1821 |
else { |
| 1822 |
_stp_error("Unable to get path (error %d) for pid %d", |
| 1823 |
rc, (int)tsk->pid); |
| 1824 |
goto stf_err; |
| 1825 |
} |
| 1826 |
} |
| 1827 |
|
| 1828 |
/* Check the thread's exe's path/pid/build-id against our list. */ |
| 1829 |
#ifdef STAPCONF_TASK_UID |
| 1830 |
tsk_euid = tsk->euid; |
| 1831 |
#else |
| 1832 |
#if defined(CONFIG_USER_NS) || (LINUX_VERSION_CODE >= KERNEL_VERSION(3,14,0)) |
| 1833 |
tsk_euid = from_kuid_munged(current_user_ns(), task_euid(tsk)); |
| 1834 |
#else |
| 1835 |
tsk_euid = task_euid(tsk); |
| 1836 |
#endif |
| 1837 |
#endif |
| 1838 |
mmpathlen = strlen(mmpath); |
| 1839 |
list_for_each(tgt_node, &__stp_task_finder_list) { |
| 1840 |
struct stap_task_finder_target *tgt; |
| 1841 |
|
| 1842 |
tgt = list_entry(tgt_node, |
| 1843 |
struct stap_task_finder_target, list); |
| 1844 |
if (tgt == NULL) |
| 1845 |
continue; |
| 1846 |
/* buildid-based target ... gets checked in __stp_tf_quiesce_worker */ |
| 1847 |
/* procname-based target */ |
| 1848 |
else if (tgt->build_id == 0 && tgt->pathlen > 0 |
| 1849 |
&& (tgt->pathlen != mmpathlen |
| 1850 |
|| strcmp(tgt->procname, mmpath) != 0)) |
| 1851 |
{ |
| 1852 |
dbug_task(2, "target path not matched: [%s] != [%s]", |
| 1853 |
tgt->procname, mmpath); |
| 1854 |
continue; |
| 1855 |
} |
| 1856 |
/* pid-based target */ |
| 1857 |
else if (tgt->pid != 0 && tgt->pid != tsk->pid) |
| 1858 |
continue; |
| 1859 |
/* Notice that "pid == 0" (which means to |
| 1860 |
* probe all threads) falls through. */ |
| 1861 |
|
| 1862 |
#if ! STP_PRIVILEGE_CONTAINS (STP_PRIVILEGE, STP_PR_STAPDEV) && \ |
| 1863 |
! STP_PRIVILEGE_CONTAINS (STP_PRIVILEGE, STP_PR_STAPSYS) |
| 1864 |
/* Make sure unprivileged users only probe their own threads. */ |
| 1865 |
if (_stp_uid != tsk_euid) { |
| 1866 |
if (tgt->pid != 0 || _stp_target) { |
| 1867 |
_stp_warn("Process %d does not belong to unprivileged user %d", |
| 1868 |
tsk->pid, _stp_uid); |
| 1869 |
} |
| 1870 |
continue; |
| 1871 |
} |
| 1872 |
#endif |
| 1873 |
|
| 1874 |
// Set up events we need for attached tasks. |
| 1875 |
rc = __stp_utrace_attach(tsk, &tgt->ops, tgt, |
| 1876 |
__STP_ATTACHED_TASK_EVENTS, |
| 1877 |
UTRACE_INTERRUPT); |
| 1878 |
dbug_task(2, "__stp_utrace_attach() for %d returned %d", tsk->tgid, |
| 1879 |
rc); |
| 1880 |
|
| 1881 |
if (rc != 0 && rc != EPERM) |
| 1882 |
goto stf_err; |
| 1883 |
rc = 0; /* ignore EPERM */ |
| 1884 |
tgt->engine_attached = 1; |
| 1885 |
} |
| 1886 |
} |
| 1887 |
stf_err: |
| 1888 |
rcu_read_unlock(); |
| 1889 |
_stp_kfree(mmpath_buf); |
| 1890 |
debug_task_finder_report(); // report at end for utrace engine counting |
| 1891 |
/* |
| 1892 |
* We need to do our own cleanup if something failed. A full task finder |
| 1893 |
* stop is needed because utrace is guaranteed to be live by this point. |
| 1894 |
*/ |
| 1895 |
if (rc) |
| 1896 |
stap_stop_task_finder(); |
| 1897 |
return rc; |
| 1898 |
} |
| 1899 |
|
| 1900 |
|
| 1901 |
static void |
| 1902 |
stap_task_finder_post_init(void) |
| 1903 |
{ |
| 1904 |
struct task_struct *grp, *tsk; |
| 1905 |
|
| 1906 |
if (atomic_read(&__stp_task_finder_state) != __STP_TF_RUNNING) { |
| 1907 |
_stp_error("task_finder not running?"); |
| 1908 |
return; |
| 1909 |
} |
| 1910 |
|
| 1911 |
dbug_task(2, "entry."); |
| 1912 |
rcu_read_lock(); |
| 1913 |
for_each_process_thread(grp, tsk) { |
| 1914 |
struct list_head *tgt_node; |
| 1915 |
|
| 1916 |
if (atomic_read(&__stp_task_finder_state) != __STP_TF_RUNNING) { |
| 1917 |
dbug_task(2, "exiting early..."); |
| 1918 |
break; |
| 1919 |
} |
| 1920 |
|
| 1921 |
/* If in stap -c/-x mode, skip over other processes. */ |
| 1922 |
if (_stp_target && tsk->tgid != _stp_target) |
| 1923 |
continue; |
| 1924 |
|
| 1925 |
/* Only "poke" thread group leaders. */ |
| 1926 |
if (tsk->tgid != tsk->pid) |
| 1927 |
continue; |
| 1928 |
|
| 1929 |
/* See if we need to "poke" this thread. */ |
| 1930 |
list_for_each(tgt_node, &__stp_task_finder_list) { |
| 1931 |
struct stap_task_finder_target *tgt; |
| 1932 |
struct utrace_engine *engine; |
| 1933 |
|
| 1934 |
tgt = list_entry(tgt_node, |
| 1935 |
struct stap_task_finder_target, list); |
| 1936 |
if (tgt == NULL || !tgt->engine_attached) |
| 1937 |
continue; |
| 1938 |
|
| 1939 |
// If we found an "interesting" task earlier, |
| 1940 |
// stop it. |
| 1941 |
engine = utrace_attach_task(tsk, |
| 1942 |
UTRACE_ATTACH_MATCH_OPS, |
| 1943 |
&tgt->ops, tgt); |
| 1944 |
if (engine != NULL && !IS_ERR(engine)) { |
| 1945 |
/* We found a target task. Stop it. */ |
| 1946 |
int rc = utrace_control(tsk, engine, |
| 1947 |
UTRACE_INTERRUPT); |
| 1948 |
/* If utrace_control() returns |
| 1949 |
* EINPROGRESS when we're |
| 1950 |
* trying to stop/interrupt, |
| 1951 |
* that means the task hasn't |
| 1952 |
* stopped quite yet, but will |
| 1953 |
* soon. Ignore this |
| 1954 |
* error. */ |
| 1955 |
if (rc != 0 && rc != -EINPROGRESS) { |
| 1956 |
_stp_error("utrace_control returned error %d on pid %d", |
| 1957 |
rc, (int)tsk->pid); |
| 1958 |
} |
| 1959 |
|
| 1960 |
dbug_task(2, "utrace_control(UTRACE_INTERRUPT) for pid %d " |
| 1961 |
"returned %d (%d)", tsk->pid, rc, -EINPROGRESS); |
| 1962 |
|
| 1963 |
utrace_engine_put(engine); |
| 1964 |
|
| 1965 |
/* Since we only need to interrupt |
| 1966 |
* the task once, not once per |
| 1967 |
* engine, get out of this loop. */ |
| 1968 |
break; |
| 1969 |
} |
| 1970 |
} |
| 1971 |
} |
| 1972 |
rcu_read_unlock(); |
| 1973 |
atomic_set(&__stp_task_finder_complete, 1); |
| 1974 |
dbug_task(2, "exit."); |
| 1975 |
return; |
| 1976 |
} |
| 1977 |
|
| 1978 |
|
| 1979 |
/* Indicates whether task_finder has complete coverage of all processes. |
| 1980 |
* e.g. uprobes prefilter can be sure it's safe to REMOVE from an unknown process. |
| 1981 |
*/ |
| 1982 |
static inline int |
| 1983 |
stap_task_finder_complete(void) |
| 1984 |
{ |
| 1985 |
return atomic_read(&__stp_task_finder_complete) != 0; |
| 1986 |
} |
| 1987 |
|
| 1988 |
|
| 1989 |
static void |
| 1990 |
stap_stop_task_finder(void) |
| 1991 |
{ |
| 1992 |
#ifdef DEBUG_TASK_FINDER |
| 1993 |
int i = 0; |
| 1994 |
|
| 1995 |
printk(KERN_ERR "%s:%d - entry\n", __FUNCTION__, __LINE__); |
| 1996 |
#endif |
| 1997 |
if (atomic_read(&__stp_task_finder_state) == __STP_TF_UNITIALIZED) |
| 1998 |
return; |
| 1999 |
|
| 2000 |
atomic_set(&__stp_task_finder_state, __STP_TF_STOPPING); |
| 2001 |
debug_task_finder_report(); |
| 2002 |
|
| 2003 |
// The utrace_shutdown() function detaches and cleans up |
| 2004 |
// everything for us - we don't have to go through each |
| 2005 |
// engine. This also means that the attach_count could end up |
| 2006 |
// > 0 (since we don't got through each engine individually). |
| 2007 |
utrace_shutdown(); |
| 2008 |
|
| 2009 |
debug_task_finder_report(); |
| 2010 |
atomic_set(&__stp_task_finder_state, __STP_TF_STOPPED); |
| 2011 |
|
| 2012 |
/* Now that all the engines are detached, make sure |
| 2013 |
* all the callbacks are finished. If they aren't, we'll |
| 2014 |
* crash the kernel when the module is removed. */ |
| 2015 |
while (atomic_read(&__stp_inuse_count) != 0) { |
| 2016 |
schedule(); |
| 2017 |
#ifdef DEBUG_TASK_FINDER |
| 2018 |
i++; |
| 2019 |
#endif |
| 2020 |
} |
| 2021 |
#ifdef DEBUG_TASK_FINDER |
| 2022 |
if (i > 0) |
| 2023 |
printk(KERN_ERR "it took %d polling loops to quit.\n", i); |
| 2024 |
#endif |
| 2025 |
debug_task_finder_report(); |
| 2026 |
|
| 2027 |
/* Make sure all pending task_work requests are canceled. */ |
| 2028 |
__stp_tf_cancel_all_task_work(); |
| 2029 |
|
| 2030 |
/* Call utrace_exit(), which also calls stp_task_work_exit() |
| 2031 |
* to wait on any running task_work items. */ |
| 2032 |
utrace_exit(); |
| 2033 |
|
| 2034 |
#ifdef DEBUG_TASK_FINDER |
| 2035 |
printk(KERN_ERR "%s:%d - exit\n", __FUNCTION__, __LINE__); |
| 2036 |
#endif |
| 2037 |
} |
| 2038 |
|
| 2039 |
#endif /* TASK_FINDER2_C */ |