Debian Codegraph @

runtime/linux/uprobes-common.c

raw
1 /* -*- linux-c -*-
2 * uprobe Functions
3 * Copyright (C) 2014 Red Hat Inc.
4 *
5 * This file is part of systemtap, and is free software. You can
6 * redistribute it and/or modify it under the terms of the GNU General
7 * Public License (GPL); either version 2, or (at your option) any
8 * later version.
9 */
10
11 #ifndef _UPROBE_COMMON_C_
12 #define _UPROBE_COMMON_C_
13
14 /* NB: Because these utrace callbacks only occur before / after
15 userspace instructions run, there is no concurrency control issue
16 between active uprobe callbacks and these registration /
17 unregistration pieces.
18
19 We protect the stap_uprobe->spec_index (which also serves as a
20 free/busy flag) value with the outer protective stap_probes_lock
21 spinlock, to protect it against concurrent registration /
22 unregistration.
23 */
24
25 static int stap_uprobe_change_plus (struct task_struct *tsk, unsigned long relocation, unsigned long length, const struct stap_uprobe_tf *stf, unsigned long offset, unsigned long vm_flags) {
26 int tfi = (stf - stap_uprobe_finders);
27 int spec_index;
28 /* iterate over stap_uprobe_spec[] that use this same stap_uprobe_tf */
29 for (spec_index=0; spec_index<sizeof(stap_uprobe_specs)/sizeof(stap_uprobe_specs[0]); spec_index++) {
30 int handled_p = 0;
31 int slotted_p = 0;
32 const struct stap_uprobe_spec *sups = &stap_uprobe_specs [spec_index];
33 struct stap_uprobe *sup;
34 pid_t sdt_sem_pid;
35 int rc = 0;
36 int i;
37 int pci;
38
39 if (likely(sups->tfi != tfi)) continue;
40 /* skip probes with an address beyond this map event; should not
41 happen unless a shlib/exec got mmapped in weirdly piecemeal */
42 if (likely((vm_flags & VM_EXEC) && sups->address >= length)) continue;
43
44 /* Found a uprobe_spec for this stap_uprobe_tf. Need to lock the
45 stap_uprobes[] array to allocate a free spot, but then we can
46 unlock and do the register_*probe subsequently. */
47
48 might_sleep();
49 mutex_lock (& stap_uprobes_lock);
50 for (i=0; i<MAXUPROBES; i++) { /* XXX: slow linear search */
51 sup = & stap_uprobes[i];
52
53 /* register new uprobe
54 We make two passes for semaphores;
55 see stap_uprobe_change_semaphore_plus */
56
57 if (sup->spec_index < 0 || (sups->sdt_sem_offset && vm_flags & VM_WRITE && sup->spec_index == spec_index)) {
58 #if (UPROBES_API_VERSION < 2)
59 /* See PR6829 comment. */
60 if (sup->spec_index == -1 && sup->up.kdata != NULL) continue;
61 else if (sup->spec_index == -2 && sup->urp.u.kdata != NULL) continue;
62 #endif
63 sup->spec_index = spec_index;
64 slotted_p = 1;
65 break;
66 }
67 }
68 mutex_unlock (& stap_uprobes_lock);
69 #ifdef DEBUG_UPROBES
70 _stp_dbug(__FUNCTION__,__LINE__, "+uprobe spec %d idx %d process %s[%d] addr %p pp %s\n", spec_index, (slotted_p ? i : -1), tsk->comm, tsk->tgid, (void*)(relocation+sups->address), sups->probe->pp);
71 #endif
72
73 /* NB: check for user-module build-id only if we have a pathname
74 at all; for a process(PID#).* probe, we may not. If at some
75 point we map process(PID#) to process("/proc/PID#/exe"), we'll
76 get a pathname. */
77 if (stf->pathname)
78 if ((rc = _stp_usermodule_check(tsk, stf->pathname, relocation)))
79 return rc;
80
81 /* Here, slotted_p implies that `i' points to the single
82 stap_uprobes[] element that has been slotted in for registration
83 or unregistration processing. !slotted_p implies that the table
84 was full (registration; MAXUPROBES) or that no matching entry was
85 found (unregistration; should not happen). */
86
87 sdt_sem_pid = (sups->return_p ? sup->urp.u.pid : sup->up.pid);
88 if (sups->sdt_sem_offset && (sdt_sem_pid != tsk->tgid || sup->sdt_sem_address == 0)) {
89 /* If the probe is in an ET_EXEC binary, then the sdt_sem_offset already
90 * is a real address. But stap_uprobe_process_found calls us in this
91 * case with relocation=offset=0, so we don't have to worry about it. */
92 sup->sdt_sem_address = (relocation - offset) + sups->sdt_sem_offset;
93 } /* sdt_sem_offset */
94
95 for (pci=0; pci < sups->perf_counters_dim; pci++) {
96 if ((sups->perf_counters)[pci] > -1)
97 _stp_perf_read_init ((sups->perf_counters)[pci], tsk);
98 }
99
100 if (slotted_p) {
101 struct stap_uprobe *sup = & stap_uprobes[i];
102 if (sups->return_p) {
103 sup->urp.u.pid = tsk->tgid;
104 sup->urp.u.vaddr = relocation + sups->address;
105 sup->urp.handler = &enter_uretprobe_probe;
106 rc = register_uretprobe (& sup->urp);
107 } else {
108 sup->up.pid = tsk->tgid;
109 sup->up.vaddr = relocation + sups->address;
110 sup->up.handler = &enter_uprobe_probe;
111 rc = register_uprobe (& sup->up);
112 }
113
114 /* The u*probe failed to register. However, if we got EEXIST,
115 * that means that the u*probe is already there, so just ignore
116 * the error. This could happen if CLONE_THREAD or CLONE_VM was
117 * used. */
118 if (rc != 0 && rc != -EEXIST) {
119 _stp_warn ("u*probe failed %s[%d] '%s' addr %p rc %d\n", tsk->comm, tsk->tgid, sups->probe->pp, (void*)(relocation + sups->address), rc);
120 /* NB: we need to release this slot,
121 so we need to borrow the mutex temporarily. */
122 might_sleep();
123 mutex_lock (& stap_uprobes_lock);
124 sup->spec_index = -1;
125 sup->sdt_sem_address = 0;
126 mutex_unlock (& stap_uprobes_lock);
127 } else {
128 handled_p = 1;
129 }
130 }
131 /* NB: handled_p implies slotted_p */
132 if (unlikely (! handled_p)) {
133 #ifdef STP_TIMING
134 atomic_inc (skipped_count_uprobe_reg());
135 #endif
136 /* NB: duplicates common_entryfn_epilogue,
137 but then this is not a probe entry fn epilogue. */
138 #ifndef STAP_SUPPRESS_HANDLER_ERRORS
139 if (unlikely (atomic_inc_return (skipped_count()) > MAXSKIPPED)) {
140 if (unlikely (pseudo_atomic_cmpxchg(session_state(), STAP_SESSION_RUNNING, STAP_SESSION_ERROR) == STAP_SESSION_RUNNING))
141 _stp_error ("Skipped too many probes, check MAXSKIPPED or try again with stap -t for more details.");
142 }
143 #endif
144 }
145 } /* close iteration over stap_uprobe_spec[] */
146 return 0; /* XXX: or rc? */
147 }
148
149 static int stap_uprobe_change_semaphore_plus (struct task_struct *tsk, unsigned long relocation, unsigned long length, const struct stap_uprobe_tf *stf) {
150 int tfi = (stf - stap_uprobe_finders);
151 int spec_index;
152 int rc = 0;
153 struct stap_uprobe *sup;
154 int i;
155
156 /* We make two passes for semaphores.
157 The first pass, stap_uprobe_change_plus, calculates the address of the
158 semaphore. If the probe is in a .so, we calculate the
159 address when the initial mmap maps the entire solib, e.g.
160 7f089885a000-7f089885b000 rw-p- libtcl.so
161 A subsequent mmap maps in the writable segment where the
162 semaphore control variable lives, e.g.
163 7f089850d000-7f0898647000 r-xp- libtcl.so
164 7f0898647000-7f0898846000 ---p libtcl.so
165 7f0898846000-7f089885b000 rw-p- libtcl.so
166 The second pass, stap_uprobe_change_semaphore_plus, sets the semaphore.
167 If the probe is in a .so this will be when the writable segment of the .so
168 is mapped in. If the task changes, then recalculate the address.
169 */
170
171 for (i=0; i<MAXUPROBES; i++) { /* XXX: slow linear search */
172 sup = & stap_uprobes[i];
173 if (sup->spec_index == -1) continue;
174 if (sup->sdt_sem_address != 0 && !(sup->up.pid == tsk->tgid && sup->sdt_sem_address >= relocation && sup->sdt_sem_address < relocation+length)) continue;
175 if (sup->sdt_sem_address) {
176 unsigned short sdt_semaphore = 0; /* NB: fixed size */
177 if ((rc = get_user (sdt_semaphore, (unsigned short __user*) sup->sdt_sem_address)) == 0) {
178 sdt_semaphore ++;
179 #ifdef DEBUG_UPROBES
180 {
181 const struct stap_uprobe_spec *sups = &stap_uprobe_specs [sup->spec_index];
182 _stp_dbug(__FUNCTION__,__LINE__, "+semaphore %#x @ %#lx spec %d idx %d task %d\n", sdt_semaphore, sup->sdt_sem_address, sup->spec_index, i, tsk->tgid);
183 }
184 #endif
185 rc = put_user (sdt_semaphore, (unsigned short __user*) sup->sdt_sem_address);
186 /* XXX: need to analyze possibility of race condition */
187 }
188 }
189 }
190 return rc;
191 }
192
193 /* Removing/unmapping a uprobe is simpler than adding one (in the
194 _plus function above). We need not care about stap_uprobe_finders
195 or anything, we just scan through stap_uprobes[] for a live probe
196 within the given address range, and kill it. */
197 static int stap_uprobe_change_minus (struct task_struct *tsk, unsigned long relocation, unsigned long length, const struct stap_uprobe_tf *stf) {
198 int i;
199
200 /* NB: it's not an error for us not to find a live uprobe within the
201 given range. We might have received a callback for a part of a
202 shlib that was unmapped and unprobed. */
203
204 for (i=0; i<MAXUPROBES; i++) { /* XXX: slow linear search */
205 struct stap_uprobe *sup = & stap_uprobes[i];
206 struct stap_uprobe_spec *sups;
207 if (sup->spec_index < 0) continue; /* skip free uprobes slot */
208 sups = (struct stap_uprobe_spec*) & stap_uprobe_specs[sup->spec_index];
209 might_sleep();
210 mutex_lock (& stap_uprobes_lock);
211
212 /* PR6829, PR9940:
213 Here we're unregistering for one of two reasons:
214 1. the process image is going away (or gone) due to exit or exec; or
215 2. the vma containing the probepoint has been unmapped.
216 In case 1, it's sort of a nop, because uprobes will notice the event
217 and dispose of the probes eventually, if it hasn't already. But by
218 calling unmap_u[ret]probe() ourselves, we free up sup right away.
219
220 In both cases, we must use unmap_u[ret]probe instead of
221 unregister_u[ret]probe, so uprobes knows not to try to restore the
222 original opcode.
223 */
224
225 /* URETPROBE */
226 if (sups->return_p && sup->urp.u.pid == tsk->tgid && sup->urp.u.vaddr >= relocation && sup->urp.u.vaddr < relocation+length) { /* in range */
227
228 #ifdef DEBUG_UPROBES
229 _stp_dbug (__FUNCTION__,__LINE__, "-uretprobe spec %d idx %d process %s[%d] addr %p pp %s\n", sup->spec_index, i, tsk->comm, tsk->tgid, (void*) sup->urp.u.vaddr, sups->probe->pp);
230 #endif
231 #if (UPROBES_API_VERSION >= 2)
232 unmap_uretprobe (& sup->urp);
233 sup->spec_index = -1;
234 sup->sdt_sem_address = 0;
235 #else
236 /* Uprobes lacks unmap_uretprobe. Before reusing sup, we must wait
237 until uprobes turns loose of the uretprobe on its own, as indicated
238 by uretprobe.kdata = NULL. */
239 sup->spec_index = -2;
240 #endif
241 /* UPROBE */
242 } else if (!sups->return_p && sup->up.pid == tsk->tgid && sup->up.vaddr >= relocation && sup->up.vaddr < relocation+length) { /* in range */
243
244 #ifdef DEBUG_UPROBES
245 _stp_dbug (__FUNCTION__,__LINE__, "-uprobe spec %d idx %d process %s[%d] reloc %p pp %s\n", sup->spec_index, i, tsk->comm, tsk->tgid, (void*) sup->up.vaddr, sups->probe->pp);
246 #endif
247 #if (UPROBES_API_VERSION >= 2)
248 unmap_uprobe (& sup->up);
249 sup->spec_index = -1;
250 sup->sdt_sem_address = 0;
251 #else
252 /* Uprobes lacks unmap_uprobe. Before reusing sup, we must wait
253 until uprobes turns loose of the uprobe on its own, as indicated
254 by uprobe.kdata = NULL. */
255 sup->spec_index = -1;
256 sup->sdt_sem_address = 0;
257 #endif
258 /* PR10655: we don't need to fidget with the ENABLED semaphore either,
259 as the process is gone, buh-bye, toodaloo, au revoir, see ya later! */
260 }
261 mutex_unlock (& stap_uprobes_lock);
262 } /* close iteration over stap_uprobes[] */
263 return 0; /* XXX: or !handled_p */
264 }
265
266 /* The task_finder_callback we use for ET_EXEC targets.
267 We used to perform uprobe insertion/removal here, but not any more.
268 (PR10524) */
269 static int stap_uprobe_process_found (struct stap_task_finder_target *tgt, struct task_struct *tsk, int register_p, int process_p) {
270 const struct stap_uprobe_tf *stf = container_of(tgt, struct stap_uprobe_tf, finder);
271 if (! process_p) return 0; /* ignore threads */
272 dbug_task_vma(1, "%cproc pid %d stf %p %p path %s\n", register_p?'+':'-', tsk->tgid, tgt, stf, stf->pathname);
273 /* ET_EXEC events are like shlib events, but with 0 relocation bases */
274 if (register_p) {
275 int rc = stap_uprobe_change_plus (tsk, 0, TASK_SIZE, stf, 0, 0);
276 stap_uprobe_change_semaphore_plus (tsk, 0, TASK_SIZE, stf);
277 return rc;
278 } else
279 return stap_uprobe_change_minus (tsk, 0, TASK_SIZE, stf);
280 }
281
282 /* The task_finder_mmap_callback */
283 static int
284 stap_uprobe_mmap_found (struct stap_task_finder_target *tgt,
285 struct task_struct *tsk, char *path,
286 struct dentry *dentry, unsigned long addr,
287 unsigned long length, unsigned long offset,
288 unsigned long vm_flags)
289 {
290 int rc = 0;
291 const struct stap_uprobe_tf *stf = container_of(tgt, struct stap_uprobe_tf, finder);
292 /* 1 - shared libraries' executable segments load from offset 0
293 * - ld.so convention offset != 0 is now allowed
294 * so stap_uprobe_change_plus can set a semaphore,
295 * i.e. a static extern, in a shared object
296 * 2 - the shared library we're interested in
297 * 3 - mapping should be executable or writable (for semaphore in .so)
298 * NB: or both, on kernels that lack noexec mapping
299 */
300 if (path == NULL || strcmp (path, stf->pathname))
301 return 0;
302
303 /* Check non-writable, executable sections for probes. */
304 if ((vm_flags & VM_EXEC) && !(vm_flags & VM_WRITE)) {
305 dbug_task_vma (1,
306 "+mmap X pid %d path %s addr %p length %u offset %p stf %p %p path %s\n",
307 tsk->tgid, path, (void *) addr, (unsigned)length, (void*) offset,
308 tgt, stf, stf->pathname);
309 rc = stap_uprobe_change_plus (tsk, addr, length, stf, offset, vm_flags);
310 }
311
312 /* Check writable sections for semaphores.
313 * NB: They may have also been executable for the check above, if we're
314 * running a kernel that lacks noexec mappings. So long as there's
315 * no error (rc == 0), we need to look for semaphores too.
316 */
317 if ((rc == 0) && (vm_flags & VM_WRITE)) {
318 dbug_task_vma (1,
319 "+mmap W pid %d path %s addr %p length %u offset %p stf %p %p path %s\n",
320 tsk->tgid, path, (void *) addr, (unsigned)length, (void*) offset,
321 tgt, stf, stf->pathname);
322 rc = stap_uprobe_change_semaphore_plus (tsk, addr, length, stf);
323 }
324
325 return rc;
326 }
327
328 /* The task_finder_munmap_callback */
329 static int stap_uprobe_munmap_found (struct stap_task_finder_target *tgt, struct task_struct *tsk, unsigned long addr, unsigned long length) {
330 const struct stap_uprobe_tf *stf = container_of(tgt, struct stap_uprobe_tf, finder);
331 dbug_task_vma (1, "-mmap pid %d addr %p length %lu stf %p %p path %s\n", tsk->tgid, (void *) addr, length, tgt, stf, stf->pathname);
332 return stap_uprobe_change_minus (tsk, addr, length, stf);
333 }
334
335 /* The task_finder_callback we use for ET_DYN targets.
336 This just forces an unmap of everything as the process exits.
337 (PR11151) */
338 static int stap_uprobe_process_munmap (struct stap_task_finder_target *tgt, struct task_struct *tsk, int register_p, int process_p) {
339 const struct stap_uprobe_tf *stf = container_of(tgt, struct stap_uprobe_tf, finder);
340 if (! process_p) return 0; /* ignore threads */
341 dbug_task_vma (1, "%cproc pid %d stf %p %p path %s\n", register_p?'+':'-', tsk->tgid, tgt, stf, stf->pathname);
342 /* Covering 0->TASK_SIZE means "unmap everything" */
343 if (!register_p)
344 return stap_uprobe_change_minus (tsk, 0, TASK_SIZE, stf);
345 return 0;
346 }
347
348 #endif /* _UPROBE_COMMON_C_ */