Debian Codegraph @

runtime/linux/loc2c-runtime.h

raw
1 /* target operations in the Linux kernel mode
2 * Copyright (C) 2005-2019 Red Hat Inc.
3 * Copyright (C) 2005-2007 Intel Corporation.
4 * Copyright (C) 2007 Quentin Barnes.
5 *
6 * This file is part of systemtap, and is free software. You can
7 * redistribute it and/or modify it under the terms of the GNU General
8 * Public License (GPL); either version 2, or (at your option) any
9 * later version.
10 */
11
12 #ifndef _LINUX_LOC2C_RUNTIME_H_
13 #define _LINUX_LOC2C_RUNTIME_H_
14
15 #ifdef STAPCONF_LINUX_UACCESS_H
16 #include <linux/uaccess.h>
17 #else
18 #include <asm/uaccess.h>
19 #endif
20 #include <linux/types.h>
21 #ifdef STAPCONF_ASM_TLBFLUSH_H
22 #include <asm/tlbflush.h>
23 #endif
24 #include <linux/nsproxy.h>
25 #define intptr_t long
26 #define uintptr_t unsigned long
27
28 #include "../loc2c-runtime.h"
29
30
31 #ifndef STAPCONF_PAGEFAULT_DISABLE /* before linux commit a866374a */
32 #define pagefault_disable() preempt_disable()
33 #define pagefault_enable() preempt_enable_no_resched()
34 #endif
35
36
37 #define check_fetch_register(regs,regno,maxregno,fn) ({ \
38 if ((regs) == 0 || (regno) < 0 || (regno) > (maxregno)) { \
39 snprintf(c->error_buffer, sizeof(c->error_buffer), \
40 STAP_MSG_LOC2C_04); \
41 c->last_error = c->error_buffer; \
42 goto deref_fault; \
43 } \
44 fn(regs, regno); \
45 })
46
47 #define check_store_register(regs,regno,maxregno,value,fn) do { \
48 if ((regs) == 0 || (regno) < 0 || (regno) > (maxregno)) { \
49 snprintf(c->error_buffer, sizeof(c->error_buffer), \
50 STAP_MSG_LOC2C_04); \
51 c->last_error = c->error_buffer; \
52 goto deref_fault; \
53 } \
54 fn(regs, regno, value); \
55 } while(0)
56
57
58 #define k_fetch_register(regno) check_fetch_register(c->kregs,regno,pt_regs_maxno,pt_regs_fetch_register)
59
60 #define k_store_register(regno,value) check_store_register(c->kregs,regno,pt_regs_maxno,value,pt_regs_store_register)
61
62
63
64 /* PR 10601: user-space (user_regset) register access.
65 Needs arch specific code, only i386 and x86_64 for now. */
66 #if ((defined(STAPCONF_REGSET)) \
67 && (defined (__i386__) || defined (__x86_64__)))
68
69 #if defined(STAPCONF_REGSET)
70 #include <linux/regset.h>
71 #endif
72
73 // PR13489, inodes-uprobes export kludge
74 #if !defined(STAPCONF_TASK_USER_REGSET_VIEW_EXPORTED)
75 // First typedef from the original decl, then #define it as a typecasted call.
76 // NB: not all archs actually have the function, but the decl is universal in regset.h
77 typedef typeof(&task_user_regset_view) task_user_regset_view_fn;
78 /* Special macro to tolerate the kallsyms function pointer being zero. */
79 #define task_user_regset_view(t) (kallsyms_task_user_regset_view ? \
80 ibt_wrapper(const struct user_regset_view *,
81 (* (task_user_regset_view_fn)(kallsyms_task_user_regset_view))((t))) : \
82 NULL)
83 #endif
84
85 struct usr_regset_lut {
86 char *name;
87 unsigned rsn;
88 unsigned pos;
89 };
90
91
92 /* DWARF register number -to- user_regset bank/offset mapping table.
93 The register numbers come from the processor-specific ELF documents.
94 The user-regset bank/offset values come from kernel $ARCH/include/asm/user*.h
95 or $ARCH/kernel/ptrace.c. */
96 #if defined (__i386__) || defined (__x86_64__)
97 static const struct usr_regset_lut url_i386[] = {
98 { "ax", NT_PRSTATUS, 6*4 },
99 { "cx", NT_PRSTATUS, 1*4 },
100 { "dx", NT_PRSTATUS, 2*4 },
101 { "bx", NT_PRSTATUS, 0*4 },
102 { "sp", NT_PRSTATUS, 15*4 },
103 { "bp", NT_PRSTATUS, 5*4 },
104 { "si", NT_PRSTATUS, 3*4 },
105 { "di", NT_PRSTATUS, 4*4 },
106 { "ip", NT_PRSTATUS, 12*4 },
107 };
108 #endif
109
110 #if defined (__x86_64__)
111 static const struct usr_regset_lut url_x86_64[] = {
112 { "rax", NT_PRSTATUS, 10*8 },
113 { "rdx", NT_PRSTATUS, 12*8 },
114 { "rcx", NT_PRSTATUS, 11*8 },
115 { "rbx", NT_PRSTATUS, 5*8 },
116 { "rsi", NT_PRSTATUS, 13*8 },
117 { "rdi", NT_PRSTATUS, 14*8 },
118 { "rbp", NT_PRSTATUS, 4*8 },
119 { "rsp", NT_PRSTATUS, 19*8 },
120 { "r8", NT_PRSTATUS, 9*8 },
121 { "r9", NT_PRSTATUS, 8*8 },
122 { "r10", NT_PRSTATUS, 7*8 },
123 { "r11", NT_PRSTATUS, 6*8 },
124 { "r12", NT_PRSTATUS, 3*8 },
125 { "r13", NT_PRSTATUS, 2*8 },
126 { "r14", NT_PRSTATUS, 1*8 },
127 { "r15", NT_PRSTATUS, 0*8 },
128 { "rip", NT_PRSTATUS, 16*8 },
129 /* XXX: SSE registers %xmm0-%xmm7 */
130 { "xmm0", NT_PRFPREG, 160+0*16}, // dwarf reg# 17 = byte #160 in PRFPREG register dump
131 { "xmm1", NT_PRFPREG, 160+1*16}, // see also gdb gdb i387-tdep.c fxsave_offset
132 { "xmm2", NT_PRFPREG, 160+2*16}, // see also intel x86-64 architecture software manual, fxsave area
133 { "xmm3", NT_PRFPREG, 160+3*16},
134 { "xmm4", NT_PRFPREG, 160+4*16},
135 { "xmm5", NT_PRFPREG, 160+5*16},
136 { "xmm6", NT_PRFPREG, 160+6*16},
137 { "xmm7", NT_PRFPREG, 160+7*16},
138 /* XXX: SSE2 registers %xmm8-%xmm15 */
139 { "xmm8", NT_PRFPREG, 160+8*16},
140 { "xmm9", NT_PRFPREG, 160+9*16},
141 { "xmm10", NT_PRFPREG, 160+10*16},
142 { "xmm11", NT_PRFPREG, 160+11*16},
143 { "xmm12", NT_PRFPREG, 160+12*16},
144 { "xmm13", NT_PRFPREG, 160+13*16},
145 { "xmm14", NT_PRFPREG, 160+14*16},
146 { "xmm15", NT_PRFPREG, 160+15*16},
147 /* XXX: FP registers %st0-%st7 */
148 /* XXX: MMX registers %mm0-%mm7 */
149 { "st0", NT_PRFPREG, 32},
150 { "st1", NT_PRFPREG, 48},
151 { "st2", NT_PRFPREG, 64},
152 { "st3", NT_PRFPREG, 80},
153 { "st4", NT_PRFPREG, 90},
154 { "st5", NT_PRFPREG, 112},
155 { "st6", NT_PRFPREG, 128},
156 { "st7", NT_PRFPREG, 144}
157 };
158 #endif
159 /* XXX: insert other architectures here. */
160
161
162 static u32 ursl_fetch32 (const struct usr_regset_lut* lut, unsigned lutsize, int e_machine, unsigned regno)
163 {
164 u32 value = ~0;
165 const struct user_regset_view *rsv = task_user_regset_view(current);
166 unsigned rsi;
167 int rc;
168 unsigned rsn;
169 unsigned pos;
170 unsigned count;
171
172 WARN_ON (!rsv);
173 if (!rsv) goto out;
174 WARN_ON (regno >= lutsize);
175 if (regno >= lutsize) goto out;
176 if (rsv->e_machine != e_machine) goto out;
177
178 rsn = lut[regno].rsn;
179 pos = lut[regno].pos;
180 count = sizeof(value);
181
182 for (rsi=0; rsi<rsv->n; rsi++)
183 if (rsv->regsets[rsi].core_note_type == rsn)
184 {
185 const struct user_regset *rs = & rsv->regsets[rsi];
186 rc = (rs->get)(current, rs, pos, count, & value, NULL);
187 WARN_ON (rc);
188 /* success */
189 goto out;
190 }
191 WARN_ON (1); /* did not find appropriate regset! */
192
193 out:
194 return value;
195 }
196
197
198 static void ursl_store32 (const struct usr_regset_lut* lut,unsigned lutsize, int e_machine, unsigned regno, u32 value)
199 {
200 const struct user_regset_view *rsv = task_user_regset_view(current);
201 unsigned rsi;
202 int rc;
203 unsigned rsn;
204 unsigned pos;
205 unsigned count;
206
207 WARN_ON (!rsv);
208 if (!rsv) goto out;
209 WARN_ON (regno >= lutsize);
210 if (regno >= lutsize) goto out;
211 if (rsv->e_machine != e_machine) goto out;
212
213 rsn = lut[regno].rsn;
214 pos = lut[regno].pos;
215 count = sizeof(value);
216
217 for (rsi=0; rsi<rsv->n; rsi++)
218 if (rsv->regsets[rsi].core_note_type == rsn)
219 {
220 const struct user_regset *rs = & rsv->regsets[rsi];
221 rc = (rs->set)(current, rs, pos, count, & value, NULL);
222 WARN_ON (rc);
223 /* success */
224 goto out;
225 }
226 WARN_ON (1); /* did not find appropriate regset! */
227
228 out:
229 return;
230 }
231
232
233 static u64 ursl_fetch64 (const struct usr_regset_lut* lut, unsigned lutsize, int e_machine, unsigned regno)
234 {
235 u64 value = ~0;
236 const struct user_regset_view *rsv = task_user_regset_view(current);
237 unsigned rsi;
238 int rc;
239 unsigned rsn;
240 unsigned pos;
241 unsigned count;
242
243 if (!rsv) goto out;
244 if (regno >= lutsize) goto out;
245 if (rsv->e_machine != e_machine) goto out;
246
247 rsn = lut[regno].rsn;
248 pos = lut[regno].pos;
249 count = sizeof(value);
250
251 for (rsi=0; rsi<rsv->n; rsi++)
252 if (rsv->regsets[rsi].core_note_type == rsn)
253 {
254 const struct user_regset *rs = & rsv->regsets[rsi];
255 rc = (rs->get)(current, rs, pos, count, & value, NULL);
256 if (rc)
257 goto out;
258 /* success */
259 return value;
260 }
261 out:
262 printk (KERN_WARNING "process %d mach %d regno %d not available for fetch.\n", current->tgid, e_machine, regno);
263 return value;
264 }
265
266
267 static void ursl_store64 (const struct usr_regset_lut* lut,unsigned lutsize, int e_machine, unsigned regno, u64 value)
268 {
269 const struct user_regset_view *rsv = task_user_regset_view(current);
270 unsigned rsi;
271 int rc;
272 unsigned rsn;
273 unsigned pos;
274 unsigned count;
275
276 WARN_ON (!rsv);
277 if (!rsv) goto out;
278 WARN_ON (regno >= lutsize);
279 if (regno >= lutsize) goto out;
280 if (rsv->e_machine != e_machine) goto out;
281
282 rsn = lut[regno].rsn;
283 pos = lut[regno].pos;
284 count = sizeof(value);
285
286 for (rsi=0; rsi<rsv->n; rsi++)
287 if (rsv->regsets[rsi].core_note_type == rsn)
288 {
289 const struct user_regset *rs = & rsv->regsets[rsi];
290 rc = (rs->set)(current, rs, pos, count, & value, NULL);
291 if (rc)
292 goto out;
293 /* success */
294 return;
295 }
296
297 out:
298 printk (KERN_WARNING "process %d mach %d regno %d not available for store.\n", current->tgid, e_machine, regno);
299 return;
300 }
301
302
303
304 #if defined (__i386__)
305
306 #define uu_fetch_register(_regs,regno) ursl_fetch32(url_i386, ARRAY_SIZE(url_i386), EM_386, regno)
307 #define uu_store_register(_regs,regno,value) ursl_store32(url_i386, ARRAY_SIZE(url_i386), EM_386, regno, value)
308
309 #define u_fetch_register(regno) check_fetch_register(c->uregs,regno,ARRAY_SIZE(url_i386),uu_fetch_register)
310 #define u_store_register(regno,value) check_store_register(c->uregs,regno,ARRAY_SIZE(url_i386),value,uu_store_register)
311
312 #elif defined (__x86_64__)
313
314 #define uu_fetch_register(_regs,regno) (_stp_is_compat_task() ? ursl_fetch32(url_i386, ARRAY_SIZE(url_i386), EM_386, regno) : ursl_fetch64(url_x86_64, ARRAY_SIZE(url_x86_64), EM_X86_64, regno))
315 #define uu_store_register(_regs,regno,value) (_stp_is_compat_task() ? ursl_store32(url_i386, ARRAY_SIZE(url_i386), EM_386, regno, value) : ursl_store64(url_x86_64, ARRAY_SIZE(url_x86_64), EM_X86_64, regno, value))
316
317 #define u_fetch_register(regno) check_fetch_register(c->uregs,regno,_stp_is_compat_task()?ARRAY_SIZE(url_i386):ARRAY_SIZE(url_x86_64),uu_fetch_register)
318 #define u_store_register(regno,value) check_store_register(c->uregs,regno,_stp_is_compat_task()?ARRAY_SIZE(url_i386):ARRAY_SIZE(url_x86_64),value,uu_store_register)
319
320 #endif
321
322 #else /* ! STAPCONF_REGSET */
323
324 /* Downgrade to pt_dwarf_register access. */
325
326 #define u_store_register(regno, value) \
327 check_store_register(c->uregs,regno,pt_regs_maxno,value,pt_regs_store_register)
328
329 /* If we're in a 32/31-bit task in a 64-bit kernel, we need to emulate
330 * 32-bitness by masking the output of pt_regs_fetch_register() */
331 #ifndef CONFIG_COMPAT
332 #define u_fetch_register(regno) \
333 check_fetch_register(c->uregs,regno,pt_regs_maxno,pt_regs_fetch_register)
334
335 #else
336 #define u_fetch_register(regno) \
337 check_fetch_register(c->uregs,regno,pt_regs_maxno,pt_regs_fetch_register) & (_stp_is_compat_task() ? 0xffffffff : ~(int64_t)0)
338 #endif
339
340 #endif /* STAPCONF_REGSET */
341
342
343 /* The deref and store_deref macros are called to safely access
344 addresses in the probe context. These macros are used only for
345 kernel addresses. The macros must handle bogus addresses here
346 gracefully (as from corrupted data structures, stale pointers,
347 etc), by doing a "goto deref_fault".
348
349 Prior to kernel 5.10, on most machines, the asm/uaccess.h macros
350 __get_user and __put_user macros do exactly the low-level work
351 we need to access memory with fault handling,
352 and are not actually specific to user-address access at all.
353
354 After kernel 5.10 on arches removing set_fs(), kernel
355 addresses should be read/written with get_kernel_nofault and
356 copy_to_kernel_nofault while user addresses are still read/written
357 with __get_user and __put_user. So we have wrapper macros
358 __stp_{get,put}_either which do the right thing on all kernel
359 versions. */
360
361 /*
362 * On most platforms, __get_user_inatomic() and __put_user_inatomic()
363 * are defined, which are the same as __get_user() and __put_user(),
364 * but without a call to might_sleep(). Since we're disabling page
365 * faults when we read, we want to use the 'inatomic' variants when
366 * available.
367 */
368 #ifdef __get_user_inatomic
369 #define __stp_get_user __get_user_inatomic
370 #else
371 #define __stp_get_user __get_user
372 #endif
373 #ifdef __put_user_inatomic
374 #define __stp_put_user __put_user_inatomic
375 #else
376 #define __stp_put_user __put_user
377 #endif
378
379 /*
380 * Some arches (like aarch64) don't have __get_user_bad() or
381 * __put_user_bad(), so use BUILD_BUG() instead.
382 */
383 #ifdef BUILD_BUG
384 #define __stp_get_user_bad BUILD_BUG
385 #define __stp_put_user_bad BUILD_BUG
386 #else
387 #define __stp_get_user_bad __get_user_bad
388 #define __stp_put_user_bad __put_user_bad
389 #endif
390
391 typedef void (*switch_task_namespaces_fn)(struct task_struct *tsk, struct nsproxy *new);
392 typedef void (*create_new_namespaces_fn)(struct task_struct *p, struct nsproxy *new);
393 typedef int (*unshare_nsproxy_namespaces_fn)(unsigned long, struct nsproxy **,
394 struct cred *, struct fs_struct *);
395 typedef void (*free_nsproxy_fn)(struct nsproxy *ns);
396 typedef bool (*proc_ns_file_fn)(const struct file *file);
397
398 /*
399 * __stp_{get,put}_either take an stp_mm_segment_t parameter
400 * and use that to decide the correct address space
401 * on post-5.10 non-set_fs() kernels.
402 */
403 #ifdef STAPCONF_SET_FS
404
405 #define __stp_get_either(v, addr, seg) __stp_get_user((v), (addr))
406 #define __stp_put_either(v, addr, seg) __stp_put_user((v), (addr))
407
408 #else
409
410 /* PR26811: Distinguish user- and kernel-space get and put operations.
411 *
412 * XXX There is slight redundancy between the size adjustments we
413 * do and the size adjustments done by {get,copy_to}_kernel_nofault. */
414
415 /* XXX copy_to_kernel_nofault is what we need, but it's not exported.
416 * First typedef from the original decl, then #define it as a typecasted call.
417 */
418 typedef typeof(&copy_to_kernel_nofault) copy_to_kernel_nofault_fn;
419 #define copy_to_kernel_nofault(dst, src, size) \
420 (kallsyms_copy_to_kernel_nofault ? \
421 ibt_wrapper(long, \
422 ((* (copy_to_kernel_nofault_fn)(kallsyms_copy_to_kernel_nofault)) \
423 ((dst),(src),(size)))) : \
424 -EFAULT)
425
426 #define __stp_get_either(v, addr, seg) \
427 (MM_SEG_IS_KERNEL((seg)) ? \
428 get_kernel_nofault((v), (addr)) : \
429 __stp_get_user((v), (addr)))
430 #define __stp_put_either(v, addr, seg) \
431 (MM_SEG_IS_KERNEL((seg)) ? \
432 ({typeof(v) _v = (v); long rc = copy_to_kernel_nofault((void *)(addr), (void *)&_v, sizeof(v)); rc;}) : \
433 __stp_put_user((v), (addr)))
434
435 #endif
436
437 /*
438 * __stp_deref_nocheck(): reads a simple type from a
439 * location with no address sanity checking.
440 *
441 * value: read the simple type into this variable
442 * size: number of bytes to read
443 * addr: address to read from
444 * seg: memory segment to use, either kernel (STP_KERNEL_DS) or user
445 * (STP_USER_DS)
446 *
447 * Note that the caller *must* check the address for validity and do
448 * any other checks necessary. This macro is designed to be used as
449 * the base for the other macros more suitable for the rest of the
450 * code to use. Note that the caller is also responsible for ensuring
451 * that the kernel doesn't pagefault while reading.
452 */
453
454 static __always_inline int __stp_deref_nocheck_(u64 *pv, size_t size,
455 void *addr, stp_mm_segment_t seg)
456 {
457 u64 v = 0;
458 int r = -EFAULT;
459
460 switch (size)
461 {
462 case 1: { u8 b; r = __stp_get_either(b, (u8 *)addr, seg); v = b; } break;
463 case 2: { u16 w; r = __stp_get_either(w, (u16 *)addr, seg); v = w; } break;
464 case 4: { u32 l; r = __stp_get_either(l, (u32 *)addr, seg); v = l; } break;
465 #if defined(__i386__) || defined(__arm__)
466 /* x86 and arm can't do 8-byte get_user, so we have to split it */
467 case 8: { union { u32 l[2]; u64 ll; } val;
468 r = __stp_get_either(val.l[0], &((u32 *)addr)[0], seg);
469 if (! r)
470 r = __stp_get_either(val.l[1], &((u32 *)addr)[1], seg);
471 if (! r)
472 v = val.ll;
473 } break;
474 #else
475 case 8: { r = __stp_get_either(v, (u64 *)addr, seg); } break;
476 #endif
477 }
478
479 *pv = v;
480 return r;
481 }
482
483 #define __stp_deref_nocheck(value, size, addr, seg) \
484 ({ \
485 u64 _v = 0; int _e = -EFAULT; \
486 switch (size) \
487 { \
488 case 1: case 2: case 4: case 8: \
489 _e = __stp_deref_nocheck_ \
490 (&_v, (size), (void *)(uintptr_t)(addr), (seg)); \
491 (value) = _v; \
492 break; \
493 default: \
494 __stp_get_user_bad(); \
495 } \
496 _e; \
497 })
498
499
500 static __always_inline bool
501 stp_is_user_ds(stp_mm_segment_t seg)
502 {
503 #ifdef STP_NUMERICAL_DS
504 return seg == STP_USER_DS;
505 #else
506 stp_mm_segment_t user_seg = STP_USER_DS;
507 return memcmp(&seg, &user_seg, sizeof(stp_mm_segment_t)) == 0;
508 #endif
509 }
510
511 static __always_inline bool
512 stp_user_access_begin(int type, const void *ptr, size_t size,
513 stp_mm_segment_t *oldfs, stp_mm_segment_t seg)
514 {
515 #ifdef STAPCONF_SET_FS
516 *oldfs = get_fs();
517 set_fs(seg);
518 return 1;
519 #elif defined(STAPCONF_USER_ACCESS_BEGIN_3_ARGS)
520 return user_access_begin(type, ptr, size);
521 #elif defined(STAPCONF_USER_ACCESS_BEGIN_2_ARGS)
522 return user_access_begin(ptr, size);
523 #else
524 /* for very old kernels */
525 return 1;
526 #endif
527 }
528
529 static __always_inline void
530 stp_user_access_end(stp_mm_segment_t oldfs)
531 {
532 #ifdef STAPCONF_SET_FS
533 set_fs(oldfs);
534 #elif defined(STAPCONF_USER_ACCESS_END)
535 user_access_end();
536 #else
537 /* do nothing for very old kernels */
538 #endif
539 }
540
541 static __always_inline bool
542 stp_mem_access_begin(int type, const void *ptr, size_t size,
543 stp_mm_segment_t *oldfs, stp_mm_segment_t seg, bool *is_user_ptr)
544 {
545 bool is_user = stp_is_user_ds(seg);
546 *is_user_ptr = is_user;
547 if (is_user)
548 return stp_user_access_begin(type, ptr, size, oldfs, seg);
549
550 /* for kernel memory accesses */
551
552 #ifdef STAPCONF_SET_FS
553 *oldfs = get_fs();
554 #endif
555 return 1;
556 }
557
558 static __always_inline void
559 stp_mem_access_end(stp_mm_segment_t oldfs, bool is_user)
560 {
561 if (is_user)
562 return stp_user_access_end(oldfs);
563
564 /* not for userland */
565
566 #ifdef STAPCONF_SET_FS
567 set_fs(oldfs);
568 #endif
569 }
570
571 /*
572 * _stp_lookup_bad_addr(): safely verify an address
573 *
574 * type: memory access type (either VERIFY_READ or VERIFY_WRITE)
575 * size: number of bytes to verify
576 * addr: address to verify
577 * seg: memory segment to use, either kernel (STP_KERNEL_DS) or user
578 * (STP_USER_DS)
579 *
580 * The macro returns 0 if the address is valid, non-zero otherwise.
581 * Note that the kernel will not pagefault when trying to verify the
582 * memory. Also note that no DEREF_FAULT will occur when verifying the
583 * memory.
584 */
585
586 static __always_inline int _stp_lookup_bad_addr_(int type, size_t size,
587 uintptr_t addr, stp_mm_segment_t seg)
588 {
589 int bad;
590 #ifdef STAPCONF_SET_FS
591 mm_segment_t oldfs = get_fs();
592
593 set_fs(seg);
594 #endif
595 pagefault_disable();
596 bad = lookup_bad_addr(type, addr, size, seg);
597 pagefault_enable();
598 #ifdef STAPCONF_SET_FS
599 set_fs(oldfs);
600 #endif
601
602 return bad;
603 }
604
605 #define _stp_lookup_bad_addr(type, size, addr, seg) \
606 _stp_lookup_bad_addr_((type), (size), (uintptr_t)(addr), (seg))
607
608
609 /*
610 * _stp_deref_nofault(): safely read a simple type from memory without
611 * a DEREF_FAULT on error
612 *
613 * value: read the simple type into this variable
614 * size: number of bytes to read
615 * addr: address to read from
616 * seg: memory segment to use, either kernel (STP_KERNEL_DS) or user
617 * (STP_USER_DS)
618 *
619 * If this macro gets an error while trying to read memory, nonzero is
620 * returned. On success, 0 is return. Note that the kernel will not
621 * pagefault when trying to read the memory.
622 */
623
624 static __always_inline int _stp_deref_nofault_(u64 *pv, size_t size, void *addr,
625 stp_mm_segment_t seg)
626 {
627 int r = -EFAULT;
628 pagefault_disable();
629 if (lookup_bad_addr(VERIFY_READ, (uintptr_t)addr, size, seg))
630 r = -EFAULT;
631 else {
632 stp_mm_segment_t oldfs;
633 bool is_user;
634
635 /* NB just to suppress -Werror=maybe-uninitialized warnings from older
636 * GCC like version 8.3 with -O3 */
637 #ifdef STP_NUMERICAL_DS
638 oldfs = 0;
639 #else
640 memset(&oldfs, 0, sizeof(stp_mm_segment_t));
641 #endif
642
643 if (!stp_mem_access_begin(VERIFY_READ, addr, size, &oldfs, seg, &is_user))
644 goto done;
645
646 r = __stp_deref_nocheck_(pv, size, addr, seg);
647
648 stp_mem_access_end(oldfs, is_user);
649 }
650 done:
651 pagefault_enable();
652 return r;
653 }
654
655 #define _stp_deref_nofault(value, size, addr, seg) \
656 ({ \
657 u64 _v = 0; int _e = -EFAULT; \
658 switch (size) \
659 { \
660 case 1: case 2: case 4: case 8: \
661 _e = _stp_deref_nofault_ \
662 (&_v, (size), (void *)(uintptr_t)(addr), (seg)); \
663 break; \
664 default: \
665 __stp_get_user_bad(); \
666 } \
667 (value) = _v; \
668 _e; \
669 })
670
671
672 /*
673 * _stp_deref(): safely read a simple type from memory
674 *
675 * size: number of bytes to read
676 * addr: address to read from
677 * seg: memory segment to use, either kernel (STP_KERNEL_DS) or user
678 * (STP_USER_DS)
679 *
680 * The macro returns the value read. If this macro gets an error while
681 * trying to read memory, a DEREF_FAULT will occur. Note that the
682 * kernel will not pagefault when trying to read the memory.
683 */
684
685 #define _stp_deref(size, addr, seg) \
686 ({ \
687 u64 _v = 0; int _e = -EFAULT; \
688 switch (size) \
689 { \
690 case 1: case 2: case 4: case 8: \
691 _e = _stp_deref_nofault_ \
692 (&_v, (size), (void *)(uintptr_t)(addr), (seg)); \
693 break; \
694 default: \
695 __stp_get_user_bad(); \
696 } \
697 if (_e) \
698 DEREF_FAULT(addr); \
699 _v; \
700 })
701
702
703 /*
704 * __stp_store_deref_nocheck(): writes a simple type to a location
705 * with no address sanity checking.
706 *
707 * size: number of bytes to write
708 * addr: address to write to
709 * value: read the simple type from this variable
710 * seg: memory segment to use, either kernel (STP_KERNEL_DS) or user
711 * (STP_USER_DS)
712 *
713 * Note that the caller *must* check the address for validity and do
714 * any other checks necessary. This macro is designed to be used as
715 * the base for the other macros more suitable for the rest of the
716 * code to use. Note that the caller is also responsible for ensuring
717 * that the kernel doesn't pagefault while writing.
718 */
719
720 static inline int __stp_store_deref_nocheck_(size_t size, void *addr,
721 u64 v, stp_mm_segment_t seg)
722 {
723 int r;
724 switch (size)
725 {
726 case 1: r = __stp_put_either((u8)v, (u8 *)addr, seg); break;
727 case 2: r = __stp_put_either((u16)v, (u16 *)addr, seg); break;
728 case 4: r = __stp_put_either((u32)v, (u32 *)addr, seg); break;
729 #if defined(__i386__) || defined(__arm__)
730 /* x86 and arm can't do 8-byte put_user, so we have to split it */
731 default: { union { u32 l[2]; u64 ll; } val;
732 val.ll = v;
733 r = __stp_put_either(val.l[0], &((u32 *)addr)[0], seg);
734 if (! r)
735 r = __stp_put_either(val.l[1], &((u32 *)addr)[1], seg);
736 } break;
737 #else
738 default: r = __stp_put_either(v, (u64 *)addr, seg); break;
739 #endif
740 }
741 return r;
742 }
743
744 #define __stp_store_deref_nocheck(size, addr, value, seg) \
745 ({ \
746 int _e = -EFAULT; \
747 switch (size) \
748 { \
749 case 1: case 2: case 4: case 8: \
750 _e = __stp_store_deref_nocheck_ \
751 ((size), (void *)(uintptr_t)(addr), (value), (seg)); \
752 break; \
753 default: \
754 __stp_put_user_bad(); \
755 } \
756 _e; \
757 })
758
759
760 /*
761 * _stp_store_deref(): safely write a simple type to memory
762 *
763 * size: number of bytes to write
764 * addr: address to write to
765 * value: read the simple type from this variable
766 * seg: memory segment to use, either kernel (STP_KERNEL_DS) or user
767 * (STP_USER_DS)
768 *
769 * The macro has no return value. If this macro gets an error while
770 * trying to write, a STORE_DEREF_FAULT will occur. Note that the
771 * kernel will not pagefault when trying to write the memory.
772 */
773
774 static inline int _stp_store_deref_(size_t size, void *addr, u64 v,
775 stp_mm_segment_t seg)
776 {
777 int r;
778 stp_mm_segment_t oldfs;
779 bool is_user;
780 pagefault_disable();
781 if (lookup_bad_addr(VERIFY_WRITE, (uintptr_t)addr, size, seg)) {
782 r = -EFAULT;
783 } else {
784 if (!stp_mem_access_begin(VERIFY_WRITE, addr, size, &oldfs, seg, &is_user)) {
785 r = -EFAULT;
786 goto done;
787 }
788
789 r = __stp_store_deref_nocheck_(size, addr, v, seg);
790 stp_mem_access_end(oldfs, is_user);
791 }
792 done:
793 pagefault_enable();
794 return r;
795 }
796
797 #define _stp_store_deref(size, addr, value, seg) \
798 ({ \
799 int _e = -EFAULT; \
800 switch (size) \
801 { \
802 case 1: case 2: case 4: case 8: \
803 _e = _stp_store_deref_ \
804 ((size), (void *)(uintptr_t)(addr), (value), (seg)); \
805 break; \
806 default: \
807 __stp_put_user_bad(); \
808 } \
809 if (_e) \
810 STORE_DEREF_FAULT(addr); \
811 (void)0; \
812 })
813
814
815 /* Map kderef/uderef to the generic segment-aware deref macros. */
816
817 #ifndef kderef
818 #define kderef(s,a) _stp_deref(s,a,STP_KERNEL_DS)
819 #endif
820
821 #ifndef store_kderef
822 #define store_kderef(s,a,v) _stp_store_deref(s,a,v,STP_KERNEL_DS)
823 #endif
824
825 #ifndef uderef
826 #define uderef(s,a) _stp_deref(s,a,STP_USER_DS)
827 #endif
828
829 #ifndef store_uderef
830 #define store_uderef(s,a,v) _stp_store_deref(s,a,v,STP_USER_DS)
831 #endif
832
833 #ifndef CONFIG_64BIT
834
835 /* The kderef/uderef macros (which is what Xderef gets set to), alway
836 * returns a 64-bit value. This causes a problem on a 32-bit system
837 * when we want to cast the 64-bit value to a 32-bit pointer - gcc
838 * gives a "cast to pointer from integer of different size" error. So,
839 * we'll cast it to a u32 before doing the final cast to the actual
840 * type. */
841
842 #define __Xread(ptr, Xderef) \
843 ((sizeof(*(ptr)) == 8) \
844 ? *(typeof(ptr))&(u64) { Xderef(sizeof(*(ptr)), (ptr)) } \
845 : *(typeof(ptr))&(u32) { (u32) Xderef(sizeof(*(ptr)), (ptr)) } )
846
847 /* For __Xwrite, we need to handle the case where 'value' is a pointer
848 * and avoid the "cast from pointer to integer of different size" gcc
849 * errors. */
850
851 #define __Xwrite(ptr, value, store_Xderef) \
852 ({ \
853 if (sizeof(*(ptr)) == 8) { \
854 union { typeof(*(ptr)) v; u64 l; } _kw; \
855 _kw.v = (typeof(*(ptr)))(value); \
856 store_Xderef(8, (ptr), _kw.l); \
857 } else \
858 store_Xderef(sizeof(*(ptr)), (ptr), (long)(typeof(*(ptr)))(value)); \
859 })
860
861 #else
862
863 #define __Xread(ptr, Xderef) \
864 ( (typeof(*(ptr))) Xderef(sizeof(*(ptr)), (ptr)) )
865 #define __Xwrite(ptr, value, store_Xderef) \
866 ( store_Xderef(sizeof(*(ptr)), (ptr), (long)(typeof(*(ptr)))(value)) )
867
868 #endif
869
870 #define kread(ptr) __Xread((ptr), kderef)
871 #define uread(ptr) __Xread((ptr), uderef)
872
873 #define kwrite(ptr, value) __Xwrite((ptr), (value), store_kderef)
874 #define uwrite(ptr, value) __Xwrite((ptr), (value), store_uderef)
875
876
877 /* Dereference a kernel buffer ADDR of size MAXBYTES. Put the bytes in
878 * address DST (which can be NULL).
879 *
880 * This function is useful for reading memory when the size isn't a
881 * size that kderef() handles. This function is very similar to
882 * kderef_string(), but kderef_buffer() doesn't quit when finding a
883 * '\0' byte or append a '\0' byte.
884 */
885
886 static inline char *kderef_buffer_(char *dst, void *addr, size_t len)
887 {
888 int err = 0;
889 size_t i;
890 #ifdef STAPCONF_SET_FS
891 mm_segment_t oldfs = get_fs();
892
893 set_fs(KERNEL_DS);
894 #endif
895 pagefault_disable();
896 if (lookup_bad_addr(VERIFY_READ, (uintptr_t)addr, len, STP_KERNEL_DS))
897 err = 1;
898 else
899 for (i = 0; i < len; ++i)
900 {
901 u8 v;
902 err = __stp_get_either(v, (u8 *)addr + i, STP_KERNEL_DS);
903 if (err)
904 break;
905 if (dst)
906 *dst++ = v;
907 }
908 pagefault_enable();
909 #ifdef STAPCONF_SET_FS
910 set_fs(oldfs);
911 #endif
912
913 return err ? (char *)-1 : dst;
914 }
915
916 #define kderef_buffer(dst, addr, maxbytes) \
917 ({ \
918 char *_r = kderef_buffer_((dst), (void *)(uintptr_t)(addr), (maxbytes)); \
919 if (_r == (char *)-1) \
920 DEREF_FAULT(addr); \
921 _r; \
922 })
923
924 /*
925 * _stp_deref_string_nofault(): safely read a string from memory
926 * without a DEREF_FAULT on error
927 *
928 * dst: read the string into this address
929 * addr: address to read from
930 * len: maximum number of bytes to store in dst, including the trailing NUL
931 * seg: memory segment to use, either kernel (STP_KERNEL_DS) or user
932 * (STP_USER_DS)
933 *
934 * If this function gets an error while trying to read memory, -EFAULT
935 * is returned. On success, the number of bytes copied is returned
936 * (not including the trailing NULL). Note that the kernel will not
937 * pagefault when trying to read the string.
938 */
939
940 static inline long _stp_deref_string_nofault(char *dst, const char *addr,
941 size_t len, stp_mm_segment_t seg)
942 {
943 int err = 0;
944 size_t i = 0;
945 stp_mm_segment_t oldfs;
946 bool is_user;
947 pagefault_disable();
948 if (lookup_bad_addr(VERIFY_READ, (uintptr_t)addr, len, seg))
949 err = 1;
950 else
951 {
952 if (!stp_mem_access_begin(VERIFY_READ, addr, len, &oldfs, seg, &is_user))
953 goto done;
954
955 /* Reduce len by 1 to leave room for '\0' terminator. */
956 for (i = 0; i + 1 < len; ++i)
957 {
958 u8 v;
959 err = __stp_get_either(v, (u8 *)addr + i, seg);
960 if (err || v == '\0')
961 break;
962 if (dst)
963 *dst++ = v;
964 }
965 if (!err && dst)
966 *dst = '\0';
967
968 stp_mem_access_end(oldfs, is_user);
969 }
970 done:
971 pagefault_enable();
972 return err ? -EFAULT : i;
973 }
974
975 #define kderef_string(dst, addr, maxbytes) \
976 ({ \
977 long _r = _stp_deref_string_nofault((dst), (void *)(uintptr_t)(addr), (maxbytes), STP_KERNEL_DS); \
978 if (_r < 0) \
979 DEREF_FAULT(addr); \
980 _r; \
981 })
982
983 /*
984 * _stp_store_deref_string(): safely write a string to memory
985 *
986 * src: source string
987 * addr: address to write to
988 * maxbytes: maximum number of bytes to write
989 * seg: memory segment to use, either kernel (STP_KERNEL_DS) or user
990 * (STP_USER_DS)
991 *
992 * The macro has no return value. If this macro gets an error while
993 * trying to write, a STORE_DEREF_FAULT will occur. Note that the
994 * kernel will not pagefault when trying to write the memory.
995 */
996
997 static inline int _stp_store_deref_string_(char *src, void *addr, size_t len,
998 stp_mm_segment_t seg)
999 {
1000 int err = 0;
1001 size_t i;
1002 stp_mm_segment_t oldfs;
1003 bool is_user;
1004
1005 pagefault_disable();
1006 if (lookup_bad_addr(VERIFY_WRITE, (uintptr_t)addr, len, seg)) {
1007 err = 1;
1008 goto done;
1009 }
1010
1011 if (!stp_mem_access_begin(VERIFY_WRITE, addr, len, &oldfs, seg, &is_user))
1012 goto done;
1013
1014 if (len > 0)
1015 {
1016 for (i = 0; i < len - 1; ++i)
1017 {
1018 if (*src == '\0')
1019 break;
1020 err = __stp_put_either(*src++, (u8 *)addr + i, seg);
1021 if (err)
1022 goto out;
1023 }
1024 /* PR31074: cast (char) '\0' to make sure right size */
1025 err = __stp_put_either((char) '\0', (u8 *)addr + i, seg);
1026 }
1027
1028 out:
1029 stp_mem_access_end(oldfs, is_user);
1030
1031 done:
1032 pagefault_enable();
1033 return err;
1034 }
1035
1036 #define _stp_store_deref_string(src, addr, maxbytes, seg) \
1037 ({ \
1038 if (_stp_store_deref_string_ \
1039 ((src), (void *)(uintptr_t)(addr), (maxbytes), (seg))) \
1040 STORE_DEREF_FAULT(addr); \
1041 (void)0; \
1042 })
1043
1044
1045 /*
1046 * store_kderef_string(): safely write a string to kernel memory
1047 *
1048 * src: source string
1049 * addr: address to write to
1050 * maxbytes: maximum number of bytes to write
1051 *
1052 * The macro has no return value. If this macro gets an error while
1053 * trying to write, a STORE_DEREF_FAULT will occur. Note that the
1054 * kernel will not pagefault when trying to write the memory.
1055 */
1056
1057 #define store_kderef_string(src, addr, maxbytes) \
1058 _stp_store_deref_string((src), (addr), (maxbytes), STP_KERNEL_DS)
1059
1060
1061 /*
1062 * store_uderef_string(): safely write a string to user memory
1063 *
1064 * src: source string
1065 * addr: address to write to
1066 * maxbytes: maximum number of bytes to write
1067 *
1068 * The macro has no return value. If this macro gets an error while
1069 * trying to write, a STORE_DEREF_FAULT will occur. Note that the
1070 * kernel will not pagefault when trying to write the memory.
1071 */
1072
1073 #define store_uderef_string(src, addr, maxbytes) \
1074 _stp_store_deref_string((src), (addr), (maxbytes), STP_USER_DS)
1075
1076 #endif /* _LINUX_LOC2C_RUNTIME_H_ */