Debian Codegraph @

runtime/linux/runtime.h

raw
1 /* main header file for Linux
2 * Copyright (C) 2005-2019 Red Hat Inc.
3 * Copyright (C) 2005-2006 Intel Corporation.
4 *
5 * This file is part of systemtap, and is free software. You can
6 * redistribute it and/or modify it under the terms of the GNU General
7 * Public License (GPL); either version 2, or (at your option) any
8 * later version.
9 */
10
11 #ifndef _LINUX_RUNTIME_H_
12 #define _LINUX_RUNTIME_H_
13
14 #include <linux/module.h>
15 #include <linux/ctype.h>
16 #include <linux/kernel.h>
17 #include <linux/miscdevice.h>
18 #include <linux/init.h>
19 #include <linux/hash.h>
20 #include <linux/string.h>
21 #include <linux/kprobes.h>
22 #include <linux/proc_fs.h>
23 #include <linux/vmalloc.h>
24 #include <linux/slab.h>
25 #include <linux/time.h>
26 #include <linux/random.h>
27 #include <linux/spinlock.h>
28 #include <linux/hardirq.h>
29 #include <asm/uaccess.h>
30 #include <linux/kallsyms.h>
31 #include <linux/version.h>
32 #if LINUX_VERSION_CODE >= KERNEL_VERSION(3,17,0)
33 #include <generated/utsrelease.h>
34 #else
35 #include <linux/vermagic.h>
36 #endif
37 #include <linux/utsname.h>
38 #include <linux/compat.h>
39 #include <linux/sched.h>
40 #include <linux/mm.h>
41 #include "timer_compatibility.h"
42 #include <linux/delay.h>
43 #include <linux/profile.h>
44 #include <linux/rcupdate.h>
45 //#include <linux/utsrelease.h> // newer kernels only
46 //#include <linux/compile.h>
47 #ifdef STAPCONF_GENERATED_COMPILE
48 #include <generated/compile.h>
49 #endif
50
51 // PR31373: Linux 6.8 kernel removed strlcpy
52 // This provides equivalent to strlcpy using strscpy
53 #if LINUX_VERSION_CODE >= KERNEL_VERSION(6,8,0)
54 #define strlcpy(dest, src, count) ({ \
55 size_t size=(count); \
56 ssize_t retval = strscpy((dest), (src), size);\
57 if (retval<0) \
58 retval = size; /* was truncated */ \
59 else \
60 retval++; /* count the NULL */ \
61 retval; \
62 })
63 #endif
64
65 // PR26811: Replace some declarations after set_fs() removal in kernel 5.10+.
66 // Should use the STP_* prefixed defines outside of STAPCONF_SET_FS.
67 #if defined(STAPCONF_SET_FS)
68 #define stp_mm_segment_t mm_segment_t
69 #define STP_KERNEL_DS KERNEL_DS
70 #define STP_USER_DS USER_DS
71 // XXX MM_SEG_IS_KERNEL only used #ifndef STAPCONF_SET_FS
72 #else
73 #define stp_mm_segment_t unsigned long
74 #define STP_KERNEL_DS 0
75 #define STP_USER_DS 1
76 #define STP_NUMERICAL_DS 1
77 #define MM_SEG_IS_KERNEL(seg) ((seg)==STP_KERNEL_DS)
78
79 // Required for kernel write operations:
80 static void *kallsyms_copy_to_kernel_nofault;
81 #endif
82
83 #ifndef STAPCONF_NMI_UACCESS_OKAY
84 /* alas. nmi_access_okay() is not exported by modern kernels */
85 static void *kallsyms_nmi_uaccess_okay = NULL;
86 #endif
87
88 static void *kallsyms_switch_task_namespaces = NULL;
89 static void *kallsyms_unshare_nsproxy_namespaces = NULL;
90 static void *kallsyms_free_nsproxy = NULL;
91 static void *kallsyms_proc_ns_file = NULL;
92
93 /* A fallthrough; macro to let the runtime survive -Wimplicit-fallthrough=5 */
94 /* from <linux/compiler_attribute.h> */
95 #ifndef fallthrough
96 #if __GNUC__ < 5
97 # define fallthrough do {} while (0) /* fallthrough */
98 #else
99 #if __has_attribute(__fallthrough__)
100 # define fallthrough __attribute__((__fallthrough__))
101 #else
102 # define fallthrough do {} while (0) /* fallthrough */
103 #endif
104 #endif
105 #endif
106
107 #include <linux/user_namespace.h>
108
109 #if !defined (CONFIG_DEBUG_FS) && !defined (CONFIG_DEBUG_FS_MODULE)
110 #error "DebugFS is required and was not found in the kernel."
111 #endif
112
113 #if defined(STAPCONF_UDELAY_SIMPLE_EXPORTED)
114 #undef udelay
115 #define udelay(x) udelay_simple(x)
116 #elif defined(STAPCONF_UDELAY_SIMPLE)
117 #undef udelay
118 static void *kallsyms_udelay_simple;
119 typedef typeof(&udelay_simple) udelay_simple_fn;
120 #define udelay(x) void_ibt_wrapper((* (udelay_simple_fn)(kallsyms_udelay_simple))((x)))
121 #endif
122
123 #ifndef clamp
124 #define clamp(val, low, high) min(max(low, val), high)
125 #endif
126
127 #ifndef clamp_t
128 #define clamp_t(type, val, low, high) min_t(type, max_t(type, low, val), high)
129 #endif
130
131 static void _stp_dbug (const char *func, int line, const char *fmt, ...) __attribute__ ((format (printf, 3, 4)));
132 static void _stp_error (const char *fmt, ...) __attribute__ ((format (printf, 1, 2)));
133 static void _stp_warn (const char *fmt, ...) __attribute__ ((format (printf, 1, 2)));
134
135 static void _stp_exit(void);
136
137
138 #ifdef STAPCONF_HLIST_4ARGS
139 #define stap_hlist_for_each_entry(a,b,c,d) hlist_for_each_entry(a,b,c,d)
140 #define stap_hlist_for_each_entry_rcu(a,b,c,d) hlist_for_each_entry_rcu(a,b,c,d)
141 #define stap_hlist_for_each_entry_safe(a,b,c,d,e) hlist_for_each_entry_safe(a,b,c,d,e)
142 #else
143 #define stap_hlist_for_each_entry(a,b,c,d) (void) b; hlist_for_each_entry(a,c,d)
144 #define stap_hlist_for_each_entry_rcu(a,b,c,d) (void) b; hlist_for_each_entry_rcu(a,c,d)
145 #define stap_hlist_for_each_entry_safe(a,b,c,d,e) (void) b; hlist_for_each_entry_safe(a,c,d,e)
146 #endif
147
148 #ifndef preempt_enable_no_resched
149 #ifdef CONFIG_PREEMPT_COUNT
150 #define preempt_enable_no_resched() do { \
151 barrier(); \
152 preempt_count_dec(); \
153 } while (0)
154 #else
155 #define preempt_enable_no_resched() barrier()
156 #endif
157 #endif
158
159 #ifndef rcu_dereference_sched
160 #define rcu_dereference_sched(p) rcu_dereference(p)
161 #endif
162
163 /* unprivileged user support */
164
165 #ifdef STAPCONF_TASK_UID
166 #define STP_CURRENT_EUID (current->euid)
167 #else
168 #if defined(CONFIG_USER_NS) || (LINUX_VERSION_CODE >= KERNEL_VERSION(3,14,0))
169 #ifndef STAPCONF_FROM_KUID_MUNGED
170 #define from_kuid_munged(user_ns, uid) ((uid))
171 #define from_kgid_munged(user_ns, gid) ((gid))
172 #endif /* !STAPCONF_FROM_KUID_MUNGED */
173 #define STP_CURRENT_EUID (from_kuid_munged(current_user_ns(), task_euid(current)))
174 #else
175 #define STP_CURRENT_EUID (task_euid(current))
176 #endif
177 #endif
178
179 #define is_myproc() (STP_CURRENT_EUID == _stp_uid)
180
181 /* Check whether the required privilege level has been attained. */
182 #define STP_PRIVILEGE_CONTAINS(actualPrivilege, requiredPrivilege) ( \
183 ((actualPrivilege) & (requiredPrivilege)) == (requiredPrivilege) \
184 )
185
186 /* Translate user privilege mask to text. */
187 static const char *privilege_to_text (int p) {
188 if (STP_PRIVILEGE_CONTAINS (p, STP_PR_STAPDEV)) return "stapdev";
189 if (STP_PRIVILEGE_CONTAINS (p, STP_PR_STAPSYS)) return "stapsys";
190 if (STP_PRIVILEGE_CONTAINS (p, STP_PR_STAPUSR)) return "stapusr";
191 return "unknown";
192 }
193
194 #if STP_PRIVILEGE_CONTAINS (STP_PRIVILEGE, STP_PR_STAPDEV) || \
195 STP_PRIVILEGE_CONTAINS (STP_PRIVILEGE, STP_PR_STAPSYS)
196 #define assert_is_myproc() do {} while (0)
197 #else
198 #define assert_is_myproc() do { \
199 if (! is_myproc()) { \
200 snprintf (CONTEXT->error_buffer, MAXSTRINGLEN, STAP_MSG_RUNTIME_H_01, \
201 current->tgid, STP_CURRENT_EUID); \
202 CONTEXT->last_error = CONTEXT->error_buffer; \
203 goto out; \
204 } } while (0)
205 #endif
206
207 #include "debug.h"
208
209 /* atomic globals */
210 static atomic_t _stp_transport_failures = ATOMIC_INIT (0);
211
212 static struct
213 {
214 atomic_t ____cacheline_aligned_in_smp seq;
215 } _stp_seq = { ATOMIC_INIT (0) };
216
217 #define _stp_seq_inc() (atomic_inc_return(&_stp_seq.seq))
218
219 /* dwarf unwinder only tested so far on arm, i386, x86_64, ppc64 and s390x.
220 Only define STP_USE_DWARF_UNWINDER when STP_NEED_UNWIND_DATA,
221 as set through a pragma:unwind in one of the [u]context-unwind.stp
222 functions. */
223 #if (defined(__arm__) || defined(__i386__) || defined(__x86_64__) || defined(__powerpc64__)) || defined (__s390x__) || defined(__aarch64__) || defined(__mips__) || defined(__riscv)
224 #ifdef STP_NEED_UNWIND_DATA
225 #ifndef STP_USE_DWARF_UNWINDER
226 #define STP_USE_DWARF_UNWINDER
227 #endif
228 #endif
229 #endif
230
231 // PR26074: Obtain kallsyms_*() addresses from relocations,
232 // call via wrapper in runtime/sym.c:
233 #if !defined(STAPCONF_KALLSYMS_LOOKUP_NAME_EXPORTED)
234 static void *_stp_kallsyms_lookup_name;
235 #endif
236 // PR11514: kallsyms_on_each_symbol() should not be used on PPC64.
237 #ifndef CONFIG_PPC64
238 #define STAPCONF_KALLSYMS_ON_EACH_SYMBOL
239 #if !defined(STAPCONF_KALLSYMS_ON_EACH_SYMBOL_EXPORTED)
240 // Not static, since it is linked into kprobes.c:
241 void *_stp_kallsyms_on_each_symbol;
242 #endif
243 #endif
244
245 // No export check and gates. This one seems simply like a non-export.
246 // Not static, since it is linked into kprobes.c:
247 void *_stp_module_kallsyms_on_each_symbol;
248
249 // PR13489, inode-uprobes sometimes lacks the necessary SYMBOL_EXPORT's.
250 #if !defined(STAPCONF_TASK_USER_REGSET_VIEW_EXPORTED)
251 static void *kallsyms_task_user_regset_view;
252 #endif
253 #if !defined(STAPCONF_UPROBE_REGISTER_EXPORTED)
254 static void *kallsyms_uprobe_register;
255 #endif
256 #if !defined(STAPCONF_UPROBE_UNREGISTER_EXPORTED)
257 static void *kallsyms_uprobe_unregister;
258 #endif
259 #if !defined(STAPCONF_UPROBE_GET_SWBP_ADDR_EXPORTED)
260 static void *kallsyms_uprobe_get_swbp_addr;
261 #endif
262
263 /* task_work functions lack the necessary SYMBOL_EXPORT's */
264 #if !defined(STAPCONF_TASK_WORK_ADD_EXPORTED)
265 static void *kallsyms_task_work_add;
266 #endif
267 #if !defined(STAPCONF_TASK_WORK_CANCEL_EXPORTED)
268 static void *kallsyms_task_work_cancel_fn;
269 #endif
270
271 #if !defined(STAPCONF_TRY_TO_WAKE_UP_EXPORTED) && !defined(STAPCONF_WAKE_UP_STATE_EXPORTED)
272 static void *kallsyms_wake_up_state;
273 #endif
274 #if !defined(STAPCONF_SIGNAL_WAKE_UP_STATE_EXPORTED)
275 static void *kallsyms_signal_wake_up_state;
276 #endif
277 #if !defined(STAPCONF_SIGNAL_WAKE_UP_EXPORTED)
278 static void *kallsyms_signal_wake_up;
279 #endif
280 #if !defined(STAPCONF___LOCK_TASK_SIGHAND_EXPORTED)
281 static void *kallsyms___lock_task_sighand;
282 #endif
283 #if !defined(STAPCONF_GET_MM_EXE_FILE_EXPORTED)
284 static void *kallsyms_get_mm_exe_file;
285 #endif
286
287 /* PR30777: Need a mechanism to temporarily turn off Intel IBT */
288 #ifdef CONFIG_X86_KERNEL_IBT
289
290 __noendbr u64 ibt_save(bool disable)
291 {
292 u64 msr = 0;
293
294 if (cpu_feature_enabled(X86_FEATURE_IBT)) {
295 rdmsrl(MSR_IA32_S_CET, msr);
296 if (disable)
297 wrmsrl(MSR_IA32_S_CET, msr & ~CET_ENDBR_EN);
298 }
299
300 return msr;
301 }
302
303 __noendbr void ibt_restore(u64 save)
304 {
305 u64 msr;
306
307 if (cpu_feature_enabled(X86_FEATURE_IBT)) {
308 rdmsrl(MSR_IA32_S_CET, msr);
309 msr &= ~CET_ENDBR_EN;
310 msr |= (save & CET_ENDBR_EN);
311 wrmsrl(MSR_IA32_S_CET, msr);
312 }
313 }
314
315 #define ibt_wrapper(rettype, function) ({ \
316 u64 ibt_value = ibt_save(true); \
317 rettype retval = (function); \
318 ibt_restore(ibt_value); \
319 retval; \
320 })
321
322 #define void_ibt_wrapper(function) ({ \
323 u64 ibt_value = ibt_save(true); \
324 (function); \
325 ibt_restore(ibt_value); \
326 0; \
327 })
328
329 #else
330 #define ibt_wrapper(rettype, function) ({ (function); })
331 #define void_ibt_wrapper(function) ({ (function); })
332 #endif
333
334 #include "access_process_vm.h"
335 #include "loc2c-runtime.h"
336
337 #include "alloc.c"
338 #include "print.h"
339 #include "stp_string.c"
340 #include "arith.c"
341 #include "copy.c"
342 #include "../regs.c"
343 #include "regs-ia64.c"
344
345 #if (defined(STAPCONF_UTRACE_VIA_TRACEPOINTS) || defined(STAPCONF_UTRACE_VIA_TRACEPOINTS2))
346 #define HAVE_TASK_FINDER
347 #include "task_finder.c"
348 #else
349 /* stub functionality that fails gracefully */
350 #include "task_finder_stubs.c"
351 #endif
352
353 #ifdef _HAVE_PERF_
354 /* Now that we've got the task_finder included (one way or another),
355 * we can include the perf stuff (if needed). */
356 #include <linux/perf_event.h>
357 #include "linux/perf.h"
358 #endif
359
360 #include "sym.c"
361 #ifdef STP_PERFMON
362 #include "perf.c"
363 #endif
364 #include "addr-map.c"
365 #include "stat.c"
366
367 /* DWARF unwinder only tested so far on i386, x86_64 and ppc64.
368 We only need to compile in the unwinder when both STP_NEED_UNWIND_DATA
369 (set when a stap script defines pragma:unwind, as done in
370 [u]context-unwind.stp) is defined and the architecture actually supports
371 dwarf unwinding (as defined by STP_USE_DWARF_UNWINDER in runtime.h). */
372 #ifdef STP_USE_DWARF_UNWINDER
373 #include "unwind.c"
374 #else
375 /* We still need unwind.h for a few structures (stp_unwind_context and
376 * stp_unwind_cache). */
377 #include "unwind/unwind.h"
378 #endif
379
380 #ifdef module_param_cb /* kernels >= 2.6.36 */
381 #define _STP_KERNEL_PARAM_ARG const struct kernel_param
382 #else
383 #define _STP_KERNEL_PARAM_ARG struct kernel_param
384 #endif
385
386 /* Support functions for int64_t module parameters. */
387 static int param_set_int64_t(const char *val, _STP_KERNEL_PARAM_ARG *kp)
388 {
389 char *endp;
390 long long ll;
391
392 if (!val)
393 return -EINVAL;
394
395 /* simple_strtoll isn't exported... */
396 if (*val == '-')
397 ll = -simple_strtoull(val+1, &endp, 0);
398 else
399 ll = simple_strtoull(val, &endp, 0);
400
401 if ((endp == val) || ((int64_t)ll != ll))
402 return -EINVAL;
403
404 *((int64_t *)kp->arg) = ll;
405 return 0;
406 }
407
408 static int param_get_int64_t(char *buffer, _STP_KERNEL_PARAM_ARG *kp)
409 {
410 return sprintf(buffer, "%lli", (long long)*((int64_t *)kp->arg));
411 }
412
413 #define param_check_int64_t(name, p) __param_check(name, p, int64_t)
414
415 #ifdef module_param_cb /* kernels >= 2.6.36 */
416 static struct kernel_param_ops param_ops_int64_t = {
417 .set = param_set_int64_t,
418 .get = param_get_int64_t,
419 };
420 #endif
421 #undef _STP_KERNEL_PARAM_ARG
422
423 #include <linux/workqueue.h>
424 struct workqueue_struct *systemtap_wq;
425
426 static inline void stp_synchronize_sched(void)
427 {
428 flush_workqueue(systemtap_wq);
429 #if defined(STAPCONF_SYNCHRONIZE_SCHED)
430 synchronize_sched();
431 #elif defined(STAPCONF_SYNCHRONIZE_RCU)
432 synchronize_rcu();
433 #elif defined(STAPCONF_SYNCHRONIZE_KERNEL)
434 synchronize_kernel();
435 #else
436 #error "No implementation for stp_synchronize_sched!"
437 #endif
438 }
439
440 /************* Module Stuff ********************/
441
442
443 static int systemtap_kernel_module_init (void);
444 static void systemtap_kernel_module_exit (void);
445
446 static unsigned long stap_hash_seed; /* Init during module startup */
447
448 static int stap_init_module (void)
449 {
450 int rc;
451 /* With deliberate hash-collision-inducing data conceivably fed to
452 stap, it is beneficial to add some runtime-random value to the
453 map hash. */
454 get_random_bytes(&stap_hash_seed, sizeof (stap_hash_seed));
455 systemtap_wq = alloc_workqueue("systemtap-wq",
456 WQ_UNBOUND, 0);
457 if (!systemtap_wq)
458 return -ENOMEM;
459 rc = systemtap_kernel_module_init();
460 if (rc){
461 destroy_workqueue(systemtap_wq);
462 return rc;
463 }
464 rc = _stp_transport_init();
465 if (rc){
466 systemtap_kernel_module_exit();
467 destroy_workqueue(systemtap_wq);
468 }
469 return rc;
470 }
471
472 module_init(stap_init_module);
473
474 void stap_cleanup_module(void);
475 void stap_cleanup_module(void)
476 {
477 _stp_transport_close();
478 /* PR19833. /proc/systemtap/$MODULENAME may be disposed-of here,
479 due to tapset-procfs.cxx cleaning up after procfs probes (such
480 as in --monitor mode). */
481 systemtap_kernel_module_exit();
482 destroy_workqueue(systemtap_wq);
483 }
484
485 module_exit(stap_cleanup_module);
486
487 #define pseudo_atomic_cmpxchg(v, old, new) ({\
488 int ret;\
489 unsigned long flags;\
490 local_irq_save(flags);\
491 ret = atomic_read(v);\
492 if (likely(ret == old))\
493 atomic_set(v, new);\
494 local_irq_restore(flags);\
495 ret; })
496
497
498 #if defined(__ia64__) && defined(__GNUC__) && (__GNUC__ < 4)
499 /* Due to http://gcc.gnu.org/PR21838, old gcc on ia64 generates calls
500 to __ia64_save_stack_nonlocal(), since our generated code uses
501 __label__ constructs since PR11004. This dummy declaration works
502 around the undefined reference. */
503 void __ia64_save_stack_nonlocal (void) { }
504 #endif
505
506 MODULE_LICENSE("GPL");
507
508 #endif /* _LINUX_RUNTIME_H_ */