Debian Codegraph @

stapbpf/stapbpf.cxx

raw
1 /* stapbpf.cxx - SystemTap BPF loader
2 *
3 * This program is free software; you can redistribute it and/or modify
4 * it under the terms of the GNU General Public License as published by
5 * the Free Software Foundation; either version 2 of the License, or
6 * (at your option) any later version.
7 *
8 * This program is distributed in the hope that it will be useful,
9 * but WITHOUT ANY WARRANTY; without even the implied warranty of
10 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
11 * GNU General Public License for more details.
12 *
13 * You should have received a copy of the GNU General Public License
14 * along with this program. If not, see <http://www.gnu.org/licenses/>.
15 *
16 * Copyright (C) 2016-2021 Red Hat, Inc.
17 *
18 */
19
20 #include <cstdio>
21 #include <cstdlib>
22 #include <cstring>
23 #include <cstdarg>
24 #include <cassert>
25 #include <csignal>
26 #include <cerrno>
27 #include <fstream>
28 #include <sstream>
29 #include <string>
30 #include <thread>
31 #include <vector>
32 #include <mutex>
33 #include <unistd.h>
34 #include <limits.h>
35 #include <inttypes.h>
36 #include <getopt.h>
37 #include <sys/fcntl.h>
38 #include <sys/ioctl.h>
39 #include <sys/stat.h>
40 #include <sys/syscall.h>
41 #include <sys/mman.h>
42 #include <sys/utsname.h>
43 #include <sys/resource.h>
44 #include <sys/wait.h>
45 #include <pwd.h>
46 #include "bpfinterp.h"
47 #include "../staputil.h"
48
49 extern "C" {
50 #include <linux/bpf.h>
51 #include <linux/perf_event.h>
52 /* Introduced in 4.1. */
53 #ifndef PERF_EVENT_IOC_SET_BPF
54 #define PERF_EVENT_IOC_SET_BPF _IOW('$', 8, __u32)
55 #endif
56 #include <libelf.h>
57 }
58
59 #include "config.h"
60 #include "../git_version.h"
61 #include "../version.h"
62 #include "../bpf-internal.h"
63
64 #ifndef EM_BPF
65 #define EM_BPF 0xeb9f
66 #endif
67 #ifndef R_BPF_MAP_FD
68 #define R_BPF_MAP_FD 1
69 #endif
70
71 using namespace std;
72
73 // XXX declarations from ../staprun/staprun.h required by start_cmd()
74 extern "C" {
75 int verbose = 0;
76 int read_stdin = 0; // TODO not used; stapbpf doesn't (yet) have input.char?
77 char *__name__ = NULL; // XXX aliased as module_name below
78 int target_pid = 0;
79 char *target_cmd = NULL;
80
81 /* ../staprun/common.c functions */
82 __attribute__ ((format (printf, 1, 2)))
83 void eprintf(const char *fmt, ...)
84 {
85 va_list va;
86 va_start(va, fmt);
87 // if (use_syslog)
88 // vsyslog(LOG_ERR, fmt, va);
89 // else
90 vfprintf(stderr, fmt, va);
91 va_end(va);
92 }
93
94 /* ../staprun/common.c functions */
95 void start_cmd(void);
96 int resume_cmd(void);
97 };
98
99 static int group_fd = -1; // ??? Need one per cpu.
100 static int target_pid_failed_p = 0;
101 static int warnings = 1;
102 static int exit_phase = 0;
103 static int interrupt_message = 0;
104 static FILE *output_f = stdout;
105 static FILE *kmsg = NULL;
106
107 #define module_name __name__
108 static const char *module_basename;
109 static const char *script_name; // name of original systemtap script
110 static const char *module_license;
111
112 static const char *user; // username
113 static std::string procfs_dir_prefix; // used to create procfs-like probe directory
114
115 static Elf *module_elf;
116
117 static uint32_t kernel_version;
118
119 // Locks for accessing procfs-like probe messages
120 std::mutex procfs_lock;
121
122 // Sized by the contents of the "maps" section.
123 static bpf_map_def *map_attrs;
124 static std::vector<int> map_fds;
125
126 // PR24543: Some perf constructs must be anchored to a single CPU.
127 // Normally we use cpu0, but it could (in very rare cases) be disabled.
128 // Initialized in mark_active_cpus() along with cpu_online.
129 static int default_cpu = 0;
130
131 // Sized by the number of CPUs:
132 static std::vector<int> perf_fds;
133 static std::vector<bool> cpu_online; // -- is CPU active?
134 static std::vector<struct perf_event_mmap_page *> perf_headers;
135 static std::vector<bpf_transport_context *> transport_contexts;
136
137 // Additional info for perf_events transport:
138 static int perf_event_page_size;
139 static int perf_event_page_count = 8;
140 static int perf_event_mmap_size;
141
142 // Table of interned strings:
143 static std::vector<std::string> interned_strings;
144
145 // Table of map id's for statistical aggregates:
146 static std::unordered_map<bpf::globals::agg_idx, bpf::globals::stats_map> aggregates;
147
148 // Table of foreach loop information:
149 static std::vector<bpf::globals::foreach_info> foreach_loop_info;
150
151 // XXX: Required static data and methods from bpf::globals, shared with translator.
152 #include "../bpf-shared-globals.h"
153
154 // Sized by the number of sections, so that we can easily
155 // look them up by st_shndx.
156 static std::vector<int> prog_fds;
157
158 // Programs to run at begin and end of execution.
159 static Elf_Data *prog_begin;
160 static Elf_Data *prog_end;
161 static Elf_Data *prog_error;
162
163 // Used to check if a hard error has occured.
164 bool error = false;
165
166 #define DEBUGFS "/sys/kernel/debug/tracing/"
167 #define KPROBE_EVENTS DEBUGFS "kprobe_events"
168 #define UPROBE_EVENTS DEBUGFS "uprobe_events"
169 #define EVENTS DEBUGFS "events"
170
171 #define CPUFS "/sys/devices/system/cpu/"
172 #define CPUS_ONLINE CPUFS "online"
173 #define CPUS_POSSIBLE CPUFS "possible"
174
175 static void unregister_kprobes(const size_t nprobes);
176
177 struct procfsprobe_data
178 {
179 std::string path;
180 uint64_t umask;
181 char type; // either 'r' (read) or 'w' (write)
182 uint64_t maxsize_val;
183 Elf_Data* read_prog;
184 std::vector<Elf_Data*> write_prog;
185
186 // ctor for read probes
187 procfsprobe_data(string path, uint64_t umask, char type, uint64_t maxsize_val, Elf_Data* prog)
188 : path(path), umask(umask), type(type), maxsize_val(maxsize_val), read_prog(prog)
189 { assert (type == 'r'); }
190
191 // ctor for write probes
192 procfsprobe_data(string path, uint64_t umask, char type, uint64_t maxsize_val, std::vector<Elf_Data*> prog)
193 : path(path), umask(umask), type(type), maxsize_val(maxsize_val), write_prog(prog)
194 { assert (type == 'w'); }
195 };
196
197
198 struct kprobe_data
199 {
200 string args;
201 char type;
202 int prog_fd;
203 int event_id;
204 int event_fd; // ??? Need one per cpu.
205
206 kprobe_data(char t, string s, int fd)
207 : args(s), type(t), prog_fd(fd), event_id(-1), event_fd(-1)
208 { }
209 };
210
211 struct uprobe_data
212 {
213 string path;
214 char type;
215 int pid;
216 unsigned long long offset;
217 int prog_fd;
218 int event_id;
219 int event_fd;
220
221 uprobe_data(string path, char t, int pid, unsigned long long off, int fd)
222 : path(path), type(t), pid(pid), offset(off), prog_fd(fd),
223 event_id(-1), event_fd(-1)
224 { }
225 };
226
227 struct timer_data
228 {
229 unsigned long period;
230 int prog_fd;
231 int event_fd;
232
233 timer_data(unsigned long period, int fd)
234 : period(period), prog_fd(fd), event_fd(-1)
235 { }
236 };
237
238 struct perf_data
239 {
240 int event_type;
241 int event_config;
242 bool has_freq;
243 unsigned long interval;
244 int prog_fd;
245 int event_fd;
246
247 perf_data(int type, int config, bool freq, unsigned long interval, int fd)
248 : event_type(type), event_config(config), has_freq(freq),
249 interval(interval), prog_fd(fd), event_fd(-1)
250 { }
251 };
252
253 struct trace_data
254 {
255 string system;
256 string name;
257 int prog_fd;
258 int event_id;
259 int event_fd;
260
261 trace_data(char *s, char *n, int fd)
262 : system(s), name(n), prog_fd(fd), event_id(-1), event_fd(-1)
263 { }
264 };
265
266 static std::vector<procfsprobe_data> procfsprobes;
267 static std::vector<kprobe_data> kprobes;
268 static std::vector<timer_data> timers;
269 static std::vector<perf_data> perf_probes;
270 static std::vector<trace_data> tracepoint_probes;
271 static std::vector<trace_data> raw_tracepoint_probes;
272 static std::vector<uprobe_data> uprobes;
273
274 // TODO: Move fatal() to bpfinterp.h and replace abort() calls in the interpreter.
275 // TODO: Add warn() option.
276 static void __attribute__((noreturn)) __attribute__ ((format (printf, 1, 2)))
277 fatal(const char *str, ...)
278 {
279 if (module_name)
280 fprintf(stderr, "Error loading %s: ", module_name);
281
282 va_list va;
283 va_start(va, str);
284 vfprintf(stderr, str, va);
285 va_end(va);
286
287 exit(1);
288 }
289
290 static void
291 fatal_sys()
292 {
293 fatal("%s\n", strerror(errno));
294 }
295
296 static void
297 fatal_elf()
298 {
299 fatal("%s\n", elf_errmsg(-1));
300 }
301
302 // XXX: based on get_online_cpus()/read_cpu_range()
303 // in bcc src/cc/common.cc
304 //
305 // PR24543: Also sets default_cpu.
306 //
307 // This is the only way I know of so far, so I have to imitate it for
308 // now. Parsing a /sys/devices diagnostic file seems a bit brittle to
309 // me, though.
310 static void
311 mark_active_cpus(unsigned ncpus)
312 {
313 std::ifstream cpu_ranges(CPUS_ONLINE);
314 std::string cpu_range;
315
316 // XXX if cpu0 is offline
317 int alternate_cpu = -1;
318 bool found_alternate = false;
319
320 cpu_online.clear();
321 for (unsigned i = 0; i < ncpus; i++)
322 cpu_online.push_back(false);
323
324 while (std::getline(cpu_ranges, cpu_range, ','))
325 {
326 size_t rangepos = cpu_range.find("-");
327 int start, end;
328 if (rangepos == std::string::npos)
329 {
330 start = end = std::stoi(cpu_range);
331 }
332 else
333 {
334 start = std::stoi(cpu_range.substr(0, rangepos));
335 end = std::stoi(cpu_range.substr(rangepos+1));
336 }
337 for (int i = start; i <= end; i++)
338 {
339 if (!found_alternate)
340 {
341 alternate_cpu = i;
342 found_alternate = true;
343 }
344 cpu_online[i] = true;
345 }
346 }
347
348 // PR24543: Make sure default_cpu is active.
349 if (!cpu_online[default_cpu] && found_alternate)
350 default_cpu = alternate_cpu;
351 }
352
353 static int
354 count_active_cpus()
355 {
356 int count = 0;
357 for (unsigned cpu = 0; cpu < cpu_online.size(); cpu++)
358 if (cpu_online[cpu])
359 count++;
360 return count;
361 }
362
363 static int
364 create_group_fds()
365 {
366 perf_event_attr peattr;
367
368 memset(&peattr, 0, sizeof(peattr));
369 peattr.size = sizeof(peattr);
370 peattr.disabled = 1;
371 peattr.type = PERF_TYPE_SOFTWARE;
372 peattr.config = PERF_COUNT_SW_DUMMY;
373
374 return group_fd = perf_event_open(&peattr, -1, default_cpu, -1, 0);
375 }
376
377 static void
378 instantiate_maps (Elf64_Shdr *shdr, Elf_Data *data)
379 {
380 if (shdr->sh_entsize != sizeof(bpf_map_def))
381 fatal("map entry size mismatch (%zu != %zu)\n",
382 (size_t)shdr->sh_entsize, sizeof(bpf_map_def));
383
384 size_t i, n = shdr->sh_size / sizeof(bpf_map_def);
385 struct bpf_map_def *attrs = static_cast<bpf_map_def *>(data->d_buf);
386
387 map_attrs = attrs;
388 map_fds.assign(n, -1);
389
390 // XXX: PR24324 -- This overhead space calculation was too
391 // conservative and caused resource exhaustion errors, disabling it
392 // until we figure out how much space we need or if the
393 // RLIM_INFINITY solution below is adequate.
394 #if 0
395 /* First, make room for the maps in this process' RLIMIT_MEMLOCK: */
396 size_t rlimit_increase = 0;
397 for (i = 0; i < n; ++i)
398 {
399 // TODO: The 58 bytes of overhead space per entry has been
400 // decided by trial and error, and may require further tweaking:
401 rlimit_increase += (58 + attrs[i].key_size + attrs[i].value_size) * attrs[i].max_entries;
402 // TODO: Note that Certain Other Tools just give up on
403 // calculating and set rlimit to the maximum possible.
404 }
405 #endif
406
407 struct rlimit curr_rlimit;
408 int rc;
409
410 rc = getrlimit(RLIMIT_MEMLOCK, &curr_rlimit);
411 if (rc < 0)
412 fatal("could not get map resource limit: %s\n",
413 strerror(errno));
414
415 rlim_t rlim_orig = curr_rlimit.rlim_cur;
416 rlim_t rlim_max_orig = curr_rlimit.rlim_max;
417 #if 0
418 curr_rlimit.rlim_cur += rlimit_increase;
419 curr_rlimit.rlim_max += rlimit_increase;
420 if (curr_rlimit.rlim_cur < rlim_orig) // handle overflow
421 curr_rlimit.rlim_cur = rlim_orig;
422 if (curr_rlimit.rlim_max < rlim_max_orig) // handle overflow
423 curr_rlimit.rlim_max = rlim_max_orig;
424 #endif
425 // TODOXXX: PR24324 -- EXPERIMENTAL fix for aggressive resource limits.
426 // Other Tools do something like this but it doesn't solve all our problems.
427 curr_rlimit.rlim_cur = RLIM_INFINITY;
428 curr_rlimit.rlim_max = RLIM_INFINITY;
429
430 rc = setrlimit(RLIMIT_MEMLOCK, &curr_rlimit);
431 if (rc < 0)
432 fatal("could not increase map resource limit -- "
433 "cur from %llu to %llu, max from %llu to %llu: %s\n",
434 (unsigned long long) rlim_orig,
435 (unsigned long long) curr_rlimit.rlim_cur,
436 (unsigned long long) rlim_max_orig,
437 (unsigned long long) curr_rlimit.rlim_max,
438 strerror(errno));
439 if (verbose > 1)
440 {
441 fprintf(stderr, "increasing map cur resource limit from %llu to %llu\n",
442 (unsigned long long) rlim_orig,
443 (unsigned long long) curr_rlimit.rlim_cur);
444 fprintf(stderr, "increasing map max resource limit from %llu to %llu\n",
445 (unsigned long long) rlim_max_orig,
446 (unsigned long long) curr_rlimit.rlim_max);
447 }
448
449 /* Now create the maps: */
450 for (i = 0; i < n; ++i)
451 {
452 /* PR22330: The perf_event_map used for message transport must
453 have max_entries equal to the number of active CPUs, which we
454 wouldn't know for sure at translate time. Set it now: */
455 bpf_map_type map_type = static_cast<bpf_map_type>(attrs[i].type);
456 if (map_type == BPF_MAP_TYPE_PERF_EVENT_ARRAY)
457 {
458 /* XXX: Assume our only perf_event_map is the percpu transport one: */
459 assert(i == bpf::globals::perf_event_map_idx);
460 assert(attrs[i].max_entries == bpf::globals::NUM_CPUS_PLACEHOLDER);
461
462 // TODO: perf_event buffers can only be created for currently
463 // active CPUs. For now we imitate Certain Other Tools and
464 // create perf_events for CPUs that are active at startup time
465 // (while sizing the perf_event_map according to total CPUs).
466 // But for full coverage, we really need to listen to CPUs
467 // coming on/offline and adjust accordingly.
468 long ncpus_ = sysconf(_SC_NPROCESSORS_CONF);
469 unsigned ncpus = ncpus_ > 0 ? ncpus_ : 1;
470 if (ncpus_ < 0)
471 fprintf(stderr, "WARNING: could not get number of CPUs, falling back to 1: %s\n", strerror(errno));
472 else if (ncpus_ == 0)
473 fprintf(stderr, "WARNING: could not get number of CPUs, falling back to 1\n"); // XXX no errno
474 //unsigned ncpus = get_nprocs_conf();
475 mark_active_cpus((unsigned)ncpus);
476 attrs[i].max_entries = ncpus;
477 }
478
479 if (verbose > 2)
480 fprintf(stderr, "creating map type %u entry %zu: key_size %u, value_size %u, "
481 "max_entries %u, map_flags %u\n", map_type, i,
482 attrs[i].key_size, attrs[i].value_size,
483 attrs[i].max_entries, attrs[i].map_flags);
484 int fd = bpf_create_map(static_cast<bpf_map_type>(attrs[i].type),
485 attrs[i].key_size, attrs[i].value_size,
486 attrs[i].max_entries, attrs[i].map_flags);
487 if (fd < 0)
488 fatal("map entry %zu: %s\n", i, strerror(errno));
489 map_fds[i] = fd;
490 }
491 }
492
493 static int
494 prog_load(Elf_Data *data, const char *name)
495 {
496 enum bpf_prog_type prog_type;
497
498 if (strncmp(name, "kprobe", 6) == 0)
499 prog_type = BPF_PROG_TYPE_KPROBE;
500 else if (strncmp(name, "kretprobe", 9) == 0)
501 prog_type = BPF_PROG_TYPE_KPROBE;
502 else if (strncmp(name, "uprobe", 6) == 0)
503 prog_type = BPF_PROG_TYPE_KPROBE;
504 else if (strncmp(name, "timer", 5) == 0)
505 prog_type = BPF_PROG_TYPE_PERF_EVENT;
506 else if (strncmp(name, "trace", 5) == 0)
507 prog_type = BPF_PROG_TYPE_TRACEPOINT;
508 #ifdef HAVE_BPF_PROG_TYPE_RAW_TRACEPOINT
509 else if (strncmp(name, "raw_trace", 9) == 0)
510 prog_type = BPF_PROG_TYPE_RAW_TRACEPOINT;
511 #endif
512 else if (strncmp(name, "perf", 4) == 0)
513 {
514 if (name[5] == '2' && name[6] == '/')
515 prog_type = BPF_PROG_TYPE_TRACEPOINT;
516 else
517 prog_type = BPF_PROG_TYPE_PERF_EVENT;
518 }
519 else
520 fatal("unhandled program type for section \"%s\"\n", name);
521
522 if (data->d_size % sizeof(bpf_insn))
523 fatal("program size not a multiple of %zu\n", sizeof(bpf_insn));
524
525 if (kmsg != NULL)
526 {
527 fprintf (kmsg, "%s (%s): stapbpf: %s, name: %s, d_size: %lu\n",
528 module_basename, script_name, VERSION, name, (unsigned long)data->d_size);
529 fflush (kmsg); // Otherwise, flush will only happen after the prog runs.
530 }
531 int fd = bpf_prog_load(prog_type, static_cast<bpf_insn *>(data->d_buf),
532 data->d_size, module_license, kernel_version);
533 if (fd < 0)
534 {
535 if (bpf_log_buf[0] != 0)
536 fatal("bpf program load failed: %s\n%s\n",
537 strerror(errno), bpf_log_buf);
538 else
539 fatal("bpf program load failed: %s\n", strerror(errno));
540 }
541 return fd;
542 }
543
544 static void
545 prog_relocate(Elf_Data *prog_data, Elf_Data *rel_data,
546 Elf_Data *sym_data, Elf_Data *str_data,
547 const char *prog_name, unsigned maps_idx, bool allocated)
548 {
549 bpf_insn *insns = static_cast<bpf_insn *>(prog_data->d_buf);
550 Elf64_Rel *rels = static_cast<Elf64_Rel *>(rel_data->d_buf);
551 Elf64_Sym *syms = static_cast<Elf64_Sym *>(sym_data->d_buf);
552
553 if (prog_data->d_size % sizeof(bpf_insn))
554 fatal("program size not a multiple of %zu\n", sizeof(bpf_insn));
555 if (rel_data->d_type != ELF_T_REL
556 || rel_data->d_size % sizeof(Elf64_Rel))
557 fatal("invalid reloc metadata\n");
558 if (sym_data->d_type != ELF_T_SYM
559 || sym_data->d_size % sizeof(Elf64_Sym))
560 fatal("invalid symbol metadata\n");
561
562 size_t psize = prog_data->d_size;
563 size_t nrels = rel_data->d_size / sizeof(Elf64_Rel);
564 size_t nsyms = sym_data->d_size / sizeof(Elf64_Sym);
565
566 for (size_t i = 0; i < nrels; ++i)
567 {
568 uint32_t sym = ELF64_R_SYM(rels[i].r_info);
569 uint32_t type = ELF64_R_TYPE(rels[i].r_info);
570 unsigned long long r_ofs = rels[i].r_offset;
571 size_t fd_idx;
572
573 if (type != R_BPF_MAP_FD)
574 fatal("invalid relocation type %u\n", type);
575 if (sym >= nsyms)
576 fatal("invalid symbol index %u\n", sym);
577 if (r_ofs >= psize || r_ofs % sizeof(bpf_insn))
578 fatal("invalid relocation offset at %s+%llu\n", prog_name, r_ofs);
579
580 if (sym >= nsyms)
581 fatal("invalid relocation symbol %u\n", sym);
582 if (syms[sym].st_shndx != maps_idx
583 || syms[sym].st_value % sizeof(bpf_map_def)
584 || (fd_idx = syms[sym].st_value / sizeof(bpf_map_def),
585 fd_idx >= map_fds.size()))
586 {
587 const char *name = "";
588 if (syms[sym].st_name < str_data->d_size)
589 name = static_cast<char *>(str_data->d_buf) + syms[sym].st_name;
590 if (*name)
591 fatal("symbol %s does not reference a map\n", name);
592 else
593 fatal("symbol %u does not reference a map\n", sym);
594 }
595
596 bpf_insn *insn = insns + (r_ofs / sizeof(bpf_insn));
597 if (insn->code != (BPF_LD | BPF_IMM | BPF_DW))
598 fatal("invalid relocation insn at %s+%llu\n", prog_name, r_ofs);
599
600 insn->src_reg = BPF_PSEUDO_MAP_FD;
601 insn->imm = (allocated ? map_fds[fd_idx] : fd_idx);
602 }
603 }
604
605 static void
606 maybe_collect_kprobe(const char *name, unsigned name_idx,
607 unsigned fd_idx, Elf64_Addr offset)
608 {
609 char type;
610 string arg;
611
612 if (strncmp(name, "kprobe/", 7) == 0)
613 {
614 string line;
615 const char *stext = NULL;
616 type = 'p';
617 name += 7;
618
619 ifstream syms("/proc/kallsyms");
620 if (!syms)
621 fatal("error opening /proc/kallsyms: %s\n", strerror(errno));
622
623 // get value of symbol _stext and add it to the offset found in name.
624 while (getline(syms, line))
625 {
626 const char *l = line.c_str();
627 if (strncmp(l + 19, "_stext", 6) == 0)
628 {
629 stext = l;
630 break;
631 }
632 }
633
634 if (stext == NULL)
635 fatal("could not find _stext in /proc/kallsyms");
636
637 unsigned long addr = strtoul(stext, NULL, 16);
638 addr += strtoul(name, NULL, 16);
639 stringstream ss;
640 ss << "0x" << hex << addr;
641 arg = ss.str();
642 }
643 else if (strncmp(name, "kretprobe/", 10) == 0)
644 type = 'r', arg = name + 10;
645 else
646 return;
647
648 int fd = -1;
649 if (fd_idx >= prog_fds.size() || (fd = prog_fds[fd_idx]) < 0)
650 fatal("probe %u section %u not loaded\n", name_idx, fd_idx);
651 if (offset != 0)
652 fatal("probe %u offset non-zero\n", name_idx);
653
654 kprobes.push_back(kprobe_data(type, arg, fd));
655 }
656
657 static void
658 collect_procfsprobe(const char *name, Elf_Data* prog)
659 {
660 unsigned long umask;
661 unsigned long maxsize_val;
662 char type;
663 char fifoname[PATH_MAX];
664
665 int res = sscanf(name, "procfsprobe/%lu/%c/%lu/%s", &umask, &type, &maxsize_val, fifoname);
666
667 if (res != 4)
668 fatal("unable to parse name of probe: %s", name);
669
670 std::string path(fifoname);
671
672 if (type == 'r')
673 procfsprobes.push_back(procfsprobe_data(path, umask, type, maxsize_val, prog));
674 else
675 {
676 // Check if a write probe with the same path already exists
677 for (unsigned i = 0; i < procfsprobes.size(); i++)
678 if (procfsprobes[i].path == string(path) && procfsprobes[i].type == 'w')
679 {
680 procfsprobes[i].write_prog.push_back(prog);
681 return;
682 }
683
684 std::vector<Elf_Data*> progs;
685 progs.push_back(prog);
686 procfsprobes.push_back(procfsprobe_data(path, umask, type, maxsize_val, progs));
687 }
688 }
689
690 static void
691 collect_uprobe(const char *name, unsigned name_idx, unsigned fd_idx)
692 {
693 char type = '\0';
694 int pid = -1;
695 unsigned long long off = 0;
696 char path[PATH_MAX];
697
698 int res = sscanf(name, "uprobe/%c/%d/%llu%s", &type, &pid, &off, path);
699
700 if (!pid)
701 pid = -1; // indicates to perf_event_open that we're tracing all processes
702
703 if (res != 4)
704 fatal("unable to parse name of probe %u section %u\n", name_idx, fd_idx);
705
706 int fd = -1;
707 if (fd_idx >= prog_fds.size() || (fd = prog_fds[fd_idx]) < 0)
708 fatal("probe %u section %u not loaded\n", name_idx, fd_idx);
709
710 uprobes.push_back(uprobe_data(std::string(path), type, pid, off, fd));
711 }
712
713 static void
714 collect_perf(const char *name, unsigned name_idx, unsigned fd_idx)
715 {
716 char has_freq;
717 int event_type;
718 int event_config;
719 unsigned long interval;
720
721 int res = sscanf(name, "perf/%d/%d/%c/%lu",
722 &event_type, &event_config, &has_freq, &interval);
723 if (res != 4)
724 fatal("unable to parse name of probe %u section %u\n", name_idx, fd_idx);
725
726 int fd = -1;
727 if (fd_idx >= prog_fds.size() || (fd = prog_fds[fd_idx]) < 0)
728 fatal("probe %u section %u not loaded\n", name_idx, fd_idx);
729
730 perf_probes.push_back(
731 perf_data(event_type, event_config, has_freq == 'f', interval, fd));
732 }
733
734 static void
735 collect_timer(const char *name, unsigned name_idx, unsigned fd_idx)
736 {
737 unsigned long period = strtoul(name + 11, NULL, 10);
738
739 if (strncmp(name + 6, "jiff/", 5) == 0)
740 {
741 long jiffies_per_sec = sysconf(_SC_CLK_TCK);
742 period *= 1e9 / jiffies_per_sec;
743 }
744
745 int fd = -1;
746 if (fd_idx >= prog_fds.size() || (fd = prog_fds[fd_idx]) < 0)
747 fatal("probe %u section %u not loaded\n", name_idx, fd_idx);
748
749 timers.push_back(timer_data(period, fd));
750 return;
751 }
752
753 static void
754 collect_tracepoint(const char *name, unsigned name_idx, unsigned fd_idx)
755 {
756 char tp_system[512];
757 char tp_name[512];
758
759 int res = sscanf(name, "trace/%[^/]/%s", tp_system, tp_name);
760 if (res != 2 || strlen(name) > 512)
761 fatal("unable to parse name of probe %u section %u\n", name_idx, fd_idx);
762
763 int fd = -1;
764 if (fd_idx >= prog_fds.size() || (fd = prog_fds[fd_idx]) < 0)
765 fatal("probe %u section %u not loaded\n", name_idx, fd_idx);
766
767 tracepoint_probes.push_back(trace_data(tp_system, tp_name, fd));
768 }
769
770 static void
771 collect_raw_tracepoint(const char *name, unsigned name_idx, unsigned fd_idx)
772 {
773 char tp_system[512];
774 char tp_name[512];
775
776 int res = sscanf(name, "raw_trace/%[^/]/%s", tp_system, tp_name);
777 if (res != 2 || strlen(name) > 512)
778 fatal("unable to parse name of probe %u section %u\n", name_idx, fd_idx);
779
780 int fd = -1;
781 if (fd_idx >= prog_fds.size() || (fd = prog_fds[fd_idx]) < 0)
782 fatal("probe %u section %u not loaded\n", name_idx, fd_idx);
783
784 raw_tracepoint_probes.push_back(trace_data(tp_system, tp_name, fd));
785 }
786
787 static void
788 kprobe_collect_from_syms(Elf_Data *sym_data, Elf_Data *str_data)
789 {
790 Elf64_Sym *syms = static_cast<Elf64_Sym *>(sym_data->d_buf);
791 size_t nsyms = sym_data->d_type / sizeof(Elf64_Sym);
792
793 if (sym_data->d_type != ELF_T_SYM
794 || sym_data->d_size % sizeof(Elf64_Sym))
795 fatal("invalid kprobes symbol metadata\n");
796
797 for (size_t i = 0; i < nsyms; ++i)
798 {
799 const char *name;
800 if (syms[i].st_name < str_data->d_size)
801 name = static_cast<char *>(str_data->d_buf) + syms[i].st_name;
802 else
803 fatal("symbol %zu has invalid string index\n", i);
804 maybe_collect_kprobe(name, i, syms[i].st_shndx, syms[i].st_value);
805 }
806 }
807
808 static void
809 unregister_uprobes(const size_t nprobes)
810 {
811 if (nprobes == 0)
812 return;
813
814 int fd = open(DEBUGFS "uprobe_events", O_WRONLY);
815 if (fd < 0)
816 return;
817
818
819 const int pid = getpid();
820 for (size_t i = 0; i < nprobes; ++i)
821 {
822 close(uprobes[i].event_fd);
823
824 char msgbuf[128];
825 ssize_t olen = snprintf(msgbuf, sizeof(msgbuf), "-:stapprobe_%d_%zu",
826 pid, i);
827 ssize_t wlen = write(fd, msgbuf, olen);
828 if (wlen < 0)
829 fprintf(stderr, "Error removing probe %zu: %s\n",
830 i, strerror(errno));
831 }
832 close(fd);
833 }
834
835 static void
836 register_uprobes()
837 {
838 size_t nprobes = uprobes.size();
839 if (nprobes == 0)
840 return;
841
842 int fd = open(UPROBE_EVENTS, O_WRONLY);
843 if (fd < 0)
844 fatal("Error opening %s: %s\n", UPROBE_EVENTS, strerror(errno));
845
846 const int pid = getpid();
847
848 for (size_t i = 0; i < nprobes; ++i)
849 {
850 uprobe_data &u = uprobes[i];
851 char msgbuf[PATH_MAX];
852
853 ssize_t olen = snprintf(msgbuf, sizeof(msgbuf), "%c:stapprobe_%d_%zu %s:0x%llx",
854 u.type, pid, i, u.path.c_str(), u.offset);
855 if ((size_t)olen >= sizeof(msgbuf))
856 {
857 fprintf(stderr, "Buffer overflow creating probe %zu\n", i);
858 if (i == 0)
859 goto fail_0;
860 nprobes = i - 1;
861 goto fail_n;
862 }
863
864 if (verbose > 1)
865 fprintf(stderr, "Associating probe %zu with uprobe %s\n", i, msgbuf);
866
867 ssize_t wlen = write(fd, msgbuf, olen);
868 if (wlen != olen)
869 {
870 fprintf(stderr, "Error creating probe %zu: %s\n",
871 i, strerror(errno));
872 if (i == 0)
873 goto fail_0;
874 nprobes = i - 1;
875 goto fail_n;
876 }
877 }
878 close(fd);
879
880 for (size_t i = 0; i < nprobes; ++i)
881 {
882 char fnbuf[PATH_MAX];
883 ssize_t len = snprintf(fnbuf, sizeof(fnbuf),
884 DEBUGFS "events/uprobes/stapprobe_%d_%zu/id", pid, i);
885 if ((size_t)len >= sizeof(bpf_log_buf))
886 {
887 fprintf(stderr, "Buffer overflow creating probe %zu\n", i);
888 goto fail_n;
889 }
890
891 fd = open(fnbuf, O_RDONLY);
892 if (fd < 0)
893 {
894 fprintf(stderr, "Error opening probe event id %zu: %s\n",
895 i, strerror(errno));
896 goto fail_n;
897 }
898
899 char msgbuf[128];
900 len = read(fd, msgbuf, sizeof(msgbuf) - 1);
901 if (len < 0)
902 {
903 fprintf(stderr, "Error reading probe event id %zu: %s\n",
904 i, strerror(errno));
905 goto fail_n;
906 }
907 close(fd);
908
909 msgbuf[len] = 0;
910 uprobes[i].event_id = atoi(msgbuf);
911 }
912
913 // ??? Iterate to enable on all cpus, each with a different group_fd.
914 {
915 perf_event_attr peattr;
916
917 memset(&peattr, 0, sizeof(peattr));
918 peattr.size = sizeof(peattr);
919 peattr.type = PERF_TYPE_TRACEPOINT;
920 peattr.sample_type = PERF_SAMPLE_RAW;
921 peattr.sample_period = 1;
922 peattr.wakeup_events = 1;
923
924 for (size_t i = 0; i < nprobes; ++i)
925 {
926 uprobe_data &u = uprobes[i];
927 peattr.config = u.event_id;
928
929 fd = perf_event_open(&peattr, u.pid, default_cpu, -1, 0);
930 if (fd < 0)
931 {
932 fprintf(stderr, "Error opening probe id %zu: %s\n",
933 i, strerror(errno));
934 goto fail_n;
935 }
936 u.event_fd = fd;
937
938 if (ioctl(fd, PERF_EVENT_IOC_SET_BPF, u.prog_fd) < 0)
939 {
940 fprintf(stderr, "Error installing bpf for probe id %zu: %s\n",
941 i, strerror(errno));
942 goto fail_n;
943 }
944 }
945 }
946 return;
947
948 fail_n:
949 unregister_uprobes(nprobes);
950 fail_0:
951 exit(1);
952 }
953
954 static void
955 register_kprobes()
956 {
957 size_t nprobes = kprobes.size();
958 if (nprobes == 0)
959 return;
960
961 int fd = open(KPROBE_EVENTS, O_WRONLY);
962 if (fd < 0)
963 fatal("Error opening %s: %s\n", KPROBE_EVENTS, strerror(errno));
964
965 const int pid = getpid();
966
967 for (size_t i = 0; i < nprobes; ++i)
968 {
969 kprobe_data &k = kprobes[i];
970 char msgbuf[128];
971
972 ssize_t olen = snprintf(msgbuf, sizeof(msgbuf), "%c:p%d_%zu %s",
973 k.type, pid, i, k.args.c_str());
974 if ((size_t)olen >= sizeof(msgbuf))
975 {
976 fprintf(stderr, "Buffer overflow creating probe %zu\n", i);
977 if (i == 0)
978 goto fail_0;
979 nprobes = i - 1;
980 goto fail_n;
981 }
982
983 if (verbose > 1)
984 fprintf(stderr, "Associating probe %zu with kprobe %s\n", i, msgbuf);
985
986 ssize_t wlen = write(fd, msgbuf, olen);
987 if (wlen != olen)
988 {
989 fprintf(stderr, "Error creating probe %zu: %s\n",
990 i, strerror(errno));
991 if (i == 0)
992 goto fail_0;
993 nprobes = i - 1;
994 goto fail_n;
995 }
996 }
997 close(fd);
998
999 for (size_t i = 0; i < nprobes; ++i)
1000 {
1001 char fnbuf[PATH_MAX];
1002 ssize_t len = snprintf(fnbuf, sizeof(fnbuf),
1003 DEBUGFS "events/kprobes/p%d_%zu/id", pid, i);
1004 if ((size_t)len >= sizeof(bpf_log_buf))
1005 {
1006 fprintf(stderr, "Buffer overflow creating probe %zu\n", i);
1007 goto fail_n;
1008 }
1009
1010 fd = open(fnbuf, O_RDONLY);
1011 if (fd < 0)
1012 {
1013 fprintf(stderr, "Error opening probe event id %zu: %s\n",
1014 i, strerror(errno));
1015 goto fail_n;
1016 }
1017
1018 char msgbuf[128];
1019 len = read(fd, msgbuf, sizeof(msgbuf) - 1);
1020 if (len < 0)
1021 {
1022 fprintf(stderr, "Error reading probe event id %zu: %s\n",
1023 i, strerror(errno));
1024 goto fail_n;
1025 }
1026 close(fd);
1027
1028 msgbuf[len] = 0;
1029 kprobes[i].event_id = atoi(msgbuf);
1030 }
1031
1032 // ??? Iterate to enable on all cpus, each with a different group_fd.
1033 {
1034 perf_event_attr peattr;
1035
1036 memset(&peattr, 0, sizeof(peattr));
1037 peattr.size = sizeof(peattr);
1038 peattr.type = PERF_TYPE_TRACEPOINT;
1039 peattr.sample_type = PERF_SAMPLE_RAW;
1040 peattr.sample_period = 1;
1041 peattr.wakeup_events = 1;
1042
1043 for (size_t i = 0; i < nprobes; ++i)
1044 {
1045 kprobe_data &k = kprobes[i];
1046 peattr.config = k.event_id;
1047
1048 fd = perf_event_open(&peattr, -1, default_cpu, group_fd, 0);
1049 if (fd < 0)
1050 {
1051 fprintf(stderr, "Error opening probe id %zu: %s\n",
1052 i, strerror(errno));
1053 goto fail_n;
1054 }
1055 k.event_fd = fd;
1056
1057 if (ioctl(fd, PERF_EVENT_IOC_SET_BPF, k.prog_fd) < 0)
1058 {
1059 fprintf(stderr, "Error installing bpf for probe id %zu: %s\n",
1060 i, strerror(errno));
1061 goto fail_n;
1062 }
1063 }
1064 }
1065 return;
1066
1067 fail_n:
1068 unregister_kprobes(nprobes);
1069 fail_0:
1070 exit(1);
1071 }
1072
1073 static void
1074 unregister_kprobes(const size_t nprobes)
1075 {
1076 if (nprobes == 0)
1077 return;
1078
1079 int fd = open(DEBUGFS "kprobe_events", O_WRONLY);
1080 if (fd < 0)
1081 return;
1082
1083
1084 const int pid = getpid();
1085 for (size_t i = 0; i < nprobes; ++i)
1086 {
1087 close(kprobes[i].event_fd);
1088
1089 char msgbuf[128];
1090 ssize_t olen = snprintf(msgbuf, sizeof(msgbuf), "-:p%d_%zu",
1091 pid, i);
1092 ssize_t wlen = write(fd, msgbuf, olen);
1093 if (wlen < 0)
1094 fprintf(stderr, "Error removing probe %zu: %s\n",
1095 i, strerror(errno));
1096 }
1097 close(fd);
1098 }
1099
1100 static void
1101 unregister_tracepoints(const size_t nprobes)
1102 {
1103 for (size_t i = 0; i < nprobes; ++i)
1104 close(tracepoint_probes[i].event_fd);
1105 }
1106
1107 static void
1108 unregister_raw_tracepoints(const size_t nprobes)
1109 {
1110 for (size_t i = 0; i < nprobes; ++i)
1111 close(raw_tracepoint_probes[i].event_fd);
1112 }
1113
1114 static void
1115 register_tracepoints()
1116 {
1117 size_t nprobes = tracepoint_probes.size();
1118 if (nprobes == 0)
1119 return;
1120
1121 for (size_t i = 0; i < nprobes; ++i)
1122 {
1123 trace_data &t = tracepoint_probes[i];
1124 char fnbuf[PATH_MAX];
1125 ssize_t len = snprintf(fnbuf, sizeof(fnbuf),
1126 DEBUGFS "events/%s/%s/id",
1127 t.system.c_str(), t.name.c_str());
1128 if ((size_t)len >= sizeof(bpf_log_buf))
1129 {
1130 fprintf(stderr, "Buffer overflow creating probe %zu\n", i);
1131 goto fail;
1132 }
1133
1134 int fd = open(fnbuf, O_RDONLY);
1135 if (fd < 0)
1136 {
1137 fprintf(stderr, "Error opening probe event id %zu: %s\n",
1138 i, strerror(errno));
1139
1140 if (errno == ENOENT)
1141 fprintf(stderr, "\"%s/%s\" could not be found in %s\n",
1142 t.system.c_str(), t.name.c_str(), EVENTS);
1143
1144 goto fail;
1145 }
1146
1147 char msgbuf[128];
1148 len = read(fd, msgbuf, sizeof(msgbuf) - 1);
1149 if (len < 0)
1150 {
1151 fprintf(stderr, "Error reading probe event id %zu: %s\n",
1152 i, strerror(errno));
1153 close(fd);
1154 goto fail;
1155 }
1156 close(fd);
1157
1158 msgbuf[len] = 0;
1159 t.event_id = atoi(msgbuf);
1160 }
1161
1162 // ??? Iterate to enable on all cpus, each with a different group_fd.
1163 {
1164 perf_event_attr peattr;
1165
1166 memset(&peattr, 0, sizeof(peattr));
1167 peattr.size = sizeof(peattr);
1168 peattr.type = PERF_TYPE_TRACEPOINT;
1169 peattr.sample_type = PERF_SAMPLE_RAW;
1170 peattr.sample_period = 1;
1171 peattr.wakeup_events = 1;
1172
1173 for (size_t i = 0; i < nprobes; ++i)
1174 {
1175 trace_data &t = tracepoint_probes[i];
1176 peattr.config = t.event_id;
1177
1178 int fd = perf_event_open(&peattr, -1, default_cpu, group_fd, 0);
1179 if (fd < 0)
1180 {
1181 fprintf(stderr, "Error opening probe id %zu: %s\n",
1182 i, strerror(errno));
1183 goto fail;
1184 }
1185 t.event_fd = fd;
1186
1187 if (ioctl(fd, PERF_EVENT_IOC_SET_BPF, t.prog_fd) < 0)
1188 {
1189 fprintf(stderr, "Error installing bpf for probe id %zu: %s\n",
1190 i, strerror(errno));
1191 goto fail;
1192 }
1193 }
1194 }
1195 return;
1196
1197 fail:
1198 unregister_tracepoints(nprobes);
1199 exit(1);
1200 }
1201
1202 static void
1203 register_raw_tracepoints()
1204 {
1205 size_t nprobes = raw_tracepoint_probes.size();
1206 if (nprobes == 0)
1207 return;
1208
1209 #ifndef HAVE_BPF_PROG_TYPE_RAW_TRACEPOINT
1210 fprintf(stderr, "BPF raw tracepoints unsupported\n");
1211 exit(1);
1212 #else
1213 {
1214 union bpf_attr peattr;
1215
1216 memset(&peattr, 0, sizeof(peattr));
1217
1218 for (size_t i = 0; i < nprobes; ++i)
1219 {
1220 trace_data &t = raw_tracepoint_probes[i];
1221 peattr.raw_tracepoint.name = ((__u64)(unsigned long) (t.name.c_str()));
1222 peattr.raw_tracepoint.prog_fd = t.prog_fd;
1223
1224 int fd = syscall(__NR_bpf, BPF_RAW_TRACEPOINT_OPEN, &peattr, sizeof(peattr));
1225 if (fd < 0)
1226 {
1227 fprintf(stderr, "Error opening probe raw tracepoint %s: %s\n",
1228 t.name.c_str(), strerror(errno));
1229 goto fail;
1230 }
1231 t.event_fd = fd;
1232
1233 }
1234 }
1235 return;
1236
1237 fail:
1238 unregister_raw_tracepoints(nprobes);
1239 exit(1);
1240 #endif
1241 }
1242
1243 static void
1244 unregister_timers(const size_t nprobes)
1245 {
1246 for (size_t i = 0; i < nprobes; ++i)
1247 close(timers[i].event_fd);
1248 }
1249
1250 static void
1251 register_timers()
1252 {
1253 perf_event_attr peattr;
1254
1255 memset(&peattr, 0, sizeof(peattr));
1256 peattr.size = sizeof(peattr);
1257 peattr.type = PERF_TYPE_SOFTWARE;
1258 peattr.config = PERF_COUNT_SW_CPU_CLOCK;
1259
1260 for (size_t i = 0; i < timers.size(); ++i)
1261 {
1262 timer_data &t = timers[i];
1263 peattr.sample_period = t.period;
1264
1265 int fd = perf_event_open(&peattr, -1, default_cpu, group_fd, 0);
1266 if (fd < 0)
1267 {
1268 int err = errno;
1269 unregister_timers(timers.size());
1270 fatal("Error opening timer probe id %zu: %s\n", i + 1, strerror(err));
1271 }
1272
1273 t.event_fd = fd;
1274 if (ioctl(fd, PERF_EVENT_IOC_SET_BPF, t.prog_fd) < 0)
1275 {
1276 int err = errno;
1277 unregister_timers(timers.size());
1278 fatal("Error installing bpf for timer probe id %zu: %s\n",
1279 i + 1, strerror(err));
1280 }
1281 }
1282
1283 return;
1284 }
1285
1286 static void
1287 unregister_perf(const size_t nprobes)
1288 {
1289 for (size_t i = 0; i < nprobes; ++i)
1290 close(perf_probes[i].event_fd);
1291 }
1292
1293 static void
1294 register_perf()
1295 {
1296 for (size_t i = 0; i < perf_probes.size(); ++i)
1297 {
1298 perf_data &p = perf_probes[i];
1299 perf_event_attr peattr;
1300
1301 memset(&peattr, 0, sizeof(peattr));
1302 peattr.size = sizeof(peattr);
1303 peattr.type = p.event_type;
1304 peattr.config = p.event_config;
1305
1306 if (p.has_freq)
1307 {
1308 peattr.freq = 1;
1309 peattr.sample_freq = p.interval;
1310 }
1311 else
1312 peattr.sample_period = p.interval;
1313
1314 // group_fd is not used since this event might have an
1315 // incompatible type/config.
1316 int fd = perf_event_open(&peattr, -1, default_cpu, -1, 0);
1317 if (fd < 0)
1318 {
1319 int err = errno;
1320 unregister_perf(perf_probes.size());
1321 fatal("Error opening perf probe id %zu: %s\n", i + 1, strerror(err));
1322 }
1323
1324 p.event_fd = fd;
1325 if (ioctl(fd, PERF_EVENT_IOC_SET_BPF, p.prog_fd) < 0)
1326 {
1327 int err = errno;
1328 unregister_perf(perf_probes.size());
1329 fatal("Error installing bpf for perf probe id %zu: %s\n",
1330 i + 1, strerror(err));
1331 }
1332 }
1333 }
1334
1335 static void
1336 init_internal_globals()
1337 {
1338 using namespace bpf;
1339
1340 std::vector<int> keys;
1341 keys.push_back(globals::EXIT);
1342 keys.push_back(globals::ERRORS);
1343
1344 int64_t val = 0;
1345
1346 for (int key: keys)
1347 if (bpf_update_elem(map_fds[globals::internal_map_idx],
1348 (void*)&key, (void*)&val, BPF_ANY) != 0)
1349 fatal("Error updating pid: %s\n", strerror(errno));
1350 }
1351
1352 // PR22330: Initialize perf_event_map and perf_fds.
1353 static void
1354 init_perf_transport()
1355 {
1356 using namespace bpf;
1357
1358 unsigned ncpus = map_attrs[globals::perf_event_map_idx].max_entries;
1359
1360 for (unsigned cpu = 0; cpu < ncpus; cpu++)
1361 {
1362 if (!cpu_online[cpu]) // -- skip inactive CPUs.
1363 {
1364 perf_fds.push_back(-1);
1365 transport_contexts.push_back(nullptr);
1366 continue;
1367 }
1368
1369 struct perf_event_attr peattr;
1370
1371 memset(&peattr, 0, sizeof(peattr));
1372 peattr.size = sizeof(peattr);
1373 peattr.sample_type = PERF_SAMPLE_RAW;
1374 peattr.type = PERF_TYPE_SOFTWARE;
1375 peattr.config = PERF_COUNT_SW_BPF_OUTPUT;
1376 peattr.sample_period = 1;
1377 peattr.wakeup_events = 1;
1378
1379 int pmu_fd = perf_event_open(&peattr, -1/*pid*/, cpu, -1/*group_fd*/, 0);
1380 if (pmu_fd < 0)
1381 fatal("Error initializing perf event for cpu %d: %s\n", cpu, strerror(errno));
1382 if (bpf_update_elem(map_fds[globals::perf_event_map_idx],
1383 (void*)&cpu, (void*)&pmu_fd, BPF_ANY) != 0)
1384 fatal("Error assigning perf event for cpu %d: %s\n", cpu, strerror(errno));
1385 ioctl(pmu_fd, PERF_EVENT_IOC_ENABLE, 0);
1386 perf_fds.push_back(pmu_fd);
1387
1388 // Create a data structure to track what's happening on each CPU:
1389 bpf_transport_context *ctx
1390 = new bpf_transport_context(cpu, pmu_fd, ncpus, map_attrs, &map_fds,
1391 output_f, &interned_strings, &aggregates,
1392 &foreach_loop_info, &error);
1393 transport_contexts.push_back(ctx);
1394 }
1395
1396 // XXX: based on perf_event_mmap_header()
1397 // in kernel tools/testing/selftests/bpf/trace_helpers.c
1398 perf_event_page_size = getpagesize();
1399 perf_event_mmap_size = perf_event_page_size * (perf_event_page_count + 1);
1400 for (unsigned cpu = 0; cpu < ncpus; cpu++)
1401 {
1402 if (!cpu_online[cpu]) // -- skip inactive CPUs.
1403 {
1404 perf_headers.push_back(nullptr);
1405 continue;
1406 }
1407
1408 int pmu_fd = perf_fds[cpu];
1409 void *base = mmap(NULL, perf_event_mmap_size,
1410 PROT_READ | PROT_WRITE, MAP_SHARED,
1411 pmu_fd, 0);
1412 if (base == MAP_FAILED)
1413 fatal("error mmapping header for perf_event fd %d\n", pmu_fd);
1414 perf_headers.push_back((perf_event_mmap_page*)base);
1415 if (verbose > 2)
1416 fprintf(stderr, "Initialized perf_event output on cpu %d\n", cpu);
1417 }
1418 }
1419
1420 static void
1421 load_bpf_file(const char *module)
1422 {
1423 string module_str(module);
1424
1425 /* Extract basename: */
1426 char *buf = (char *)malloc(BPF_MAXSTRINGLEN * sizeof(char));
1427 // NB: If module doesn't contain a single '/', then the behaviour
1428 // of rfind (-1) and substr (-1 + 1) will default to module_str.
1429 string module_basename_str
1430 = module_str.substr(module_str.rfind('/')+1); // basename
1431 size_t len = module_basename_str.copy(buf, BPF_MAXSTRINGLEN-1);
1432 buf[len] = '\0';
1433 module_basename = buf;
1434
1435 /* Extract name: */
1436 buf = (char*) malloc(BPF_MAXSTRINGLEN * sizeof(char));
1437 string suffix = ".bo";
1438 string module_name_str
1439 = module_basename_str.substr(0, module_basename_str.rfind(suffix)); // name
1440 len = module_name_str.copy(buf, BPF_MAXSTRINGLEN-1);
1441 buf[len] = '\0';
1442 module_name = buf;
1443
1444 int fd = open(module, O_RDONLY);
1445 if (fd < 0)
1446 fatal_sys();
1447
1448 elf_version(EV_CURRENT);
1449
1450 Elf *elf = elf_begin(fd, ELF_C_READ_MMAP_PRIVATE, NULL);
1451 if (elf == NULL)
1452 fatal_elf();
1453 module_elf = elf;
1454
1455 Elf64_Ehdr *ehdr = elf64_getehdr(elf);
1456 if (ehdr == NULL)
1457 fatal_elf();
1458
1459 // Get username and set procfs_dir_prefix
1460 struct passwd *p = getpwuid(geteuid()); // effective uid
1461 if (p)
1462 user = p->pw_name;
1463 if (!user)
1464 fatal("an error occured while retrieving username. %s.\n", strerror(errno));
1465
1466 procfs_dir_prefix = "/var/tmp/systemtap-" + string(user) + "/" + module_name_str + "/";
1467
1468 // Byte order should match the host, since we're loading locally.
1469 {
1470 const char *end_str;
1471 switch (ehdr->e_ident[EI_DATA])
1472 {
1473 case ELFDATA2MSB:
1474 if (__BYTE_ORDER == __BIG_ENDIAN)
1475 break;
1476 end_str = "MSB";
1477 goto err_endian;
1478 case ELFDATA2LSB:
1479 if (__BYTE_ORDER == __LITTLE_ENDIAN)
1480 break;
1481 end_str = "LSB";
1482 goto err_endian;
1483 case ELFCLASSNONE:
1484 end_str = "none";
1485 goto err_endian;
1486 default:
1487 end_str = "unknown";
1488 err_endian:
1489 fatal("incorrect byte ordering: %s\n", end_str);
1490 }
1491 }
1492
1493 // Tiny bit of sanity checking on the rest of the header. Since LLVM
1494 // began by producing files with EM_NONE, accept that too.
1495 if (ehdr->e_machine != EM_NONE && ehdr->e_machine != EM_BPF)
1496 fatal("incorrect machine type: %d\n", ehdr->e_machine);
1497
1498 unsigned shnum = ehdr->e_shnum;
1499 prog_fds.assign(shnum, -1);
1500
1501 std::vector<Elf64_Shdr *> shdrs(shnum, NULL);
1502 std::vector<Elf_Data *> sh_data(shnum, NULL);
1503 std::vector<const char *> sh_name(shnum, NULL);
1504 unsigned maps_idx = 0;
1505 unsigned version_idx = 0;
1506 unsigned license_idx = 0;
1507 unsigned script_name_idx = 0;
1508 unsigned interned_strings_idx = 0;
1509 unsigned aggregates_idx = 0;
1510 unsigned foreach_loop_info_idx = 0;
1511 unsigned kprobes_idx = 0;
1512 unsigned begin_idx = 0;
1513 unsigned end_idx = 0;
1514 unsigned error_idx = 0;
1515
1516 std::vector<unsigned> procfsprobes_idx;
1517
1518 // First pass to identify special sections, and make sure
1519 // all data is readable.
1520 for (unsigned i = 1; i < shnum; ++i)
1521 {
1522 Elf_Scn *scn = elf_getscn(elf, i);
1523 if (!scn)
1524 fatal_elf();
1525
1526 Elf64_Shdr *shdr = elf64_getshdr(scn);
1527 if (!shdr)
1528 fatal_elf();
1529
1530 const char *shname = elf_strptr(elf, ehdr->e_shstrndx, shdr->sh_name);
1531 if (!shname)
1532 fatal_elf();
1533
1534 // We need not consider any empty sections.
1535 if (shdr->sh_size == 0 || !*shname)
1536 continue;
1537
1538 Elf_Data *data = elf_getdata(scn, NULL);
1539 if (data == NULL)
1540 fatal_elf();
1541
1542 shdrs[i] = shdr;
1543 sh_name[i] = shname;
1544 sh_data[i] = data;
1545
1546 if (strcmp(shname, "license") == 0)
1547 license_idx = i;
1548 else if (strcmp(shname, "stapbpf_script_name") == 0)
1549 script_name_idx = i;
1550 else if (strcmp(shname, "stapbpf_interned_strings") == 0)
1551 interned_strings_idx = i;
1552 else if (strcmp(shname, "stapbpf_aggregates") == 0)
1553 aggregates_idx = i;
1554 else if (strcmp(shname, "stapbpf_foreach_loop_info") == 0)
1555 foreach_loop_info_idx = i;
1556 else if (strcmp(shname, "version") == 0)
1557 version_idx = i;
1558 else if (strcmp(shname, "maps") == 0)
1559 maps_idx = i;
1560 else if (strcmp(shname, "kprobes") == 0)
1561 kprobes_idx = i;
1562 else if (strcmp(shname, "stap_begin") == 0)
1563 begin_idx = i;
1564 else if (strcmp(shname, "stap_end") == 0)
1565 end_idx = i;
1566 else if (strcmp(shname, "stap_error") == 0)
1567 error_idx = i;
1568 else if (strncmp(shname, "procfs", strlen("procfs")) == 0) {
1569 // procfs probes have a "procfs" prefix in their names, we don't
1570 // use normal strcmp as the full shname includes args
1571 procfsprobes_idx.push_back(i);
1572 }
1573 }
1574
1575 // Two special sections are not optional.
1576 if (license_idx != 0)
1577 module_license = static_cast<char *>(sh_data[license_idx]->d_buf);
1578 else
1579 fatal("missing license section\n");
1580 if (script_name_idx != 0)
1581 script_name = static_cast<char *>(sh_data[script_name_idx]->d_buf);
1582 else
1583 script_name = "<unknown>";
1584
1585 if (version_idx != 0)
1586 {
1587 unsigned long long size = shdrs[version_idx]->sh_size;
1588 if (size != 4)
1589 fatal("invalid version size (%llu)\n", size);
1590 memcpy(&kernel_version, sh_data[version_idx]->d_buf, 4);
1591 }
1592 else
1593 fatal("missing version section\n");
1594
1595 // Create bpf maps as required.
1596 if (maps_idx != 0)
1597 instantiate_maps(shdrs[maps_idx], sh_data[maps_idx]);
1598
1599 // Create interned strings as required.
1600 if (interned_strings_idx != 0)
1601 {
1602 // XXX: Whatever the type used by the translator, this section
1603 // just holds a blob of NUL-terminated strings we parse as follows:
1604 char *strtab = static_cast<char *>(sh_data[interned_strings_idx]->d_buf);
1605 unsigned long long strtab_size = shdrs[interned_strings_idx]->sh_size;
1606 unsigned ofs = 0;
1607 bool found_hdr = false;
1608 while (ofs < strtab_size)
1609 {
1610 // XXX: Potentially vulnerable to NUL byte in string constant.
1611 std::string str(strtab+ofs); // XXX: will slurp up to NUL byte
1612 if (str.size() == 0 && !found_hdr)
1613 found_hdr = true; // section *may* start with an extra NUL byte
1614 else
1615 interned_strings.push_back(str);
1616 ofs += str.size() + 1;
1617 }
1618 }
1619
1620 // PR23476: Initialize table of statistical aggregates.
1621 if (aggregates_idx != 0)
1622 {
1623 uint64_t *aggtab = static_cast<uint64_t *>(sh_data[aggregates_idx]->d_buf);
1624 unsigned long long aggtab_size = shdrs[aggregates_idx]->sh_size;
1625 unsigned ofs = 0; unsigned i = 0;
1626 while (ofs < aggtab_size)
1627 {
1628 bpf::globals::agg_idx agg_id = (bpf::globals::agg_idx)aggtab[i];
1629 bpf::globals::interned_stats_map ism;
1630 for (unsigned j = 0; j < bpf::globals::stat_fields.size(); j++)
1631 {
1632 ism.push_back(aggtab[i+1+j]);
1633 }
1634 aggregates[agg_id] = bpf::globals::deintern_stats_map(ism);
1635 i += 1 + bpf::globals::stat_fields.size();
1636 ofs = sizeof(uint64_t) * i;
1637 }
1638 }
1639
1640 // PR23478: Initialize table of foreach loop information.
1641 if (foreach_loop_info_idx != 0)
1642 {
1643 uint64_t *foreachtab = static_cast<uint64_t *>(sh_data[foreach_loop_info_idx]->d_buf);
1644 unsigned long long foreachtab_size = shdrs[foreach_loop_info_idx]->sh_size;
1645 unsigned ofs = 0; unsigned i = 0;
1646 while (ofs < foreachtab_size)
1647 {
1648 bpf::globals::interned_foreach_info ifi;
1649 for (unsigned j = 0; j < bpf::globals::n_foreach_info_fields; j++)
1650 {
1651 ifi.push_back(foreachtab[i+j]);
1652 }
1653 foreach_loop_info.push_back(bpf::globals::deintern_foreach_info(ifi));
1654 i += bpf::globals::n_foreach_info_fields;
1655 ofs = sizeof(uint64_t) * i;
1656 }
1657 }
1658
1659 // Relocate all programs that require it.
1660 for (unsigned i = 1; i < shnum; ++i)
1661 {
1662 Elf64_Shdr *rel_hdr = shdrs[i];
1663 if (rel_hdr == NULL || rel_hdr->sh_type != SHT_REL)
1664 continue;
1665
1666 unsigned progi = rel_hdr->sh_info;
1667 if (progi == 0 || progi >= shnum)
1668 fatal("invalid section info %u->%u\n", i, progi);
1669 Elf64_Shdr *prog_hdr = shdrs[progi];
1670
1671 unsigned symi = rel_hdr->sh_link;
1672 if (symi == 0 || symi >= shnum)
1673 fatal("invalid section link %u->%u\n", i, symi);
1674 Elf64_Shdr *sym_hdr = shdrs[symi];
1675
1676 unsigned stri = sym_hdr->sh_link;
1677 if (stri == 0 || stri >= shnum)
1678 fatal("invalid section link %u->%u\n", symi, stri);
1679
1680 if (prog_hdr->sh_flags & SHF_EXECINSTR)
1681 prog_relocate(sh_data[progi], sh_data[i], sh_data[symi],
1682 sh_data[stri], sh_name[progi], maps_idx,
1683 prog_hdr->sh_flags & SHF_ALLOC);
1684 }
1685
1686 // Load all programs that require it.
1687 for (unsigned i = 1; i < shnum; ++i)
1688 {
1689 Elf64_Shdr *shdr = shdrs[i];
1690 if ((shdr->sh_flags & SHF_ALLOC) && (shdr->sh_flags & SHF_EXECINSTR))
1691 prog_fds[i] = prog_load(sh_data[i], sh_name[i]);
1692 }
1693
1694 // Remember begin, end, error and procfs-like probes.
1695 if (begin_idx)
1696 {
1697 Elf64_Shdr *shdr = shdrs[begin_idx];
1698 if (shdr->sh_flags & SHF_EXECINSTR)
1699 prog_begin = sh_data[begin_idx];
1700 }
1701 if (end_idx)
1702 {
1703 Elf64_Shdr *shdr = shdrs[end_idx];
1704 if (shdr->sh_flags & SHF_EXECINSTR)
1705 prog_end = sh_data[end_idx];
1706 }
1707 if (error_idx)
1708 {
1709 Elf64_Shdr *shdr = shdrs[error_idx];
1710 if (shdr->sh_flags & SHF_EXECINSTR)
1711 prog_error = sh_data[error_idx];
1712 }
1713
1714 for (unsigned i = 0; i < procfsprobes_idx.size(); ++i)
1715 {
1716 unsigned actual_idx = procfsprobes_idx[i];
1717
1718 Elf64_Shdr *shdr = shdrs[actual_idx];
1719 if (shdr->sh_flags & SHF_EXECINSTR)
1720 collect_procfsprobe(sh_name[actual_idx], sh_data[actual_idx]);
1721 }
1722
1723 // Record all kprobes.
1724 if (kprobes_idx != 0)
1725 {
1726 // The Preferred Systemtap Way puts kprobe strings into a symbol
1727 // table, so that multiple kprobes can reference the same program.
1728
1729 // ??? We don't really have to have a separate kprobe symbol table;
1730 // we could pull kprobes out of the main symbol table too. This
1731 // would probably make it easier for llvm-bpf folks to transition.
1732 // One would only need to create symbol aliases with custom asm names.
1733
1734 Elf64_Shdr *sym_hdr = shdrs[kprobes_idx];
1735 if (sym_hdr->sh_type != SHT_SYMTAB)
1736 fatal("invalid section type for kprobes section\n");
1737
1738 unsigned stri = sym_hdr->sh_link;
1739 if (stri == 0 || stri >= shnum)
1740 fatal("invalid section link %u->%u\n", kprobes_idx, stri);
1741
1742 kprobe_collect_from_syms(sh_data[kprobes_idx], sh_data[stri]);
1743 }
1744 else
1745 {
1746 // The original llvm-bpf way puts kprobe strings into the
1747 // section name. Each kprobe has its own program.
1748 for (unsigned i = 1; i < shnum; ++i)
1749 maybe_collect_kprobe(sh_name[i], i, i, 0);
1750 }
1751
1752 // Record all other probes
1753 for (unsigned i = 1; i < shnum; ++i) {
1754 if (strncmp(sh_name[i], "uprobe", 6) == 0)
1755 collect_uprobe(sh_name[i], i, i);
1756 if (strncmp(sh_name[i], "trace", 5) == 0)
1757 collect_tracepoint(sh_name[i], i, i);
1758 if (strncmp(sh_name[i], "raw_trace", 9) == 0)
1759 collect_raw_tracepoint(sh_name[i], i, i);
1760 if (strncmp(sh_name[i], "perf", 4) == 0)
1761 collect_perf(sh_name[i], i, i);
1762 if (strncmp(sh_name[i], "timer", 5) == 0)
1763 collect_timer(sh_name[i], i, i);
1764 }
1765 }
1766
1767 static int
1768 get_exit_status()
1769 {
1770 int key = bpf::globals::EXIT;
1771 int64_t val = 0;
1772
1773 if (bpf_lookup_elem
1774 (map_fds[bpf::globals::internal_map_idx], &key, &val) != 0)
1775 fatal("error during bpf map lookup: %s\n", strerror(errno));
1776
1777 return val;
1778 }
1779
1780 static int
1781 get_error_count()
1782 {
1783 int key = bpf::globals::ERRORS;
1784 int64_t val = 0;
1785
1786 if (bpf_lookup_elem
1787 (map_fds[bpf::globals::internal_map_idx], &key, &val) != 0)
1788 fatal("error during bpf map lookup: %s\n", strerror(errno));
1789
1790 return val;
1791 }
1792
1793 // XXX: based on perf_event_sample
1794 // in kernel tools/testing/selftests/bpf/trace_helpers.c
1795 struct perf_event_sample {
1796 struct perf_event_header header;
1797 __u32 size;
1798 char data[];
1799 };
1800
1801 static enum bpf_perf_event_ret
1802 perf_event_handle(struct perf_event_header *hdr, void *private_data)
1803 {
1804 // XXX: based on bpf_perf_event_print
1805 // in kernel tools/testing/selftests/bpf/trace_helpers.c
1806
1807 struct perf_event_sample *e = (struct perf_event_sample *)hdr;
1808 bpf_transport_context *ctx = (bpf_transport_context *)private_data;
1809 bpf_perf_event_ret ret;
1810
1811 // Make sure we weren't passed a userspace context by accident.
1812 assert(ctx->pmu_fd >= 0);
1813
1814 if (e->header.type == PERF_RECORD_SAMPLE)
1815 {
1816 __u32 actual_size = e->size - sizeof(e->size);
1817 ret = bpf_handle_transport_msg(e->data, actual_size, ctx);
1818 if (ret != LIBBPF_PERF_EVENT_CONT)
1819 return ret;
1820 }
1821 else if (e->header.type == PERF_RECORD_LOST)
1822 {
1823 struct lost_events {
1824 struct perf_event_header header;
1825 __u64 id;
1826 __u64 lost;
1827 };
1828 struct lost_events *lost = (lost_events *) e;
1829 fprintf(stderr, "WARNING: lost %lld perf_events on cpu %d\n",
1830 (long long)lost->lost, ctx->cpu);
1831 }
1832 else
1833 {
1834 fprintf(stderr, "WARNING: unknown perf_event type=%d size=%d on cpu %d\n",
1835 e->header.type, e->header.size, ctx->cpu);
1836 }
1837 return LIBBPF_PERF_EVENT_CONT;
1838 }
1839
1840 // PR22330: Listen for perf_events.
1841 static void
1842 perf_event_loop(pthread_t main_thread)
1843 {
1844 // XXX: based on perf_event_poller_multi()
1845 // in kernel tools/testing/selftests/bpf/trace_helpers.c
1846
1847 enum bpf_perf_event_ret ret;
1848 void *data = NULL;
1849 size_t len = 0;
1850
1851 unsigned ncpus
1852 = map_attrs[bpf::globals::perf_event_map_idx].max_entries;
1853 unsigned n_active_cpus
1854 = count_active_cpus();
1855 struct pollfd *pmu_fds
1856 = (struct pollfd *)malloc(n_active_cpus * sizeof(struct pollfd));
1857 vector<unsigned> cpuids;
1858
1859 assert(ncpus == perf_fds.size());
1860 unsigned i = 0;
1861 for (unsigned cpu = 0; cpu < ncpus; cpu++)
1862 {
1863 if (!cpu_online[cpu]) continue; // -- skip inactive CPUs.
1864
1865 pmu_fds[i].fd = perf_fds[cpu];
1866 pmu_fds[i].events = POLLIN;
1867 cpuids.push_back(cpu);
1868 i++;
1869 }
1870 assert(n_active_cpus == cpuids.size());
1871
1872 // Avoid multiple warnings about errors reading from an fd:
1873 std::set<int> already_warned;
1874
1875 for (;;)
1876 {
1877 if (verbose > 3)
1878 fprintf(stderr, "Polling for perf_event data on %d cpus...\n", n_active_cpus);
1879 int ready = poll(pmu_fds, n_active_cpus, 1000); // XXX: Consider setting timeout -1 (unlimited).
1880 if (ready < 0 && errno == EINTR)
1881 goto signal_exit;
1882 if (ready < 0)
1883 fatal("Error checking for perf events: %s\n", strerror(errno));
1884 for (unsigned i = 0; i < n_active_cpus; i++)
1885 {
1886 if (pmu_fds[i].revents <= 0)
1887 continue;
1888 if (verbose > 3)
1889 fprintf(stderr, "Saw perf_event on fd %d\n", pmu_fds[i].fd);
1890
1891 ready --;
1892 unsigned cpu = cpuids[i];
1893 ret = bpf_perf_event_read_simple
1894 (perf_headers[cpu],
1895 perf_event_page_count * perf_event_page_size,
1896 perf_event_page_size,
1897 &data, &len,
1898 perf_event_handle, transport_contexts[cpu]);
1899
1900 if (ret == LIBBPF_PERF_EVENT_DONE)
1901 {
1902 // Saw STP_EXIT message. If the exit flag is set,
1903 // wake up main thread to begin program shutdown.
1904 if (get_exit_status())
1905 goto signal_exit;
1906 continue;
1907 }
1908 if (ret != LIBBPF_PERF_EVENT_CONT)
1909 if (already_warned.count(pmu_fds[i].fd) == 0)
1910 {
1911 fprintf(stderr, "WARNING: could not read from perf_event buffer on fd %d\n", pmu_fds[i].fd);
1912 already_warned.insert(pmu_fds[i].fd);
1913 }
1914 }
1915 assert(ready == 0);
1916 }
1917
1918 signal_exit:
1919 pthread_kill(main_thread, SIGINT);
1920 free(pmu_fds);
1921 return;
1922 }
1923
1924
1925 static void
1926 procfs_read_event_loop (procfsprobe_data* data, bpf_transport_context* uctx)
1927 {
1928 std::string path_s = procfs_dir_prefix + data->path;
1929 const char* path = path_s.c_str();
1930
1931 Elf_Data* prog = data->read_prog;
1932
1933 while (true)
1934 {
1935 int fd = open(path, O_WRONLY);
1936
1937 if (fd == -1)
1938 {
1939 if (errno == ENOENT)
1940 fatal("an error occured while opening procfs fifo (%s). %s.\n", path, strerror(errno));
1941
1942 fprintf(stderr, "WARNING: an error occurred while opening procfs fifo (%s). %s.\n",
1943 path, strerror(errno));
1944 continue;
1945 }
1946
1947 procfs_lock.lock();
1948
1949 // Run the probe and collect the message.
1950 bpf_interpret(prog->d_size / sizeof(bpf_insn), static_cast<bpf_insn *>(prog->d_buf), uctx);
1951
1952 // Make a copy of the message and reset it.
1953 std::string msg = uctx->procfs_msg;
1954 uctx->procfs_msg.clear();
1955
1956 procfs_lock.unlock();
1957
1958 if (data->maxsize_val && (msg.size() > data->maxsize_val - 1))
1959 fprintf(stderr, "WARNING: procfs message size (%lu) exceeds specified maximum size (%lu).\n",
1960 (unsigned long) msg.size() + 1, (unsigned long) data->maxsize_val);
1961
1962 if (write(fd, msg.data(), msg.size()) == -1)
1963 {
1964 fprintf(stderr, "WARNING: an error occurred while writing to procfs fifo (%s). %s.\n",
1965 path, strerror(errno));
1966 (void) close(fd);
1967 continue;
1968 }
1969
1970 (void) close(fd);
1971
1972 // We're not sure at this point whether the read end of the pipe has closed. We
1973 // perform a small open hack to spin until read end of the pipe has closed.
1974
1975 do {
1976
1977 fd = open(path, O_WRONLY | O_NONBLOCK);
1978
1979 if (fd != -1) close(fd);
1980
1981 } while (fd != -1);
1982 }
1983 }
1984
1985
1986 static void
1987 procfs_write_event_loop (procfsprobe_data* data, bpf_transport_context* uctx)
1988 {
1989 std::string path_s = procfs_dir_prefix + data->path;
1990 const char* path = path_s.c_str();
1991
1992 std::vector<Elf_Data*> prog = data->write_prog;
1993
1994 while (true)
1995 {
1996 int fd = open(path, O_RDONLY);
1997
1998 if (fd == -1)
1999 {
2000 if (errno == ENOENT)
2001 fatal("an error occured while opening procfs fifo (%s). %s.\n", path, strerror(errno));
2002
2003 fprintf(stderr, "WARNING: an error occurred while opening procfs fifo (%s). %s.\n",
2004 path, strerror(errno));
2005 continue;
2006 }
2007
2008 std::string msg;
2009
2010 const int read_size = 1024;
2011 int bytes_read;
2012
2013 do {
2014
2015 char buffer_feed[read_size];
2016 bytes_read = read(fd, buffer_feed, read_size);
2017
2018 if (bytes_read == -1)
2019 fprintf(stderr, "WARNING: an error occurred while reading from procfs fifo (%s). %s.\n",
2020 path, strerror(errno));
2021
2022 if (bytes_read > 0)
2023 msg.append(buffer_feed, bytes_read);
2024
2025 } while (bytes_read > 0);
2026
2027 (void) close(fd);
2028
2029 procfs_lock.lock();
2030
2031 uctx->procfs_msg = msg;
2032
2033 // Now that we have the message, run the probes serially.
2034 for (unsigned i = 0; i < prog.size(); ++i)
2035 bpf_interpret(prog[i]->d_size / sizeof(bpf_insn), static_cast<bpf_insn *>(prog[i]->d_buf), uctx);
2036
2037 procfs_lock.unlock();
2038 }
2039 }
2040
2041
2042 static void
2043 procfs_cleanup()
2044 {
2045 // Delete files and directories created for procfs-like probes.
2046 for (size_t k = 0; k < procfsprobes.size(); ++k)
2047 {
2048 std::string file_s = procfs_dir_prefix + procfsprobes[k].path;
2049 const char* file = file_s.c_str();
2050 if (remove_file_or_dir(file))
2051 fprintf(stderr, "WARNING: an error occurred while deleting a file (%s). %s.\n", file, strerror(errno));
2052 }
2053
2054 const char* dir = procfs_dir_prefix.c_str();
2055 if (procfsprobes.size() > 0 && remove_file_or_dir(dir))
2056 fprintf(stderr, "WARNING: an error ocurred while deleting a directory (%s). %s.\n", dir, strerror(errno));
2057
2058 if (verbose)
2059 fprintf(stderr, "removed fifo directory %s\n", dir);
2060 }
2061
2062
2063 static void
2064 procfs_spawn(bpf_transport_context* uctx)
2065 {
2066 // Enable cleanup routine.
2067 if (atexit(procfs_cleanup))
2068 fatal("an error occurred while setting up procfs cleaner. %s.\n", strerror(errno));
2069
2070 // Create directory for procfs-like probes.
2071 if (procfsprobes.size() > 0 && create_dir(procfs_dir_prefix.c_str()))
2072 fatal("an error occurred while making procfs directory. %s.\n", strerror(errno));
2073
2074 // Create all of the fifos used for procfs-like probes and spawn threads.
2075 for (size_t k =0; k < procfsprobes.size(); ++k)
2076 {
2077 procfsprobe_data* data = &procfsprobes[k];
2078
2079 std::string path = procfs_dir_prefix + data->path;
2080
2081 uint64_t cmask = umask(data->umask);
2082
2083 mode_t mode = (data->type == 'r') ? 0444 : 0222;
2084 mode &= ~cmask; // NB: not umask(2)
2085
2086 if ((mkfifo(path.c_str(), mode) == -1))
2087 fatal("an error occured while making procfs fifos. %s.\n", strerror(errno));
2088
2089 if (verbose)
2090 fprintf(stderr, "created %c fifo %s\n", data->type, path.c_str());
2091
2092 // TODO: Could set the owner/group of the fifo to the effective user. or real?
2093
2094 if (data->type == 'r')
2095 std::thread(procfs_read_event_loop, data, uctx).detach();
2096 else
2097 std::thread(procfs_write_event_loop, data, uctx).detach();
2098 }
2099 }
2100
2101
2102 static void
2103 usage(const char *argv0)
2104 {
2105 printf(_("Usage: %s [-v][-w][-V][-h] [-c cmd] [-x pid] [-o FILE] <bpf-file>\n"), argv0);
2106 printf(_("-h, --help Show this help text.\n"
2107 "-v, --verbose Increase verbosity.\n"
2108 "-V, --version Show version.\n"
2109 "-w Suppress warnings.\n"
2110 "-c cmd Command \'cmd\' will be run and stapbpf will\n"
2111 " exit when it does. The '_stp_target' variable\n"
2112 " will contain the pid for the command.\n"
2113 "-x pid Sets the '_stp_target' variable to pid.\n"
2114 "-o FILE Send output to FILE.\n"));
2115 }
2116
2117
2118 void
2119 sigint(int s)
2120 {
2121 // suppress any subsequent SIGINTs that may come from stap parent process
2122 signal(s, SIG_IGN);
2123
2124 // during the exit phase, ^C should exit immediately
2125 if (exit_phase)
2126 {
2127 if (!interrupt_message) // avoid duplicate message
2128 fprintf(stderr, "received interrupt during exit probe\n");
2129 interrupt_message = 1;
2130 abort();
2131 }
2132
2133 // set exit flag
2134 int key = bpf::globals::EXIT;
2135 int64_t val = 1;
2136
2137 if (bpf_update_elem
2138 (map_fds[bpf::globals::internal_map_idx], &key, &val, 0) != 0)
2139 fatal("error during bpf map update: %s\n", strerror(errno));
2140 }
2141
2142 // XXX similar to chld_proc in ../staprun/mainloop.c
2143 void
2144 sigchld(int signum)
2145 {
2146 int chld_stat = 0;
2147 if (verbose > 2)
2148 fprintf(stderr, "sigchld %d (%s)\n", signum, strsignal(signum));
2149 pid_t pid = waitpid(-1, &chld_stat, WNOHANG);
2150 if (pid != target_pid) {
2151 return;
2152 }
2153
2154 if (chld_stat) {
2155 // our child exited with a non-zero status
2156 if (WIFSIGNALED(chld_stat)) {
2157 fprintf(stderr, _("WARNING: Child process exited with signal %d (%s)\n"),
2158 WTERMSIG(chld_stat), strsignal(WTERMSIG(chld_stat)));
2159 target_pid_failed_p = 1;
2160 }
2161 if (WIFEXITED(chld_stat) && WEXITSTATUS(chld_stat)) {
2162 fprintf(stderr, _("WARNING: Child process exited with status %d\n"),
2163 WEXITSTATUS(chld_stat));
2164 target_pid_failed_p = 1;
2165 }
2166 }
2167
2168 // set exit flag
2169 int key = bpf::globals::EXIT;
2170 int64_t val = 1;
2171
2172 if (bpf_update_elem
2173 (map_fds[bpf::globals::internal_map_idx], &key, &val, 0) != 0)
2174 fatal("error during bpf map update: %s\n", strerror(errno));
2175 }
2176
2177 int
2178 main(int argc, char **argv)
2179 {
2180 static const option long_opts[] = {
2181 { "help", 0, NULL, 'h' },
2182 { "verbose", 0, NULL, 'v' },
2183 { "version", 0, NULL, 'V' },
2184 };
2185
2186 int rc;
2187
2188 while ((rc = getopt_long(argc, argv, "hvVwc:x:o:", long_opts, NULL)) >= 0)
2189 switch (rc)
2190 {
2191 case 'v':
2192 verbose++;
2193 break;
2194 case 'w':
2195 warnings = 0;
2196 break;
2197 case 'c':
2198 target_cmd = optarg;
2199 break;
2200 case 'x':
2201 target_pid = atoi(optarg);
2202 break;
2203
2204 case 'o':
2205 output_f = fopen(optarg, "w");
2206 if (output_f == NULL)
2207 {
2208 fprintf(stderr, "Error opening %s for output: %s\n",
2209 optarg, strerror(errno));
2210 return 1;
2211 }
2212 break;
2213
2214 case 'V':
2215 printf("Systemtap BPF loader/runner (version %s, %s)\n"
2216 "Copyright (C) 2016-2025 Red Hat, Inc. and others\n" // PRERELEASE
2217 "This is free software; "
2218 "see the source for copying conditions.\n",
2219 VERSION, STAP_EXTENDED_VERSION);
2220 return 0;
2221
2222 case 'h':
2223 usage(argv[0]);
2224 return 0;
2225
2226 default:
2227 do_usage:
2228 usage(argv[0]);
2229 return 1;
2230 }
2231 if (optind != argc - 1)
2232 goto do_usage;
2233 if (target_cmd && target_pid) {
2234 fprintf(stderr, _("ERROR: You can't specify the '-c' and '-x' options together.\n"));
2235 goto do_usage;
2236 }
2237
2238 // Be sure dmesg mentions that we are loading bpf programs:
2239 kmsg = fopen("/dev/kmsg", "w");
2240 if (kmsg == NULL)
2241 fprintf(stderr, _("WARNING: Could not open /dev/kmsg for diagnostics: %s\n"), strerror(errno));
2242
2243 load_bpf_file(argv[optind]); // <- XXX initializes cpus online, PR24543 initializes default_cpu
2244 init_internal_globals();
2245 init_perf_transport();
2246
2247 // PR25177: For '-c' option, start the command now so its pid is available:
2248 if (target_cmd)
2249 start_cmd();
2250 // XXX Done before begin probes, after load_bpf_file() sets __name__.
2251
2252 // Create a bpf_transport_context for userspace programs:
2253 unsigned ncpus = map_attrs[bpf::globals::perf_event_map_idx].max_entries;
2254 bpf_transport_context uctx(default_cpu, -1/*pmu_fd*/, ncpus,
2255 map_attrs, &map_fds, output_f,
2256 &interned_strings, &aggregates,
2257 &foreach_loop_info, &error);
2258
2259 if (create_group_fds() < 0)
2260 fatal("Error creating perf event group: %s\n", strerror(errno));
2261
2262 register_kprobes();
2263 register_uprobes();
2264 register_timers();
2265 register_tracepoints();
2266 register_raw_tracepoints();
2267 register_perf();
2268
2269 // Run the begin probes.
2270 if (prog_begin)
2271 bpf_interpret(prog_begin->d_size / sizeof(bpf_insn),
2272 static_cast<bpf_insn *>(prog_begin->d_buf),
2273 &uctx);
2274
2275 // Wait for ^C; read BPF_OUTPUT events, copying them to output_f.
2276 signal(SIGINT, (sighandler_t)sigint);
2277 signal(SIGTERM, (sighandler_t)sigint);
2278
2279 // perf_event listener is not yet active.
2280 bool perf_ioc_enabled = false;
2281
2282 // PR25177: Signal the '-c' command to start running.
2283 int target_rc = 0;
2284 if (target_cmd)
2285 target_rc = resume_cmd();
2286 if (target_rc < 0)
2287 goto cleanup_and_exit;
2288
2289 // PR25177: This is to notify when our child process (-c) ends.
2290 if (target_cmd)
2291 signal(SIGCHLD, (sighandler_t)sigchld);
2292
2293 // PR26109: Only continue startup if begin probes did not exit.
2294 if (!get_exit_status()) { // may already be set by begin probe
2295 // PR22330: Listen for perf_events:
2296 std::thread(perf_event_loop, pthread_self()).detach();
2297
2298 // Spawn all procfs threads.
2299 procfs_spawn(&uctx);
2300
2301 // Now that the begin probe has run and the perf_event listener is active, enable the kprobes.
2302 ioctl(group_fd, PERF_EVENT_IOC_ENABLE, 0);
2303 perf_ioc_enabled = true;
2304 }
2305
2306 // Wait for STP_EXIT message:
2307 while (!get_exit_status()) {
2308 pause();
2309 }
2310
2311 cleanup_and_exit:
2312 // Disable the kprobes before deregistering and running exit probes.
2313 if (perf_ioc_enabled)
2314 ioctl(group_fd, PERF_EVENT_IOC_DISABLE, 0);
2315 close(group_fd);
2316
2317 // Unregister all probes.
2318 unregister_kprobes(kprobes.size());
2319 unregister_uprobes(uprobes.size());
2320 unregister_timers(timers.size());
2321 unregister_perf(perf_probes.size());
2322 unregister_tracepoints(tracepoint_probes.size());
2323 unregister_raw_tracepoints(raw_tracepoint_probes.size());
2324
2325 // Clean procfs-like probe files.
2326 procfs_cleanup();
2327
2328 // We are now running exit probes, so ^C should exit immediately:
2329 exit_phase = 1;
2330 signal(SIGINT, (sighandler_t)sigint); // restore previously ignored signal
2331 signal(SIGTERM, (sighandler_t)sigint);
2332 signal(SIGCHLD, SIG_IGN); // assume child is no longer relevant
2333
2334 // PR25177: Skip exit probes if target_cmd did not resume correctly.
2335 if (target_rc >= 0)
2336 {
2337 // Run the end probes.
2338 if (prog_end && !error)
2339 bpf_interpret(prog_end->d_size / sizeof(bpf_insn),
2340 static_cast<bpf_insn *>(prog_end->d_buf),
2341 &uctx);
2342
2343 // Run the error probes.
2344 if (prog_error && error)
2345 bpf_interpret(prog_error->d_size / sizeof(bpf_insn),
2346 static_cast<bpf_insn *>(prog_error->d_buf),
2347 &uctx);
2348 }
2349
2350 // Clean up transport layer allocations:
2351 for (std::vector<bpf_transport_context *>::iterator it = transport_contexts.begin();
2352 it != transport_contexts.end(); it++)
2353 delete *it;
2354
2355 elf_end(module_elf);
2356 fclose(kmsg);
2357
2358 int error_count = get_error_count();
2359
2360 if (error_count > 0) {
2361 // TODO: Need better color configuration. Borrow staprun's dbug, warn, err macros?
2362 fprintf(stderr, "\033[0;33m" "WARNING:" "\033[0m" " Number of errors: %d\n", error_count);
2363 return 1;
2364 }
2365
2366 if (target_pid_failed_p)
2367 return 1;
2368
2369 return 0;
2370 }