Debian Codegraph @

stapbpf/bpfinterp.cxx

raw
1 /* bpfinterp.cxx - SystemTap BPF interpreter
2 *
3 * This program is free software; you can redistribute it and/or modify
4 * it under the terms of the GNU General Public License as published by
5 * the Free Software Foundation; either version 2 of the License, or
6 * (at your option) any later version.
7 *
8 * This program is distributed in the hope that it will be useful,
9 * but WITHOUT ANY WARRANTY; without even the implied warranty of
10 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
11 * GNU General Public License for more details.
12 *
13 * You should have received a copy of the GNU General Public License
14 * along with this program. If not, see <http://www.gnu.org/licenses/>.
15 *
16 * Copyright (C) 2016-2021 Red Hat, Inc.
17 *
18 */
19
20 #include <sys/time.h>
21 #include <cstdio>
22 #include <cstring>
23 #include <cstdlib>
24 #include <cerrno>
25 #include <vector>
26 #include <deque>
27 #include <algorithm>
28 #include <type_traits>
29 #include <inttypes.h>
30 #include "bpfinterp.h"
31 #include "libbpf.h"
32 #include "../bpf-internal.h"
33 #include "../staputil.h"
34
35 #define stapbpf_abort(reason) \
36 ({ fprintf(stderr, _("bpfinterp.cxx:%d: %s\n"), \
37 __LINE__, (reason)); \
38 abort(); })
39 #define stapbpf_just_abort() stapbpf_abort("bpf userspace interpreter error")
40
41 inline uintptr_t
42 as_int(void *ptr)
43 {
44 return reinterpret_cast<uintptr_t>(ptr);
45 }
46
47 inline uintptr_t
48 as_int(uint64_t *ptr)
49 {
50 return reinterpret_cast<uintptr_t>(ptr);
51 }
52
53 inline void *
54 as_ptr(uintptr_t ptr)
55 {
56 return reinterpret_cast<void *>(ptr);
57 }
58
59 inline void *
60 as_ptr(uint64_t *ptr)
61 {
62 return reinterpret_cast<void *>(ptr);
63 }
64
65 inline void *
66 as_ptr(char *ptr)
67 {
68 return reinterpret_cast<void *>(ptr);
69 }
70
71 inline char *
72 as_str(uintptr_t ptr)
73 {
74 return reinterpret_cast<char *>(ptr);
75 }
76
77 const std::string
78 remove_tag(const char *fstr)
79 {
80 while (*(++fstr) != '>' && *fstr != '\0');
81 if (*fstr == '\0') return ""; // avoid segfault
82 ++fstr;
83 const char *end = fstr + strlen(fstr);
84 while (*(--end) != '<' && end >= fstr);
85 assert(end >= fstr);
86 return std::string(fstr, end - fstr);
87 }
88
89 // Represents an in-progress foreach loop, including sorted keys/vals:
90 struct foreach_state {
91 uint64_t foreach_id;
92
93 // Used to store and sort (col,key) or (val,key) pairs for
94 // iteration, where col is a sort column of the key:
95 std::deque<std::pair<std::string, uint64_t *>> str_sorted;
96 std::deque<std::pair<int64_t, uint64_t *>> int_sorted;
97
98 // Used to track keys for deallocation by foreach_state_cleanup:
99 std::vector<uint64_t *> keys;
100 };
101
102 // The interpreter maintains a stack of foreach_states,
103 // with each nested foreach loop pushing onto the stack:
104 typedef std::vector<foreach_state> foreach_stack;
105
106 #define foreach_info bpf::globals::foreach_info
107
108 void
109 foreach_state_add(const foreach_info &fi, foreach_state &s,
110 uint64_t *kp, uint64_t *colp,
111 bool scalar_long)
112 {
113 // extract col_str, col_long from colp
114 std::string col_str;
115 bool use_str_col = false;
116 int64_t col_long;
117 //bool use_long_col = false;
118 if (scalar_long)
119 {
120 // colp points to a scalar long value
121 //use_long_col = true;
122 col_long = *colp;
123 }
124 else if (fi.sort_column == 0 /* use_value */
125 || fi.sort_column_ofs == -1 /* !key_composite */)
126 {
127 // colp is a single string
128 use_str_col = true;
129 std::string col_str2((char *)colp, BPF_MAXSTRINGLEN);
130 col_str = col_str2;
131 }
132 else
133 {
134 // colp contains a column at fi.sort_column_ofs
135 colp = (uint64_t *)(((char *)colp)+fi.sort_column_ofs);
136 if (fi.sort_column_size == BPF_MAXSTRINGLEN)
137 {
138 use_str_col = true;
139 std::string col_str3((char *)colp, BPF_MAXSTRINGLEN);
140 col_str = col_str3;
141 }
142 else
143 {
144 //use_long_col = true;
145 col_long = *colp;
146 }
147 }
148
149 // copy and save key
150 uint64_t *kp2 = (uint64_t *)malloc(fi.keysize);
151 memcpy(kp2, kp, fi.keysize);
152 s.keys.push_back(kp2);
153
154 // save (col,key) pair
155 if (use_str_col)
156 {
157 std::pair<std::string, uint64_t *> item_str(col_str, kp2);
158 s.str_sorted.push_back(item_str);
159 }
160 else // use_long_col
161 {
162 std::pair<int64_t, uint64_t *> item_long(col_long, kp2);
163 s.int_sorted.push_back(item_long);
164 }
165 }
166
167 bool
168 foreach_cmp_str(const std::pair<std::string, void *> &a,
169 const std::pair<std::string, void *> &b)
170 {
171 return a.first < b.first;
172 }
173
174 bool
175 foreach_cmp_int(const std::pair<int64_t, void *> &a,
176 const std::pair<int64_t, void *> &b)
177 {
178 return a.first < b.first;
179 }
180
181 void
182 foreach_state_sort(foreach_state &s)
183 {
184 if (!s.str_sorted.empty())
185 {
186 std::stable_sort(s.str_sorted.begin(), s.str_sorted.end(),
187 foreach_cmp_str);
188 }
189 if (!s.int_sorted.empty())
190 {
191 std::stable_sort(s.int_sorted.begin(), s.int_sorted.end(),
192 foreach_cmp_int);
193 }
194 }
195
196 bool
197 foreach_state_empty(const foreach_state &s)
198 {
199 return s.str_sorted.empty() && s.int_sorted.empty();
200 }
201
202 void
203 convert_key(const foreach_info &fi,
204 uint64_t *kp, uint64_t *next_kp,
205 std::vector<std::string> &strings,
206 std::vector<uint64_t *> &map_values)
207 {
208 bool scalar_long = fi.sort_column_ofs == -1 /* !key_composite */
209 && fi.keysize != BPF_MAXSTRINGLEN /* key_long */;
210 bool scalar_str = fi.sort_column_ofs == -1 /* !key_composite */
211 && fi.keysize == BPF_MAXSTRINGLEN /* key_str */;
212 if (scalar_long)
213 {
214 // handle scalar long keys being passed directly
215 *next_kp = *kp;
216 return;
217 }
218 if (scalar_str)
219 {
220 // handle string keys being passed as pointers
221 // TODO: Could merge strings into map_values:
222 std::string str((char*)kp, BPF_MAXSTRINGLEN);
223 strings.push_back(str);
224 *next_kp = reinterpret_cast<uint64_t>(strings.back().c_str());
225 return;
226 }
227
228 // handle string composite keys being passed as pointers
229 // allocate correctly sized buffer and store it in map_values:
230 uint64_t *lookup_tmp = (uint64_t*)malloc(fi.keysize);
231 memcpy(lookup_tmp, kp, fi.keysize);
232 map_values.push_back(lookup_tmp);
233 *next_kp = reinterpret_cast<uint64_t>(map_values.back());
234 }
235
236 template<typename T>
237 int
238 _foreach_state_next(const foreach_info &fi, /* XXX foreach_state &s, */
239 std::deque<std::pair<T,uint64_t*>> &sorted,
240 int64_t key, int64_t next_key,
241 std::vector<std::string> &strings,
242 std::vector<uint64_t *> &map_values)
243 {
244 (void)key; // XXX unused; see comment in foreach_state_next()
245
246 if (sorted.empty())
247 return -1;
248
249 if (fi.sort_direction > 0)
250 {
251 std::pair<T,uint64_t*> item = sorted.front();
252 convert_key(fi, item.second, (uint64_t *)next_key,
253 strings, map_values);
254 sorted.pop_front();
255 }
256 else // sort_direction < 0
257 {
258 std::pair<T,uint64_t*> item = sorted.back();
259 convert_key(fi, item.second, (uint64_t *)next_key,
260 strings, map_values);
261 sorted.pop_back();
262 }
263 return 0;
264 }
265
266 int
267 foreach_state_next(const foreach_info &fi, foreach_state &s,
268 int64_t key, int64_t next_key,
269 std::vector<std::string> &strings,
270 std::vector<uint64_t *> &map_values)
271 {
272 // TODO: Sanity check that we are continuing the same iteration?
273 // Would require storing key in foreach_state.
274 //
275 // XXX Otherwise the code assumes that the BPF probe exactly follows
276 // a foreach pattern, always passing the returned key as the next
277 // key and not trying to 'start iteration in the middle'.
278 if (!s.str_sorted.empty())
279 {
280 return _foreach_state_next<std::string>(fi, /* XXX s, */
281 s.str_sorted,
282 key, next_key,
283 strings, map_values);
284 }
285 if (!s.int_sorted.empty())
286 {
287 return _foreach_state_next<int64_t>(fi, /* XXX s, */
288 s.int_sorted,
289 key, next_key,
290 strings, map_values);
291 }
292 return -1;
293 }
294
295 void
296 foreach_state_cleanup(foreach_state &s)
297 {
298 for (uint64_t *ptr : s.keys)
299 free(ptr);
300 }
301
302 // Wrapper for bpf_get_next_key that includes logic for accessing
303 // keys in ascending or descending order, or
304 // (PR23858) in ascending or descending order by value.
305 int
306 map_get_next_key(int fd_idx, int64_t key, int64_t next_key,
307 uint64_t foreach_id, int64_t limit,
308 bpf_transport_context *ctx, foreach_stack &foreach_ctx,
309 std::vector<std::string> &strings,
310 std::vector<uint64_t *> &map_values)
311 {
312 int fd = (*ctx->map_fds)[fd_idx];
313
314 // Retrieve foreach loop info
315 foreach_info fi;
316 bool have_fi = !ctx->foreach_loop_info->empty()
317 && ctx->foreach_loop_info->size() > foreach_id;
318 fi.sort_column = 1;
319 fi.sort_direction = 0;
320 fi.keysize = 0;
321 fi.sort_column_size = 0;
322 fi.sort_column_ofs = 0;
323 if (!have_fi)
324 {
325 // XXX Backwards compatibility for older .bo's using
326 // sort_flags and no foreach_loop_info table.
327 uint64_t sort_flags = foreach_id;
328 fi.sort_column = GET_SORT_COLUMN(sort_flags);
329 fi.sort_direction = GET_SORT_DIRECTION(sort_flags);
330
331 // XXX The entire key is a single column.
332 fi.keysize = ctx->map_attrs[fd_idx].key_size;
333 fi.sort_column_size = fi.keysize;
334 fi.sort_column_ofs = -1; // XXX use entire key
335 }
336 else
337 {
338 fi = ctx->foreach_loop_info->operator[](foreach_id);
339 //assert(fi.keysize == ctx->map_attrs[fd_idx].key_size);
340 // TODO PR24528: also handle s->sort_aggr for stat aggregates
341 }
342 //fprintf(stderr, "DEBUG called map_get_next_key fd=%d sort_column=%u sort_direction=%d key=%lx next_key=%lx limit=%ld\n", fd, fi.sort_column, fi.sort_direction, key, next_key, limit);
343
344 // Identify type of key, value
345 bool use_val = fi.sort_column == 0;
346 bool key_composite = fi.sort_column_ofs != -1;
347 bool key_long = false;
348 bool key_str = false;
349 if (!key_composite)
350 {
351 // XXX If the key has one column, it is either
352 // a string (BPF_MAXSTRINGLEN) or a scalar long.
353 key_long = fi.keysize != BPF_MAXSTRINGLEN;
354 key_str = fi.keysize == BPF_MAXSTRINGLEN;
355 }
356 bool val_long = ctx->map_attrs[fd_idx].value_size != BPF_MAXSTRINGLEN;
357 //bool val_str = !val_long;
358
359 // Check iteration limit
360 if (limit == 0)
361 {
362 // Final iteration, therefore foreach_ctx.back() is no longer needed
363
364 // PR24811 att1: If key is not set, the map is empty and the
365 // context wasn't created. Only pop with a nonzero key.
366 //
367 // XXX: A malformed .bo could still mess with this check
368 // by issuing map_get_next_key(key=?,limit=0) calls
369 // outside the usual foreach pattern.
370 //if (!key)
371 // return -1;
372
373 // PR24811 att2: Check if the context for this foreach_id was created.
374 // It might not have been if the limit is zero on the first call.
375 if (foreach_ctx.empty()
376 || foreach_ctx.back().foreach_id != foreach_id)
377 return -1;
378
379 foreach_state_cleanup(foreach_ctx.back());
380 foreach_ctx.pop_back();
381 return -1;
382 }
383
384 // Handle fi.sort_direction==0, where no foreach_ctx is needed
385 if (fi.sort_direction == 0)
386 {
387 // handle scalar long values being passed directly
388 if (key_long)
389 return bpf_get_next_key(fd, as_ptr(key), as_ptr(next_key));
390
391 // handle string and composite keys being passed as pointers
392 char _n[BPF_MAXKEYLEN_PLUS];
393 uint64_t *kp = key == 0x0 ? (uint64_t *)0x0 : *(uint64_t **)key;
394 uint64_t *np = (uint64_t *)_n;
395 int rc = bpf_get_next_key(fd, as_ptr(kp), as_ptr(np));
396 if (!rc && key_str)
397 {
398 // TODO: Could merge strings into map_values:
399 std::string next_key2(_n, BPF_MAXSTRINGLEN);
400 strings.push_back(next_key2);
401 *(uint64_t *)next_key =
402 reinterpret_cast<uint64_t>(strings.back().c_str());
403 }
404 else if (!rc)
405 {
406 // allocate correctly sized buffer and store it in map_values:
407 uint64_t *lookup_tmp = (uint64_t*)malloc(fi.keysize);
408 memcpy(lookup_tmp, _n, fi.keysize);
409 map_values.push_back(lookup_tmp);
410 *(uint64_t *)next_key =
411 reinterpret_cast<uint64_t>(map_values.back());
412 }
413 return rc;
414 }
415
416 // Sort the map on initial iteration
417 if (!key)
418 {
419 // handle both uint64_t and string column types
420 char _k[BPF_MAXKEYLEN_PLUS], _n[BPF_MAXKEYLEN_PLUS];
421 _k[BPF_MAXSTRINGLEN] = _k[BPF_MAXKEYLEN] = '\0';
422 _n[BPF_MAXSTRINGLEN] = _n[BPF_MAXKEYLEN] = '\0';
423 uint64_t *kp = (uint64_t *)_k;
424 uint64_t *np = (uint64_t *)_n;
425 foreach_state s;
426
427 int rc = bpf_get_next_key(fd, 0, as_ptr(np));
428 while (!rc)
429 {
430 if (use_val)
431 {
432 char _v[BPF_MAXKEYLEN_PLUS];
433 _v[BPF_MAXSTRINGLEN] = _v[BPF_MAXKEYLEN] = '\0';
434 uint64_t *vp = (uint64_t *)_v;
435 int res = bpf_lookup_elem(fd, as_ptr(np), as_ptr(vp));
436 if (res) // element could not be found
437 stapbpf_abort("bpf_map_get_next_key BUG: could not find key " \
438 "returned by bpf_get_next_key");
439 foreach_state_add(fi, s, np, vp, val_long);
440 }
441 else
442 {
443 // foreach_state_add extracts the column from np
444 foreach_state_add(fi, s, np, np, key_long);
445 }
446 memcpy(kp, np, fi.keysize);
447 rc = bpf_get_next_key(fd, as_ptr(kp), as_ptr(np));
448 }
449 foreach_state_sort(s);
450 if (foreach_state_empty(s))
451 return -1;
452 foreach_ctx.push_back(s);
453 }
454
455 if (foreach_ctx.empty())
456 stapbpf_abort("bpf_map_get_next_key BUG: called outside a foreach loop");
457
458 // Get next value from sorted data
459 int rc = foreach_state_next(fi, foreach_ctx.back(), key, next_key,
460 strings, map_values);
461 if (rc < 0) // no more elements to return
462 {
463 foreach_state_cleanup(foreach_ctx.back());
464 foreach_ctx.pop_back();
465 return -1;
466 }
467 return 0;
468 }
469
470 // TODO: Adapt to MAXPRINTFARGS == 32.
471 uint64_t
472 bpf_sprintf(std::vector<std::string> &strings, char *fstr,
473 uint64_t arg1, uint64_t arg2, uint64_t arg3)
474 {
475 char s[256]; // TODO: configure maximum length setting e.g. BPF_MAXSPRINTFLEN
476 #pragma GCC diagnostic push
477 #pragma GCC diagnostic ignored "-Wformat-nonliteral"
478 snprintf(s, 256, fstr, arg1, arg2, arg3);
479 #pragma GCC diagnostic pop
480 std::string str(s, 256);
481 strings.push_back(str);
482
483 // Elements of "strings" should not be mutated to avoid
484 // invalidating c_str() pointers.
485 return reinterpret_cast<uint64_t>(strings.back().c_str());
486 }
487
488 uint64_t
489 bpf_text_str(std::vector<std::string> &strings, char* input, bool quoted)
490 {
491 std::string str(input);
492 std::stringstream stream;
493
494 for (std::string::iterator it = str.begin(); it != str.end(); ++it)
495 {
496 unsigned char c = *it;
497 unsigned int i = c;
498
499 if (std::isprint(c) && i < 128 && c != '\\' && c != '"')
500 stream << c;
501 else
502 {
503 stream << '\\';
504 switch (c) {
505 case '\0': stream << '0'; break; // Not handled by translate_escapes
506 case '\a': stream << 'a'; break; // Not handled by translate_escapes
507 case '\b': stream << 'b'; break; // Not handled by translate_escapes
508 case '\f': stream << 'f'; break;
509 case '\n': stream << 'n'; break;
510 case '\r': stream << 'r'; break;
511 case '\t': stream << 't'; break;
512 case '\v': stream << 'v'; break;
513 case '"': stream << '"'; break;
514 case '\\': stream << '\\'; break;
515 default:
516 stream << "x" << std::setfill('0') << std::setw(2) << std::hex << i;
517 break;
518 }
519 }
520 }
521
522 if (quoted)
523 strings.push_back("\"" + stream.str() + "\"");
524 else
525 strings.push_back(stream.str());
526
527 return reinterpret_cast<uint64_t>(strings.back().c_str());
528 }
529
530 uint64_t
531 bpf_str_concat(std::vector<std::string> &strings, char* left, char* right)
532 {
533 std::string concat;
534 concat += left;
535 concat += right;
536
537 strings.push_back(concat);
538
539 // After adding the new string into the vector, the c_str() pointer value
540 // will be valid and safe to use
541 return reinterpret_cast<uint64_t>(strings.back().c_str());
542 }
543
544 // Allocates and returns a buffer of percpu data for a stat field:
545 uint64_t *
546 stapbpf_stat_get_percpu(bpf::globals::map_idx map, uint64_t idx,
547 bpf_transport_context *ctx)
548 {
549 uint64_t *ret = (uint64_t *)calloc(ctx->ncpus, sizeof(uint64_t));
550 int res = bpf_lookup_elem((*ctx->map_fds)[map], as_ptr(idx), ret);
551 if (res) {
552 // element could not be found
553 free(ret);
554 return 0;
555 }
556 else
557 return ret;
558 }
559
560 // XXX Based on _stp_stat_get in runtime/stat.c.
561 // There might be a clever way to avoid code duplication later,
562 // but right now the code format is too different. Just reimplement.
563 uint64_t
564 stapbpf_stat_get(bpf::globals::agg_idx agg_id, uint64_t idx,
565 stat_component_type sc_op,
566 bpf_transport_context *ctx)
567 {
568 if (ctx->aggregates->find(agg_id) == ctx->aggregates->end())
569 stapbpf_abort("aggregate could not be found");
570 bpf::globals::stats_map sd = (*ctx->aggregates)[agg_id];
571
572 // XXX Based on struct stat_data in runtime/stat.h:
573 struct stapbpf_stat_data {
574 int shift;
575 int64_t count;
576 int64_t sum;
577 int64_t avg_s;
578 // TODO PR23476: Add more fields.
579 } agg;
580 // TODO: Consider caching each agg for the duration of userspace program execution.
581
582 // Retrieve the fields that we are going to aggregate.
583 //
584 // XXX: This took a while to figure out.
585 // bpf_map_lookup_elem() for percpu map returns an array.
586 uint64_t *count_data = stapbpf_stat_get_percpu(sd["count"], idx, ctx);
587 uint64_t *sum_data = stapbpf_stat_get_percpu(sd["sum"], idx, ctx);
588
589 // TODO PR23476: Simplified code for now.
590 agg.shift = 0;
591 agg.count = 0;
592 agg.sum = 0;
593
594 // XXX for_each_possible_cpu(i)
595 if (count_data) {
596 for (unsigned i = 0; i < ctx->ncpus; i++)
597 agg.count += count_data[i];
598 free(count_data);
599 }
600
601 // XXX for_each_possible_cpu(i)
602 if (sum_data) {
603 for (unsigned i = 0; i < ctx->ncpus; i++)
604 agg.sum += sum_data[i];
605 free(sum_data);
606 }
607
608 // XXX Simplified version of _stp_div64():
609 if (agg.count == 0)
610 agg.avg_s = 0;
611 else
612 agg.avg_s = (agg.sum << agg.shift) / agg.count;
613
614 switch (sc_op)
615 {
616 case sc_average:
617 if (agg.count == 0)
618 stapbpf_abort("empty aggregate"); // TODO: Should produce proper error.
619 return agg.avg_s;
620
621 case sc_count:
622 return agg.count;
623
624 case sc_sum:
625 return agg.sum;
626
627 case sc_none:
628 // should not happen, as sc_none is only used in foreach slots
629 stapbpf_abort("unexpected sc_none");
630
631 // TODO PR23476: Not yet implemented.
632 case sc_min:
633 case sc_max:
634 case sc_variance:
635 default:
636 stapbpf_abort("unsupported aggregate");
637 }
638 }
639
640 uint64_t
641 bpf_ktime_get_ns()
642 {
643 struct timespec t;
644 clock_gettime (CLOCK_BOOTTIME, &t);
645 return (t.tv_sec * 1000000000) + t.tv_nsec;
646 }
647
648 uint64_t
649 bpf_gettimeofday_ns()
650 {
651 struct timeval t;
652 gettimeofday (&t, NULL);
653 return (((t.tv_sec * 1000000) + t.tv_usec) * 1000);
654 }
655
656 uint64_t
657 bpf_get_target()
658 {
659 return target_pid;
660 }
661
662 uint64_t
663 bpf_set_procfs_value(char* msg, bpf_transport_context* ctx)
664 {
665 assert(msg != nullptr);
666
667 ctx->procfs_msg = std::string(msg);
668
669 return 0;
670 }
671
672 uint64_t
673 bpf_append_procfs_value(char* msg, bpf_transport_context* ctx)
674 {
675 assert(msg != nullptr);
676
677 ctx->procfs_msg.append(std::string(msg));
678
679 return 0;
680 }
681
682 uint64_t
683 bpf_get_procfs_value(bpf_transport_context* ctx)
684 {
685 return (uint64_t) (ctx->procfs_msg.data());
686 }
687
688 enum bpf_perf_event_ret
689 bpf_handle_transport_msg(void *buf, size_t size,
690 bpf_transport_context *ctx)
691 {
692 // Unpack transport message:
693 struct bpf_transport_msg {
694 BPF_TRANSPORT_VAL type;
695 BPF_TRANSPORT_ARG content_start;
696 };
697 bpf_transport_msg *_msg = (bpf_transport_msg *) buf;
698 bpf::globals::perf_event_type msg_type = (bpf::globals::perf_event_type)_msg->type;
699 void *msg_content = (void*)&_msg->content_start;
700 size_t msg_size = size - sizeof(BPF_TRANSPORT_ARG);
701
702 // Used for bpf::globals::STP_EXIT:
703 int exit_key = bpf::globals::EXIT;
704 long exit_val = 1;
705
706 // Used for bpf::globals::STP_FORMAT_ARG:
707 void *arg;
708
709 switch (msg_type)
710 {
711 case bpf::globals::STP_STORE_ERROR_MSG:
712 // Store error message for future printing.
713 ctx->error_message.push(std::string((char*) msg_content));
714 break;
715
716 case bpf::globals::STP_PRINT_ERROR_MSG:
717 // Print error message that was stored previously.
718 assert(!ctx->error_message.empty());
719 // TODO: Need better color configuration.
720 std::cout << "\033[1m\033[31m" << "ERROR: " << "\033[0m" << ctx->error_message.front() << std::endl;
721 ctx->error_message.pop();
722 break;
723
724 case bpf::globals::STP_ERROR:
725 // Signal an exit from the program and communicate a hard error:
726 if (bpf_update_elem((*ctx->map_fds)[bpf::globals::internal_map_idx],
727 &exit_key, &exit_val, BPF_ANY) != 0)
728 stapbpf_abort("could not set exit status");
729 *ctx->error = true;
730 return LIBBPF_PERF_EVENT_DONE;
731
732 case bpf::globals::STP_EXIT:
733 // Signal an exit from the program:
734 if (bpf_update_elem((*ctx->map_fds)[bpf::globals::internal_map_idx],
735 &exit_key, &exit_val, BPF_ANY) != 0)
736 stapbpf_abort("could not set exit status");
737 return LIBBPF_PERF_EVENT_DONE;
738
739 case bpf::globals::STP_PRINTF_START:
740 if (ctx->in_printf)
741 stapbpf_abort("printf already started");
742 if (msg_size != sizeof(BPF_TRANSPORT_ARG))
743 stapbpf_abort("wrong argument size");
744 ctx->in_printf = true; ctx->format_no = -1;
745 ctx->expected_args = *(BPF_TRANSPORT_ARG*)msg_content;
746 break;
747
748 case bpf::globals::STP_PRINTF_END:
749 if (!ctx->in_printf)
750 stapbpf_abort("printf not started");
751 if (ctx->format_no < 0 || ctx->format_no >= (int)ctx->interned_strings->size())
752 stapbpf_abort("printf format is missing");
753 if (ctx->printf_args.size() != ctx->expected_args)
754 stapbpf_abort("wrong number of printf args");
755
756 // TODO: Check this code on 32-bit systems after fixing PR24358.
757 //
758 // XXX: Surprisingly, it is not easy to pass an array to a
759 // printf-type function. The best I can do for now is hardcode a
760 // call to fprintf with BPF_MAXPRINTFARGS arguments:
761 {
762 std::string &format_str = (*ctx->interned_strings)[ctx->format_no];
763 void *fargs[BPF_MAXPRINTFARGS];
764 for (unsigned i = 0; i < BPF_MAXPRINTFARGS; i++)
765 if (i < ctx->printf_args.size()
766 && ctx->printf_arg_types[i] == bpf::globals::STP_PRINTF_ARG_LONG)
767 fargs[i] = (void *)*(uint64_t*)ctx->printf_args[i];
768 else if (i < ctx->printf_args.size())
769 fargs[i] = ctx->printf_args[i];
770 else
771 fargs[i] = NULL;
772 assert(BPF_MAXPRINTFARGS == 32); // XXX: Change the fprintf() call if this changes.
773 #pragma GCC diagnostic push
774 #pragma GCC diagnostic ignored "-Wformat-nonliteral"
775 fprintf(ctx->output_f, format_str.c_str(),
776 fargs[0], fargs[1], fargs[2], fargs[3], fargs[4], fargs[5], fargs[6], fargs[7],
777 fargs[8], fargs[9], fargs[10], fargs[11], fargs[12], fargs[13], fargs[14], fargs[15],
778 fargs[16], fargs[17], fargs[18], fargs[19], fargs[20], fargs[21], fargs[22], fargs[23],
779 fargs[24], fargs[25], fargs[26], fargs[27], fargs[28], fargs[29], fargs[30], fargs[31]);
780 fflush(ctx->output_f);
781 #pragma GCC diagnostic pop
782 }
783
784 // Deallocate accumulated format+args:
785 ctx->in_printf = false; ctx->format_no = -1;
786 for (unsigned i = 0; i < ctx->printf_args.size(); i++)
787 free(ctx->printf_args[i]);
788 ctx->printf_args.clear();
789 ctx->printf_arg_types.clear();
790 break;
791
792 case bpf::globals::STP_PRINTF_FORMAT:
793 if (!ctx->in_printf)
794 stapbpf_abort("printf not started");
795 if (ctx->format_no != -1)
796 stapbpf_abort("printf already has format");
797 if (msg_size != sizeof(BPF_TRANSPORT_ARG))
798 stapbpf_abort("wrong argument size");
799 ctx->format_no = *(BPF_TRANSPORT_ARG*)msg_content;
800 break;
801
802 // XXX: Could save spurious mallocs by storing ARG_LONG as the void * itself.
803 case bpf::globals::STP_PRINTF_ARG_LONG:
804 case bpf::globals::STP_PRINTF_ARG_STR:
805 if (!ctx->in_printf)
806 stapbpf_abort("printf not started");
807 arg = malloc(msg_size);
808 memcpy(arg, msg_content, msg_size);
809 ctx->printf_args.push_back(arg);
810 ctx->printf_arg_types.push_back(msg_type);
811 break;
812
813 default:
814 stapbpf_abort("unknown transport message");
815 }
816 return LIBBPF_PERF_EVENT_CONT;
817 }
818
819 uint64_t
820 bpf_interpret(size_t ninsns, const struct bpf_insn insns[],
821 bpf_transport_context *ctx)
822 {
823 uint64_t result = 0; // return value
824 uint64_t stack[65536 / 8]; // see MAX_BPF_USER_STACK in bpf-internal.h
825 uint64_t regs[MAX_BPF_REG];
826 memset(regs, 0x0, sizeof(uint64_t) * MAX_BPF_REG);
827 const struct bpf_insn *i = insns;
828 static std::vector<uint64_t *> map_values;
829
830 // Multiple threads accessing strings can cause concurrency issues for
831 // procfs_probes. However, the procfs_lock should prevent this and thus,
832 // clearing it on exit is unecessary for now.
833 static std::vector<std::string> strings;
834
835 bpf_map_def *map_attrs = ctx->map_attrs;
836 std::vector<int> &map_fds = *ctx->map_fds;
837 FILE *output_f = ctx->output_f;
838
839 std::vector<foreach_stack> foreach_ctxs(map_fds.size());
840
841 map_values.clear(); // XXX: avoid double free
842
843 regs[BPF_REG_10] = (uintptr_t)stack + sizeof(stack);
844
845 while ((size_t)(i - insns) < ninsns)
846 {
847 uint64_t dr, sr, si, s1;
848 bpf_perf_event_ret tr;
849
850 dr = regs[i->dst_reg];
851 sr = regs[i->src_reg];
852 si = i->imm;
853 s1 = i->code & BPF_X ? sr : si;
854
855 switch (i->code)
856 {
857 case BPF_LDX | BPF_MEM | BPF_B:
858 dr = *(uint8_t *)((uintptr_t)sr + i->off);
859 break;
860 case BPF_LDX | BPF_MEM | BPF_H:
861 dr = *(uint16_t *)((uintptr_t)sr + i->off);
862 break;
863 case BPF_LDX | BPF_MEM | BPF_W:
864 dr = *(uint32_t *)((uintptr_t)sr + i->off);
865 break;
866 case BPF_LDX | BPF_MEM | BPF_DW:
867 dr = *(uint64_t *)((uintptr_t)sr + i->off);
868 break;
869
870 case BPF_ST | BPF_MEM | BPF_B:
871 sr = si;
872 /* Fallthrough */
873 case BPF_STX | BPF_MEM | BPF_B:
874 *(uint8_t *)((uintptr_t)dr + i->off) = sr;
875 goto nowrite;
876 case BPF_ST | BPF_MEM | BPF_H:
877 sr = si;
878 /* Fallthrough */
879 case BPF_STX | BPF_MEM | BPF_H:
880 *(uint16_t *)((uintptr_t)dr + i->off) = sr;
881 goto nowrite;
882 case BPF_ST | BPF_MEM | BPF_W:
883 sr = si;
884 /* Fallthrough */
885 case BPF_STX | BPF_MEM | BPF_W:
886 *(uint32_t *)((uintptr_t)dr + i->off) = sr;
887 goto nowrite;
888 case BPF_ST | BPF_MEM | BPF_DW:
889 sr = si;
890 /* Fallthrough */
891 case BPF_STX | BPF_MEM | BPF_DW:
892 *(uint64_t *)((uintptr_t)dr + i->off) = sr;
893 goto nowrite;
894
895 case BPF_ALU64 | BPF_ADD | BPF_X:
896 case BPF_ALU64 | BPF_ADD | BPF_K: dr += s1; break;
897 case BPF_ALU64 | BPF_SUB | BPF_X:
898 case BPF_ALU64 | BPF_SUB | BPF_K: dr -= s1; break;
899 case BPF_ALU64 | BPF_AND | BPF_X:
900 case BPF_ALU64 | BPF_AND | BPF_K: dr &= s1; break;
901 case BPF_ALU64 | BPF_OR | BPF_X:
902 case BPF_ALU64 | BPF_OR | BPF_K: dr |= s1; break;
903 case BPF_ALU64 | BPF_LSH | BPF_X:
904 case BPF_ALU64 | BPF_LSH | BPF_K: dr <<= s1; break;
905 case BPF_ALU64 | BPF_RSH | BPF_X:
906 case BPF_ALU64 | BPF_RSH | BPF_K: dr >>= s1; break;
907 case BPF_ALU64 | BPF_XOR | BPF_X:
908 case BPF_ALU64 | BPF_XOR | BPF_K: dr ^= s1; break;
909 case BPF_ALU64 | BPF_MUL | BPF_X:
910 case BPF_ALU64 | BPF_MUL | BPF_K: dr *= s1; break;
911 case BPF_ALU64 | BPF_MOV | BPF_X:
912 case BPF_ALU64 | BPF_MOV | BPF_K: dr = s1; break;
913 case BPF_ALU64 | BPF_ARSH | BPF_X:
914 case BPF_ALU64 | BPF_ARSH | BPF_K: dr = (int64_t)dr >> s1; break;
915 case BPF_ALU64 | BPF_NEG: dr = -sr; break;
916 case BPF_ALU64 | BPF_DIV | BPF_X:
917 case BPF_ALU64 | BPF_DIV | BPF_K:
918 if (s1 == 0)
919 {
920 // TODO: Signal a proper error.
921 result = 0; goto cleanup;
922 }
923 dr /= s1;
924 break;
925 case BPF_ALU64 | BPF_MOD | BPF_X:
926 case BPF_ALU64 | BPF_MOD | BPF_K:
927 if (s1 == 0)
928 {
929 // TODO: Signal a proper error.
930 result = 0; goto cleanup;
931 }
932 dr %= s1;
933 break;
934
935 case BPF_ALU | BPF_ADD | BPF_X:
936 case BPF_ALU | BPF_ADD | BPF_K: dr = (uint32_t)(dr + s1); break;
937 case BPF_ALU | BPF_SUB | BPF_X:
938 case BPF_ALU | BPF_SUB | BPF_K: dr = (uint32_t)(dr - s1); break;
939 case BPF_ALU | BPF_AND | BPF_X:
940 case BPF_ALU | BPF_AND | BPF_K: dr = (uint32_t)(dr & s1); break;
941 case BPF_ALU | BPF_OR | BPF_X:
942 case BPF_ALU | BPF_OR | BPF_K: dr = (uint32_t)(dr | s1); break;
943 case BPF_ALU | BPF_LSH | BPF_X:
944 case BPF_ALU | BPF_LSH | BPF_K:
945 // signal to coverity that we really do want a 32-bit result
946 // coverity[overflow_before_widen:SUPPRESS]
947 dr = (uint64_t)((uint32_t)dr << s1); break;
948 case BPF_ALU | BPF_RSH | BPF_X:
949 case BPF_ALU | BPF_RSH | BPF_K: dr = (uint32_t)dr >> s1; break;
950 case BPF_ALU | BPF_XOR | BPF_X:
951 case BPF_ALU | BPF_XOR | BPF_K: dr = (uint32_t)(dr ^ s1); break;
952 case BPF_ALU | BPF_MUL | BPF_X:
953 case BPF_ALU | BPF_MUL | BPF_K: dr = (uint32_t)(dr * s1); break;
954 case BPF_ALU | BPF_MOV | BPF_X:
955 case BPF_ALU | BPF_MOV | BPF_K: dr = (uint32_t)s1; break;
956 case BPF_ALU | BPF_ARSH | BPF_X:
957 case BPF_ALU | BPF_ARSH | BPF_K: dr = (int32_t)dr >> s1; break;
958 case BPF_ALU | BPF_NEG: dr = -(uint32_t)sr; break;
959 case BPF_ALU | BPF_DIV | BPF_X:
960 case BPF_ALU | BPF_DIV | BPF_K:
961 if ((uint32_t)s1 == 0)
962 {
963 // TODO: Signal a proper error.
964 result = 0; goto cleanup;
965 }
966 dr = (uint32_t)dr / (uint32_t)s1;
967 break;
968 case BPF_ALU | BPF_MOD | BPF_X:
969 case BPF_ALU | BPF_MOD | BPF_K:
970 if ((uint32_t)s1 == 0)
971 {
972 // TODO: Signal a proper error.
973 result = 0; goto cleanup;
974 }
975 dr = (uint32_t)dr % (uint32_t)s1;
976 break;
977
978 case BPF_LD | BPF_IMM | BPF_DW:
979 switch (i->src_reg)
980 {
981 case 0:
982 dr = (uint32_t)si | ((uint64_t)i[1].imm << 32);
983 break;
984 case BPF_PSEUDO_MAP_FD:
985 if (si >= map_fds.size())
986 {
987 // TODO: Signal a proper error.
988 result = 0;
989 goto cleanup;
990 }
991 dr = si;
992 break;
993 default:
994 stapbpf_just_abort();
995 }
996 regs[i->dst_reg] = dr;
997 i += 2;
998 continue;
999
1000 case BPF_JMP | BPF_JEQ | BPF_X:
1001 case BPF_JMP | BPF_JEQ | BPF_K:
1002 if (dr == s1)
1003 goto dojmp;
1004 goto nowrite;
1005 case BPF_JMP | BPF_JNE | BPF_X:
1006 case BPF_JMP | BPF_JNE | BPF_K:
1007 if (dr != s1)
1008 goto dojmp;
1009 goto nowrite;
1010 case BPF_JMP | BPF_JGT | BPF_X:
1011 case BPF_JMP | BPF_JGT | BPF_K:
1012 if (dr > s1)
1013 goto dojmp;
1014 goto nowrite;
1015 case BPF_JMP | BPF_JGE | BPF_X:
1016 case BPF_JMP | BPF_JGE | BPF_K:
1017 if (dr >= s1)
1018 goto dojmp;
1019 goto nowrite;
1020 case BPF_JMP | BPF_JSGT | BPF_X:
1021 case BPF_JMP | BPF_JSGT | BPF_K:
1022 if ((int64_t)dr > (int64_t)s1)
1023 goto dojmp;
1024 goto nowrite;
1025 case BPF_JMP | BPF_JSGE | BPF_X:
1026 case BPF_JMP | BPF_JSGE | BPF_K:
1027 if ((int64_t)dr >= (int64_t)s1)
1028 goto dojmp;
1029 goto nowrite;
1030 case BPF_JMP | BPF_JSET | BPF_X:
1031 case BPF_JMP | BPF_JSET | BPF_K:
1032 if (dr & s1)
1033 goto dojmp;
1034 goto nowrite;
1035 case BPF_JMP | BPF_JA:
1036 dojmp:
1037 i += 1 + i->off;
1038 continue;
1039
1040 case BPF_JMP | BPF_CALL:
1041 switch (si)
1042 {
1043 case BPF_FUNC_map_lookup_elem:
1044 {
1045 // allocate correctly sized buffer and store it in map_values
1046 uint64_t *lookup_tmp = (uint64_t *)malloc(map_attrs[regs[1]].value_size);
1047 map_values.push_back(lookup_tmp);
1048
1049 int res = bpf_lookup_elem(map_fds[regs[1]], as_ptr(regs[2]),
1050 as_ptr(lookup_tmp));
1051
1052 if (res)
1053 // element could not be found
1054 dr = 0;
1055 else
1056 dr = as_int(lookup_tmp);
1057 }
1058 break;
1059 case BPF_FUNC_map_update_elem:
1060 dr = bpf_update_elem(map_fds[regs[1]], as_ptr(regs[2]),
1061 as_ptr(regs[3]), regs[4]);
1062 break;
1063 case BPF_FUNC_map_delete_elem:
1064 dr = bpf_delete_elem(map_fds[regs[1]], as_ptr(regs[2]));
1065 break;
1066 case BPF_FUNC_ktime_get_ns:
1067 dr = bpf_ktime_get_ns();
1068 break;
1069 case BPF_FUNC_perf_event_output:
1070 /* XXX ignored, but could be checked: regs[1], regs[2], regs[3] */
1071 tr = bpf_handle_transport_msg
1072 ((void *)regs[4], (size_t)regs[5], ctx);
1073 /* Normalize return value to match the helper API.
1074 XXX: May want to look at errno as well? */
1075 dr = (tr != LIBBPF_PERF_EVENT_ERROR) ? 0 : -1;
1076 break;
1077 case BPF_FUNC_trace_printk:
1078 /* XXX no longer need this code after PR22330 */
1079 #pragma GCC diagnostic push
1080 #pragma GCC diagnostic ignored "-Wformat-nonliteral"
1081 // regs[2] is the strlen(regs[1]) - not used by printf(3);
1082 // instead we assume regs[1] string is \0 terminated
1083 dr = fprintf(output_f, remove_tag(as_str(regs[1])).c_str(),
1084 /*regs[2],*/ regs[3], regs[4], regs[5]);
1085 fflush(output_f);
1086 #pragma GCC diagnostic pop
1087 break;
1088 case bpf::BPF_FUNC_sprintf:
1089 dr = bpf_sprintf(strings, as_str(regs[1]),
1090 regs[3], regs[4], regs[5]);
1091 break;
1092 case bpf::BPF_FUNC_text_str:
1093 dr = bpf_text_str(strings, as_str(regs[1]), false);
1094 break;
1095 case bpf::BPF_FUNC_string_quoted:
1096 dr = bpf_text_str(strings, as_str(regs[1]), true);
1097 break;
1098 case bpf::BPF_FUNC_str_concat:
1099 dr = bpf_str_concat(strings, as_str(regs[1]),
1100 as_str(regs[2]));
1101 break;
1102 case bpf::BPF_FUNC_map_get_next_key:
1103 dr = map_get_next_key(regs[1], regs[2], regs[3],
1104 regs[4], regs[5],
1105 ctx, foreach_ctxs[regs[1]],
1106 strings, map_values);
1107 break;
1108 case bpf::BPF_FUNC_stapbpf_stat_get:
1109 dr = stapbpf_stat_get((bpf::globals::agg_idx)regs[1], regs[2],
1110 bpf::globals::deintern_sc_type(regs[3]), ctx);
1111 break;
1112 case bpf::BPF_FUNC_gettimeofday_ns:
1113 dr = bpf_gettimeofday_ns();
1114 break;
1115 case bpf::BPF_FUNC_get_target:
1116 dr = bpf_get_target();
1117 break;
1118 case bpf::BPF_FUNC_set_procfs_value:
1119 dr = bpf_set_procfs_value(as_str(regs[1]), ctx);
1120 break;
1121 case bpf::BPF_FUNC_append_procfs_value:
1122 dr = bpf_append_procfs_value(as_str(regs[1]), ctx);
1123 break;
1124 case bpf::BPF_FUNC_get_procfs_value:
1125 dr = bpf_get_procfs_value(ctx);
1126 break;
1127 default:
1128 stapbpf_abort("unknown helper function");
1129 }
1130 regs[0] = dr;
1131 regs[1] = 0xea7bee75;
1132 regs[2] = 0xea7bee75;
1133 regs[3] = 0xea7bee75;
1134 regs[4] = 0xea7bee75;
1135 regs[5] = 0xea7bee75;
1136 goto nowrite;
1137
1138 case BPF_JMP | BPF_EXIT:
1139 result = regs[0];
1140 goto cleanup;
1141
1142 default:
1143 stapbpf_abort("unknown bpf opcode");
1144 }
1145
1146 regs[i->dst_reg] = dr;
1147 nowrite:
1148 i++;
1149 }
1150 result = 0;
1151 cleanup:
1152 for (uint64_t *ptr : map_values)
1153 free(ptr);
1154 map_values.clear(); // XXX: avoid double free
1155
1156 return result;
1157 }