/* bpfinterp.cxx - SystemTap BPF interpreter * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by * the Free Software Foundation; either version 2 of the License, or * (at your option) any later version. * * This program is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * GNU General Public License for more details. * * You should have received a copy of the GNU General Public License * along with this program. If not, see . * * Copyright (C) 2016-2021 Red Hat, Inc. * */ #include #include #include #include #include #include #include #include #include #include #include "bpfinterp.h" #include "libbpf.h" #include "../bpf-internal.h" #include "../staputil.h" #define stapbpf_abort(reason) \ ({ fprintf(stderr, _("bpfinterp.cxx:%d: %s\n"), \ __LINE__, (reason)); \ abort(); }) #define stapbpf_just_abort() stapbpf_abort("bpf userspace interpreter error") inline uintptr_t as_int(void *ptr) { return reinterpret_cast(ptr); } inline uintptr_t as_int(uint64_t *ptr) { return reinterpret_cast(ptr); } inline void * as_ptr(uintptr_t ptr) { return reinterpret_cast(ptr); } inline void * as_ptr(uint64_t *ptr) { return reinterpret_cast(ptr); } inline void * as_ptr(char *ptr) { return reinterpret_cast(ptr); } inline char * as_str(uintptr_t ptr) { return reinterpret_cast(ptr); } const std::string remove_tag(const char *fstr) { while (*(++fstr) != '>' && *fstr != '\0'); if (*fstr == '\0') return ""; // avoid segfault ++fstr; const char *end = fstr + strlen(fstr); while (*(--end) != '<' && end >= fstr); assert(end >= fstr); return std::string(fstr, end - fstr); } // Represents an in-progress foreach loop, including sorted keys/vals: struct foreach_state { uint64_t foreach_id; // Used to store and sort (col,key) or (val,key) pairs for // iteration, where col is a sort column of the key: std::deque> str_sorted; std::deque> int_sorted; // Used to track keys for deallocation by foreach_state_cleanup: std::vector keys; }; // The interpreter maintains a stack of foreach_states, // with each nested foreach loop pushing onto the stack: typedef std::vector foreach_stack; #define foreach_info bpf::globals::foreach_info void foreach_state_add(const foreach_info &fi, foreach_state &s, uint64_t *kp, uint64_t *colp, bool scalar_long) { // extract col_str, col_long from colp std::string col_str; bool use_str_col = false; int64_t col_long; //bool use_long_col = false; if (scalar_long) { // colp points to a scalar long value //use_long_col = true; col_long = *colp; } else if (fi.sort_column == 0 /* use_value */ || fi.sort_column_ofs == -1 /* !key_composite */) { // colp is a single string use_str_col = true; std::string col_str2((char *)colp, BPF_MAXSTRINGLEN); col_str = col_str2; } else { // colp contains a column at fi.sort_column_ofs colp = (uint64_t *)(((char *)colp)+fi.sort_column_ofs); if (fi.sort_column_size == BPF_MAXSTRINGLEN) { use_str_col = true; std::string col_str3((char *)colp, BPF_MAXSTRINGLEN); col_str = col_str3; } else { //use_long_col = true; col_long = *colp; } } // copy and save key uint64_t *kp2 = (uint64_t *)malloc(fi.keysize); memcpy(kp2, kp, fi.keysize); s.keys.push_back(kp2); // save (col,key) pair if (use_str_col) { std::pair item_str(col_str, kp2); s.str_sorted.push_back(item_str); } else // use_long_col { std::pair item_long(col_long, kp2); s.int_sorted.push_back(item_long); } } bool foreach_cmp_str(const std::pair &a, const std::pair &b) { return a.first < b.first; } bool foreach_cmp_int(const std::pair &a, const std::pair &b) { return a.first < b.first; } void foreach_state_sort(foreach_state &s) { if (!s.str_sorted.empty()) { std::stable_sort(s.str_sorted.begin(), s.str_sorted.end(), foreach_cmp_str); } if (!s.int_sorted.empty()) { std::stable_sort(s.int_sorted.begin(), s.int_sorted.end(), foreach_cmp_int); } } bool foreach_state_empty(const foreach_state &s) { return s.str_sorted.empty() && s.int_sorted.empty(); } void convert_key(const foreach_info &fi, uint64_t *kp, uint64_t *next_kp, std::vector &strings, std::vector &map_values) { bool scalar_long = fi.sort_column_ofs == -1 /* !key_composite */ && fi.keysize != BPF_MAXSTRINGLEN /* key_long */; bool scalar_str = fi.sort_column_ofs == -1 /* !key_composite */ && fi.keysize == BPF_MAXSTRINGLEN /* key_str */; if (scalar_long) { // handle scalar long keys being passed directly *next_kp = *kp; return; } if (scalar_str) { // handle string keys being passed as pointers // TODO: Could merge strings into map_values: std::string str((char*)kp, BPF_MAXSTRINGLEN); strings.push_back(str); *next_kp = reinterpret_cast(strings.back().c_str()); return; } // handle string composite keys being passed as pointers // allocate correctly sized buffer and store it in map_values: uint64_t *lookup_tmp = (uint64_t*)malloc(fi.keysize); memcpy(lookup_tmp, kp, fi.keysize); map_values.push_back(lookup_tmp); *next_kp = reinterpret_cast(map_values.back()); } template int _foreach_state_next(const foreach_info &fi, /* XXX foreach_state &s, */ std::deque> &sorted, int64_t key, int64_t next_key, std::vector &strings, std::vector &map_values) { (void)key; // XXX unused; see comment in foreach_state_next() if (sorted.empty()) return -1; if (fi.sort_direction > 0) { std::pair item = sorted.front(); convert_key(fi, item.second, (uint64_t *)next_key, strings, map_values); sorted.pop_front(); } else // sort_direction < 0 { std::pair item = sorted.back(); convert_key(fi, item.second, (uint64_t *)next_key, strings, map_values); sorted.pop_back(); } return 0; } int foreach_state_next(const foreach_info &fi, foreach_state &s, int64_t key, int64_t next_key, std::vector &strings, std::vector &map_values) { // TODO: Sanity check that we are continuing the same iteration? // Would require storing key in foreach_state. // // XXX Otherwise the code assumes that the BPF probe exactly follows // a foreach pattern, always passing the returned key as the next // key and not trying to 'start iteration in the middle'. if (!s.str_sorted.empty()) { return _foreach_state_next(fi, /* XXX s, */ s.str_sorted, key, next_key, strings, map_values); } if (!s.int_sorted.empty()) { return _foreach_state_next(fi, /* XXX s, */ s.int_sorted, key, next_key, strings, map_values); } return -1; } void foreach_state_cleanup(foreach_state &s) { for (uint64_t *ptr : s.keys) free(ptr); } // Wrapper for bpf_get_next_key that includes logic for accessing // keys in ascending or descending order, or // (PR23858) in ascending or descending order by value. int map_get_next_key(int fd_idx, int64_t key, int64_t next_key, uint64_t foreach_id, int64_t limit, bpf_transport_context *ctx, foreach_stack &foreach_ctx, std::vector &strings, std::vector &map_values) { int fd = (*ctx->map_fds)[fd_idx]; // Retrieve foreach loop info foreach_info fi; bool have_fi = !ctx->foreach_loop_info->empty() && ctx->foreach_loop_info->size() > foreach_id; fi.sort_column = 1; fi.sort_direction = 0; fi.keysize = 0; fi.sort_column_size = 0; fi.sort_column_ofs = 0; if (!have_fi) { // XXX Backwards compatibility for older .bo's using // sort_flags and no foreach_loop_info table. uint64_t sort_flags = foreach_id; fi.sort_column = GET_SORT_COLUMN(sort_flags); fi.sort_direction = GET_SORT_DIRECTION(sort_flags); // XXX The entire key is a single column. fi.keysize = ctx->map_attrs[fd_idx].key_size; fi.sort_column_size = fi.keysize; fi.sort_column_ofs = -1; // XXX use entire key } else { fi = ctx->foreach_loop_info->operator[](foreach_id); //assert(fi.keysize == ctx->map_attrs[fd_idx].key_size); // TODO PR24528: also handle s->sort_aggr for stat aggregates } //fprintf(stderr, "DEBUG called map_get_next_key fd=%d sort_column=%u sort_direction=%d key=%lx next_key=%lx limit=%ld\n", fd, fi.sort_column, fi.sort_direction, key, next_key, limit); // Identify type of key, value bool use_val = fi.sort_column == 0; bool key_composite = fi.sort_column_ofs != -1; bool key_long = false; bool key_str = false; if (!key_composite) { // XXX If the key has one column, it is either // a string (BPF_MAXSTRINGLEN) or a scalar long. key_long = fi.keysize != BPF_MAXSTRINGLEN; key_str = fi.keysize == BPF_MAXSTRINGLEN; } bool val_long = ctx->map_attrs[fd_idx].value_size != BPF_MAXSTRINGLEN; //bool val_str = !val_long; // Check iteration limit if (limit == 0) { // Final iteration, therefore foreach_ctx.back() is no longer needed // PR24811 att1: If key is not set, the map is empty and the // context wasn't created. Only pop with a nonzero key. // // XXX: A malformed .bo could still mess with this check // by issuing map_get_next_key(key=?,limit=0) calls // outside the usual foreach pattern. //if (!key) // return -1; // PR24811 att2: Check if the context for this foreach_id was created. // It might not have been if the limit is zero on the first call. if (foreach_ctx.empty() || foreach_ctx.back().foreach_id != foreach_id) return -1; foreach_state_cleanup(foreach_ctx.back()); foreach_ctx.pop_back(); return -1; } // Handle fi.sort_direction==0, where no foreach_ctx is needed if (fi.sort_direction == 0) { // handle scalar long values being passed directly if (key_long) return bpf_get_next_key(fd, as_ptr(key), as_ptr(next_key)); // handle string and composite keys being passed as pointers char _n[BPF_MAXKEYLEN_PLUS]; uint64_t *kp = key == 0x0 ? (uint64_t *)0x0 : *(uint64_t **)key; uint64_t *np = (uint64_t *)_n; int rc = bpf_get_next_key(fd, as_ptr(kp), as_ptr(np)); if (!rc && key_str) { // TODO: Could merge strings into map_values: std::string next_key2(_n, BPF_MAXSTRINGLEN); strings.push_back(next_key2); *(uint64_t *)next_key = reinterpret_cast(strings.back().c_str()); } else if (!rc) { // allocate correctly sized buffer and store it in map_values: uint64_t *lookup_tmp = (uint64_t*)malloc(fi.keysize); memcpy(lookup_tmp, _n, fi.keysize); map_values.push_back(lookup_tmp); *(uint64_t *)next_key = reinterpret_cast(map_values.back()); } return rc; } // Sort the map on initial iteration if (!key) { // handle both uint64_t and string column types char _k[BPF_MAXKEYLEN_PLUS], _n[BPF_MAXKEYLEN_PLUS]; _k[BPF_MAXSTRINGLEN] = _k[BPF_MAXKEYLEN] = '\0'; _n[BPF_MAXSTRINGLEN] = _n[BPF_MAXKEYLEN] = '\0'; uint64_t *kp = (uint64_t *)_k; uint64_t *np = (uint64_t *)_n; foreach_state s; int rc = bpf_get_next_key(fd, 0, as_ptr(np)); while (!rc) { if (use_val) { char _v[BPF_MAXKEYLEN_PLUS]; _v[BPF_MAXSTRINGLEN] = _v[BPF_MAXKEYLEN] = '\0'; uint64_t *vp = (uint64_t *)_v; int res = bpf_lookup_elem(fd, as_ptr(np), as_ptr(vp)); if (res) // element could not be found stapbpf_abort("bpf_map_get_next_key BUG: could not find key " \ "returned by bpf_get_next_key"); foreach_state_add(fi, s, np, vp, val_long); } else { // foreach_state_add extracts the column from np foreach_state_add(fi, s, np, np, key_long); } memcpy(kp, np, fi.keysize); rc = bpf_get_next_key(fd, as_ptr(kp), as_ptr(np)); } foreach_state_sort(s); if (foreach_state_empty(s)) return -1; foreach_ctx.push_back(s); } if (foreach_ctx.empty()) stapbpf_abort("bpf_map_get_next_key BUG: called outside a foreach loop"); // Get next value from sorted data int rc = foreach_state_next(fi, foreach_ctx.back(), key, next_key, strings, map_values); if (rc < 0) // no more elements to return { foreach_state_cleanup(foreach_ctx.back()); foreach_ctx.pop_back(); return -1; } return 0; } // TODO: Adapt to MAXPRINTFARGS == 32. uint64_t bpf_sprintf(std::vector &strings, char *fstr, uint64_t arg1, uint64_t arg2, uint64_t arg3) { char s[256]; // TODO: configure maximum length setting e.g. BPF_MAXSPRINTFLEN #pragma GCC diagnostic push #pragma GCC diagnostic ignored "-Wformat-nonliteral" snprintf(s, 256, fstr, arg1, arg2, arg3); #pragma GCC diagnostic pop std::string str(s, 256); strings.push_back(str); // Elements of "strings" should not be mutated to avoid // invalidating c_str() pointers. return reinterpret_cast(strings.back().c_str()); } uint64_t bpf_text_str(std::vector &strings, char* input, bool quoted) { std::string str(input); std::stringstream stream; for (std::string::iterator it = str.begin(); it != str.end(); ++it) { unsigned char c = *it; unsigned int i = c; if (std::isprint(c) && i < 128 && c != '\\' && c != '"') stream << c; else { stream << '\\'; switch (c) { case '\0': stream << '0'; break; // Not handled by translate_escapes case '\a': stream << 'a'; break; // Not handled by translate_escapes case '\b': stream << 'b'; break; // Not handled by translate_escapes case '\f': stream << 'f'; break; case '\n': stream << 'n'; break; case '\r': stream << 'r'; break; case '\t': stream << 't'; break; case '\v': stream << 'v'; break; case '"': stream << '"'; break; case '\\': stream << '\\'; break; default: stream << "x" << std::setfill('0') << std::setw(2) << std::hex << i; break; } } } if (quoted) strings.push_back("\"" + stream.str() + "\""); else strings.push_back(stream.str()); return reinterpret_cast(strings.back().c_str()); } uint64_t bpf_str_concat(std::vector &strings, char* left, char* right) { std::string concat; concat += left; concat += right; strings.push_back(concat); // After adding the new string into the vector, the c_str() pointer value // will be valid and safe to use return reinterpret_cast(strings.back().c_str()); } // Allocates and returns a buffer of percpu data for a stat field: uint64_t * stapbpf_stat_get_percpu(bpf::globals::map_idx map, uint64_t idx, bpf_transport_context *ctx) { uint64_t *ret = (uint64_t *)calloc(ctx->ncpus, sizeof(uint64_t)); int res = bpf_lookup_elem((*ctx->map_fds)[map], as_ptr(idx), ret); if (res) { // element could not be found free(ret); return 0; } else return ret; } // XXX Based on _stp_stat_get in runtime/stat.c. // There might be a clever way to avoid code duplication later, // but right now the code format is too different. Just reimplement. uint64_t stapbpf_stat_get(bpf::globals::agg_idx agg_id, uint64_t idx, stat_component_type sc_op, bpf_transport_context *ctx) { if (ctx->aggregates->find(agg_id) == ctx->aggregates->end()) stapbpf_abort("aggregate could not be found"); bpf::globals::stats_map sd = (*ctx->aggregates)[agg_id]; // XXX Based on struct stat_data in runtime/stat.h: struct stapbpf_stat_data { int shift; int64_t count; int64_t sum; int64_t avg_s; // TODO PR23476: Add more fields. } agg; // TODO: Consider caching each agg for the duration of userspace program execution. // Retrieve the fields that we are going to aggregate. // // XXX: This took a while to figure out. // bpf_map_lookup_elem() for percpu map returns an array. uint64_t *count_data = stapbpf_stat_get_percpu(sd["count"], idx, ctx); uint64_t *sum_data = stapbpf_stat_get_percpu(sd["sum"], idx, ctx); // TODO PR23476: Simplified code for now. agg.shift = 0; agg.count = 0; agg.sum = 0; // XXX for_each_possible_cpu(i) if (count_data) { for (unsigned i = 0; i < ctx->ncpus; i++) agg.count += count_data[i]; free(count_data); } // XXX for_each_possible_cpu(i) if (sum_data) { for (unsigned i = 0; i < ctx->ncpus; i++) agg.sum += sum_data[i]; free(sum_data); } // XXX Simplified version of _stp_div64(): if (agg.count == 0) agg.avg_s = 0; else agg.avg_s = (agg.sum << agg.shift) / agg.count; switch (sc_op) { case sc_average: if (agg.count == 0) stapbpf_abort("empty aggregate"); // TODO: Should produce proper error. return agg.avg_s; case sc_count: return agg.count; case sc_sum: return agg.sum; case sc_none: // should not happen, as sc_none is only used in foreach slots stapbpf_abort("unexpected sc_none"); // TODO PR23476: Not yet implemented. case sc_min: case sc_max: case sc_variance: default: stapbpf_abort("unsupported aggregate"); } } uint64_t bpf_ktime_get_ns() { struct timespec t; clock_gettime (CLOCK_BOOTTIME, &t); return (t.tv_sec * 1000000000) + t.tv_nsec; } uint64_t bpf_gettimeofday_ns() { struct timeval t; gettimeofday (&t, NULL); return (((t.tv_sec * 1000000) + t.tv_usec) * 1000); } uint64_t bpf_get_target() { return target_pid; } uint64_t bpf_set_procfs_value(char* msg, bpf_transport_context* ctx) { assert(msg != nullptr); ctx->procfs_msg = std::string(msg); return 0; } uint64_t bpf_append_procfs_value(char* msg, bpf_transport_context* ctx) { assert(msg != nullptr); ctx->procfs_msg.append(std::string(msg)); return 0; } uint64_t bpf_get_procfs_value(bpf_transport_context* ctx) { return (uint64_t) (ctx->procfs_msg.data()); } enum bpf_perf_event_ret bpf_handle_transport_msg(void *buf, size_t size, bpf_transport_context *ctx) { // Unpack transport message: struct bpf_transport_msg { BPF_TRANSPORT_VAL type; BPF_TRANSPORT_ARG content_start; }; bpf_transport_msg *_msg = (bpf_transport_msg *) buf; bpf::globals::perf_event_type msg_type = (bpf::globals::perf_event_type)_msg->type; void *msg_content = (void*)&_msg->content_start; size_t msg_size = size - sizeof(BPF_TRANSPORT_ARG); // Used for bpf::globals::STP_EXIT: int exit_key = bpf::globals::EXIT; long exit_val = 1; // Used for bpf::globals::STP_FORMAT_ARG: void *arg; switch (msg_type) { case bpf::globals::STP_STORE_ERROR_MSG: // Store error message for future printing. ctx->error_message.push(std::string((char*) msg_content)); break; case bpf::globals::STP_PRINT_ERROR_MSG: // Print error message that was stored previously. assert(!ctx->error_message.empty()); // TODO: Need better color configuration. std::cout << "\033[1m\033[31m" << "ERROR: " << "\033[0m" << ctx->error_message.front() << std::endl; ctx->error_message.pop(); break; case bpf::globals::STP_ERROR: // Signal an exit from the program and communicate a hard error: if (bpf_update_elem((*ctx->map_fds)[bpf::globals::internal_map_idx], &exit_key, &exit_val, BPF_ANY) != 0) stapbpf_abort("could not set exit status"); *ctx->error = true; return LIBBPF_PERF_EVENT_DONE; case bpf::globals::STP_EXIT: // Signal an exit from the program: if (bpf_update_elem((*ctx->map_fds)[bpf::globals::internal_map_idx], &exit_key, &exit_val, BPF_ANY) != 0) stapbpf_abort("could not set exit status"); return LIBBPF_PERF_EVENT_DONE; case bpf::globals::STP_PRINTF_START: if (ctx->in_printf) stapbpf_abort("printf already started"); if (msg_size != sizeof(BPF_TRANSPORT_ARG)) stapbpf_abort("wrong argument size"); ctx->in_printf = true; ctx->format_no = -1; ctx->expected_args = *(BPF_TRANSPORT_ARG*)msg_content; break; case bpf::globals::STP_PRINTF_END: if (!ctx->in_printf) stapbpf_abort("printf not started"); if (ctx->format_no < 0 || ctx->format_no >= (int)ctx->interned_strings->size()) stapbpf_abort("printf format is missing"); if (ctx->printf_args.size() != ctx->expected_args) stapbpf_abort("wrong number of printf args"); // TODO: Check this code on 32-bit systems after fixing PR24358. // // XXX: Surprisingly, it is not easy to pass an array to a // printf-type function. The best I can do for now is hardcode a // call to fprintf with BPF_MAXPRINTFARGS arguments: { std::string &format_str = (*ctx->interned_strings)[ctx->format_no]; void *fargs[BPF_MAXPRINTFARGS]; for (unsigned i = 0; i < BPF_MAXPRINTFARGS; i++) if (i < ctx->printf_args.size() && ctx->printf_arg_types[i] == bpf::globals::STP_PRINTF_ARG_LONG) fargs[i] = (void *)*(uint64_t*)ctx->printf_args[i]; else if (i < ctx->printf_args.size()) fargs[i] = ctx->printf_args[i]; else fargs[i] = NULL; assert(BPF_MAXPRINTFARGS == 32); // XXX: Change the fprintf() call if this changes. #pragma GCC diagnostic push #pragma GCC diagnostic ignored "-Wformat-nonliteral" fprintf(ctx->output_f, format_str.c_str(), fargs[0], fargs[1], fargs[2], fargs[3], fargs[4], fargs[5], fargs[6], fargs[7], fargs[8], fargs[9], fargs[10], fargs[11], fargs[12], fargs[13], fargs[14], fargs[15], fargs[16], fargs[17], fargs[18], fargs[19], fargs[20], fargs[21], fargs[22], fargs[23], fargs[24], fargs[25], fargs[26], fargs[27], fargs[28], fargs[29], fargs[30], fargs[31]); fflush(ctx->output_f); #pragma GCC diagnostic pop } // Deallocate accumulated format+args: ctx->in_printf = false; ctx->format_no = -1; for (unsigned i = 0; i < ctx->printf_args.size(); i++) free(ctx->printf_args[i]); ctx->printf_args.clear(); ctx->printf_arg_types.clear(); break; case bpf::globals::STP_PRINTF_FORMAT: if (!ctx->in_printf) stapbpf_abort("printf not started"); if (ctx->format_no != -1) stapbpf_abort("printf already has format"); if (msg_size != sizeof(BPF_TRANSPORT_ARG)) stapbpf_abort("wrong argument size"); ctx->format_no = *(BPF_TRANSPORT_ARG*)msg_content; break; // XXX: Could save spurious mallocs by storing ARG_LONG as the void * itself. case bpf::globals::STP_PRINTF_ARG_LONG: case bpf::globals::STP_PRINTF_ARG_STR: if (!ctx->in_printf) stapbpf_abort("printf not started"); arg = malloc(msg_size); memcpy(arg, msg_content, msg_size); ctx->printf_args.push_back(arg); ctx->printf_arg_types.push_back(msg_type); break; default: stapbpf_abort("unknown transport message"); } return LIBBPF_PERF_EVENT_CONT; } uint64_t bpf_interpret(size_t ninsns, const struct bpf_insn insns[], bpf_transport_context *ctx) { uint64_t result = 0; // return value uint64_t stack[65536 / 8]; // see MAX_BPF_USER_STACK in bpf-internal.h uint64_t regs[MAX_BPF_REG]; memset(regs, 0x0, sizeof(uint64_t) * MAX_BPF_REG); const struct bpf_insn *i = insns; static std::vector map_values; // Multiple threads accessing strings can cause concurrency issues for // procfs_probes. However, the procfs_lock should prevent this and thus, // clearing it on exit is unecessary for now. static std::vector strings; bpf_map_def *map_attrs = ctx->map_attrs; std::vector &map_fds = *ctx->map_fds; FILE *output_f = ctx->output_f; std::vector foreach_ctxs(map_fds.size()); map_values.clear(); // XXX: avoid double free regs[BPF_REG_10] = (uintptr_t)stack + sizeof(stack); while ((size_t)(i - insns) < ninsns) { uint64_t dr, sr, si, s1; bpf_perf_event_ret tr; dr = regs[i->dst_reg]; sr = regs[i->src_reg]; si = i->imm; s1 = i->code & BPF_X ? sr : si; switch (i->code) { case BPF_LDX | BPF_MEM | BPF_B: dr = *(uint8_t *)((uintptr_t)sr + i->off); break; case BPF_LDX | BPF_MEM | BPF_H: dr = *(uint16_t *)((uintptr_t)sr + i->off); break; case BPF_LDX | BPF_MEM | BPF_W: dr = *(uint32_t *)((uintptr_t)sr + i->off); break; case BPF_LDX | BPF_MEM | BPF_DW: dr = *(uint64_t *)((uintptr_t)sr + i->off); break; case BPF_ST | BPF_MEM | BPF_B: sr = si; /* Fallthrough */ case BPF_STX | BPF_MEM | BPF_B: *(uint8_t *)((uintptr_t)dr + i->off) = sr; goto nowrite; case BPF_ST | BPF_MEM | BPF_H: sr = si; /* Fallthrough */ case BPF_STX | BPF_MEM | BPF_H: *(uint16_t *)((uintptr_t)dr + i->off) = sr; goto nowrite; case BPF_ST | BPF_MEM | BPF_W: sr = si; /* Fallthrough */ case BPF_STX | BPF_MEM | BPF_W: *(uint32_t *)((uintptr_t)dr + i->off) = sr; goto nowrite; case BPF_ST | BPF_MEM | BPF_DW: sr = si; /* Fallthrough */ case BPF_STX | BPF_MEM | BPF_DW: *(uint64_t *)((uintptr_t)dr + i->off) = sr; goto nowrite; case BPF_ALU64 | BPF_ADD | BPF_X: case BPF_ALU64 | BPF_ADD | BPF_K: dr += s1; break; case BPF_ALU64 | BPF_SUB | BPF_X: case BPF_ALU64 | BPF_SUB | BPF_K: dr -= s1; break; case BPF_ALU64 | BPF_AND | BPF_X: case BPF_ALU64 | BPF_AND | BPF_K: dr &= s1; break; case BPF_ALU64 | BPF_OR | BPF_X: case BPF_ALU64 | BPF_OR | BPF_K: dr |= s1; break; case BPF_ALU64 | BPF_LSH | BPF_X: case BPF_ALU64 | BPF_LSH | BPF_K: dr <<= s1; break; case BPF_ALU64 | BPF_RSH | BPF_X: case BPF_ALU64 | BPF_RSH | BPF_K: dr >>= s1; break; case BPF_ALU64 | BPF_XOR | BPF_X: case BPF_ALU64 | BPF_XOR | BPF_K: dr ^= s1; break; case BPF_ALU64 | BPF_MUL | BPF_X: case BPF_ALU64 | BPF_MUL | BPF_K: dr *= s1; break; case BPF_ALU64 | BPF_MOV | BPF_X: case BPF_ALU64 | BPF_MOV | BPF_K: dr = s1; break; case BPF_ALU64 | BPF_ARSH | BPF_X: case BPF_ALU64 | BPF_ARSH | BPF_K: dr = (int64_t)dr >> s1; break; case BPF_ALU64 | BPF_NEG: dr = -sr; break; case BPF_ALU64 | BPF_DIV | BPF_X: case BPF_ALU64 | BPF_DIV | BPF_K: if (s1 == 0) { // TODO: Signal a proper error. result = 0; goto cleanup; } dr /= s1; break; case BPF_ALU64 | BPF_MOD | BPF_X: case BPF_ALU64 | BPF_MOD | BPF_K: if (s1 == 0) { // TODO: Signal a proper error. result = 0; goto cleanup; } dr %= s1; break; case BPF_ALU | BPF_ADD | BPF_X: case BPF_ALU | BPF_ADD | BPF_K: dr = (uint32_t)(dr + s1); break; case BPF_ALU | BPF_SUB | BPF_X: case BPF_ALU | BPF_SUB | BPF_K: dr = (uint32_t)(dr - s1); break; case BPF_ALU | BPF_AND | BPF_X: case BPF_ALU | BPF_AND | BPF_K: dr = (uint32_t)(dr & s1); break; case BPF_ALU | BPF_OR | BPF_X: case BPF_ALU | BPF_OR | BPF_K: dr = (uint32_t)(dr | s1); break; case BPF_ALU | BPF_LSH | BPF_X: case BPF_ALU | BPF_LSH | BPF_K: // signal to coverity that we really do want a 32-bit result // coverity[overflow_before_widen:SUPPRESS] dr = (uint64_t)((uint32_t)dr << s1); break; case BPF_ALU | BPF_RSH | BPF_X: case BPF_ALU | BPF_RSH | BPF_K: dr = (uint32_t)dr >> s1; break; case BPF_ALU | BPF_XOR | BPF_X: case BPF_ALU | BPF_XOR | BPF_K: dr = (uint32_t)(dr ^ s1); break; case BPF_ALU | BPF_MUL | BPF_X: case BPF_ALU | BPF_MUL | BPF_K: dr = (uint32_t)(dr * s1); break; case BPF_ALU | BPF_MOV | BPF_X: case BPF_ALU | BPF_MOV | BPF_K: dr = (uint32_t)s1; break; case BPF_ALU | BPF_ARSH | BPF_X: case BPF_ALU | BPF_ARSH | BPF_K: dr = (int32_t)dr >> s1; break; case BPF_ALU | BPF_NEG: dr = -(uint32_t)sr; break; case BPF_ALU | BPF_DIV | BPF_X: case BPF_ALU | BPF_DIV | BPF_K: if ((uint32_t)s1 == 0) { // TODO: Signal a proper error. result = 0; goto cleanup; } dr = (uint32_t)dr / (uint32_t)s1; break; case BPF_ALU | BPF_MOD | BPF_X: case BPF_ALU | BPF_MOD | BPF_K: if ((uint32_t)s1 == 0) { // TODO: Signal a proper error. result = 0; goto cleanup; } dr = (uint32_t)dr % (uint32_t)s1; break; case BPF_LD | BPF_IMM | BPF_DW: switch (i->src_reg) { case 0: dr = (uint32_t)si | ((uint64_t)i[1].imm << 32); break; case BPF_PSEUDO_MAP_FD: if (si >= map_fds.size()) { // TODO: Signal a proper error. result = 0; goto cleanup; } dr = si; break; default: stapbpf_just_abort(); } regs[i->dst_reg] = dr; i += 2; continue; case BPF_JMP | BPF_JEQ | BPF_X: case BPF_JMP | BPF_JEQ | BPF_K: if (dr == s1) goto dojmp; goto nowrite; case BPF_JMP | BPF_JNE | BPF_X: case BPF_JMP | BPF_JNE | BPF_K: if (dr != s1) goto dojmp; goto nowrite; case BPF_JMP | BPF_JGT | BPF_X: case BPF_JMP | BPF_JGT | BPF_K: if (dr > s1) goto dojmp; goto nowrite; case BPF_JMP | BPF_JGE | BPF_X: case BPF_JMP | BPF_JGE | BPF_K: if (dr >= s1) goto dojmp; goto nowrite; case BPF_JMP | BPF_JSGT | BPF_X: case BPF_JMP | BPF_JSGT | BPF_K: if ((int64_t)dr > (int64_t)s1) goto dojmp; goto nowrite; case BPF_JMP | BPF_JSGE | BPF_X: case BPF_JMP | BPF_JSGE | BPF_K: if ((int64_t)dr >= (int64_t)s1) goto dojmp; goto nowrite; case BPF_JMP | BPF_JSET | BPF_X: case BPF_JMP | BPF_JSET | BPF_K: if (dr & s1) goto dojmp; goto nowrite; case BPF_JMP | BPF_JA: dojmp: i += 1 + i->off; continue; case BPF_JMP | BPF_CALL: switch (si) { case BPF_FUNC_map_lookup_elem: { // allocate correctly sized buffer and store it in map_values uint64_t *lookup_tmp = (uint64_t *)malloc(map_attrs[regs[1]].value_size); map_values.push_back(lookup_tmp); int res = bpf_lookup_elem(map_fds[regs[1]], as_ptr(regs[2]), as_ptr(lookup_tmp)); if (res) // element could not be found dr = 0; else dr = as_int(lookup_tmp); } break; case BPF_FUNC_map_update_elem: dr = bpf_update_elem(map_fds[regs[1]], as_ptr(regs[2]), as_ptr(regs[3]), regs[4]); break; case BPF_FUNC_map_delete_elem: dr = bpf_delete_elem(map_fds[regs[1]], as_ptr(regs[2])); break; case BPF_FUNC_ktime_get_ns: dr = bpf_ktime_get_ns(); break; case BPF_FUNC_perf_event_output: /* XXX ignored, but could be checked: regs[1], regs[2], regs[3] */ tr = bpf_handle_transport_msg ((void *)regs[4], (size_t)regs[5], ctx); /* Normalize return value to match the helper API. XXX: May want to look at errno as well? */ dr = (tr != LIBBPF_PERF_EVENT_ERROR) ? 0 : -1; break; case BPF_FUNC_trace_printk: /* XXX no longer need this code after PR22330 */ #pragma GCC diagnostic push #pragma GCC diagnostic ignored "-Wformat-nonliteral" // regs[2] is the strlen(regs[1]) - not used by printf(3); // instead we assume regs[1] string is \0 terminated dr = fprintf(output_f, remove_tag(as_str(regs[1])).c_str(), /*regs[2],*/ regs[3], regs[4], regs[5]); fflush(output_f); #pragma GCC diagnostic pop break; case bpf::BPF_FUNC_sprintf: dr = bpf_sprintf(strings, as_str(regs[1]), regs[3], regs[4], regs[5]); break; case bpf::BPF_FUNC_text_str: dr = bpf_text_str(strings, as_str(regs[1]), false); break; case bpf::BPF_FUNC_string_quoted: dr = bpf_text_str(strings, as_str(regs[1]), true); break; case bpf::BPF_FUNC_str_concat: dr = bpf_str_concat(strings, as_str(regs[1]), as_str(regs[2])); break; case bpf::BPF_FUNC_map_get_next_key: dr = map_get_next_key(regs[1], regs[2], regs[3], regs[4], regs[5], ctx, foreach_ctxs[regs[1]], strings, map_values); break; case bpf::BPF_FUNC_stapbpf_stat_get: dr = stapbpf_stat_get((bpf::globals::agg_idx)regs[1], regs[2], bpf::globals::deintern_sc_type(regs[3]), ctx); break; case bpf::BPF_FUNC_gettimeofday_ns: dr = bpf_gettimeofday_ns(); break; case bpf::BPF_FUNC_get_target: dr = bpf_get_target(); break; case bpf::BPF_FUNC_set_procfs_value: dr = bpf_set_procfs_value(as_str(regs[1]), ctx); break; case bpf::BPF_FUNC_append_procfs_value: dr = bpf_append_procfs_value(as_str(regs[1]), ctx); break; case bpf::BPF_FUNC_get_procfs_value: dr = bpf_get_procfs_value(ctx); break; default: stapbpf_abort("unknown helper function"); } regs[0] = dr; regs[1] = 0xea7bee75; regs[2] = 0xea7bee75; regs[3] = 0xea7bee75; regs[4] = 0xea7bee75; regs[5] = 0xea7bee75; goto nowrite; case BPF_JMP | BPF_EXIT: result = regs[0]; goto cleanup; default: stapbpf_abort("unknown bpf opcode"); } regs[i->dst_reg] = dr; nowrite: i++; } result = 0; cleanup: for (uint64_t *ptr : map_values) free(ptr); map_values.clear(); // XXX: avoid double free return result; }