| 1 |
// SPDX-License-Identifier: (LGPL-2.1 OR BSD-2-Clause) |
| 2 |
/* Copyright (c) 2022 Meta Platforms, Inc. and affiliates. */ |
| 3 |
#include <ctype.h> |
| 4 |
#include <stdio.h> |
| 5 |
#include <stdlib.h> |
| 6 |
#include <string.h> |
| 7 |
#include <libelf.h> |
| 8 |
#include <gelf.h> |
| 9 |
#include <unistd.h> |
| 10 |
#include <linux/ptrace.h> |
| 11 |
#include <linux/kernel.h> |
| 12 |
|
| 13 |
/* s8 will be marked as poison while it's a reg of riscv */ |
| 14 |
#if defined(__riscv) |
| 15 |
#define rv_s8 s8 |
| 16 |
#endif |
| 17 |
|
| 18 |
#include "bpf.h" |
| 19 |
#include "libbpf.h" |
| 20 |
#include "libbpf_common.h" |
| 21 |
#include "libbpf_internal.h" |
| 22 |
#include "hashmap.h" |
| 23 |
|
| 24 |
/* libbpf's USDT support consists of BPF-side state/code and user-space |
| 25 |
* state/code working together in concert. BPF-side parts are defined in |
| 26 |
* usdt.bpf.h header library. User-space state is encapsulated by struct |
| 27 |
* usdt_manager and all the supporting code centered around usdt_manager. |
| 28 |
* |
| 29 |
* usdt.bpf.h defines two BPF maps that usdt_manager expects: USDT spec map |
| 30 |
* and IP-to-spec-ID map, which is auxiliary map necessary for kernels that |
| 31 |
* don't support BPF cookie (see below). These two maps are implicitly |
| 32 |
* embedded into user's end BPF object file when user's code included |
| 33 |
* usdt.bpf.h. This means that libbpf doesn't do anything special to create |
| 34 |
* these USDT support maps. They are created by normal libbpf logic of |
| 35 |
* instantiating BPF maps when opening and loading BPF object. |
| 36 |
* |
| 37 |
* As such, libbpf is basically unaware of the need to do anything |
| 38 |
* USDT-related until the very first call to bpf_program__attach_usdt(), which |
| 39 |
* can be called by user explicitly or happen automatically during skeleton |
| 40 |
* attach (or, equivalently, through generic bpf_program__attach() call). At |
| 41 |
* this point, libbpf will instantiate and initialize struct usdt_manager and |
| 42 |
* store it in bpf_object. USDT manager is per-BPF object construct, as each |
| 43 |
* independent BPF object might or might not have USDT programs, and thus all |
| 44 |
* the expected USDT-related state. There is no coordination between two |
| 45 |
* bpf_object in parts of USDT attachment, they are oblivious of each other's |
| 46 |
* existence and libbpf is just oblivious, dealing with bpf_object-specific |
| 47 |
* USDT state. |
| 48 |
* |
| 49 |
* Quick crash course on USDTs. |
| 50 |
* |
| 51 |
* From user-space application's point of view, USDT is essentially just |
| 52 |
* a slightly special function call that normally has zero overhead, unless it |
| 53 |
* is being traced by some external entity (e.g, BPF-based tool). Here's how |
| 54 |
* a typical application can trigger USDT probe: |
| 55 |
* |
| 56 |
* #include <sys/sdt.h> // provided by systemtap-sdt-devel package |
| 57 |
* // folly also provide similar functionality in folly/tracing/StaticTracepoint.h |
| 58 |
* |
| 59 |
* STAP_PROBE3(my_usdt_provider, my_usdt_probe_name, 123, x, &y); |
| 60 |
* |
| 61 |
* USDT is identified by its <provider-name>:<probe-name> pair of names. Each |
| 62 |
* individual USDT has a fixed number of arguments (3 in the above example) |
| 63 |
* and specifies values of each argument as if it was a function call. |
| 64 |
* |
| 65 |
* USDT call is actually not a function call, but is instead replaced by |
| 66 |
* a single NOP instruction (thus zero overhead, effectively). But in addition |
| 67 |
* to that, those USDT macros generate special SHT_NOTE ELF records in |
| 68 |
* .note.stapsdt ELF section. Here's an example USDT definition as emitted by |
| 69 |
* `readelf -n <binary>`: |
| 70 |
* |
| 71 |
* stapsdt 0x00000089 NT_STAPSDT (SystemTap probe descriptors) |
| 72 |
* Provider: test |
| 73 |
* Name: usdt12 |
| 74 |
* Location: 0x0000000000549df3, Base: 0x00000000008effa4, Semaphore: 0x0000000000a4606e |
| 75 |
* Arguments: -4@-1204(%rbp) -4@%edi -8@-1216(%rbp) -8@%r8 -4@$5 -8@%r9 8@%rdx 8@%r10 -4@$-9 -2@%cx -2@%ax -1@%sil |
| 76 |
* |
| 77 |
* In this case we have USDT test:usdt12 with 12 arguments. |
| 78 |
* |
| 79 |
* Location and base are offsets used to calculate absolute IP address of that |
| 80 |
* NOP instruction that kernel can replace with an interrupt instruction to |
| 81 |
* trigger instrumentation code (BPF program for all that we care about). |
| 82 |
* |
| 83 |
* Semaphore above is an optional feature. It records an address of a 2-byte |
| 84 |
* refcount variable (normally in '.probes' ELF section) used for signaling if |
| 85 |
* there is anything that is attached to USDT. This is useful for user |
| 86 |
* applications if, for example, they need to prepare some arguments that are |
| 87 |
* passed only to USDTs and preparation is expensive. By checking if USDT is |
| 88 |
* "activated", an application can avoid paying those costs unnecessarily. |
| 89 |
* Recent enough kernel has built-in support for automatically managing this |
| 90 |
* refcount, which libbpf expects and relies on. If USDT is defined without |
| 91 |
* associated semaphore, this value will be zero. See selftests for semaphore |
| 92 |
* examples. |
| 93 |
* |
| 94 |
* Arguments is the most interesting part. This USDT specification string is |
| 95 |
* providing information about all the USDT arguments and their locations. The |
| 96 |
* part before @ sign defined byte size of the argument (1, 2, 4, or 8) and |
| 97 |
* whether the argument is signed or unsigned (negative size means signed). |
| 98 |
* The part after @ sign is assembly-like definition of argument location |
| 99 |
* (see [0] for more details). Technically, assembler can provide some pretty |
| 100 |
* advanced definitions, but libbpf is currently supporting three most common |
| 101 |
* cases: |
| 102 |
* 1) immediate constant, see 5th and 9th args above (-4@$5 and -4@-9); |
| 103 |
* 2) register value, e.g., 8@%rdx, which means "unsigned 8-byte integer |
| 104 |
* whose value is in register %rdx"; |
| 105 |
* 3) memory dereference addressed by register, e.g., -4@-1204(%rbp), which |
| 106 |
* specifies signed 32-bit integer stored at offset -1204 bytes from |
| 107 |
* memory address stored in %rbp. |
| 108 |
* |
| 109 |
* [0] https://sourceware.org/systemtap/wiki/UserSpaceProbeImplementation |
| 110 |
* |
| 111 |
* During attachment, libbpf parses all the relevant USDT specifications and |
| 112 |
* prepares `struct usdt_spec` (USDT spec), which is then provided to BPF-side |
| 113 |
* code through spec map. This allows BPF applications to quickly fetch the |
| 114 |
* actual value at runtime using a simple BPF-side code. |
| 115 |
* |
| 116 |
* With basics out of the way, let's go over less immediately obvious aspects |
| 117 |
* of supporting USDTs. |
| 118 |
* |
| 119 |
* First, there is no special USDT BPF program type. It is actually just |
| 120 |
* a uprobe BPF program (which for kernel, at least currently, is just a kprobe |
| 121 |
* program, so BPF_PROG_TYPE_KPROBE program type). With the only difference |
| 122 |
* that uprobe is usually attached at the function entry, while USDT will |
| 123 |
* normally be somewhere inside the function. But it should always be |
| 124 |
* pointing to NOP instruction, which makes such uprobes the fastest uprobe |
| 125 |
* kind. |
| 126 |
* |
| 127 |
* Second, it's important to realize that such STAP_PROBEn(provider, name, ...) |
| 128 |
* macro invocations can end up being inlined many-many times, depending on |
| 129 |
* specifics of each individual user application. So single conceptual USDT |
| 130 |
* (identified by provider:name pair of identifiers) is, generally speaking, |
| 131 |
* multiple uprobe locations (USDT call sites) in different places in user |
| 132 |
* application. Further, again due to inlining, each USDT call site might end |
| 133 |
* up having the same argument #N be located in a different place. In one call |
| 134 |
* site it could be a constant, in another will end up in a register, and in |
| 135 |
* yet another could be some other register or even somewhere on the stack. |
| 136 |
* |
| 137 |
* As such, "attaching to USDT" means (in general case) attaching the same |
| 138 |
* uprobe BPF program to multiple target locations in user application, each |
| 139 |
* potentially having a completely different USDT spec associated with it. |
| 140 |
* To wire all this up together libbpf allocates a unique integer spec ID for |
| 141 |
* each unique USDT spec. Spec IDs are allocated as sequential small integers |
| 142 |
* so that they can be used as keys in array BPF map (for performance reasons). |
| 143 |
* Spec ID allocation and accounting is big part of what usdt_manager is |
| 144 |
* about. This state has to be maintained per-BPF object and coordinate |
| 145 |
* between different USDT attachments within the same BPF object. |
| 146 |
* |
| 147 |
* Spec ID is the key in spec BPF map, value is the actual USDT spec layed out |
| 148 |
* as struct usdt_spec. Each invocation of BPF program at runtime needs to |
| 149 |
* know its associated spec ID. It gets it either through BPF cookie, which |
| 150 |
* libbpf sets to spec ID during attach time, or, if kernel is too old to |
| 151 |
* support BPF cookie, through IP-to-spec-ID map that libbpf maintains in such |
| 152 |
* case. The latter means that some modes of operation can't be supported |
| 153 |
* without BPF cookie. Such a mode is attaching to shared library "generically", |
| 154 |
* without specifying target process. In such case, it's impossible to |
| 155 |
* calculate absolute IP addresses for IP-to-spec-ID map, and thus such mode |
| 156 |
* is not supported without BPF cookie support. |
| 157 |
* |
| 158 |
* Note that libbpf is using BPF cookie functionality for its own internal |
| 159 |
* needs, so user itself can't rely on BPF cookie feature. To that end, libbpf |
| 160 |
* provides conceptually equivalent USDT cookie support. It's still u64 |
| 161 |
* user-provided value that can be associated with USDT attachment. Note that |
| 162 |
* this will be the same value for all USDT call sites within the same single |
| 163 |
* *logical* USDT attachment. This makes sense because to user attaching to |
| 164 |
* USDT is a single BPF program triggered for singular USDT probe. The fact |
| 165 |
* that this is done at multiple actual locations is a mostly hidden |
| 166 |
* implementation details. This USDT cookie value can be fetched with |
| 167 |
* bpf_usdt_cookie(ctx) API provided by usdt.bpf.h |
| 168 |
* |
| 169 |
* Lastly, while single USDT can have tons of USDT call sites, it doesn't |
| 170 |
* necessarily have that many different USDT specs. It very well might be |
| 171 |
* that 1000 USDT call sites only need 5 different USDT specs, because all the |
| 172 |
* arguments are typically contained in a small set of registers or stack |
| 173 |
* locations. As such, it's wasteful to allocate as many USDT spec IDs as |
| 174 |
* there are USDT call sites. So libbpf tries to be frugal and performs |
| 175 |
* on-the-fly deduplication during a single USDT attachment to only allocate |
| 176 |
* the minimal required amount of unique USDT specs (and thus spec IDs). This |
| 177 |
* is trivially achieved by using USDT spec string (Arguments string from USDT |
| 178 |
* note) as a lookup key in a hashmap. USDT spec string uniquely defines |
| 179 |
* everything about how to fetch USDT arguments, so two USDT call sites |
| 180 |
* sharing USDT spec string can safely share the same USDT spec and spec ID. |
| 181 |
* Note, this spec string deduplication is happening only during the same USDT |
| 182 |
* attachment, so each USDT spec shares the same USDT cookie value. This is |
| 183 |
* not generally true for other USDT attachments within the same BPF object, |
| 184 |
* as even if USDT spec string is the same, USDT cookie value can be |
| 185 |
* different. It was deemed excessive to try to deduplicate across independent |
| 186 |
* USDT attachments by taking into account USDT spec string *and* USDT cookie |
| 187 |
* value, which would complicate spec ID accounting significantly for little |
| 188 |
* gain. |
| 189 |
*/ |
| 190 |
|
| 191 |
#define USDT_BASE_SEC ".stapsdt.base" |
| 192 |
#define USDT_SEMA_SEC ".probes" |
| 193 |
#define USDT_NOTE_SEC ".note.stapsdt" |
| 194 |
#define USDT_NOTE_TYPE 3 |
| 195 |
#define USDT_NOTE_NAME "stapsdt" |
| 196 |
|
| 197 |
/* should match exactly enum __bpf_usdt_arg_type from usdt.bpf.h */ |
| 198 |
enum usdt_arg_type { |
| 199 |
USDT_ARG_CONST, |
| 200 |
USDT_ARG_REG, |
| 201 |
USDT_ARG_REG_DEREF, |
| 202 |
USDT_ARG_SIB, |
| 203 |
}; |
| 204 |
|
| 205 |
/* should match exactly struct __bpf_usdt_arg_spec from usdt.bpf.h */ |
| 206 |
struct usdt_arg_spec { |
| 207 |
__u64 val_off; |
| 208 |
#if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__ |
| 209 |
enum usdt_arg_type arg_type: 8; |
| 210 |
__u16 idx_reg_off: 12; |
| 211 |
__u16 scale_bitshift: 4; |
| 212 |
__u8 __reserved: 8; /* keep reg_off offset stable */ |
| 213 |
#else |
| 214 |
__u8 __reserved: 8; /* keep reg_off offset stable */ |
| 215 |
__u16 idx_reg_off: 12; |
| 216 |
__u16 scale_bitshift: 4; |
| 217 |
enum usdt_arg_type arg_type: 8; |
| 218 |
#endif |
| 219 |
short reg_off; |
| 220 |
bool arg_signed; |
| 221 |
char arg_bitshift; |
| 222 |
}; |
| 223 |
|
| 224 |
/* should match BPF_USDT_MAX_ARG_CNT in usdt.bpf.h */ |
| 225 |
#define USDT_MAX_ARG_CNT 12 |
| 226 |
|
| 227 |
/* should match struct __bpf_usdt_spec from usdt.bpf.h */ |
| 228 |
struct usdt_spec { |
| 229 |
struct usdt_arg_spec args[USDT_MAX_ARG_CNT]; |
| 230 |
__u64 usdt_cookie; |
| 231 |
short arg_cnt; |
| 232 |
}; |
| 233 |
|
| 234 |
struct usdt_note { |
| 235 |
const char *provider; |
| 236 |
const char *name; |
| 237 |
/* USDT args specification string, e.g.: |
| 238 |
* "-4@%esi -4@-24(%rbp) -4@%ecx 2@%ax 8@%rdx" |
| 239 |
*/ |
| 240 |
const char *args; |
| 241 |
long loc_addr; |
| 242 |
long base_addr; |
| 243 |
long sema_addr; |
| 244 |
}; |
| 245 |
|
| 246 |
struct usdt_target { |
| 247 |
long abs_ip; |
| 248 |
long rel_ip; |
| 249 |
long sema_off; |
| 250 |
struct usdt_spec spec; |
| 251 |
const char *spec_str; |
| 252 |
}; |
| 253 |
|
| 254 |
struct usdt_manager { |
| 255 |
struct bpf_map *specs_map; |
| 256 |
struct bpf_map *ip_to_spec_id_map; |
| 257 |
|
| 258 |
int *free_spec_ids; |
| 259 |
size_t free_spec_cnt; |
| 260 |
size_t next_free_spec_id; |
| 261 |
|
| 262 |
bool has_bpf_cookie; |
| 263 |
bool has_sema_refcnt; |
| 264 |
bool has_uprobe_multi; |
| 265 |
}; |
| 266 |
|
| 267 |
struct usdt_manager *usdt_manager_new(struct bpf_object *obj) |
| 268 |
{ |
| 269 |
static const char *ref_ctr_sysfs_path = "/sys/bus/event_source/devices/uprobe/format/ref_ctr_offset"; |
| 270 |
struct usdt_manager *man; |
| 271 |
struct bpf_map *specs_map, *ip_to_spec_id_map; |
| 272 |
|
| 273 |
specs_map = bpf_object__find_map_by_name(obj, "__bpf_usdt_specs"); |
| 274 |
ip_to_spec_id_map = bpf_object__find_map_by_name(obj, "__bpf_usdt_ip_to_spec_id"); |
| 275 |
if (!specs_map || !ip_to_spec_id_map) { |
| 276 |
pr_warn("usdt: failed to find USDT support BPF maps, did you forget to include bpf/usdt.bpf.h?\n"); |
| 277 |
return ERR_PTR(-ESRCH); |
| 278 |
} |
| 279 |
|
| 280 |
man = calloc(1, sizeof(*man)); |
| 281 |
if (!man) |
| 282 |
return ERR_PTR(-ENOMEM); |
| 283 |
|
| 284 |
man->specs_map = specs_map; |
| 285 |
man->ip_to_spec_id_map = ip_to_spec_id_map; |
| 286 |
|
| 287 |
/* Detect if BPF cookie is supported for kprobes. |
| 288 |
* We don't need IP-to-ID mapping if we can use BPF cookies. |
| 289 |
* Added in: 7adfc6c9b315 ("bpf: Add bpf_get_attach_cookie() BPF helper to access bpf_cookie value") |
| 290 |
*/ |
| 291 |
man->has_bpf_cookie = kernel_supports(obj, FEAT_BPF_COOKIE); |
| 292 |
|
| 293 |
/* Detect kernel support for automatic refcounting of USDT semaphore. |
| 294 |
* If this is not supported, USDTs with semaphores will not be supported. |
| 295 |
* Added in: a6ca88b241d5 ("trace_uprobe: support reference counter in fd-based uprobe") |
| 296 |
*/ |
| 297 |
man->has_sema_refcnt = faccessat(AT_FDCWD, ref_ctr_sysfs_path, F_OK, AT_EACCESS) == 0; |
| 298 |
|
| 299 |
/* |
| 300 |
* Detect kernel support for uprobe multi link to be used for attaching |
| 301 |
* usdt probes. |
| 302 |
*/ |
| 303 |
man->has_uprobe_multi = kernel_supports(obj, FEAT_UPROBE_MULTI_LINK); |
| 304 |
return man; |
| 305 |
} |
| 306 |
|
| 307 |
void usdt_manager_free(struct usdt_manager *man) |
| 308 |
{ |
| 309 |
if (IS_ERR_OR_NULL(man)) |
| 310 |
return; |
| 311 |
|
| 312 |
free(man->free_spec_ids); |
| 313 |
free(man); |
| 314 |
} |
| 315 |
|
| 316 |
static int sanity_check_usdt_elf(Elf *elf, const char *path) |
| 317 |
{ |
| 318 |
GElf_Ehdr ehdr; |
| 319 |
int endianness; |
| 320 |
|
| 321 |
if (elf_kind(elf) != ELF_K_ELF) { |
| 322 |
pr_warn("usdt: unrecognized ELF kind %d for '%s'\n", elf_kind(elf), path); |
| 323 |
return -EBADF; |
| 324 |
} |
| 325 |
|
| 326 |
switch (gelf_getclass(elf)) { |
| 327 |
case ELFCLASS64: |
| 328 |
if (sizeof(void *) != 8) { |
| 329 |
pr_warn("usdt: attaching to 64-bit ELF binary '%s' is not supported\n", path); |
| 330 |
return -EBADF; |
| 331 |
} |
| 332 |
break; |
| 333 |
case ELFCLASS32: |
| 334 |
if (sizeof(void *) != 4) { |
| 335 |
pr_warn("usdt: attaching to 32-bit ELF binary '%s' is not supported\n", path); |
| 336 |
return -EBADF; |
| 337 |
} |
| 338 |
break; |
| 339 |
default: |
| 340 |
pr_warn("usdt: unsupported ELF class for '%s'\n", path); |
| 341 |
return -EBADF; |
| 342 |
} |
| 343 |
|
| 344 |
if (!gelf_getehdr(elf, &ehdr)) |
| 345 |
return -EINVAL; |
| 346 |
|
| 347 |
if (ehdr.e_type != ET_EXEC && ehdr.e_type != ET_DYN) { |
| 348 |
pr_warn("usdt: unsupported type of ELF binary '%s' (%d), only ET_EXEC and ET_DYN are supported\n", |
| 349 |
path, ehdr.e_type); |
| 350 |
return -EBADF; |
| 351 |
} |
| 352 |
|
| 353 |
#if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__ |
| 354 |
endianness = ELFDATA2LSB; |
| 355 |
#elif __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ |
| 356 |
endianness = ELFDATA2MSB; |
| 357 |
#else |
| 358 |
# error "Unrecognized __BYTE_ORDER__" |
| 359 |
#endif |
| 360 |
if (endianness != ehdr.e_ident[EI_DATA]) { |
| 361 |
pr_warn("usdt: ELF endianness mismatch for '%s'\n", path); |
| 362 |
return -EBADF; |
| 363 |
} |
| 364 |
|
| 365 |
return 0; |
| 366 |
} |
| 367 |
|
| 368 |
static int find_elf_sec_by_name(Elf *elf, const char *sec_name, GElf_Shdr *shdr, Elf_Scn **scn) |
| 369 |
{ |
| 370 |
Elf_Scn *sec = NULL; |
| 371 |
size_t shstrndx; |
| 372 |
|
| 373 |
if (elf_getshdrstrndx(elf, &shstrndx)) |
| 374 |
return -EINVAL; |
| 375 |
|
| 376 |
/* check if ELF is corrupted and avoid calling elf_strptr if yes */ |
| 377 |
if (!elf_rawdata(elf_getscn(elf, shstrndx), NULL)) |
| 378 |
return -EINVAL; |
| 379 |
|
| 380 |
while ((sec = elf_nextscn(elf, sec)) != NULL) { |
| 381 |
char *name; |
| 382 |
|
| 383 |
if (!gelf_getshdr(sec, shdr)) |
| 384 |
return -EINVAL; |
| 385 |
|
| 386 |
name = elf_strptr(elf, shstrndx, shdr->sh_name); |
| 387 |
if (name && strcmp(sec_name, name) == 0) { |
| 388 |
*scn = sec; |
| 389 |
return 0; |
| 390 |
} |
| 391 |
} |
| 392 |
|
| 393 |
return -ENOENT; |
| 394 |
} |
| 395 |
|
| 396 |
struct elf_seg { |
| 397 |
long start; |
| 398 |
long end; |
| 399 |
long offset; |
| 400 |
bool is_exec; |
| 401 |
}; |
| 402 |
|
| 403 |
static int cmp_elf_segs(const void *_a, const void *_b) |
| 404 |
{ |
| 405 |
const struct elf_seg *a = _a; |
| 406 |
const struct elf_seg *b = _b; |
| 407 |
|
| 408 |
return a->start < b->start ? -1 : 1; |
| 409 |
} |
| 410 |
|
| 411 |
static int parse_elf_segs(Elf *elf, const char *path, struct elf_seg **segs, size_t *seg_cnt) |
| 412 |
{ |
| 413 |
GElf_Phdr phdr; |
| 414 |
size_t n; |
| 415 |
int i, err; |
| 416 |
struct elf_seg *seg; |
| 417 |
void *tmp; |
| 418 |
|
| 419 |
*seg_cnt = 0; |
| 420 |
|
| 421 |
if (elf_getphdrnum(elf, &n)) { |
| 422 |
err = -errno; |
| 423 |
return err; |
| 424 |
} |
| 425 |
|
| 426 |
for (i = 0; i < n; i++) { |
| 427 |
if (!gelf_getphdr(elf, i, &phdr)) { |
| 428 |
err = -errno; |
| 429 |
return err; |
| 430 |
} |
| 431 |
|
| 432 |
pr_debug("usdt: discovered PHDR #%d in '%s': vaddr 0x%lx memsz 0x%lx offset 0x%lx type 0x%lx flags 0x%lx\n", |
| 433 |
i, path, (long)phdr.p_vaddr, (long)phdr.p_memsz, (long)phdr.p_offset, |
| 434 |
(long)phdr.p_type, (long)phdr.p_flags); |
| 435 |
if (phdr.p_type != PT_LOAD) |
| 436 |
continue; |
| 437 |
|
| 438 |
tmp = libbpf_reallocarray(*segs, *seg_cnt + 1, sizeof(**segs)); |
| 439 |
if (!tmp) |
| 440 |
return -ENOMEM; |
| 441 |
|
| 442 |
*segs = tmp; |
| 443 |
seg = *segs + *seg_cnt; |
| 444 |
(*seg_cnt)++; |
| 445 |
|
| 446 |
seg->start = phdr.p_vaddr; |
| 447 |
seg->end = phdr.p_vaddr + phdr.p_memsz; |
| 448 |
seg->offset = phdr.p_offset; |
| 449 |
seg->is_exec = phdr.p_flags & PF_X; |
| 450 |
} |
| 451 |
|
| 452 |
if (*seg_cnt == 0) { |
| 453 |
pr_warn("usdt: failed to find PT_LOAD program headers in '%s'\n", path); |
| 454 |
return -ESRCH; |
| 455 |
} |
| 456 |
|
| 457 |
qsort(*segs, *seg_cnt, sizeof(**segs), cmp_elf_segs); |
| 458 |
return 0; |
| 459 |
} |
| 460 |
|
| 461 |
static int parse_vma_segs(int pid, const char *lib_path, struct elf_seg **segs, size_t *seg_cnt) |
| 462 |
{ |
| 463 |
char path[PATH_MAX], line[PATH_MAX], mode[16]; |
| 464 |
size_t seg_start, seg_end, seg_off; |
| 465 |
struct elf_seg *seg; |
| 466 |
int tmp_pid, i, err; |
| 467 |
FILE *f; |
| 468 |
|
| 469 |
*seg_cnt = 0; |
| 470 |
|
| 471 |
/* Handle containerized binaries only accessible from |
| 472 |
* /proc/<pid>/root/<path>. They will be reported as just /<path> in |
| 473 |
* /proc/<pid>/maps. |
| 474 |
*/ |
| 475 |
if (sscanf(lib_path, "/proc/%d/root%s", &tmp_pid, path) == 2 && pid == tmp_pid) |
| 476 |
goto proceed; |
| 477 |
|
| 478 |
if (!realpath(lib_path, path)) { |
| 479 |
pr_warn("usdt: failed to get absolute path of '%s' (err %s), using path as is...\n", |
| 480 |
lib_path, errstr(-errno)); |
| 481 |
libbpf_strlcpy(path, lib_path, sizeof(path)); |
| 482 |
} |
| 483 |
|
| 484 |
proceed: |
| 485 |
sprintf(line, "/proc/%d/maps", pid); |
| 486 |
f = fopen(line, "re"); |
| 487 |
if (!f) { |
| 488 |
err = -errno; |
| 489 |
pr_warn("usdt: failed to open '%s' to get base addr of '%s': %s\n", |
| 490 |
line, lib_path, errstr(err)); |
| 491 |
return err; |
| 492 |
} |
| 493 |
|
| 494 |
/* We need to handle lines with no path at the end: |
| 495 |
* |
| 496 |
* 7f5c6f5d1000-7f5c6f5d3000 rw-p 001c7000 08:04 21238613 /usr/lib64/libc-2.17.so |
| 497 |
* 7f5c6f5d3000-7f5c6f5d8000 rw-p 00000000 00:00 0 |
| 498 |
* 7f5c6f5d8000-7f5c6f5d9000 r-xp 00000000 103:01 362990598 /data/users/andriin/linux/tools/bpf/usdt/libhello_usdt.so |
| 499 |
*/ |
| 500 |
while (fscanf(f, "%zx-%zx %s %zx %*s %*d%[^\n]\n", |
| 501 |
&seg_start, &seg_end, mode, &seg_off, line) == 5) { |
| 502 |
void *tmp; |
| 503 |
|
| 504 |
/* to handle no path case (see above) we need to capture line |
| 505 |
* without skipping any whitespaces. So we need to strip |
| 506 |
* leading whitespaces manually here |
| 507 |
*/ |
| 508 |
i = 0; |
| 509 |
while (isblank(line[i])) |
| 510 |
i++; |
| 511 |
if (strcmp(line + i, path) != 0) |
| 512 |
continue; |
| 513 |
|
| 514 |
pr_debug("usdt: discovered segment for lib '%s': addrs %zx-%zx mode %s offset %zx\n", |
| 515 |
path, seg_start, seg_end, mode, seg_off); |
| 516 |
|
| 517 |
/* ignore non-executable sections for shared libs */ |
| 518 |
if (mode[2] != 'x') |
| 519 |
continue; |
| 520 |
|
| 521 |
tmp = libbpf_reallocarray(*segs, *seg_cnt + 1, sizeof(**segs)); |
| 522 |
if (!tmp) { |
| 523 |
err = -ENOMEM; |
| 524 |
goto err_out; |
| 525 |
} |
| 526 |
|
| 527 |
*segs = tmp; |
| 528 |
seg = *segs + *seg_cnt; |
| 529 |
*seg_cnt += 1; |
| 530 |
|
| 531 |
seg->start = seg_start; |
| 532 |
seg->end = seg_end; |
| 533 |
seg->offset = seg_off; |
| 534 |
seg->is_exec = true; |
| 535 |
} |
| 536 |
|
| 537 |
if (*seg_cnt == 0) { |
| 538 |
pr_warn("usdt: failed to find '%s' (resolved to '%s') within PID %d memory mappings\n", |
| 539 |
lib_path, path, pid); |
| 540 |
err = -ESRCH; |
| 541 |
goto err_out; |
| 542 |
} |
| 543 |
|
| 544 |
qsort(*segs, *seg_cnt, sizeof(**segs), cmp_elf_segs); |
| 545 |
err = 0; |
| 546 |
err_out: |
| 547 |
fclose(f); |
| 548 |
return err; |
| 549 |
} |
| 550 |
|
| 551 |
static struct elf_seg *find_elf_seg(struct elf_seg *segs, size_t seg_cnt, long virtaddr) |
| 552 |
{ |
| 553 |
struct elf_seg *seg; |
| 554 |
int i; |
| 555 |
|
| 556 |
/* for ELF binaries (both executables and shared libraries), we are |
| 557 |
* given virtual address (absolute for executables, relative for |
| 558 |
* libraries) which should match address range of [seg_start, seg_end) |
| 559 |
*/ |
| 560 |
for (i = 0, seg = segs; i < seg_cnt; i++, seg++) { |
| 561 |
if (seg->start <= virtaddr && virtaddr < seg->end) |
| 562 |
return seg; |
| 563 |
} |
| 564 |
return NULL; |
| 565 |
} |
| 566 |
|
| 567 |
static struct elf_seg *find_vma_seg(struct elf_seg *segs, size_t seg_cnt, long offset) |
| 568 |
{ |
| 569 |
struct elf_seg *seg; |
| 570 |
int i; |
| 571 |
|
| 572 |
/* for VMA segments from /proc/<pid>/maps file, provided "address" is |
| 573 |
* actually a file offset, so should be fall within logical |
| 574 |
* offset-based range of [offset_start, offset_end) |
| 575 |
*/ |
| 576 |
for (i = 0, seg = segs; i < seg_cnt; i++, seg++) { |
| 577 |
if (seg->offset <= offset && offset < seg->offset + (seg->end - seg->start)) |
| 578 |
return seg; |
| 579 |
} |
| 580 |
return NULL; |
| 581 |
} |
| 582 |
|
| 583 |
static int parse_usdt_note(GElf_Nhdr *nhdr, const char *data, size_t name_off, |
| 584 |
size_t desc_off, struct usdt_note *usdt_note); |
| 585 |
|
| 586 |
static int parse_usdt_spec(struct usdt_spec *spec, const struct usdt_note *note, __u64 usdt_cookie); |
| 587 |
|
| 588 |
static int collect_usdt_targets(struct usdt_manager *man, Elf *elf, const char *path, pid_t pid, |
| 589 |
const char *usdt_provider, const char *usdt_name, __u64 usdt_cookie, |
| 590 |
struct usdt_target **out_targets, size_t *out_target_cnt) |
| 591 |
{ |
| 592 |
size_t off, name_off, desc_off, seg_cnt = 0, vma_seg_cnt = 0, target_cnt = 0; |
| 593 |
struct elf_seg *segs = NULL, *vma_segs = NULL; |
| 594 |
struct usdt_target *targets = NULL, *target; |
| 595 |
long base_addr = 0; |
| 596 |
Elf_Scn *notes_scn, *base_scn; |
| 597 |
GElf_Shdr base_shdr, notes_shdr; |
| 598 |
GElf_Ehdr ehdr; |
| 599 |
GElf_Nhdr nhdr; |
| 600 |
Elf_Data *data; |
| 601 |
int err; |
| 602 |
|
| 603 |
*out_targets = NULL; |
| 604 |
*out_target_cnt = 0; |
| 605 |
|
| 606 |
err = find_elf_sec_by_name(elf, USDT_NOTE_SEC, ¬es_shdr, ¬es_scn); |
| 607 |
if (err) { |
| 608 |
pr_warn("usdt: no USDT notes section (%s) found in '%s'\n", USDT_NOTE_SEC, path); |
| 609 |
return err; |
| 610 |
} |
| 611 |
|
| 612 |
if (notes_shdr.sh_type != SHT_NOTE || !gelf_getehdr(elf, &ehdr)) { |
| 613 |
pr_warn("usdt: invalid USDT notes section (%s) in '%s'\n", USDT_NOTE_SEC, path); |
| 614 |
return -EINVAL; |
| 615 |
} |
| 616 |
|
| 617 |
err = parse_elf_segs(elf, path, &segs, &seg_cnt); |
| 618 |
if (err) { |
| 619 |
pr_warn("usdt: failed to process ELF program segments for '%s': %s\n", |
| 620 |
path, errstr(err)); |
| 621 |
goto err_out; |
| 622 |
} |
| 623 |
|
| 624 |
/* .stapsdt.base ELF section is optional, but is used for prelink |
| 625 |
* offset compensation (see a big comment further below) |
| 626 |
*/ |
| 627 |
if (find_elf_sec_by_name(elf, USDT_BASE_SEC, &base_shdr, &base_scn) == 0) |
| 628 |
base_addr = base_shdr.sh_addr; |
| 629 |
|
| 630 |
data = elf_getdata(notes_scn, 0); |
| 631 |
off = 0; |
| 632 |
while ((off = gelf_getnote(data, off, &nhdr, &name_off, &desc_off)) > 0) { |
| 633 |
long usdt_abs_ip, usdt_rel_ip, usdt_sema_off = 0; |
| 634 |
struct usdt_note note; |
| 635 |
struct elf_seg *seg = NULL; |
| 636 |
void *tmp; |
| 637 |
|
| 638 |
err = parse_usdt_note(&nhdr, data->d_buf, name_off, desc_off, ¬e); |
| 639 |
if (err) |
| 640 |
goto err_out; |
| 641 |
|
| 642 |
if (strcmp(note.provider, usdt_provider) != 0 || strcmp(note.name, usdt_name) != 0) |
| 643 |
continue; |
| 644 |
|
| 645 |
/* We need to compensate "prelink effect". See [0] for details, |
| 646 |
* relevant parts quoted here: |
| 647 |
* |
| 648 |
* Each SDT probe also expands into a non-allocated ELF note. You can |
| 649 |
* find this by looking at SHT_NOTE sections and decoding the format; |
| 650 |
* see below for details. Because the note is non-allocated, it means |
| 651 |
* there is no runtime cost, and also preserved in both stripped files |
| 652 |
* and .debug files. |
| 653 |
* |
| 654 |
* However, this means that prelink won't adjust the note's contents |
| 655 |
* for address offsets. Instead, this is done via the .stapsdt.base |
| 656 |
* section. This is a special section that is added to the text. We |
| 657 |
* will only ever have one of these sections in a final link and it |
| 658 |
* will only ever be one byte long. Nothing about this section itself |
| 659 |
* matters, we just use it as a marker to detect prelink address |
| 660 |
* adjustments. |
| 661 |
* |
| 662 |
* Each probe note records the link-time address of the .stapsdt.base |
| 663 |
* section alongside the probe PC address. The decoder compares the |
| 664 |
* base address stored in the note with the .stapsdt.base section's |
| 665 |
* sh_addr. Initially these are the same, but the section header will |
| 666 |
* be adjusted by prelink. So the decoder applies the difference to |
| 667 |
* the probe PC address to get the correct prelinked PC address; the |
| 668 |
* same adjustment is applied to the semaphore address, if any. |
| 669 |
* |
| 670 |
* [0] https://sourceware.org/systemtap/wiki/UserSpaceProbeImplementation |
| 671 |
*/ |
| 672 |
usdt_abs_ip = note.loc_addr; |
| 673 |
if (base_addr && note.base_addr) |
| 674 |
usdt_abs_ip += base_addr - note.base_addr; |
| 675 |
|
| 676 |
/* When attaching uprobes (which is what USDTs basically are) |
| 677 |
* kernel expects file offset to be specified, not a relative |
| 678 |
* virtual address, so we need to translate virtual address to |
| 679 |
* file offset, for both ET_EXEC and ET_DYN binaries. |
| 680 |
*/ |
| 681 |
seg = find_elf_seg(segs, seg_cnt, usdt_abs_ip); |
| 682 |
if (!seg) { |
| 683 |
err = -ESRCH; |
| 684 |
pr_warn("usdt: failed to find ELF program segment for '%s:%s' in '%s' at IP 0x%lx\n", |
| 685 |
usdt_provider, usdt_name, path, usdt_abs_ip); |
| 686 |
goto err_out; |
| 687 |
} |
| 688 |
if (!seg->is_exec) { |
| 689 |
err = -ESRCH; |
| 690 |
pr_warn("usdt: matched ELF binary '%s' segment [0x%lx, 0x%lx) for '%s:%s' at IP 0x%lx is not executable\n", |
| 691 |
path, seg->start, seg->end, usdt_provider, usdt_name, |
| 692 |
usdt_abs_ip); |
| 693 |
goto err_out; |
| 694 |
} |
| 695 |
/* translate from virtual address to file offset */ |
| 696 |
usdt_rel_ip = usdt_abs_ip - seg->start + seg->offset; |
| 697 |
|
| 698 |
if (ehdr.e_type == ET_DYN && !man->has_bpf_cookie) { |
| 699 |
/* If we don't have BPF cookie support but need to |
| 700 |
* attach to a shared library, we'll need to know and |
| 701 |
* record absolute addresses of attach points due to |
| 702 |
* the need to lookup USDT spec by absolute IP of |
| 703 |
* triggered uprobe. Doing this resolution is only |
| 704 |
* possible when we have a specific PID of the process |
| 705 |
* that's using specified shared library. BPF cookie |
| 706 |
* removes the absolute address limitation as we don't |
| 707 |
* need to do this lookup (we just use BPF cookie as |
| 708 |
* an index of USDT spec), so for newer kernels with |
| 709 |
* BPF cookie support libbpf supports USDT attachment |
| 710 |
* to shared libraries with no PID filter. |
| 711 |
*/ |
| 712 |
if (pid < 0) { |
| 713 |
pr_warn("usdt: attaching to shared libraries without specific PID is not supported on current kernel\n"); |
| 714 |
err = -ENOTSUP; |
| 715 |
goto err_out; |
| 716 |
} |
| 717 |
|
| 718 |
/* vma_segs are lazily initialized only if necessary */ |
| 719 |
if (vma_seg_cnt == 0) { |
| 720 |
err = parse_vma_segs(pid, path, &vma_segs, &vma_seg_cnt); |
| 721 |
if (err) { |
| 722 |
pr_warn("usdt: failed to get memory segments in PID %d for shared library '%s': %s\n", |
| 723 |
pid, path, errstr(err)); |
| 724 |
goto err_out; |
| 725 |
} |
| 726 |
} |
| 727 |
|
| 728 |
seg = find_vma_seg(vma_segs, vma_seg_cnt, usdt_rel_ip); |
| 729 |
if (!seg) { |
| 730 |
err = -ESRCH; |
| 731 |
pr_warn("usdt: failed to find shared lib memory segment for '%s:%s' in '%s' at relative IP 0x%lx\n", |
| 732 |
usdt_provider, usdt_name, path, usdt_rel_ip); |
| 733 |
goto err_out; |
| 734 |
} |
| 735 |
|
| 736 |
usdt_abs_ip = seg->start - seg->offset + usdt_rel_ip; |
| 737 |
} |
| 738 |
|
| 739 |
pr_debug("usdt: probe for '%s:%s' in %s '%s': addr 0x%lx base 0x%lx (resolved abs_ip 0x%lx rel_ip 0x%lx) args '%s' in segment [0x%lx, 0x%lx) at offset 0x%lx\n", |
| 740 |
usdt_provider, usdt_name, ehdr.e_type == ET_EXEC ? "exec" : "lib ", path, |
| 741 |
note.loc_addr, note.base_addr, usdt_abs_ip, usdt_rel_ip, note.args, |
| 742 |
seg ? seg->start : 0, seg ? seg->end : 0, seg ? seg->offset : 0); |
| 743 |
|
| 744 |
/* Adjust semaphore address to be a file offset */ |
| 745 |
if (note.sema_addr) { |
| 746 |
if (!man->has_sema_refcnt) { |
| 747 |
pr_warn("usdt: kernel doesn't support USDT semaphore refcounting for '%s:%s' in '%s'\n", |
| 748 |
usdt_provider, usdt_name, path); |
| 749 |
err = -ENOTSUP; |
| 750 |
goto err_out; |
| 751 |
} |
| 752 |
|
| 753 |
seg = find_elf_seg(segs, seg_cnt, note.sema_addr); |
| 754 |
if (!seg) { |
| 755 |
err = -ESRCH; |
| 756 |
pr_warn("usdt: failed to find ELF loadable segment with semaphore of '%s:%s' in '%s' at 0x%lx\n", |
| 757 |
usdt_provider, usdt_name, path, note.sema_addr); |
| 758 |
goto err_out; |
| 759 |
} |
| 760 |
if (seg->is_exec) { |
| 761 |
err = -ESRCH; |
| 762 |
pr_warn("usdt: matched ELF binary '%s' segment [0x%lx, 0x%lx] for semaphore of '%s:%s' at 0x%lx is executable\n", |
| 763 |
path, seg->start, seg->end, usdt_provider, usdt_name, |
| 764 |
note.sema_addr); |
| 765 |
goto err_out; |
| 766 |
} |
| 767 |
|
| 768 |
usdt_sema_off = note.sema_addr - seg->start + seg->offset; |
| 769 |
|
| 770 |
pr_debug("usdt: sema for '%s:%s' in %s '%s': addr 0x%lx base 0x%lx (resolved 0x%lx) in segment [0x%lx, 0x%lx] at offset 0x%lx\n", |
| 771 |
usdt_provider, usdt_name, ehdr.e_type == ET_EXEC ? "exec" : "lib ", |
| 772 |
path, note.sema_addr, note.base_addr, usdt_sema_off, |
| 773 |
seg->start, seg->end, seg->offset); |
| 774 |
} |
| 775 |
|
| 776 |
/* Record adjusted addresses and offsets and parse USDT spec */ |
| 777 |
tmp = libbpf_reallocarray(targets, target_cnt + 1, sizeof(*targets)); |
| 778 |
if (!tmp) { |
| 779 |
err = -ENOMEM; |
| 780 |
goto err_out; |
| 781 |
} |
| 782 |
targets = tmp; |
| 783 |
|
| 784 |
target = &targets[target_cnt]; |
| 785 |
memset(target, 0, sizeof(*target)); |
| 786 |
|
| 787 |
target->abs_ip = usdt_abs_ip; |
| 788 |
target->rel_ip = usdt_rel_ip; |
| 789 |
target->sema_off = usdt_sema_off; |
| 790 |
|
| 791 |
/* notes.args references strings from ELF itself, so they can |
| 792 |
* be referenced safely until elf_end() call |
| 793 |
*/ |
| 794 |
target->spec_str = note.args; |
| 795 |
|
| 796 |
err = parse_usdt_spec(&target->spec, ¬e, usdt_cookie); |
| 797 |
if (err) |
| 798 |
goto err_out; |
| 799 |
|
| 800 |
target_cnt++; |
| 801 |
} |
| 802 |
|
| 803 |
*out_targets = targets; |
| 804 |
*out_target_cnt = target_cnt; |
| 805 |
err = target_cnt; |
| 806 |
|
| 807 |
err_out: |
| 808 |
free(segs); |
| 809 |
free(vma_segs); |
| 810 |
if (err < 0) |
| 811 |
free(targets); |
| 812 |
return err; |
| 813 |
} |
| 814 |
|
| 815 |
struct bpf_link_usdt { |
| 816 |
struct bpf_link link; |
| 817 |
|
| 818 |
struct usdt_manager *usdt_man; |
| 819 |
|
| 820 |
size_t spec_cnt; |
| 821 |
int *spec_ids; |
| 822 |
|
| 823 |
size_t uprobe_cnt; |
| 824 |
struct { |
| 825 |
long abs_ip; |
| 826 |
struct bpf_link *link; |
| 827 |
} *uprobes; |
| 828 |
|
| 829 |
struct bpf_link *multi_link; |
| 830 |
}; |
| 831 |
|
| 832 |
static int bpf_link_usdt_detach(struct bpf_link *link) |
| 833 |
{ |
| 834 |
struct bpf_link_usdt *usdt_link = container_of(link, struct bpf_link_usdt, link); |
| 835 |
struct usdt_manager *man = usdt_link->usdt_man; |
| 836 |
int i; |
| 837 |
|
| 838 |
bpf_link__destroy(usdt_link->multi_link); |
| 839 |
|
| 840 |
/* When having multi_link, uprobe_cnt is 0 */ |
| 841 |
for (i = 0; i < usdt_link->uprobe_cnt; i++) { |
| 842 |
/* detach underlying uprobe link */ |
| 843 |
bpf_link__destroy(usdt_link->uprobes[i].link); |
| 844 |
/* there is no need to update specs map because it will be |
| 845 |
* unconditionally overwritten on subsequent USDT attaches, |
| 846 |
* but if BPF cookies are not used we need to remove entry |
| 847 |
* from ip_to_spec_id map, otherwise we'll run into false |
| 848 |
* conflicting IP errors |
| 849 |
*/ |
| 850 |
if (!man->has_bpf_cookie) { |
| 851 |
/* not much we can do about errors here */ |
| 852 |
(void)bpf_map_delete_elem(bpf_map__fd(man->ip_to_spec_id_map), |
| 853 |
&usdt_link->uprobes[i].abs_ip); |
| 854 |
} |
| 855 |
} |
| 856 |
|
| 857 |
/* try to return the list of previously used spec IDs to usdt_manager |
| 858 |
* for future reuse for subsequent USDT attaches |
| 859 |
*/ |
| 860 |
if (!man->free_spec_ids) { |
| 861 |
/* if there were no free spec IDs yet, just transfer our IDs */ |
| 862 |
man->free_spec_ids = usdt_link->spec_ids; |
| 863 |
man->free_spec_cnt = usdt_link->spec_cnt; |
| 864 |
usdt_link->spec_ids = NULL; |
| 865 |
} else { |
| 866 |
/* otherwise concat IDs */ |
| 867 |
size_t new_cnt = man->free_spec_cnt + usdt_link->spec_cnt; |
| 868 |
int *new_free_ids; |
| 869 |
|
| 870 |
new_free_ids = libbpf_reallocarray(man->free_spec_ids, new_cnt, |
| 871 |
sizeof(*new_free_ids)); |
| 872 |
/* If we couldn't resize free_spec_ids, we'll just leak |
| 873 |
* a bunch of free IDs; this is very unlikely to happen and if |
| 874 |
* system is so exhausted on memory, it's the least of user's |
| 875 |
* concerns, probably. |
| 876 |
* So just do our best here to return those IDs to usdt_manager. |
| 877 |
* Another edge case when we can legitimately get NULL is when |
| 878 |
* new_cnt is zero, which can happen in some edge cases, so we |
| 879 |
* need to be careful about that. |
| 880 |
*/ |
| 881 |
if (new_free_ids || new_cnt == 0) { |
| 882 |
memcpy(new_free_ids + man->free_spec_cnt, usdt_link->spec_ids, |
| 883 |
usdt_link->spec_cnt * sizeof(*usdt_link->spec_ids)); |
| 884 |
man->free_spec_ids = new_free_ids; |
| 885 |
man->free_spec_cnt = new_cnt; |
| 886 |
} |
| 887 |
} |
| 888 |
|
| 889 |
return 0; |
| 890 |
} |
| 891 |
|
| 892 |
static void bpf_link_usdt_dealloc(struct bpf_link *link) |
| 893 |
{ |
| 894 |
struct bpf_link_usdt *usdt_link = container_of(link, struct bpf_link_usdt, link); |
| 895 |
|
| 896 |
free(usdt_link->spec_ids); |
| 897 |
free(usdt_link->uprobes); |
| 898 |
free(usdt_link); |
| 899 |
} |
| 900 |
|
| 901 |
static size_t specs_hash_fn(long key, void *ctx) |
| 902 |
{ |
| 903 |
return str_hash((char *)key); |
| 904 |
} |
| 905 |
|
| 906 |
static bool specs_equal_fn(long key1, long key2, void *ctx) |
| 907 |
{ |
| 908 |
return strcmp((char *)key1, (char *)key2) == 0; |
| 909 |
} |
| 910 |
|
| 911 |
static int allocate_spec_id(struct usdt_manager *man, struct hashmap *specs_hash, |
| 912 |
struct bpf_link_usdt *link, struct usdt_target *target, |
| 913 |
int *spec_id, bool *is_new) |
| 914 |
{ |
| 915 |
long tmp; |
| 916 |
void *new_ids; |
| 917 |
int err; |
| 918 |
|
| 919 |
/* check if we already allocated spec ID for this spec string */ |
| 920 |
if (hashmap__find(specs_hash, target->spec_str, &tmp)) { |
| 921 |
*spec_id = tmp; |
| 922 |
*is_new = false; |
| 923 |
return 0; |
| 924 |
} |
| 925 |
|
| 926 |
/* otherwise it's a new ID that needs to be set up in specs map and |
| 927 |
* returned back to usdt_manager when USDT link is detached |
| 928 |
*/ |
| 929 |
new_ids = libbpf_reallocarray(link->spec_ids, link->spec_cnt + 1, sizeof(*link->spec_ids)); |
| 930 |
if (!new_ids) |
| 931 |
return -ENOMEM; |
| 932 |
link->spec_ids = new_ids; |
| 933 |
|
| 934 |
/* get next free spec ID, giving preference to free list, if not empty */ |
| 935 |
if (man->free_spec_cnt) { |
| 936 |
*spec_id = man->free_spec_ids[man->free_spec_cnt - 1]; |
| 937 |
|
| 938 |
/* cache spec ID for current spec string for future lookups */ |
| 939 |
err = hashmap__add(specs_hash, target->spec_str, *spec_id); |
| 940 |
if (err) |
| 941 |
return err; |
| 942 |
|
| 943 |
man->free_spec_cnt--; |
| 944 |
} else { |
| 945 |
/* don't allocate spec ID bigger than what fits in specs map */ |
| 946 |
if (man->next_free_spec_id >= bpf_map__max_entries(man->specs_map)) |
| 947 |
return -E2BIG; |
| 948 |
|
| 949 |
*spec_id = man->next_free_spec_id; |
| 950 |
|
| 951 |
/* cache spec ID for current spec string for future lookups */ |
| 952 |
err = hashmap__add(specs_hash, target->spec_str, *spec_id); |
| 953 |
if (err) |
| 954 |
return err; |
| 955 |
|
| 956 |
man->next_free_spec_id++; |
| 957 |
} |
| 958 |
|
| 959 |
/* remember new spec ID in the link for later return back to free list on detach */ |
| 960 |
link->spec_ids[link->spec_cnt] = *spec_id; |
| 961 |
link->spec_cnt++; |
| 962 |
*is_new = true; |
| 963 |
return 0; |
| 964 |
} |
| 965 |
|
| 966 |
struct bpf_link *usdt_manager_attach_usdt(struct usdt_manager *man, const struct bpf_program *prog, |
| 967 |
pid_t pid, const char *path, |
| 968 |
const char *usdt_provider, const char *usdt_name, |
| 969 |
__u64 usdt_cookie) |
| 970 |
{ |
| 971 |
unsigned long *offsets = NULL, *ref_ctr_offsets = NULL; |
| 972 |
int i, err, spec_map_fd, ip_map_fd; |
| 973 |
LIBBPF_OPTS(bpf_uprobe_opts, opts); |
| 974 |
struct hashmap *specs_hash = NULL; |
| 975 |
struct bpf_link_usdt *link = NULL; |
| 976 |
struct usdt_target *targets = NULL; |
| 977 |
__u64 *cookies = NULL; |
| 978 |
struct elf_fd elf_fd; |
| 979 |
size_t target_cnt; |
| 980 |
|
| 981 |
spec_map_fd = bpf_map__fd(man->specs_map); |
| 982 |
ip_map_fd = bpf_map__fd(man->ip_to_spec_id_map); |
| 983 |
|
| 984 |
err = elf_open(path, &elf_fd); |
| 985 |
if (err) |
| 986 |
return libbpf_err_ptr(err); |
| 987 |
|
| 988 |
err = sanity_check_usdt_elf(elf_fd.elf, path); |
| 989 |
if (err) |
| 990 |
goto err_out; |
| 991 |
|
| 992 |
/* normalize PID filter */ |
| 993 |
if (pid < 0) |
| 994 |
pid = -1; |
| 995 |
else if (pid == 0) |
| 996 |
pid = getpid(); |
| 997 |
|
| 998 |
/* discover USDT in given binary, optionally limiting |
| 999 |
* activations to a given PID, if pid > 0 |
| 1000 |
*/ |
| 1001 |
err = collect_usdt_targets(man, elf_fd.elf, path, pid, usdt_provider, usdt_name, |
| 1002 |
usdt_cookie, &targets, &target_cnt); |
| 1003 |
if (err <= 0) { |
| 1004 |
err = (err == 0) ? -ENOENT : err; |
| 1005 |
goto err_out; |
| 1006 |
} |
| 1007 |
|
| 1008 |
specs_hash = hashmap__new(specs_hash_fn, specs_equal_fn, NULL); |
| 1009 |
if (IS_ERR(specs_hash)) { |
| 1010 |
err = PTR_ERR(specs_hash); |
| 1011 |
goto err_out; |
| 1012 |
} |
| 1013 |
|
| 1014 |
link = calloc(1, sizeof(*link)); |
| 1015 |
if (!link) { |
| 1016 |
err = -ENOMEM; |
| 1017 |
goto err_out; |
| 1018 |
} |
| 1019 |
|
| 1020 |
link->usdt_man = man; |
| 1021 |
link->link.detach = &bpf_link_usdt_detach; |
| 1022 |
link->link.dealloc = &bpf_link_usdt_dealloc; |
| 1023 |
|
| 1024 |
if (man->has_uprobe_multi) { |
| 1025 |
offsets = calloc(target_cnt, sizeof(*offsets)); |
| 1026 |
cookies = calloc(target_cnt, sizeof(*cookies)); |
| 1027 |
ref_ctr_offsets = calloc(target_cnt, sizeof(*ref_ctr_offsets)); |
| 1028 |
|
| 1029 |
if (!offsets || !ref_ctr_offsets || !cookies) { |
| 1030 |
err = -ENOMEM; |
| 1031 |
goto err_out; |
| 1032 |
} |
| 1033 |
} else { |
| 1034 |
link->uprobes = calloc(target_cnt, sizeof(*link->uprobes)); |
| 1035 |
if (!link->uprobes) { |
| 1036 |
err = -ENOMEM; |
| 1037 |
goto err_out; |
| 1038 |
} |
| 1039 |
} |
| 1040 |
|
| 1041 |
for (i = 0; i < target_cnt; i++) { |
| 1042 |
struct usdt_target *target = &targets[i]; |
| 1043 |
struct bpf_link *uprobe_link; |
| 1044 |
bool is_new; |
| 1045 |
int spec_id; |
| 1046 |
|
| 1047 |
/* Spec ID can be either reused or newly allocated. If it is |
| 1048 |
* newly allocated, we'll need to fill out spec map, otherwise |
| 1049 |
* entire spec should be valid and can be just used by a new |
| 1050 |
* uprobe. We reuse spec when USDT arg spec is identical. We |
| 1051 |
* also never share specs between two different USDT |
| 1052 |
* attachments ("links"), so all the reused specs already |
| 1053 |
* share USDT cookie value implicitly. |
| 1054 |
*/ |
| 1055 |
err = allocate_spec_id(man, specs_hash, link, target, &spec_id, &is_new); |
| 1056 |
if (err) |
| 1057 |
goto err_out; |
| 1058 |
|
| 1059 |
if (is_new && bpf_map_update_elem(spec_map_fd, &spec_id, &target->spec, BPF_ANY)) { |
| 1060 |
err = -errno; |
| 1061 |
pr_warn("usdt: failed to set USDT spec #%d for '%s:%s' in '%s': %s\n", |
| 1062 |
spec_id, usdt_provider, usdt_name, path, errstr(err)); |
| 1063 |
goto err_out; |
| 1064 |
} |
| 1065 |
if (!man->has_bpf_cookie && |
| 1066 |
bpf_map_update_elem(ip_map_fd, &target->abs_ip, &spec_id, BPF_NOEXIST)) { |
| 1067 |
err = -errno; |
| 1068 |
if (err == -EEXIST) { |
| 1069 |
pr_warn("usdt: IP collision detected for spec #%d for '%s:%s' in '%s'\n", |
| 1070 |
spec_id, usdt_provider, usdt_name, path); |
| 1071 |
} else { |
| 1072 |
pr_warn("usdt: failed to map IP 0x%lx to spec #%d for '%s:%s' in '%s': %s\n", |
| 1073 |
target->abs_ip, spec_id, usdt_provider, usdt_name, |
| 1074 |
path, errstr(err)); |
| 1075 |
} |
| 1076 |
goto err_out; |
| 1077 |
} |
| 1078 |
|
| 1079 |
if (man->has_uprobe_multi) { |
| 1080 |
offsets[i] = target->rel_ip; |
| 1081 |
ref_ctr_offsets[i] = target->sema_off; |
| 1082 |
cookies[i] = spec_id; |
| 1083 |
} else { |
| 1084 |
opts.ref_ctr_offset = target->sema_off; |
| 1085 |
opts.bpf_cookie = man->has_bpf_cookie ? spec_id : 0; |
| 1086 |
uprobe_link = bpf_program__attach_uprobe_opts(prog, pid, path, |
| 1087 |
target->rel_ip, &opts); |
| 1088 |
err = libbpf_get_error(uprobe_link); |
| 1089 |
if (err) { |
| 1090 |
pr_warn("usdt: failed to attach uprobe #%d for '%s:%s' in '%s': %s\n", |
| 1091 |
i, usdt_provider, usdt_name, path, errstr(err)); |
| 1092 |
goto err_out; |
| 1093 |
} |
| 1094 |
|
| 1095 |
link->uprobes[i].link = uprobe_link; |
| 1096 |
link->uprobes[i].abs_ip = target->abs_ip; |
| 1097 |
link->uprobe_cnt++; |
| 1098 |
} |
| 1099 |
} |
| 1100 |
|
| 1101 |
if (man->has_uprobe_multi) { |
| 1102 |
LIBBPF_OPTS(bpf_uprobe_multi_opts, opts_multi, |
| 1103 |
.ref_ctr_offsets = ref_ctr_offsets, |
| 1104 |
.offsets = offsets, |
| 1105 |
.cookies = cookies, |
| 1106 |
.cnt = target_cnt, |
| 1107 |
); |
| 1108 |
|
| 1109 |
link->multi_link = bpf_program__attach_uprobe_multi(prog, pid, path, |
| 1110 |
NULL, &opts_multi); |
| 1111 |
if (!link->multi_link) { |
| 1112 |
err = -errno; |
| 1113 |
pr_warn("usdt: failed to attach uprobe multi for '%s:%s' in '%s': %s\n", |
| 1114 |
usdt_provider, usdt_name, path, errstr(err)); |
| 1115 |
goto err_out; |
| 1116 |
} |
| 1117 |
|
| 1118 |
free(offsets); |
| 1119 |
free(ref_ctr_offsets); |
| 1120 |
free(cookies); |
| 1121 |
} |
| 1122 |
|
| 1123 |
free(targets); |
| 1124 |
hashmap__free(specs_hash); |
| 1125 |
elf_close(&elf_fd); |
| 1126 |
return &link->link; |
| 1127 |
|
| 1128 |
err_out: |
| 1129 |
free(offsets); |
| 1130 |
free(ref_ctr_offsets); |
| 1131 |
free(cookies); |
| 1132 |
|
| 1133 |
if (link) |
| 1134 |
bpf_link__destroy(&link->link); |
| 1135 |
free(targets); |
| 1136 |
hashmap__free(specs_hash); |
| 1137 |
elf_close(&elf_fd); |
| 1138 |
return libbpf_err_ptr(err); |
| 1139 |
} |
| 1140 |
|
| 1141 |
/* Parse out USDT ELF note from '.note.stapsdt' section. |
| 1142 |
* Logic inspired by perf's code. |
| 1143 |
*/ |
| 1144 |
static int parse_usdt_note(GElf_Nhdr *nhdr, const char *data, size_t name_off, size_t desc_off, |
| 1145 |
struct usdt_note *note) |
| 1146 |
{ |
| 1147 |
const char *provider, *name, *args; |
| 1148 |
long addrs[3]; |
| 1149 |
size_t len; |
| 1150 |
|
| 1151 |
/* sanity check USDT note name and type first */ |
| 1152 |
if (strncmp(data + name_off, USDT_NOTE_NAME, nhdr->n_namesz) != 0) |
| 1153 |
return -EINVAL; |
| 1154 |
if (nhdr->n_type != USDT_NOTE_TYPE) |
| 1155 |
return -EINVAL; |
| 1156 |
|
| 1157 |
/* sanity check USDT note contents ("description" in ELF terminology) */ |
| 1158 |
len = nhdr->n_descsz; |
| 1159 |
data = data + desc_off; |
| 1160 |
|
| 1161 |
/* +3 is the very minimum required to store three empty strings */ |
| 1162 |
if (len < sizeof(addrs) + 3) |
| 1163 |
return -EINVAL; |
| 1164 |
|
| 1165 |
/* get location, base, and semaphore addrs */ |
| 1166 |
memcpy(&addrs, data, sizeof(addrs)); |
| 1167 |
|
| 1168 |
/* parse string fields: provider, name, args */ |
| 1169 |
provider = data + sizeof(addrs); |
| 1170 |
|
| 1171 |
name = (const char *)memchr(provider, '\0', data + len - provider); |
| 1172 |
if (!name) /* non-zero-terminated provider */ |
| 1173 |
return -EINVAL; |
| 1174 |
name++; |
| 1175 |
if (name >= data + len || *name == '\0') /* missing or empty name */ |
| 1176 |
return -EINVAL; |
| 1177 |
|
| 1178 |
args = memchr(name, '\0', data + len - name); |
| 1179 |
if (!args) /* non-zero-terminated name */ |
| 1180 |
return -EINVAL; |
| 1181 |
++args; |
| 1182 |
if (args >= data + len) /* missing arguments spec */ |
| 1183 |
return -EINVAL; |
| 1184 |
|
| 1185 |
note->provider = provider; |
| 1186 |
note->name = name; |
| 1187 |
if (*args == '\0' || *args == ':') |
| 1188 |
note->args = ""; |
| 1189 |
else |
| 1190 |
note->args = args; |
| 1191 |
note->loc_addr = addrs[0]; |
| 1192 |
note->base_addr = addrs[1]; |
| 1193 |
note->sema_addr = addrs[2]; |
| 1194 |
|
| 1195 |
return 0; |
| 1196 |
} |
| 1197 |
|
| 1198 |
static int parse_usdt_arg(const char *arg_str, int arg_num, struct usdt_arg_spec *arg, int *arg_sz); |
| 1199 |
|
| 1200 |
static int parse_usdt_spec(struct usdt_spec *spec, const struct usdt_note *note, __u64 usdt_cookie) |
| 1201 |
{ |
| 1202 |
struct usdt_arg_spec *arg; |
| 1203 |
const char *s; |
| 1204 |
int arg_sz, len; |
| 1205 |
|
| 1206 |
spec->usdt_cookie = usdt_cookie; |
| 1207 |
spec->arg_cnt = 0; |
| 1208 |
|
| 1209 |
s = note->args; |
| 1210 |
while (s[0]) { |
| 1211 |
if (spec->arg_cnt >= USDT_MAX_ARG_CNT) { |
| 1212 |
pr_warn("usdt: too many USDT arguments (> %d) for '%s:%s' with args spec '%s'\n", |
| 1213 |
USDT_MAX_ARG_CNT, note->provider, note->name, note->args); |
| 1214 |
return -E2BIG; |
| 1215 |
} |
| 1216 |
|
| 1217 |
arg = &spec->args[spec->arg_cnt]; |
| 1218 |
len = parse_usdt_arg(s, spec->arg_cnt, arg, &arg_sz); |
| 1219 |
if (len < 0) |
| 1220 |
return len; |
| 1221 |
|
| 1222 |
arg->arg_signed = arg_sz < 0; |
| 1223 |
if (arg_sz < 0) |
| 1224 |
arg_sz = -arg_sz; |
| 1225 |
|
| 1226 |
switch (arg_sz) { |
| 1227 |
case 1: case 2: case 4: case 8: |
| 1228 |
arg->arg_bitshift = 64 - arg_sz * 8; |
| 1229 |
break; |
| 1230 |
default: |
| 1231 |
pr_warn("usdt: unsupported arg #%d (spec '%s') size: %d\n", |
| 1232 |
spec->arg_cnt, s, arg_sz); |
| 1233 |
return -EINVAL; |
| 1234 |
} |
| 1235 |
|
| 1236 |
s += len; |
| 1237 |
spec->arg_cnt++; |
| 1238 |
} |
| 1239 |
|
| 1240 |
return 0; |
| 1241 |
} |
| 1242 |
|
| 1243 |
/* Architecture-specific logic for parsing USDT argument location specs */ |
| 1244 |
|
| 1245 |
#if defined(__x86_64__) || defined(__i386__) |
| 1246 |
|
| 1247 |
static int calc_pt_regs_off(const char *reg_name) |
| 1248 |
{ |
| 1249 |
static struct { |
| 1250 |
const char *names[4]; |
| 1251 |
size_t pt_regs_off; |
| 1252 |
} reg_map[] = { |
| 1253 |
#ifdef __x86_64__ |
| 1254 |
#define reg_off(reg64, reg32) offsetof(struct pt_regs, reg64) |
| 1255 |
#else |
| 1256 |
#define reg_off(reg64, reg32) offsetof(struct pt_regs, reg32) |
| 1257 |
#endif |
| 1258 |
{ {"rip", "eip", "", ""}, reg_off(rip, eip) }, |
| 1259 |
{ {"rax", "eax", "ax", "al"}, reg_off(rax, eax) }, |
| 1260 |
{ {"rbx", "ebx", "bx", "bl"}, reg_off(rbx, ebx) }, |
| 1261 |
{ {"rcx", "ecx", "cx", "cl"}, reg_off(rcx, ecx) }, |
| 1262 |
{ {"rdx", "edx", "dx", "dl"}, reg_off(rdx, edx) }, |
| 1263 |
{ {"rsi", "esi", "si", "sil"}, reg_off(rsi, esi) }, |
| 1264 |
{ {"rdi", "edi", "di", "dil"}, reg_off(rdi, edi) }, |
| 1265 |
{ {"rbp", "ebp", "bp", "bpl"}, reg_off(rbp, ebp) }, |
| 1266 |
{ {"rsp", "esp", "sp", "spl"}, reg_off(rsp, esp) }, |
| 1267 |
#undef reg_off |
| 1268 |
#ifdef __x86_64__ |
| 1269 |
{ {"r8", "r8d", "r8w", "r8b"}, offsetof(struct pt_regs, r8) }, |
| 1270 |
{ {"r9", "r9d", "r9w", "r9b"}, offsetof(struct pt_regs, r9) }, |
| 1271 |
{ {"r10", "r10d", "r10w", "r10b"}, offsetof(struct pt_regs, r10) }, |
| 1272 |
{ {"r11", "r11d", "r11w", "r11b"}, offsetof(struct pt_regs, r11) }, |
| 1273 |
{ {"r12", "r12d", "r12w", "r12b"}, offsetof(struct pt_regs, r12) }, |
| 1274 |
{ {"r13", "r13d", "r13w", "r13b"}, offsetof(struct pt_regs, r13) }, |
| 1275 |
{ {"r14", "r14d", "r14w", "r14b"}, offsetof(struct pt_regs, r14) }, |
| 1276 |
{ {"r15", "r15d", "r15w", "r15b"}, offsetof(struct pt_regs, r15) }, |
| 1277 |
#endif |
| 1278 |
}; |
| 1279 |
int i, j; |
| 1280 |
|
| 1281 |
for (i = 0; i < ARRAY_SIZE(reg_map); i++) { |
| 1282 |
for (j = 0; j < ARRAY_SIZE(reg_map[i].names); j++) { |
| 1283 |
if (strcmp(reg_name, reg_map[i].names[j]) == 0) |
| 1284 |
return reg_map[i].pt_regs_off; |
| 1285 |
} |
| 1286 |
} |
| 1287 |
|
| 1288 |
pr_warn("usdt: unrecognized register '%s'\n", reg_name); |
| 1289 |
return -ENOENT; |
| 1290 |
} |
| 1291 |
|
| 1292 |
static int parse_usdt_arg(const char *arg_str, int arg_num, struct usdt_arg_spec *arg, int *arg_sz) |
| 1293 |
{ |
| 1294 |
char reg_name[16] = {0}, idx_reg_name[16] = {0}; |
| 1295 |
int len, reg_off, idx_reg_off, scale = 1; |
| 1296 |
long off = 0; |
| 1297 |
|
| 1298 |
if (sscanf(arg_str, " %d @ %ld ( %%%15[^,] , %%%15[^,] , %d ) %n", |
| 1299 |
arg_sz, &off, reg_name, idx_reg_name, &scale, &len) == 5 || |
| 1300 |
sscanf(arg_str, " %d @ ( %%%15[^,] , %%%15[^,] , %d ) %n", |
| 1301 |
arg_sz, reg_name, idx_reg_name, &scale, &len) == 4 || |
| 1302 |
sscanf(arg_str, " %d @ %ld ( %%%15[^,] , %%%15[^)] ) %n", |
| 1303 |
arg_sz, &off, reg_name, idx_reg_name, &len) == 4 || |
| 1304 |
sscanf(arg_str, " %d @ ( %%%15[^,] , %%%15[^)] ) %n", |
| 1305 |
arg_sz, reg_name, idx_reg_name, &len) == 3 |
| 1306 |
) { |
| 1307 |
/* |
| 1308 |
* Scale Index Base case: |
| 1309 |
* 1@-96(%rbp,%rax,8) |
| 1310 |
* 1@(%rbp,%rax,8) |
| 1311 |
* 1@-96(%rbp,%rax) |
| 1312 |
* 1@(%rbp,%rax) |
| 1313 |
*/ |
| 1314 |
arg->arg_type = USDT_ARG_SIB; |
| 1315 |
arg->val_off = off; |
| 1316 |
|
| 1317 |
reg_off = calc_pt_regs_off(reg_name); |
| 1318 |
if (reg_off < 0) |
| 1319 |
return reg_off; |
| 1320 |
arg->reg_off = reg_off; |
| 1321 |
|
| 1322 |
idx_reg_off = calc_pt_regs_off(idx_reg_name); |
| 1323 |
if (idx_reg_off < 0) |
| 1324 |
return idx_reg_off; |
| 1325 |
arg->idx_reg_off = idx_reg_off; |
| 1326 |
|
| 1327 |
/* validate scale factor and set fields directly */ |
| 1328 |
switch (scale) { |
| 1329 |
case 1: arg->scale_bitshift = 0; break; |
| 1330 |
case 2: arg->scale_bitshift = 1; break; |
| 1331 |
case 4: arg->scale_bitshift = 2; break; |
| 1332 |
case 8: arg->scale_bitshift = 3; break; |
| 1333 |
default: |
| 1334 |
pr_warn("usdt: invalid SIB scale %d, expected 1, 2, 4, 8\n", scale); |
| 1335 |
return -EINVAL; |
| 1336 |
} |
| 1337 |
} else if (sscanf(arg_str, " %d @ %ld ( %%%15[^)] ) %n", |
| 1338 |
arg_sz, &off, reg_name, &len) == 3) { |
| 1339 |
/* Memory dereference case, e.g., -4@-20(%rbp) */ |
| 1340 |
arg->arg_type = USDT_ARG_REG_DEREF; |
| 1341 |
arg->val_off = off; |
| 1342 |
reg_off = calc_pt_regs_off(reg_name); |
| 1343 |
if (reg_off < 0) |
| 1344 |
return reg_off; |
| 1345 |
arg->reg_off = reg_off; |
| 1346 |
} else if (sscanf(arg_str, " %d @ ( %%%15[^)] ) %n", arg_sz, reg_name, &len) == 2) { |
| 1347 |
/* Memory dereference case without offset, e.g., 8@(%rsp) */ |
| 1348 |
arg->arg_type = USDT_ARG_REG_DEREF; |
| 1349 |
arg->val_off = 0; |
| 1350 |
reg_off = calc_pt_regs_off(reg_name); |
| 1351 |
if (reg_off < 0) |
| 1352 |
return reg_off; |
| 1353 |
arg->reg_off = reg_off; |
| 1354 |
} else if (sscanf(arg_str, " %d @ %%%15s %n", arg_sz, reg_name, &len) == 2) { |
| 1355 |
/* Register read case, e.g., -4@%eax */ |
| 1356 |
arg->arg_type = USDT_ARG_REG; |
| 1357 |
/* register read has no memory offset */ |
| 1358 |
arg->val_off = 0; |
| 1359 |
|
| 1360 |
reg_off = calc_pt_regs_off(reg_name); |
| 1361 |
if (reg_off < 0) |
| 1362 |
return reg_off; |
| 1363 |
arg->reg_off = reg_off; |
| 1364 |
} else if (sscanf(arg_str, " %d @ $%ld %n", arg_sz, &off, &len) == 2) { |
| 1365 |
/* Constant value case, e.g., 4@$71 */ |
| 1366 |
arg->arg_type = USDT_ARG_CONST; |
| 1367 |
arg->val_off = off; |
| 1368 |
arg->reg_off = 0; |
| 1369 |
} else { |
| 1370 |
pr_warn("usdt: unrecognized arg #%d spec '%s'\n", arg_num, arg_str); |
| 1371 |
return -EINVAL; |
| 1372 |
} |
| 1373 |
|
| 1374 |
return len; |
| 1375 |
} |
| 1376 |
|
| 1377 |
#elif defined(__s390x__) |
| 1378 |
|
| 1379 |
static int parse_usdt_arg(const char *arg_str, int arg_num, struct usdt_arg_spec *arg, int *arg_sz) |
| 1380 |
{ |
| 1381 |
unsigned int reg; |
| 1382 |
int len; |
| 1383 |
long off; |
| 1384 |
|
| 1385 |
if (sscanf(arg_str, " %d @ %ld ( %%r%u ) %n", arg_sz, &off, ®, &len) == 3) { |
| 1386 |
/* Memory dereference case, e.g., -2@-28(%r15) */ |
| 1387 |
arg->arg_type = USDT_ARG_REG_DEREF; |
| 1388 |
arg->val_off = off; |
| 1389 |
if (reg > 15) { |
| 1390 |
pr_warn("usdt: unrecognized register '%%r%u'\n", reg); |
| 1391 |
return -EINVAL; |
| 1392 |
} |
| 1393 |
arg->reg_off = offsetof(user_pt_regs, gprs[reg]); |
| 1394 |
} else if (sscanf(arg_str, " %d @ %%r%u %n", arg_sz, ®, &len) == 2) { |
| 1395 |
/* Register read case, e.g., -8@%r0 */ |
| 1396 |
arg->arg_type = USDT_ARG_REG; |
| 1397 |
arg->val_off = 0; |
| 1398 |
if (reg > 15) { |
| 1399 |
pr_warn("usdt: unrecognized register '%%r%u'\n", reg); |
| 1400 |
return -EINVAL; |
| 1401 |
} |
| 1402 |
arg->reg_off = offsetof(user_pt_regs, gprs[reg]); |
| 1403 |
} else if (sscanf(arg_str, " %d @ %ld %n", arg_sz, &off, &len) == 2) { |
| 1404 |
/* Constant value case, e.g., 4@71 */ |
| 1405 |
arg->arg_type = USDT_ARG_CONST; |
| 1406 |
arg->val_off = off; |
| 1407 |
arg->reg_off = 0; |
| 1408 |
} else { |
| 1409 |
pr_warn("usdt: unrecognized arg #%d spec '%s'\n", arg_num, arg_str); |
| 1410 |
return -EINVAL; |
| 1411 |
} |
| 1412 |
|
| 1413 |
return len; |
| 1414 |
} |
| 1415 |
|
| 1416 |
#elif defined(__aarch64__) |
| 1417 |
|
| 1418 |
static int calc_pt_regs_off(const char *reg_name) |
| 1419 |
{ |
| 1420 |
int reg_num; |
| 1421 |
|
| 1422 |
if (sscanf(reg_name, "x%d", ®_num) == 1) { |
| 1423 |
if (reg_num >= 0 && reg_num < 31) |
| 1424 |
return offsetof(struct user_pt_regs, regs[reg_num]); |
| 1425 |
} else if (strcmp(reg_name, "sp") == 0) { |
| 1426 |
return offsetof(struct user_pt_regs, sp); |
| 1427 |
} |
| 1428 |
pr_warn("usdt: unrecognized register '%s'\n", reg_name); |
| 1429 |
return -ENOENT; |
| 1430 |
} |
| 1431 |
|
| 1432 |
static int parse_usdt_arg(const char *arg_str, int arg_num, struct usdt_arg_spec *arg, int *arg_sz) |
| 1433 |
{ |
| 1434 |
char reg_name[16]; |
| 1435 |
int len, reg_off; |
| 1436 |
long off; |
| 1437 |
|
| 1438 |
if (sscanf(arg_str, " %d @ \[ %15[a-z0-9] , %ld ] %n", arg_sz, reg_name, &off, &len) == 3) { |
| 1439 |
/* Memory dereference case, e.g., -4@[sp, 96] */ |
| 1440 |
arg->arg_type = USDT_ARG_REG_DEREF; |
| 1441 |
arg->val_off = off; |
| 1442 |
reg_off = calc_pt_regs_off(reg_name); |
| 1443 |
if (reg_off < 0) |
| 1444 |
return reg_off; |
| 1445 |
arg->reg_off = reg_off; |
| 1446 |
} else if (sscanf(arg_str, " %d @ \[ %15[a-z0-9] ] %n", arg_sz, reg_name, &len) == 2) { |
| 1447 |
/* Memory dereference case, e.g., -4@[sp] */ |
| 1448 |
arg->arg_type = USDT_ARG_REG_DEREF; |
| 1449 |
arg->val_off = 0; |
| 1450 |
reg_off = calc_pt_regs_off(reg_name); |
| 1451 |
if (reg_off < 0) |
| 1452 |
return reg_off; |
| 1453 |
arg->reg_off = reg_off; |
| 1454 |
} else if (sscanf(arg_str, " %d @ %ld %n", arg_sz, &off, &len) == 2) { |
| 1455 |
/* Constant value case, e.g., 4@5 */ |
| 1456 |
arg->arg_type = USDT_ARG_CONST; |
| 1457 |
arg->val_off = off; |
| 1458 |
arg->reg_off = 0; |
| 1459 |
} else if (sscanf(arg_str, " %d @ %15[a-z0-9] %n", arg_sz, reg_name, &len) == 2) { |
| 1460 |
/* Register read case, e.g., -8@x4 */ |
| 1461 |
arg->arg_type = USDT_ARG_REG; |
| 1462 |
arg->val_off = 0; |
| 1463 |
reg_off = calc_pt_regs_off(reg_name); |
| 1464 |
if (reg_off < 0) |
| 1465 |
return reg_off; |
| 1466 |
arg->reg_off = reg_off; |
| 1467 |
} else { |
| 1468 |
pr_warn("usdt: unrecognized arg #%d spec '%s'\n", arg_num, arg_str); |
| 1469 |
return -EINVAL; |
| 1470 |
} |
| 1471 |
|
| 1472 |
return len; |
| 1473 |
} |
| 1474 |
|
| 1475 |
#elif defined(__riscv) |
| 1476 |
|
| 1477 |
static int calc_pt_regs_off(const char *reg_name) |
| 1478 |
{ |
| 1479 |
static struct { |
| 1480 |
const char *name; |
| 1481 |
size_t pt_regs_off; |
| 1482 |
} reg_map[] = { |
| 1483 |
{ "ra", offsetof(struct user_regs_struct, ra) }, |
| 1484 |
{ "sp", offsetof(struct user_regs_struct, sp) }, |
| 1485 |
{ "gp", offsetof(struct user_regs_struct, gp) }, |
| 1486 |
{ "tp", offsetof(struct user_regs_struct, tp) }, |
| 1487 |
{ "a0", offsetof(struct user_regs_struct, a0) }, |
| 1488 |
{ "a1", offsetof(struct user_regs_struct, a1) }, |
| 1489 |
{ "a2", offsetof(struct user_regs_struct, a2) }, |
| 1490 |
{ "a3", offsetof(struct user_regs_struct, a3) }, |
| 1491 |
{ "a4", offsetof(struct user_regs_struct, a4) }, |
| 1492 |
{ "a5", offsetof(struct user_regs_struct, a5) }, |
| 1493 |
{ "a6", offsetof(struct user_regs_struct, a6) }, |
| 1494 |
{ "a7", offsetof(struct user_regs_struct, a7) }, |
| 1495 |
{ "s0", offsetof(struct user_regs_struct, s0) }, |
| 1496 |
{ "s1", offsetof(struct user_regs_struct, s1) }, |
| 1497 |
{ "s2", offsetof(struct user_regs_struct, s2) }, |
| 1498 |
{ "s3", offsetof(struct user_regs_struct, s3) }, |
| 1499 |
{ "s4", offsetof(struct user_regs_struct, s4) }, |
| 1500 |
{ "s5", offsetof(struct user_regs_struct, s5) }, |
| 1501 |
{ "s6", offsetof(struct user_regs_struct, s6) }, |
| 1502 |
{ "s7", offsetof(struct user_regs_struct, s7) }, |
| 1503 |
{ "s8", offsetof(struct user_regs_struct, rv_s8) }, |
| 1504 |
{ "s9", offsetof(struct user_regs_struct, s9) }, |
| 1505 |
{ "s10", offsetof(struct user_regs_struct, s10) }, |
| 1506 |
{ "s11", offsetof(struct user_regs_struct, s11) }, |
| 1507 |
{ "t0", offsetof(struct user_regs_struct, t0) }, |
| 1508 |
{ "t1", offsetof(struct user_regs_struct, t1) }, |
| 1509 |
{ "t2", offsetof(struct user_regs_struct, t2) }, |
| 1510 |
{ "t3", offsetof(struct user_regs_struct, t3) }, |
| 1511 |
{ "t4", offsetof(struct user_regs_struct, t4) }, |
| 1512 |
{ "t5", offsetof(struct user_regs_struct, t5) }, |
| 1513 |
{ "t6", offsetof(struct user_regs_struct, t6) }, |
| 1514 |
}; |
| 1515 |
int i; |
| 1516 |
|
| 1517 |
for (i = 0; i < ARRAY_SIZE(reg_map); i++) { |
| 1518 |
if (strcmp(reg_name, reg_map[i].name) == 0) |
| 1519 |
return reg_map[i].pt_regs_off; |
| 1520 |
} |
| 1521 |
|
| 1522 |
pr_warn("usdt: unrecognized register '%s'\n", reg_name); |
| 1523 |
return -ENOENT; |
| 1524 |
} |
| 1525 |
|
| 1526 |
static int parse_usdt_arg(const char *arg_str, int arg_num, struct usdt_arg_spec *arg, int *arg_sz) |
| 1527 |
{ |
| 1528 |
char reg_name[16]; |
| 1529 |
int len, reg_off; |
| 1530 |
long off; |
| 1531 |
|
| 1532 |
if (sscanf(arg_str, " %d @ %ld ( %15[a-z0-9] ) %n", arg_sz, &off, reg_name, &len) == 3) { |
| 1533 |
/* Memory dereference case, e.g., -8@-88(s0) */ |
| 1534 |
arg->arg_type = USDT_ARG_REG_DEREF; |
| 1535 |
arg->val_off = off; |
| 1536 |
reg_off = calc_pt_regs_off(reg_name); |
| 1537 |
if (reg_off < 0) |
| 1538 |
return reg_off; |
| 1539 |
arg->reg_off = reg_off; |
| 1540 |
} else if (sscanf(arg_str, " %d @ %ld %n", arg_sz, &off, &len) == 2) { |
| 1541 |
/* Constant value case, e.g., 4@5 */ |
| 1542 |
arg->arg_type = USDT_ARG_CONST; |
| 1543 |
arg->val_off = off; |
| 1544 |
arg->reg_off = 0; |
| 1545 |
} else if (sscanf(arg_str, " %d @ %15[a-z0-9] %n", arg_sz, reg_name, &len) == 2) { |
| 1546 |
/* Register read case, e.g., -8@a1 */ |
| 1547 |
arg->arg_type = USDT_ARG_REG; |
| 1548 |
arg->val_off = 0; |
| 1549 |
reg_off = calc_pt_regs_off(reg_name); |
| 1550 |
if (reg_off < 0) |
| 1551 |
return reg_off; |
| 1552 |
arg->reg_off = reg_off; |
| 1553 |
} else { |
| 1554 |
pr_warn("usdt: unrecognized arg #%d spec '%s'\n", arg_num, arg_str); |
| 1555 |
return -EINVAL; |
| 1556 |
} |
| 1557 |
|
| 1558 |
return len; |
| 1559 |
} |
| 1560 |
|
| 1561 |
#elif defined(__arm__) |
| 1562 |
|
| 1563 |
static int calc_pt_regs_off(const char *reg_name) |
| 1564 |
{ |
| 1565 |
static struct { |
| 1566 |
const char *name; |
| 1567 |
size_t pt_regs_off; |
| 1568 |
} reg_map[] = { |
| 1569 |
{ "r0", offsetof(struct pt_regs, uregs[0]) }, |
| 1570 |
{ "r1", offsetof(struct pt_regs, uregs[1]) }, |
| 1571 |
{ "r2", offsetof(struct pt_regs, uregs[2]) }, |
| 1572 |
{ "r3", offsetof(struct pt_regs, uregs[3]) }, |
| 1573 |
{ "r4", offsetof(struct pt_regs, uregs[4]) }, |
| 1574 |
{ "r5", offsetof(struct pt_regs, uregs[5]) }, |
| 1575 |
{ "r6", offsetof(struct pt_regs, uregs[6]) }, |
| 1576 |
{ "r7", offsetof(struct pt_regs, uregs[7]) }, |
| 1577 |
{ "r8", offsetof(struct pt_regs, uregs[8]) }, |
| 1578 |
{ "r9", offsetof(struct pt_regs, uregs[9]) }, |
| 1579 |
{ "r10", offsetof(struct pt_regs, uregs[10]) }, |
| 1580 |
{ "fp", offsetof(struct pt_regs, uregs[11]) }, |
| 1581 |
{ "ip", offsetof(struct pt_regs, uregs[12]) }, |
| 1582 |
{ "sp", offsetof(struct pt_regs, uregs[13]) }, |
| 1583 |
{ "lr", offsetof(struct pt_regs, uregs[14]) }, |
| 1584 |
{ "pc", offsetof(struct pt_regs, uregs[15]) }, |
| 1585 |
}; |
| 1586 |
int i; |
| 1587 |
|
| 1588 |
for (i = 0; i < ARRAY_SIZE(reg_map); i++) { |
| 1589 |
if (strcmp(reg_name, reg_map[i].name) == 0) |
| 1590 |
return reg_map[i].pt_regs_off; |
| 1591 |
} |
| 1592 |
|
| 1593 |
pr_warn("usdt: unrecognized register '%s'\n", reg_name); |
| 1594 |
return -ENOENT; |
| 1595 |
} |
| 1596 |
|
| 1597 |
static int parse_usdt_arg(const char *arg_str, int arg_num, struct usdt_arg_spec *arg, int *arg_sz) |
| 1598 |
{ |
| 1599 |
char reg_name[16]; |
| 1600 |
int len, reg_off; |
| 1601 |
long off; |
| 1602 |
|
| 1603 |
if (sscanf(arg_str, " %d @ \[ %15[a-z0-9] , #%ld ] %n", |
| 1604 |
arg_sz, reg_name, &off, &len) == 3) { |
| 1605 |
/* Memory dereference case, e.g., -4@[fp, #96] */ |
| 1606 |
arg->arg_type = USDT_ARG_REG_DEREF; |
| 1607 |
arg->val_off = off; |
| 1608 |
reg_off = calc_pt_regs_off(reg_name); |
| 1609 |
if (reg_off < 0) |
| 1610 |
return reg_off; |
| 1611 |
arg->reg_off = reg_off; |
| 1612 |
} else if (sscanf(arg_str, " %d @ \[ %15[a-z0-9] ] %n", arg_sz, reg_name, &len) == 2) { |
| 1613 |
/* Memory dereference case, e.g., -4@[sp] */ |
| 1614 |
arg->arg_type = USDT_ARG_REG_DEREF; |
| 1615 |
arg->val_off = 0; |
| 1616 |
reg_off = calc_pt_regs_off(reg_name); |
| 1617 |
if (reg_off < 0) |
| 1618 |
return reg_off; |
| 1619 |
arg->reg_off = reg_off; |
| 1620 |
} else if (sscanf(arg_str, " %d @ #%ld %n", arg_sz, &off, &len) == 2) { |
| 1621 |
/* Constant value case, e.g., 4@#5 */ |
| 1622 |
arg->arg_type = USDT_ARG_CONST; |
| 1623 |
arg->val_off = off; |
| 1624 |
arg->reg_off = 0; |
| 1625 |
} else if (sscanf(arg_str, " %d @ %15[a-z0-9] %n", arg_sz, reg_name, &len) == 2) { |
| 1626 |
/* Register read case, e.g., -8@r4 */ |
| 1627 |
arg->arg_type = USDT_ARG_REG; |
| 1628 |
arg->val_off = 0; |
| 1629 |
reg_off = calc_pt_regs_off(reg_name); |
| 1630 |
if (reg_off < 0) |
| 1631 |
return reg_off; |
| 1632 |
arg->reg_off = reg_off; |
| 1633 |
} else { |
| 1634 |
pr_warn("usdt: unrecognized arg #%d spec '%s'\n", arg_num, arg_str); |
| 1635 |
return -EINVAL; |
| 1636 |
} |
| 1637 |
|
| 1638 |
return len; |
| 1639 |
} |
| 1640 |
|
| 1641 |
#else |
| 1642 |
|
| 1643 |
static int parse_usdt_arg(const char *arg_str, int arg_num, struct usdt_arg_spec *arg, int *arg_sz) |
| 1644 |
{ |
| 1645 |
pr_warn("usdt: libbpf doesn't support USDTs on current architecture\n"); |
| 1646 |
return -ENOTSUP; |
| 1647 |
} |
| 1648 |
|
| 1649 |
#endif |