Debian Codegraph @

src/shared/bpf-program.c

raw
1 /* SPDX-License-Identifier: LGPL-2.1-or-later */
2
3 #include <fcntl.h>
4 #include <linux/bpf.h>
5 #include <linux/bpf_insn.h>
6 #include <sys/bpf.h>
7 #include <unistd.h>
8
9 #include "alloc-util.h"
10 #include "bpf-program.h"
11 #include "errno-util.h"
12 #include "escape.h"
13 #include "extract-word.h"
14 #include "fd-util.h"
15 #include "fdset.h"
16 #include "log.h"
17 #include "memory-util.h"
18 #include "parse-util.h"
19 #include "path-util.h"
20 #include "serialize.h"
21 #include "set.h"
22 #include "string-table.h"
23 #include "string-util.h"
24
25 static const char *const bpf_cgroup_attach_type_table[__MAX_BPF_ATTACH_TYPE] = {
26 [BPF_CGROUP_INET_INGRESS] = "ingress",
27 [BPF_CGROUP_INET_EGRESS] = "egress",
28 [BPF_CGROUP_INET_SOCK_CREATE] = "sock_create",
29 [BPF_CGROUP_SOCK_OPS] = "sock_ops",
30 [BPF_CGROUP_DEVICE] = "device",
31 [BPF_CGROUP_INET4_BIND] = "bind4",
32 [BPF_CGROUP_INET6_BIND] = "bind6",
33 [BPF_CGROUP_INET4_CONNECT] = "connect4",
34 [BPF_CGROUP_INET6_CONNECT] = "connect6",
35 [BPF_CGROUP_INET4_POST_BIND] = "post_bind4",
36 [BPF_CGROUP_INET6_POST_BIND] = "post_bind6",
37 [BPF_CGROUP_UDP4_SENDMSG] = "sendmsg4",
38 [BPF_CGROUP_UDP6_SENDMSG] = "sendmsg6",
39 [BPF_CGROUP_SYSCTL] = "sysctl",
40 [BPF_CGROUP_UDP4_RECVMSG] = "recvmsg4",
41 [BPF_CGROUP_UDP6_RECVMSG] = "recvmsg6",
42 [BPF_CGROUP_GETSOCKOPT] = "getsockopt",
43 [BPF_CGROUP_SETSOCKOPT] = "setsockopt",
44 };
45
46 DEFINE_STRING_TABLE_LOOKUP(bpf_cgroup_attach_type, int);
47
48 DEFINE_HASH_OPS_WITH_KEY_DESTRUCTOR(bpf_program_hash_ops, void, trivial_hash_func, trivial_compare_func, bpf_program_free);
49
50 int bpf_program_supported(void) {
51 static int cached = 0;
52 int r;
53
54 if (cached != 0)
55 return cached;
56
57 /* Currently, we only use the following three types:
58 * - BPF_PROG_TYPE_CGROUP_SKB, supported since kernel v4.10 (0e33661de493db325435d565a4a722120ae4cbf3),
59 * - BPF_PROG_TYPE_CGROUP_DEVICE, supported since kernel v4.15 (ebc614f687369f9df99828572b1d85a7c2de3d92),
60 * - BPF_PROG_TYPE_CGROUP_SOCK_ADDR, supported since kernel v4.17 (4fbac77d2d092b475dda9eea66da674369665427).
61 * As our baseline on the kernel is v5.4, it is enough to check if one BPF program can be created and loaded. */
62
63 _cleanup_(bpf_program_freep) BPFProgram *program = NULL;
64 r = bpf_program_new(BPF_PROG_TYPE_CGROUP_SKB, /* prog_name= */ NULL, &program);
65 if (r < 0)
66 return cached = log_debug_errno(r, "Can't allocate CGROUP SKB BPF program, assuming BPF is not supported: %m");
67
68 static const struct bpf_insn trivial[] = {
69 BPF_MOV64_IMM(BPF_REG_0, 1),
70 BPF_EXIT_INSN()
71 };
72 r = bpf_program_add_instructions(program, trivial, ELEMENTSOF(trivial));
73 if (r < 0)
74 return cached = log_debug_errno(r, "Can't add trivial instructions to CGROUP SKB BPF program, assuming BPF is not supported: %m");
75
76 r = bpf_program_load_kernel(program, /* log_buf= */ NULL, /* log_size= */ 0);
77 if (r < 0)
78 return cached = log_debug_errno(r, "Can't load kernel CGROUP SKB BPF program, assuming BPF is not supported: %m");
79
80 /* Unfortunately the kernel allows us to create BPF_PROG_TYPE_CGROUP_SKB (maybe also other types)
81 * programs even when CONFIG_CGROUP_BPF is turned off at kernel compilation time. This sucks of course:
82 * why does it allow us to create a cgroup BPF program if we can't do a thing with it later?
83 *
84 * We detect this case by issuing the BPF_PROG_DETACH bpf() call with invalid file descriptors: if
85 * CONFIG_CGROUP_BPF is turned off, then the call will fail early with EINVAL. If it is turned on the
86 * parameters are validated however, and that'll fail with EBADF then.
87 *
88 * The check seems also important when we are running with sanitizers. With sanitizers (at least with
89 * LLVM v20), the following check and other bpf() calls fails even if the kernel supports BPF. To
90 * avoid unexpected fail when running with sanitizers, let's explicitly check if bpf() syscall works. */
91
92 /* Clang and GCC (>=15) do not 0-pad with structured initialization, causing the kernel to reject the
93 * bpf_attr as invalid. See: https://github.com/torvalds/linux/blob/v5.9/kernel/bpf/syscall.c#L65
94 * Hence, we cannot use structured initialization here, and need to clear the structure with zero
95 * explicitly before use. */
96 union bpf_attr attr;
97 zero(attr);
98 attr.attach_type = BPF_CGROUP_INET_EGRESS; /* since kernel v4.10 (0e33661de493db325435d565a4a722120ae4cbf3) */
99 attr.target_fd = -EBADF;
100 attr.attach_bpf_fd = -EBADF;
101
102 if (bpf(BPF_PROG_DETACH, &attr, sizeof(attr)) < 0) {
103 if (errno == EBADF) /* YAY! */
104 return cached = true;
105
106 return cached = log_debug_errno(errno, "Didn't get EBADF from invalid BPF_PROG_DETACH call: %m");
107 }
108
109 return cached = log_debug_errno(SYNTHETIC_ERRNO(EBADE),
110 "Wut? Kernel accepted our invalid BPF_PROG_DETACH call? Something is weird, assuming BPF is broken and hence not supported.");
111 }
112
113 BPFProgram *bpf_program_free(BPFProgram *p) {
114 if (!p)
115 return NULL;
116 /* Unfortunately, the kernel currently doesn't implicitly detach BPF programs from their cgroups when the last
117 * fd to the BPF program is closed. This has nasty side-effects since this means that abnormally terminated
118 * programs that attached one of their BPF programs to a cgroup will leave this program pinned for good with
119 * zero chance of recovery, until the cgroup is removed. This is particularly problematic if the cgroup in
120 * question is the root cgroup (or any other cgroup belonging to a service that cannot be restarted during
121 * operation, such as dbus), as the memory for the BPF program can only be reclaimed through a reboot. To
122 * counter this, we track closely to which cgroup a program was attached to and will detach it on our own
123 * whenever we close the BPF fd. */
124 (void) bpf_program_cgroup_detach(p);
125
126 safe_close(p->kernel_fd);
127 free(p->prog_name);
128 free(p->instructions);
129 free(p->attached_path);
130
131 return mfree(p);
132 }
133
134 /* struct bpf_prog_info info must be initialized since its value is both input and output
135 * for BPF_OBJ_GET_INFO_BY_FD syscall. */
136 static int bpf_program_get_info_by_fd(int prog_fd, struct bpf_prog_info *info, uint32_t info_len) {
137 union bpf_attr attr;
138
139 /* Explicitly memset to zero since some compilers may produce non-zero-initialized padding when
140 * structured initialization is used.
141 * Refer to https://github.com/systemd/systemd/issues/18164
142 */
143 zero(attr);
144 attr.info.bpf_fd = prog_fd;
145 attr.info.info_len = info_len;
146 attr.info.info = PTR_TO_UINT64(info);
147
148 return RET_NERRNO(bpf(BPF_OBJ_GET_INFO_BY_FD, &attr, sizeof(attr)));
149 }
150
151 int bpf_program_new(uint32_t prog_type, const char *prog_name, BPFProgram **ret) {
152 _cleanup_(bpf_program_freep) BPFProgram *p = NULL;
153 _cleanup_free_ char *name = NULL;
154
155 assert(ret);
156
157 if (prog_name) {
158 if (strlen(prog_name) >= BPF_OBJ_NAME_LEN)
159 return -ENAMETOOLONG;
160
161 name = strdup(prog_name);
162 if (!name)
163 return -ENOMEM;
164 }
165
166 p = new(BPFProgram, 1);
167 if (!p)
168 return -ENOMEM;
169
170 *p = (BPFProgram) {
171 .prog_type = prog_type,
172 .kernel_fd = -EBADF,
173 .prog_name = TAKE_PTR(name),
174 };
175
176 *ret = TAKE_PTR(p);
177
178 return 0;
179 }
180
181 int bpf_program_new_from_bpffs_path(const char *path, BPFProgram **ret) {
182 _cleanup_(bpf_program_freep) BPFProgram *p = NULL;
183 struct bpf_prog_info info = {};
184 int r;
185
186 assert(path);
187 assert(ret);
188
189 p = new(BPFProgram, 1);
190 if (!p)
191 return -ENOMEM;
192
193 *p = (BPFProgram) {
194 .prog_type = BPF_PROG_TYPE_UNSPEC,
195 .kernel_fd = -EBADF,
196 };
197
198 r = bpf_program_load_from_bpf_fs(p, path);
199 if (r < 0)
200 return r;
201
202 r = bpf_program_get_info_by_fd(p->kernel_fd, &info, sizeof(info));
203 if (r < 0)
204 return r;
205
206 p->prog_type = info.type;
207 *ret = TAKE_PTR(p);
208
209 return 0;
210 }
211
212 int bpf_program_add_instructions(BPFProgram *p, const struct bpf_insn *instructions, size_t count) {
213
214 assert(p);
215
216 if (p->kernel_fd >= 0) /* don't allow modification after we uploaded things to the kernel */
217 return -EBUSY;
218
219 if (!GREEDY_REALLOC(p->instructions, p->n_instructions + count))
220 return -ENOMEM;
221
222 memcpy(p->instructions + p->n_instructions, instructions, sizeof(struct bpf_insn) * count);
223 p->n_instructions += count;
224
225 return 0;
226 }
227
228 int bpf_program_load_kernel(BPFProgram *p, char *log_buf, size_t log_size) {
229 union bpf_attr attr;
230
231 assert(p);
232
233 if (p->kernel_fd >= 0) { /* make this idempotent */
234 memzero(log_buf, log_size);
235 return 0;
236 }
237
238 // FIXME: Clang doesn't 0-pad with structured initialization, causing
239 // the kernel to reject the bpf_attr as invalid. See:
240 // https://github.com/torvalds/linux/blob/v5.9/kernel/bpf/syscall.c#L65
241 // Ideally it should behave like GCC, so that we can remove these workarounds.
242 zero(attr);
243 attr.prog_type = p->prog_type;
244 attr.insns = PTR_TO_UINT64(p->instructions);
245 attr.insn_cnt = p->n_instructions;
246 attr.license = PTR_TO_UINT64("GPL");
247 attr.log_buf = PTR_TO_UINT64(log_buf);
248 attr.log_level = !!log_buf;
249 attr.log_size = log_size;
250 if (p->prog_name)
251 strncpy(attr.prog_name, p->prog_name, BPF_OBJ_NAME_LEN - 1);
252
253 p->kernel_fd = bpf(BPF_PROG_LOAD, &attr, sizeof(attr));
254 if (p->kernel_fd < 0)
255 return -errno;
256
257 return 0;
258 }
259
260 int bpf_program_load_from_bpf_fs(BPFProgram *p, const char *path) {
261 union bpf_attr attr;
262
263 assert(p);
264
265 if (p->kernel_fd >= 0) /* don't overwrite an assembled or loaded program */
266 return -EBUSY;
267
268 zero(attr);
269 attr.pathname = PTR_TO_UINT64(path);
270
271 p->kernel_fd = bpf(BPF_OBJ_GET, &attr, sizeof(attr));
272 if (p->kernel_fd < 0)
273 return -errno;
274
275 return 0;
276 }
277
278 int bpf_program_cgroup_attach(BPFProgram *p, int type, const char *path, uint32_t flags) {
279 _cleanup_free_ char *copy = NULL;
280 _cleanup_close_ int fd = -EBADF;
281 union bpf_attr attr;
282 int r;
283
284 assert(p);
285 assert(type >= 0);
286 assert(path);
287
288 if (!IN_SET(flags, 0, BPF_F_ALLOW_OVERRIDE, BPF_F_ALLOW_MULTI))
289 return -EINVAL;
290
291 /* We need to track which cgroup the program is attached to, and we can only track one attachment, hence let's
292 * refuse this early. */
293 if (p->attached_path) {
294 if (!path_equal(p->attached_path, path))
295 return -EBUSY;
296 if (p->attached_type != type)
297 return -EBUSY;
298 if (p->attached_flags != flags)
299 return -EBUSY;
300
301 /* Here's a shortcut: if we previously attached this program already, then we don't have to do so
302 * again. Well, with one exception: if we are in BPF_F_ALLOW_OVERRIDE mode then someone else might have
303 * replaced our program since the last time, hence let's reattach it again, just to be safe. In flags
304 * == 0 mode this is not an issue since nobody else can replace our program in that case, and in flags
305 * == BPF_F_ALLOW_MULTI mode any other's program would be installed in addition to ours hence ours
306 * would remain in effect. */
307 if (flags != BPF_F_ALLOW_OVERRIDE)
308 return 0;
309 }
310
311 /* Ensure we have a kernel object for this. */
312 r = bpf_program_load_kernel(p, NULL, 0);
313 if (r < 0)
314 return r;
315
316 copy = strdup(path);
317 if (!copy)
318 return -ENOMEM;
319
320 fd = open(path, O_DIRECTORY|O_RDONLY|O_CLOEXEC);
321 if (fd < 0)
322 return -errno;
323
324 zero(attr);
325 attr.attach_type = type;
326 attr.target_fd = fd;
327 attr.attach_bpf_fd = p->kernel_fd;
328 attr.attach_flags = flags;
329
330 if (bpf(BPF_PROG_ATTACH, &attr, sizeof(attr)) < 0)
331 return -errno;
332
333 free_and_replace(p->attached_path, copy);
334 p->attached_type = type;
335 p->attached_flags = flags;
336
337 return 0;
338 }
339
340 int bpf_program_cgroup_detach(BPFProgram *p) {
341 _cleanup_close_ int fd = -EBADF;
342
343 assert(p);
344
345 if (!p->attached_path)
346 return -EUNATCH;
347
348 fd = open(p->attached_path, O_DIRECTORY|O_RDONLY|O_CLOEXEC);
349 if (fd < 0) {
350 if (errno != ENOENT)
351 return -errno;
352
353 /* If the cgroup does not exist anymore, then we don't have to explicitly detach, it got detached
354 * implicitly by the removal, hence don't complain */
355
356 } else {
357 union bpf_attr attr;
358
359 zero(attr);
360 attr.attach_type = p->attached_type;
361 attr.target_fd = fd;
362 attr.attach_bpf_fd = p->kernel_fd;
363
364 if (bpf(BPF_PROG_DETACH, &attr, sizeof(attr)) < 0)
365 return -errno;
366 }
367
368 p->attached_path = mfree(p->attached_path);
369
370 return 0;
371 }
372
373 int bpf_map_new(
374 const char *name,
375 enum bpf_map_type type,
376 size_t key_size,
377 size_t value_size,
378 size_t max_entries,
379 uint32_t flags) {
380
381 union bpf_attr attr;
382 const char *n = name;
383
384 zero(attr);
385 attr.map_type = type;
386 attr.key_size = key_size;
387 attr.value_size = value_size;
388 attr.max_entries = max_entries;
389 attr.map_flags = flags;
390
391 /* The map name is primarily informational for debugging purposes, and typically too short
392 * to carry the full unit name, hence we employ a trivial lossy escaping to make it fit
393 * (truncation + only alphanumerical, "." and "_" are allowed as per
394 * https://docs.kernel.org/bpf/maps.html#usage-notes) */
395 for (size_t i = 0; i < sizeof(attr.map_name) - 1 && *n; i++, n++)
396 attr.map_name[i] = strchr(ALPHANUMERICAL ".", *n) ? *n : '_';
397
398 return RET_NERRNO(bpf(BPF_MAP_CREATE, &attr, sizeof(attr)));
399 }
400
401 int bpf_map_update_element(int fd, const void *key, void *value) {
402 union bpf_attr attr;
403
404 zero(attr);
405 attr.map_fd = fd;
406 attr.key = PTR_TO_UINT64(key);
407 attr.value = PTR_TO_UINT64(value);
408
409 return RET_NERRNO(bpf(BPF_MAP_UPDATE_ELEM, &attr, sizeof(attr)));
410 }
411
412 int bpf_map_lookup_element(int fd, const void *key, void *value) {
413 union bpf_attr attr;
414
415 zero(attr);
416 attr.map_fd = fd;
417 attr.key = PTR_TO_UINT64(key);
418 attr.value = PTR_TO_UINT64(value);
419
420 return RET_NERRNO(bpf(BPF_MAP_LOOKUP_ELEM, &attr, sizeof(attr)));
421 }
422
423 int bpf_program_pin(int prog_fd, const char *bpffs_path) {
424 union bpf_attr attr;
425
426 zero(attr);
427 attr.pathname = PTR_TO_UINT64((void *) bpffs_path);
428 attr.bpf_fd = prog_fd;
429
430 return RET_NERRNO(bpf(BPF_OBJ_PIN, &attr, sizeof(attr)));
431 }
432
433 int bpf_program_get_id_by_fd(int prog_fd, uint32_t *ret_id) {
434 struct bpf_prog_info info = {};
435 int r;
436
437 assert(ret_id);
438
439 r = bpf_program_get_info_by_fd(prog_fd, &info, sizeof(info));
440 if (r < 0)
441 return r;
442
443 *ret_id = info.id;
444
445 return 0;
446 };
447
448 int bpf_program_serialize_attachment(
449 FILE *f,
450 FDSet *fds,
451 const char *key,
452 BPFProgram *p) {
453
454 _cleanup_free_ char *escaped = NULL;
455 int copy, r;
456
457 if (!p || !p->attached_path)
458 return 0;
459
460 assert(p->kernel_fd >= 0);
461
462 escaped = cescape(p->attached_path);
463 if (!escaped)
464 return -ENOMEM;
465
466 copy = fdset_put_dup(fds, p->kernel_fd);
467 if (copy < 0)
468 return log_error_errno(copy, "Failed to add BPF kernel fd to serialize: %m");
469
470 r = serialize_item_format(
471 f,
472 key,
473 "%i %s %s",
474 copy,
475 bpf_cgroup_attach_type_to_string(p->attached_type),
476 escaped);
477 if (r < 0)
478 return r;
479
480 /* After serialization, let's forget the fact that this program is attached. The attachment — if you
481 * so will — is now 'owned' by the serialization, and not us anymore. Why does that matter? Because
482 * of BPF's less-than-ideal lifecycle handling: to detach a program from a cgroup we have to
483 * explicitly do so, it's not done implicitly on close(). Now, since we are serializing here we don't
484 * want the program to be detached while freeing things, so that the attachment can be retained after
485 * deserializing again. bpf_program_free() implicitly detaches things, if attached_path is non-NULL,
486 * hence we set it to NULL here. */
487
488 p->attached_path = mfree(p->attached_path);
489 return 0;
490 }
491
492 int bpf_program_serialize_attachment_set(FILE *f, FDSet *fds, const char *key, Set *set) {
493 BPFProgram *p;
494 int r;
495
496 SET_FOREACH(p, set) {
497 r = bpf_program_serialize_attachment(f, fds, key, p);
498 if (r < 0)
499 return r;
500 }
501
502 return 0;
503 }
504
505 int bpf_program_deserialize_attachment(const char *v, FDSet *fds, BPFProgram **bpfp) {
506 _cleanup_free_ char *sfd = NULL, *sat = NULL, *unescaped = NULL;
507 _cleanup_(bpf_program_freep) BPFProgram *p = NULL;
508 _cleanup_close_ int fd = -EBADF;
509 ssize_t l;
510 int ifd, at, r;
511
512 assert(v);
513 assert(bpfp);
514
515 /* Extract first word: the fd number */
516 r = extract_first_word(&v, &sfd, NULL, 0);
517 if (r < 0)
518 return r;
519 if (r == 0)
520 return -EINVAL;
521
522 ifd = parse_fd(sfd);
523 if (ifd < 0)
524 return r;
525
526 /* Extract second word: the attach type */
527 r = extract_first_word(&v, &sat, NULL, 0);
528 if (r < 0)
529 return r;
530 if (r == 0)
531 return -EINVAL;
532
533 at = bpf_cgroup_attach_type_from_string(sat);
534 if (at < 0)
535 return at;
536
537 /* The rest is the path */
538 if (isempty(v))
539 return -EINVAL;
540
541 l = cunescape(v, 0, &unescaped);
542 if (l < 0)
543 return l;
544
545 fd = fdset_remove(fds, ifd);
546 if (fd < 0)
547 return fd;
548
549 p = new(BPFProgram, 1);
550 if (!p)
551 return -ENOMEM;
552
553 *p = (BPFProgram) {
554 .kernel_fd = TAKE_FD(fd),
555 .prog_type = BPF_PROG_TYPE_UNSPEC,
556 .attached_path = TAKE_PTR(unescaped),
557 .attached_type = at,
558 };
559
560 if (*bpfp)
561 bpf_program_free(*bpfp);
562
563 *bpfp = TAKE_PTR(p);
564 return 0;
565 }
566
567 int bpf_program_deserialize_attachment_set(const char *v, FDSet *fds, Set **bpfsetp) {
568 BPFProgram *p = NULL;
569 int r;
570
571 assert(v);
572 assert(bpfsetp);
573
574 r = bpf_program_deserialize_attachment(v, fds, &p);
575 if (r < 0)
576 return r;
577
578 r = set_ensure_consume(bpfsetp, &bpf_program_hash_ops, p);
579 if (r < 0)
580 return r;
581
582 return 0;
583 }