Debian Codegraph @

tools/lib/bpf/btf.c

raw
1 // SPDX-License-Identifier: (LGPL-2.1 OR BSD-2-Clause)
2 /* Copyright (c) 2018 Facebook */
3
4 #include <byteswap.h>
5 #include <endian.h>
6 #include <stdio.h>
7 #include <stdlib.h>
8 #include <string.h>
9 #include <fcntl.h>
10 #include <unistd.h>
11 #include <errno.h>
12 #include <sys/utsname.h>
13 #include <sys/param.h>
14 #include <sys/stat.h>
15 #include <sys/mman.h>
16 #include <linux/kernel.h>
17 #include <linux/err.h>
18 #include <linux/btf.h>
19 #include <gelf.h>
20 #include "btf.h"
21 #include "bpf.h"
22 #include "libbpf.h"
23 #include "libbpf_internal.h"
24 #include "hashmap.h"
25 #include "strset.h"
26
27 #define BTF_MAX_NR_TYPES 0x7fffffffU
28 #define BTF_MAX_STR_OFFSET 0x7fffffffU
29
30 static struct btf_type btf_void;
31
32 struct btf {
33 /* raw BTF data in native endianness */
34 void *raw_data;
35 /* raw BTF data in non-native endianness */
36 void *raw_data_swapped;
37 __u32 raw_size;
38 /* whether target endianness differs from the native one */
39 bool swapped_endian;
40
41 /*
42 * When BTF is loaded from an ELF or raw memory it is stored
43 * in a contiguous memory block. The hdr, type_data, and, strs_data
44 * point inside that memory region to their respective parts of BTF
45 * representation:
46 *
47 * +--------------------------------+
48 * | Header | Types | Strings |
49 * +--------------------------------+
50 * ^ ^ ^
51 * | | |
52 * hdr | |
53 * types_data-+ |
54 * strs_data------------+
55 *
56 * If BTF data is later modified, e.g., due to types added or
57 * removed, BTF deduplication performed, etc, this contiguous
58 * representation is broken up into three independently allocated
59 * memory regions to be able to modify them independently.
60 * raw_data is nulled out at that point, but can be later allocated
61 * and cached again if user calls btf__raw_data(), at which point
62 * raw_data will contain a contiguous copy of header, types, and
63 * strings:
64 *
65 * +----------+ +---------+ +-----------+
66 * | Header | | Types | | Strings |
67 * +----------+ +---------+ +-----------+
68 * ^ ^ ^
69 * | | |
70 * hdr | |
71 * types_data----+ |
72 * strset__data(strs_set)-----+
73 *
74 * +----------+---------+-----------+
75 * | Header | Types | Strings |
76 * raw_data----->+----------+---------+-----------+
77 */
78 struct btf_header *hdr;
79
80 void *types_data;
81 size_t types_data_cap; /* used size stored in hdr->type_len */
82
83 /* type ID to `struct btf_type *` lookup index
84 * type_offs[0] corresponds to the first non-VOID type:
85 * - for base BTF it's type [1];
86 * - for split BTF it's the first non-base BTF type.
87 */
88 __u32 *type_offs;
89 size_t type_offs_cap;
90 /* number of types in this BTF instance:
91 * - doesn't include special [0] void type;
92 * - for split BTF counts number of types added on top of base BTF.
93 */
94 __u32 nr_types;
95 /* the start IDs of named types in sorted BTF */
96 int named_start_id;
97 /* if not NULL, points to the base BTF on top of which the current
98 * split BTF is based
99 */
100 struct btf *base_btf;
101 /* BTF type ID of the first type in this BTF instance:
102 * - for base BTF it's equal to 1;
103 * - for split BTF it's equal to biggest type ID of base BTF plus 1.
104 */
105 int start_id;
106 /* logical string offset of this BTF instance:
107 * - for base BTF it's equal to 0;
108 * - for split BTF it's equal to total size of base BTF's string section size.
109 */
110 int start_str_off;
111
112 /* only one of strs_data or strs_set can be non-NULL, depending on
113 * whether BTF is in a modifiable state (strs_set is used) or not
114 * (strs_data points inside raw_data)
115 */
116 void *strs_data;
117 /* a set of unique strings */
118 struct strset *strs_set;
119 /* whether strings are already deduplicated */
120 bool strs_deduped;
121
122 /* whether base_btf should be freed in btf_free for this instance */
123 bool owns_base;
124
125 /* whether raw_data is a (read-only) mmap */
126 bool raw_data_is_mmap;
127
128 /* BTF object FD, if loaded into kernel */
129 int fd;
130
131 /* Pointer size (in bytes) for a target architecture of this BTF */
132 int ptr_sz;
133 };
134
135 static inline __u64 ptr_to_u64(const void *ptr)
136 {
137 return (__u64) (unsigned long) ptr;
138 }
139
140 /* Ensure given dynamically allocated memory region pointed to by *data* with
141 * capacity of *cap_cnt* elements each taking *elem_sz* bytes has enough
142 * memory to accommodate *add_cnt* new elements, assuming *cur_cnt* elements
143 * are already used. At most *max_cnt* elements can be ever allocated.
144 * If necessary, memory is reallocated and all existing data is copied over,
145 * new pointer to the memory region is stored at *data, new memory region
146 * capacity (in number of elements) is stored in *cap.
147 * On success, memory pointer to the beginning of unused memory is returned.
148 * On error, NULL is returned.
149 */
150 void *libbpf_add_mem(void **data, size_t *cap_cnt, size_t elem_sz,
151 size_t cur_cnt, size_t max_cnt, size_t add_cnt)
152 {
153 size_t new_cnt;
154 void *new_data;
155
156 if (cur_cnt + add_cnt <= *cap_cnt)
157 return *data + cur_cnt * elem_sz;
158
159 /* requested more than the set limit */
160 if (cur_cnt + add_cnt > max_cnt)
161 return NULL;
162
163 new_cnt = *cap_cnt;
164 new_cnt += new_cnt / 4; /* expand by 25% */
165 if (new_cnt < 16) /* but at least 16 elements */
166 new_cnt = 16;
167 if (new_cnt > max_cnt) /* but not exceeding a set limit */
168 new_cnt = max_cnt;
169 if (new_cnt < cur_cnt + add_cnt) /* also ensure we have enough memory */
170 new_cnt = cur_cnt + add_cnt;
171
172 new_data = libbpf_reallocarray(*data, new_cnt, elem_sz);
173 if (!new_data)
174 return NULL;
175
176 /* zero out newly allocated portion of memory */
177 memset(new_data + (*cap_cnt) * elem_sz, 0, (new_cnt - *cap_cnt) * elem_sz);
178
179 *data = new_data;
180 *cap_cnt = new_cnt;
181 return new_data + cur_cnt * elem_sz;
182 }
183
184 /* Ensure given dynamically allocated memory region has enough allocated space
185 * to accommodate *need_cnt* elements of size *elem_sz* bytes each
186 */
187 int libbpf_ensure_mem(void **data, size_t *cap_cnt, size_t elem_sz, size_t need_cnt)
188 {
189 void *p;
190
191 if (need_cnt <= *cap_cnt)
192 return 0;
193
194 p = libbpf_add_mem(data, cap_cnt, elem_sz, *cap_cnt, SIZE_MAX, need_cnt - *cap_cnt);
195 if (!p)
196 return -ENOMEM;
197
198 return 0;
199 }
200
201 static void *btf_add_type_offs_mem(struct btf *btf, size_t add_cnt)
202 {
203 return libbpf_add_mem((void **)&btf->type_offs, &btf->type_offs_cap, sizeof(__u32),
204 btf->nr_types, BTF_MAX_NR_TYPES, add_cnt);
205 }
206
207 static int btf_add_type_idx_entry(struct btf *btf, __u32 type_off)
208 {
209 __u32 *p;
210
211 p = btf_add_type_offs_mem(btf, 1);
212 if (!p)
213 return -ENOMEM;
214
215 *p = type_off;
216 return 0;
217 }
218
219 static void btf_bswap_hdr(struct btf_header *h)
220 {
221 h->magic = bswap_16(h->magic);
222 h->hdr_len = bswap_32(h->hdr_len);
223 h->type_off = bswap_32(h->type_off);
224 h->type_len = bswap_32(h->type_len);
225 h->str_off = bswap_32(h->str_off);
226 h->str_len = bswap_32(h->str_len);
227 }
228
229 static int btf_parse_hdr(struct btf *btf)
230 {
231 struct btf_header *hdr = btf->hdr;
232 __u32 meta_left;
233
234 if (btf->raw_size < sizeof(struct btf_header)) {
235 pr_debug("BTF header not found\n");
236 return -EINVAL;
237 }
238
239 if (hdr->magic == bswap_16(BTF_MAGIC)) {
240 btf->swapped_endian = true;
241 if (bswap_32(hdr->hdr_len) != sizeof(struct btf_header)) {
242 pr_warn("Can't load BTF with non-native endianness due to unsupported header length %u\n",
243 bswap_32(hdr->hdr_len));
244 return -ENOTSUP;
245 }
246 btf_bswap_hdr(hdr);
247 } else if (hdr->magic != BTF_MAGIC) {
248 pr_debug("Invalid BTF magic: %x\n", hdr->magic);
249 return -EINVAL;
250 }
251
252 if (btf->raw_size < hdr->hdr_len) {
253 pr_debug("BTF header len %u larger than data size %u\n",
254 hdr->hdr_len, btf->raw_size);
255 return -EINVAL;
256 }
257
258 meta_left = btf->raw_size - hdr->hdr_len;
259 if (meta_left < (long long)hdr->str_off + hdr->str_len) {
260 pr_debug("Invalid BTF total size: %u\n", btf->raw_size);
261 return -EINVAL;
262 }
263
264 if ((long long)hdr->type_off + hdr->type_len > hdr->str_off) {
265 pr_debug("Invalid BTF data sections layout: type data at %u + %u, strings data at %u + %u\n",
266 hdr->type_off, hdr->type_len, hdr->str_off, hdr->str_len);
267 return -EINVAL;
268 }
269
270 if (hdr->type_off % 4) {
271 pr_debug("BTF type section is not aligned to 4 bytes\n");
272 return -EINVAL;
273 }
274
275 return 0;
276 }
277
278 static int btf_parse_str_sec(struct btf *btf)
279 {
280 const struct btf_header *hdr = btf->hdr;
281 const char *start = btf->strs_data;
282 const char *end = start + btf->hdr->str_len;
283
284 if (btf->base_btf && hdr->str_len == 0)
285 return 0;
286 if (!hdr->str_len || hdr->str_len - 1 > BTF_MAX_STR_OFFSET || end[-1]) {
287 pr_debug("Invalid BTF string section\n");
288 return -EINVAL;
289 }
290 if (!btf->base_btf && start[0]) {
291 pr_debug("Malformed BTF string section, did you forget to provide base BTF?\n");
292 return -EINVAL;
293 }
294 return 0;
295 }
296
297 static int btf_type_size(const struct btf_type *t)
298 {
299 const int base_size = sizeof(struct btf_type);
300 __u16 vlen = btf_vlen(t);
301
302 switch (btf_kind(t)) {
303 case BTF_KIND_FWD:
304 case BTF_KIND_CONST:
305 case BTF_KIND_VOLATILE:
306 case BTF_KIND_RESTRICT:
307 case BTF_KIND_PTR:
308 case BTF_KIND_TYPEDEF:
309 case BTF_KIND_FUNC:
310 case BTF_KIND_FLOAT:
311 case BTF_KIND_TYPE_TAG:
312 return base_size;
313 case BTF_KIND_INT:
314 return base_size + sizeof(__u32);
315 case BTF_KIND_ENUM:
316 return base_size + vlen * sizeof(struct btf_enum);
317 case BTF_KIND_ENUM64:
318 return base_size + vlen * sizeof(struct btf_enum64);
319 case BTF_KIND_ARRAY:
320 return base_size + sizeof(struct btf_array);
321 case BTF_KIND_STRUCT:
322 case BTF_KIND_UNION:
323 return base_size + vlen * sizeof(struct btf_member);
324 case BTF_KIND_FUNC_PROTO:
325 return base_size + vlen * sizeof(struct btf_param);
326 case BTF_KIND_VAR:
327 return base_size + sizeof(struct btf_var);
328 case BTF_KIND_DATASEC:
329 return base_size + vlen * sizeof(struct btf_var_secinfo);
330 case BTF_KIND_DECL_TAG:
331 return base_size + sizeof(struct btf_decl_tag);
332 default:
333 pr_debug("Unsupported BTF_KIND:%u\n", btf_kind(t));
334 return -EINVAL;
335 }
336 }
337
338 static void btf_bswap_type_base(struct btf_type *t)
339 {
340 t->name_off = bswap_32(t->name_off);
341 t->info = bswap_32(t->info);
342 t->type = bswap_32(t->type);
343 }
344
345 static int btf_bswap_type_rest(struct btf_type *t)
346 {
347 struct btf_var_secinfo *v;
348 struct btf_enum64 *e64;
349 struct btf_member *m;
350 struct btf_array *a;
351 struct btf_param *p;
352 struct btf_enum *e;
353 __u16 vlen = btf_vlen(t);
354 int i;
355
356 switch (btf_kind(t)) {
357 case BTF_KIND_FWD:
358 case BTF_KIND_CONST:
359 case BTF_KIND_VOLATILE:
360 case BTF_KIND_RESTRICT:
361 case BTF_KIND_PTR:
362 case BTF_KIND_TYPEDEF:
363 case BTF_KIND_FUNC:
364 case BTF_KIND_FLOAT:
365 case BTF_KIND_TYPE_TAG:
366 return 0;
367 case BTF_KIND_INT:
368 *(__u32 *)(t + 1) = bswap_32(*(__u32 *)(t + 1));
369 return 0;
370 case BTF_KIND_ENUM:
371 for (i = 0, e = btf_enum(t); i < vlen; i++, e++) {
372 e->name_off = bswap_32(e->name_off);
373 e->val = bswap_32(e->val);
374 }
375 return 0;
376 case BTF_KIND_ENUM64:
377 for (i = 0, e64 = btf_enum64(t); i < vlen; i++, e64++) {
378 e64->name_off = bswap_32(e64->name_off);
379 e64->val_lo32 = bswap_32(e64->val_lo32);
380 e64->val_hi32 = bswap_32(e64->val_hi32);
381 }
382 return 0;
383 case BTF_KIND_ARRAY:
384 a = btf_array(t);
385 a->type = bswap_32(a->type);
386 a->index_type = bswap_32(a->index_type);
387 a->nelems = bswap_32(a->nelems);
388 return 0;
389 case BTF_KIND_STRUCT:
390 case BTF_KIND_UNION:
391 for (i = 0, m = btf_members(t); i < vlen; i++, m++) {
392 m->name_off = bswap_32(m->name_off);
393 m->type = bswap_32(m->type);
394 m->offset = bswap_32(m->offset);
395 }
396 return 0;
397 case BTF_KIND_FUNC_PROTO:
398 for (i = 0, p = btf_params(t); i < vlen; i++, p++) {
399 p->name_off = bswap_32(p->name_off);
400 p->type = bswap_32(p->type);
401 }
402 return 0;
403 case BTF_KIND_VAR:
404 btf_var(t)->linkage = bswap_32(btf_var(t)->linkage);
405 return 0;
406 case BTF_KIND_DATASEC:
407 for (i = 0, v = btf_var_secinfos(t); i < vlen; i++, v++) {
408 v->type = bswap_32(v->type);
409 v->offset = bswap_32(v->offset);
410 v->size = bswap_32(v->size);
411 }
412 return 0;
413 case BTF_KIND_DECL_TAG:
414 btf_decl_tag(t)->component_idx = bswap_32(btf_decl_tag(t)->component_idx);
415 return 0;
416 default:
417 pr_debug("Unsupported BTF_KIND:%u\n", btf_kind(t));
418 return -EINVAL;
419 }
420 }
421
422 static int btf_parse_type_sec(struct btf *btf)
423 {
424 struct btf_header *hdr = btf->hdr;
425 void *next_type = btf->types_data;
426 void *end_type = next_type + hdr->type_len;
427 int err, type_size;
428
429 while (next_type + sizeof(struct btf_type) <= end_type) {
430 if (btf->swapped_endian)
431 btf_bswap_type_base(next_type);
432
433 type_size = btf_type_size(next_type);
434 if (type_size < 0)
435 return type_size;
436 if (next_type + type_size > end_type) {
437 pr_warn("BTF type [%d] is malformed\n", btf->start_id + btf->nr_types);
438 return -EINVAL;
439 }
440
441 if (btf->swapped_endian && btf_bswap_type_rest(next_type))
442 return -EINVAL;
443
444 err = btf_add_type_idx_entry(btf, next_type - btf->types_data);
445 if (err)
446 return err;
447
448 next_type += type_size;
449 btf->nr_types++;
450 }
451
452 if (next_type != end_type) {
453 pr_warn("BTF types data is malformed\n");
454 return -EINVAL;
455 }
456
457 return 0;
458 }
459
460 static int btf_validate_str(const struct btf *btf, __u32 str_off, const char *what, __u32 type_id)
461 {
462 const char *s;
463
464 s = btf__str_by_offset(btf, str_off);
465 if (!s) {
466 pr_warn("btf: type [%u]: invalid %s (string offset %u)\n", type_id, what, str_off);
467 return -EINVAL;
468 }
469
470 return 0;
471 }
472
473 static int btf_validate_id(const struct btf *btf, __u32 id, __u32 ctx_id)
474 {
475 const struct btf_type *t;
476
477 t = btf__type_by_id(btf, id);
478 if (!t) {
479 pr_warn("btf: type [%u]: invalid referenced type ID %u\n", ctx_id, id);
480 return -EINVAL;
481 }
482
483 return 0;
484 }
485
486 static int btf_validate_type(const struct btf *btf, const struct btf_type *t, __u32 id)
487 {
488 __u32 kind = btf_kind(t);
489 int err, i, n;
490
491 err = btf_validate_str(btf, t->name_off, "type name", id);
492 if (err)
493 return err;
494
495 switch (kind) {
496 case BTF_KIND_UNKN:
497 case BTF_KIND_INT:
498 case BTF_KIND_FWD:
499 case BTF_KIND_FLOAT:
500 break;
501 case BTF_KIND_PTR:
502 case BTF_KIND_TYPEDEF:
503 case BTF_KIND_VOLATILE:
504 case BTF_KIND_CONST:
505 case BTF_KIND_RESTRICT:
506 case BTF_KIND_VAR:
507 case BTF_KIND_DECL_TAG:
508 case BTF_KIND_TYPE_TAG:
509 err = btf_validate_id(btf, t->type, id);
510 if (err)
511 return err;
512 break;
513 case BTF_KIND_ARRAY: {
514 const struct btf_array *a = btf_array(t);
515
516 err = btf_validate_id(btf, a->type, id);
517 err = err ?: btf_validate_id(btf, a->index_type, id);
518 if (err)
519 return err;
520 break;
521 }
522 case BTF_KIND_STRUCT:
523 case BTF_KIND_UNION: {
524 const struct btf_member *m = btf_members(t);
525
526 n = btf_vlen(t);
527 for (i = 0; i < n; i++, m++) {
528 err = btf_validate_str(btf, m->name_off, "field name", id);
529 err = err ?: btf_validate_id(btf, m->type, id);
530 if (err)
531 return err;
532 }
533 break;
534 }
535 case BTF_KIND_ENUM: {
536 const struct btf_enum *m = btf_enum(t);
537
538 n = btf_vlen(t);
539 for (i = 0; i < n; i++, m++) {
540 err = btf_validate_str(btf, m->name_off, "enum name", id);
541 if (err)
542 return err;
543 }
544 break;
545 }
546 case BTF_KIND_ENUM64: {
547 const struct btf_enum64 *m = btf_enum64(t);
548
549 n = btf_vlen(t);
550 for (i = 0; i < n; i++, m++) {
551 err = btf_validate_str(btf, m->name_off, "enum name", id);
552 if (err)
553 return err;
554 }
555 break;
556 }
557 case BTF_KIND_FUNC: {
558 const struct btf_type *ft;
559
560 err = btf_validate_id(btf, t->type, id);
561 if (err)
562 return err;
563 ft = btf__type_by_id(btf, t->type);
564 if (btf_kind(ft) != BTF_KIND_FUNC_PROTO) {
565 pr_warn("btf: type [%u]: referenced type [%u] is not FUNC_PROTO\n", id, t->type);
566 return -EINVAL;
567 }
568 break;
569 }
570 case BTF_KIND_FUNC_PROTO: {
571 const struct btf_param *m = btf_params(t);
572
573 n = btf_vlen(t);
574 for (i = 0; i < n; i++, m++) {
575 err = btf_validate_str(btf, m->name_off, "param name", id);
576 err = err ?: btf_validate_id(btf, m->type, id);
577 if (err)
578 return err;
579 }
580 break;
581 }
582 case BTF_KIND_DATASEC: {
583 const struct btf_var_secinfo *m = btf_var_secinfos(t);
584
585 n = btf_vlen(t);
586 for (i = 0; i < n; i++, m++) {
587 err = btf_validate_id(btf, m->type, id);
588 if (err)
589 return err;
590 }
591 break;
592 }
593 default:
594 pr_warn("btf: type [%u]: unrecognized kind %u\n", id, kind);
595 return -EINVAL;
596 }
597 return 0;
598 }
599
600 /* Validate basic sanity of BTF. It's intentionally less thorough than
601 * kernel's validation and validates only properties of BTF that libbpf relies
602 * on to be correct (e.g., valid type IDs, valid string offsets, etc)
603 */
604 static int btf_sanity_check(const struct btf *btf)
605 {
606 const struct btf_type *t;
607 __u32 i, n = btf__type_cnt(btf);
608 int err;
609
610 for (i = btf->start_id; i < n; i++) {
611 t = btf_type_by_id(btf, i);
612 err = btf_validate_type(btf, t, i);
613 if (err)
614 return err;
615 }
616 return 0;
617 }
618
619 __u32 btf__type_cnt(const struct btf *btf)
620 {
621 return btf->start_id + btf->nr_types;
622 }
623
624 const struct btf *btf__base_btf(const struct btf *btf)
625 {
626 return btf->base_btf;
627 }
628
629 /* internal helper returning non-const pointer to a type */
630 struct btf_type *btf_type_by_id(const struct btf *btf, __u32 type_id)
631 {
632 if (type_id == 0)
633 return &btf_void;
634 if (type_id < btf->start_id)
635 return btf_type_by_id(btf->base_btf, type_id);
636 return btf->types_data + btf->type_offs[type_id - btf->start_id];
637 }
638
639 const struct btf_type *btf__type_by_id(const struct btf *btf, __u32 type_id)
640 {
641 if (type_id >= btf->start_id + btf->nr_types)
642 return errno = EINVAL, NULL;
643 return btf_type_by_id((struct btf *)btf, type_id);
644 }
645
646 static int determine_ptr_size(const struct btf *btf)
647 {
648 static const char * const long_aliases[] = {
649 "long",
650 "long int",
651 "int long",
652 "unsigned long",
653 "long unsigned",
654 "unsigned long int",
655 "unsigned int long",
656 "long unsigned int",
657 "long int unsigned",
658 "int unsigned long",
659 "int long unsigned",
660 };
661 const struct btf_type *t;
662 const char *name;
663 int i, j, n;
664
665 if (btf->base_btf && btf->base_btf->ptr_sz > 0)
666 return btf->base_btf->ptr_sz;
667
668 n = btf__type_cnt(btf);
669 for (i = 1; i < n; i++) {
670 t = btf__type_by_id(btf, i);
671 if (!btf_is_int(t))
672 continue;
673
674 if (t->size != 4 && t->size != 8)
675 continue;
676
677 name = btf__name_by_offset(btf, t->name_off);
678 if (!name)
679 continue;
680
681 for (j = 0; j < ARRAY_SIZE(long_aliases); j++) {
682 if (strcmp(name, long_aliases[j]) == 0)
683 return t->size;
684 }
685 }
686
687 return -1;
688 }
689
690 static size_t btf_ptr_sz(const struct btf *btf)
691 {
692 if (!btf->ptr_sz)
693 ((struct btf *)btf)->ptr_sz = determine_ptr_size(btf);
694 return btf->ptr_sz < 0 ? sizeof(void *) : btf->ptr_sz;
695 }
696
697 /* Return pointer size this BTF instance assumes. The size is heuristically
698 * determined by looking for 'long' or 'unsigned long' integer type and
699 * recording its size in bytes. If BTF type information doesn't have any such
700 * type, this function returns 0. In the latter case, native architecture's
701 * pointer size is assumed, so will be either 4 or 8, depending on
702 * architecture that libbpf was compiled for. It's possible to override
703 * guessed value by using btf__set_pointer_size() API.
704 */
705 size_t btf__pointer_size(const struct btf *btf)
706 {
707 if (!btf->ptr_sz)
708 ((struct btf *)btf)->ptr_sz = determine_ptr_size(btf);
709
710 if (btf->ptr_sz < 0)
711 /* not enough BTF type info to guess */
712 return 0;
713
714 return btf->ptr_sz;
715 }
716
717 /* Override or set pointer size in bytes. Only values of 4 and 8 are
718 * supported.
719 */
720 int btf__set_pointer_size(struct btf *btf, size_t ptr_sz)
721 {
722 if (ptr_sz != 4 && ptr_sz != 8)
723 return libbpf_err(-EINVAL);
724 btf->ptr_sz = ptr_sz;
725 return 0;
726 }
727
728 static bool is_host_big_endian(void)
729 {
730 #if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__
731 return false;
732 #elif __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__
733 return true;
734 #else
735 # error "Unrecognized __BYTE_ORDER__"
736 #endif
737 }
738
739 enum btf_endianness btf__endianness(const struct btf *btf)
740 {
741 if (is_host_big_endian())
742 return btf->swapped_endian ? BTF_LITTLE_ENDIAN : BTF_BIG_ENDIAN;
743 else
744 return btf->swapped_endian ? BTF_BIG_ENDIAN : BTF_LITTLE_ENDIAN;
745 }
746
747 int btf__set_endianness(struct btf *btf, enum btf_endianness endian)
748 {
749 if (endian != BTF_LITTLE_ENDIAN && endian != BTF_BIG_ENDIAN)
750 return libbpf_err(-EINVAL);
751
752 btf->swapped_endian = is_host_big_endian() != (endian == BTF_BIG_ENDIAN);
753 if (!btf->swapped_endian) {
754 free(btf->raw_data_swapped);
755 btf->raw_data_swapped = NULL;
756 }
757 return 0;
758 }
759
760 static bool btf_type_is_void(const struct btf_type *t)
761 {
762 return t == &btf_void || btf_is_fwd(t);
763 }
764
765 static bool btf_type_is_void_or_null(const struct btf_type *t)
766 {
767 return !t || btf_type_is_void(t);
768 }
769
770 #define MAX_RESOLVE_DEPTH 32
771
772 __s64 btf__resolve_size(const struct btf *btf, __u32 type_id)
773 {
774 const struct btf_array *array;
775 const struct btf_type *t;
776 __u32 nelems = 1;
777 __s64 size = -1;
778 int i;
779
780 t = btf__type_by_id(btf, type_id);
781 for (i = 0; i < MAX_RESOLVE_DEPTH && !btf_type_is_void_or_null(t); i++) {
782 switch (btf_kind(t)) {
783 case BTF_KIND_INT:
784 case BTF_KIND_STRUCT:
785 case BTF_KIND_UNION:
786 case BTF_KIND_ENUM:
787 case BTF_KIND_ENUM64:
788 case BTF_KIND_DATASEC:
789 case BTF_KIND_FLOAT:
790 size = t->size;
791 goto done;
792 case BTF_KIND_PTR:
793 size = btf_ptr_sz(btf);
794 goto done;
795 case BTF_KIND_TYPEDEF:
796 case BTF_KIND_VOLATILE:
797 case BTF_KIND_CONST:
798 case BTF_KIND_RESTRICT:
799 case BTF_KIND_VAR:
800 case BTF_KIND_DECL_TAG:
801 case BTF_KIND_TYPE_TAG:
802 type_id = t->type;
803 break;
804 case BTF_KIND_ARRAY:
805 array = btf_array(t);
806 if (nelems && array->nelems > UINT32_MAX / nelems)
807 return libbpf_err(-E2BIG);
808 nelems *= array->nelems;
809 type_id = array->type;
810 break;
811 default:
812 return libbpf_err(-EINVAL);
813 }
814
815 t = btf__type_by_id(btf, type_id);
816 }
817
818 done:
819 if (size < 0)
820 return libbpf_err(-EINVAL);
821 if (nelems && size > UINT32_MAX / nelems)
822 return libbpf_err(-E2BIG);
823
824 return nelems * size;
825 }
826
827 int btf__align_of(const struct btf *btf, __u32 id)
828 {
829 const struct btf_type *t = btf__type_by_id(btf, id);
830 __u16 kind = btf_kind(t);
831
832 switch (kind) {
833 case BTF_KIND_INT:
834 case BTF_KIND_ENUM:
835 case BTF_KIND_ENUM64:
836 case BTF_KIND_FLOAT:
837 return min(btf_ptr_sz(btf), (size_t)t->size);
838 case BTF_KIND_PTR:
839 return btf_ptr_sz(btf);
840 case BTF_KIND_TYPEDEF:
841 case BTF_KIND_VOLATILE:
842 case BTF_KIND_CONST:
843 case BTF_KIND_RESTRICT:
844 case BTF_KIND_TYPE_TAG:
845 return btf__align_of(btf, t->type);
846 case BTF_KIND_ARRAY:
847 return btf__align_of(btf, btf_array(t)->type);
848 case BTF_KIND_STRUCT:
849 case BTF_KIND_UNION: {
850 const struct btf_member *m = btf_members(t);
851 __u16 vlen = btf_vlen(t);
852 int i, max_align = 1, align;
853
854 for (i = 0; i < vlen; i++, m++) {
855 align = btf__align_of(btf, m->type);
856 if (align <= 0)
857 return libbpf_err(align);
858 max_align = max(max_align, align);
859
860 /* if field offset isn't aligned according to field
861 * type's alignment, then struct must be packed
862 */
863 if (btf_member_bitfield_size(t, i) == 0 &&
864 (m->offset % (8 * align)) != 0)
865 return 1;
866 }
867
868 /* if struct/union size isn't a multiple of its alignment,
869 * then struct must be packed
870 */
871 if ((t->size % max_align) != 0)
872 return 1;
873
874 return max_align;
875 }
876 default:
877 pr_warn("unsupported BTF_KIND:%u\n", btf_kind(t));
878 return errno = EINVAL, 0;
879 }
880 }
881
882 int btf__resolve_type(const struct btf *btf, __u32 type_id)
883 {
884 const struct btf_type *t;
885 int depth = 0;
886
887 t = btf__type_by_id(btf, type_id);
888 while (depth < MAX_RESOLVE_DEPTH &&
889 !btf_type_is_void_or_null(t) &&
890 (btf_is_mod(t) || btf_is_typedef(t) || btf_is_var(t))) {
891 type_id = t->type;
892 t = btf__type_by_id(btf, type_id);
893 depth++;
894 }
895
896 if (depth == MAX_RESOLVE_DEPTH || btf_type_is_void_or_null(t))
897 return libbpf_err(-EINVAL);
898
899 return type_id;
900 }
901
902 static void btf_check_sorted(struct btf *btf)
903 {
904 __u32 i, n, named_start_id = 0;
905
906 n = btf__type_cnt(btf);
907 for (i = btf->start_id + 1; i < n; i++) {
908 struct btf_type *ta = btf_type_by_id(btf, i - 1);
909 struct btf_type *tb = btf_type_by_id(btf, i);
910 const char *na = btf__str_by_offset(btf, ta->name_off);
911 const char *nb = btf__str_by_offset(btf, tb->name_off);
912
913 if (strcmp(na, nb) > 0)
914 return;
915
916 if (named_start_id == 0 && na[0] != '\0')
917 named_start_id = i - 1;
918 if (named_start_id == 0 && nb[0] != '\0')
919 named_start_id = i;
920 }
921
922 if (named_start_id)
923 btf->named_start_id = named_start_id;
924 }
925
926 static __s32 btf_find_type_by_name_bsearch(const struct btf *btf, const char *name,
927 __s32 start_id)
928 {
929 const struct btf_type *t;
930 const char *tname;
931 __s32 l, r, m;
932
933 l = start_id;
934 r = btf__type_cnt(btf) - 1;
935 while (l <= r) {
936 m = l + (r - l) / 2;
937 t = btf_type_by_id(btf, m);
938 tname = btf__str_by_offset(btf, t->name_off);
939 if (strcmp(tname, name) >= 0) {
940 if (l == r)
941 return r;
942 r = m;
943 } else {
944 l = m + 1;
945 }
946 }
947
948 return btf__type_cnt(btf);
949 }
950
951 static __s32 btf_find_by_name_kind(const struct btf *btf, int start_id,
952 const char *type_name, __s32 kind)
953 {
954 __u32 nr_types = btf__type_cnt(btf);
955 const struct btf_type *t;
956 const char *tname;
957 __s32 id;
958
959 if (start_id < btf->start_id) {
960 id = btf_find_by_name_kind(btf->base_btf, start_id,
961 type_name, kind);
962 if (id >= 0)
963 return id;
964 start_id = btf->start_id;
965 }
966
967 if (kind == BTF_KIND_UNKN || strcmp(type_name, "void") == 0)
968 return 0;
969
970 if (btf->named_start_id > 0 && type_name[0]) {
971 start_id = max(start_id, btf->named_start_id);
972 id = btf_find_type_by_name_bsearch(btf, type_name, start_id);
973 for (; id < nr_types; id++) {
974 t = btf__type_by_id(btf, id);
975 tname = btf__str_by_offset(btf, t->name_off);
976 if (strcmp(tname, type_name) != 0)
977 return libbpf_err(-ENOENT);
978 if (kind < 0 || btf_kind(t) == kind)
979 return id;
980 }
981 } else {
982 for (id = start_id; id < nr_types; id++) {
983 t = btf_type_by_id(btf, id);
984 if (kind > 0 && btf_kind(t) != kind)
985 continue;
986 tname = btf__str_by_offset(btf, t->name_off);
987 if (strcmp(tname, type_name) == 0)
988 return id;
989 }
990 }
991
992 return libbpf_err(-ENOENT);
993 }
994
995 /* the kind value of -1 indicates that kind matching should be skipped */
996 __s32 btf__find_by_name(const struct btf *btf, const char *type_name)
997 {
998 return btf_find_by_name_kind(btf, 1, type_name, -1);
999 }
1000
1001 __s32 btf__find_by_name_kind_own(const struct btf *btf, const char *type_name,
1002 __u32 kind)
1003 {
1004 return btf_find_by_name_kind(btf, btf->start_id, type_name, kind);
1005 }
1006
1007 __s32 btf__find_by_name_kind(const struct btf *btf, const char *type_name,
1008 __u32 kind)
1009 {
1010 return btf_find_by_name_kind(btf, 1, type_name, kind);
1011 }
1012
1013 static bool btf_is_modifiable(const struct btf *btf)
1014 {
1015 return (void *)btf->hdr != btf->raw_data;
1016 }
1017
1018 static void btf_free_raw_data(struct btf *btf)
1019 {
1020 if (btf->raw_data_is_mmap) {
1021 munmap(btf->raw_data, btf->raw_size);
1022 btf->raw_data_is_mmap = false;
1023 } else {
1024 free(btf->raw_data);
1025 }
1026 btf->raw_data = NULL;
1027 }
1028
1029 void btf__free(struct btf *btf)
1030 {
1031 if (IS_ERR_OR_NULL(btf))
1032 return;
1033
1034 if (btf->fd >= 0)
1035 close(btf->fd);
1036
1037 if (btf_is_modifiable(btf)) {
1038 /* if BTF was modified after loading, it will have a split
1039 * in-memory representation for header, types, and strings
1040 * sections, so we need to free all of them individually. It
1041 * might still have a cached contiguous raw data present,
1042 * which will be unconditionally freed below.
1043 */
1044 free(btf->hdr);
1045 free(btf->types_data);
1046 strset__free(btf->strs_set);
1047 }
1048 btf_free_raw_data(btf);
1049 free(btf->raw_data_swapped);
1050 free(btf->type_offs);
1051 if (btf->owns_base)
1052 btf__free(btf->base_btf);
1053 free(btf);
1054 }
1055
1056 static struct btf *btf_new_empty(struct btf *base_btf)
1057 {
1058 struct btf *btf;
1059
1060 btf = calloc(1, sizeof(*btf));
1061 if (!btf)
1062 return ERR_PTR(-ENOMEM);
1063
1064 btf->nr_types = 0;
1065 btf->start_id = 1;
1066 btf->start_str_off = 0;
1067 btf->fd = -1;
1068 btf->ptr_sz = sizeof(void *);
1069 btf->swapped_endian = false;
1070 btf->named_start_id = 0;
1071
1072 if (base_btf) {
1073 btf->base_btf = base_btf;
1074 btf->start_id = btf__type_cnt(base_btf);
1075 btf->start_str_off = base_btf->hdr->str_len + base_btf->start_str_off;
1076 btf->swapped_endian = base_btf->swapped_endian;
1077 }
1078
1079 /* +1 for empty string at offset 0 */
1080 btf->raw_size = sizeof(struct btf_header) + (base_btf ? 0 : 1);
1081 btf->raw_data = calloc(1, btf->raw_size);
1082 if (!btf->raw_data) {
1083 free(btf);
1084 return ERR_PTR(-ENOMEM);
1085 }
1086
1087 btf->hdr = btf->raw_data;
1088 btf->hdr->hdr_len = sizeof(struct btf_header);
1089 btf->hdr->magic = BTF_MAGIC;
1090 btf->hdr->version = BTF_VERSION;
1091
1092 btf->types_data = btf->raw_data + btf->hdr->hdr_len;
1093 btf->strs_data = btf->raw_data + btf->hdr->hdr_len;
1094 btf->hdr->str_len = base_btf ? 0 : 1; /* empty string at offset 0 */
1095
1096 return btf;
1097 }
1098
1099 struct btf *btf__new_empty(void)
1100 {
1101 return libbpf_ptr(btf_new_empty(NULL));
1102 }
1103
1104 struct btf *btf__new_empty_split(struct btf *base_btf)
1105 {
1106 return libbpf_ptr(btf_new_empty(base_btf));
1107 }
1108
1109 static struct btf *btf_new(const void *data, __u32 size, struct btf *base_btf, bool is_mmap)
1110 {
1111 struct btf *btf;
1112 int err;
1113
1114 btf = calloc(1, sizeof(struct btf));
1115 if (!btf)
1116 return ERR_PTR(-ENOMEM);
1117
1118 btf->nr_types = 0;
1119 btf->start_id = 1;
1120 btf->start_str_off = 0;
1121 btf->fd = -1;
1122 btf->named_start_id = 0;
1123
1124 if (base_btf) {
1125 btf->base_btf = base_btf;
1126 btf->start_id = btf__type_cnt(base_btf);
1127 btf->start_str_off = base_btf->hdr->str_len + base_btf->start_str_off;
1128 }
1129
1130 if (is_mmap) {
1131 btf->raw_data = (void *)data;
1132 btf->raw_data_is_mmap = true;
1133 } else {
1134 btf->raw_data = malloc(size);
1135 if (!btf->raw_data) {
1136 err = -ENOMEM;
1137 goto done;
1138 }
1139 memcpy(btf->raw_data, data, size);
1140 }
1141
1142 btf->raw_size = size;
1143
1144 btf->hdr = btf->raw_data;
1145 err = btf_parse_hdr(btf);
1146 if (err)
1147 goto done;
1148
1149 btf->strs_data = btf->raw_data + btf->hdr->hdr_len + btf->hdr->str_off;
1150 btf->types_data = btf->raw_data + btf->hdr->hdr_len + btf->hdr->type_off;
1151
1152 err = btf_parse_str_sec(btf);
1153 err = err ?: btf_parse_type_sec(btf);
1154 err = err ?: btf_sanity_check(btf);
1155 if (err)
1156 goto done;
1157 btf_check_sorted(btf);
1158
1159 done:
1160 if (err) {
1161 btf__free(btf);
1162 return ERR_PTR(err);
1163 }
1164
1165 return btf;
1166 }
1167
1168 struct btf *btf__new(const void *data, __u32 size)
1169 {
1170 return libbpf_ptr(btf_new(data, size, NULL, false));
1171 }
1172
1173 struct btf *btf__new_split(const void *data, __u32 size, struct btf *base_btf)
1174 {
1175 return libbpf_ptr(btf_new(data, size, base_btf, false));
1176 }
1177
1178 struct btf_elf_secs {
1179 Elf_Data *btf_data;
1180 Elf_Data *btf_ext_data;
1181 Elf_Data *btf_base_data;
1182 };
1183
1184 static int btf_find_elf_sections(Elf *elf, const char *path, struct btf_elf_secs *secs)
1185 {
1186 Elf_Scn *scn = NULL;
1187 Elf_Data *data;
1188 GElf_Ehdr ehdr;
1189 size_t shstrndx;
1190 int idx = 0;
1191
1192 if (!gelf_getehdr(elf, &ehdr)) {
1193 pr_warn("failed to get EHDR from %s\n", path);
1194 goto err;
1195 }
1196
1197 if (elf_getshdrstrndx(elf, &shstrndx)) {
1198 pr_warn("failed to get section names section index for %s\n",
1199 path);
1200 goto err;
1201 }
1202
1203 if (!elf_rawdata(elf_getscn(elf, shstrndx), NULL)) {
1204 pr_warn("failed to get e_shstrndx from %s\n", path);
1205 goto err;
1206 }
1207
1208 while ((scn = elf_nextscn(elf, scn)) != NULL) {
1209 Elf_Data **field;
1210 GElf_Shdr sh;
1211 char *name;
1212
1213 idx++;
1214 if (gelf_getshdr(scn, &sh) != &sh) {
1215 pr_warn("failed to get section(%d) header from %s\n",
1216 idx, path);
1217 goto err;
1218 }
1219 name = elf_strptr(elf, shstrndx, sh.sh_name);
1220 if (!name) {
1221 pr_warn("failed to get section(%d) name from %s\n",
1222 idx, path);
1223 goto err;
1224 }
1225
1226 if (strcmp(name, BTF_ELF_SEC) == 0)
1227 field = &secs->btf_data;
1228 else if (strcmp(name, BTF_EXT_ELF_SEC) == 0)
1229 field = &secs->btf_ext_data;
1230 else if (strcmp(name, BTF_BASE_ELF_SEC) == 0)
1231 field = &secs->btf_base_data;
1232 else
1233 continue;
1234
1235 if (sh.sh_type != SHT_PROGBITS) {
1236 pr_warn("unexpected section type (%d) of section(%d, %s) from %s\n",
1237 sh.sh_type, idx, name, path);
1238 goto err;
1239 }
1240
1241 data = elf_getdata(scn, 0);
1242 if (!data) {
1243 pr_warn("failed to get section(%d, %s) data from %s\n",
1244 idx, name, path);
1245 goto err;
1246 }
1247 *field = data;
1248 }
1249
1250 return 0;
1251
1252 err:
1253 return -LIBBPF_ERRNO__FORMAT;
1254 }
1255
1256 static struct btf *btf_parse_elf(const char *path, struct btf *base_btf,
1257 struct btf_ext **btf_ext)
1258 {
1259 struct btf_elf_secs secs = {};
1260 struct btf *dist_base_btf = NULL;
1261 struct btf *btf = NULL;
1262 int err = 0, fd = -1;
1263 Elf *elf = NULL;
1264
1265 if (elf_version(EV_CURRENT) == EV_NONE) {
1266 pr_warn("failed to init libelf for %s\n", path);
1267 return ERR_PTR(-LIBBPF_ERRNO__LIBELF);
1268 }
1269
1270 fd = open(path, O_RDONLY | O_CLOEXEC);
1271 if (fd < 0) {
1272 err = -errno;
1273 pr_warn("failed to open %s: %s\n", path, errstr(err));
1274 return ERR_PTR(err);
1275 }
1276
1277 elf = elf_begin(fd, ELF_C_READ, NULL);
1278 if (!elf) {
1279 err = -LIBBPF_ERRNO__FORMAT;
1280 pr_warn("failed to open %s as ELF file\n", path);
1281 goto done;
1282 }
1283
1284 err = btf_find_elf_sections(elf, path, &secs);
1285 if (err)
1286 goto done;
1287
1288 if (!secs.btf_data) {
1289 pr_warn("failed to find '%s' ELF section in %s\n", BTF_ELF_SEC, path);
1290 err = -ENODATA;
1291 goto done;
1292 }
1293
1294 if (secs.btf_base_data) {
1295 dist_base_btf = btf_new(secs.btf_base_data->d_buf, secs.btf_base_data->d_size,
1296 NULL, false);
1297 if (IS_ERR(dist_base_btf)) {
1298 err = PTR_ERR(dist_base_btf);
1299 dist_base_btf = NULL;
1300 goto done;
1301 }
1302 }
1303
1304 btf = btf_new(secs.btf_data->d_buf, secs.btf_data->d_size,
1305 dist_base_btf ?: base_btf, false);
1306 if (IS_ERR(btf)) {
1307 err = PTR_ERR(btf);
1308 goto done;
1309 }
1310 if (dist_base_btf && base_btf) {
1311 err = btf__relocate(btf, base_btf);
1312 if (err)
1313 goto done;
1314 btf__free(dist_base_btf);
1315 dist_base_btf = NULL;
1316 }
1317
1318 if (dist_base_btf)
1319 btf->owns_base = true;
1320
1321 switch (gelf_getclass(elf)) {
1322 case ELFCLASS32:
1323 btf__set_pointer_size(btf, 4);
1324 break;
1325 case ELFCLASS64:
1326 btf__set_pointer_size(btf, 8);
1327 break;
1328 default:
1329 pr_warn("failed to get ELF class (bitness) for %s\n", path);
1330 break;
1331 }
1332
1333 if (btf_ext && secs.btf_ext_data) {
1334 *btf_ext = btf_ext__new(secs.btf_ext_data->d_buf, secs.btf_ext_data->d_size);
1335 if (IS_ERR(*btf_ext)) {
1336 err = PTR_ERR(*btf_ext);
1337 goto done;
1338 }
1339 } else if (btf_ext) {
1340 *btf_ext = NULL;
1341 }
1342 done:
1343 if (elf)
1344 elf_end(elf);
1345 close(fd);
1346
1347 if (!err)
1348 return btf;
1349
1350 if (btf_ext)
1351 btf_ext__free(*btf_ext);
1352 btf__free(dist_base_btf);
1353 btf__free(btf);
1354
1355 return ERR_PTR(err);
1356 }
1357
1358 struct btf *btf__parse_elf(const char *path, struct btf_ext **btf_ext)
1359 {
1360 return libbpf_ptr(btf_parse_elf(path, NULL, btf_ext));
1361 }
1362
1363 struct btf *btf__parse_elf_split(const char *path, struct btf *base_btf)
1364 {
1365 return libbpf_ptr(btf_parse_elf(path, base_btf, NULL));
1366 }
1367
1368 static struct btf *btf_parse_raw(const char *path, struct btf *base_btf)
1369 {
1370 struct btf *btf = NULL;
1371 void *data = NULL;
1372 FILE *f = NULL;
1373 __u16 magic;
1374 int err = 0;
1375 long sz;
1376
1377 f = fopen(path, "rbe");
1378 if (!f) {
1379 err = -errno;
1380 goto err_out;
1381 }
1382
1383 /* check BTF magic */
1384 if (fread(&magic, 1, sizeof(magic), f) < sizeof(magic)) {
1385 err = -EIO;
1386 goto err_out;
1387 }
1388 if (magic != BTF_MAGIC && magic != bswap_16(BTF_MAGIC)) {
1389 /* definitely not a raw BTF */
1390 err = -EPROTO;
1391 goto err_out;
1392 }
1393
1394 /* get file size */
1395 if (fseek(f, 0, SEEK_END)) {
1396 err = -errno;
1397 goto err_out;
1398 }
1399 sz = ftell(f);
1400 if (sz < 0) {
1401 err = -errno;
1402 goto err_out;
1403 }
1404 /* rewind to the start */
1405 if (fseek(f, 0, SEEK_SET)) {
1406 err = -errno;
1407 goto err_out;
1408 }
1409
1410 /* pre-alloc memory and read all of BTF data */
1411 data = malloc(sz);
1412 if (!data) {
1413 err = -ENOMEM;
1414 goto err_out;
1415 }
1416 if (fread(data, 1, sz, f) < sz) {
1417 err = -EIO;
1418 goto err_out;
1419 }
1420
1421 /* finally parse BTF data */
1422 btf = btf_new(data, sz, base_btf, false);
1423
1424 err_out:
1425 free(data);
1426 if (f)
1427 fclose(f);
1428 return err ? ERR_PTR(err) : btf;
1429 }
1430
1431 struct btf *btf__parse_raw(const char *path)
1432 {
1433 return libbpf_ptr(btf_parse_raw(path, NULL));
1434 }
1435
1436 struct btf *btf__parse_raw_split(const char *path, struct btf *base_btf)
1437 {
1438 return libbpf_ptr(btf_parse_raw(path, base_btf));
1439 }
1440
1441 static struct btf *btf_parse_raw_mmap(const char *path, struct btf *base_btf)
1442 {
1443 struct stat st;
1444 void *data;
1445 struct btf *btf;
1446 int fd, err;
1447
1448 fd = open(path, O_RDONLY);
1449 if (fd < 0)
1450 return ERR_PTR(-errno);
1451
1452 if (fstat(fd, &st) < 0) {
1453 err = -errno;
1454 close(fd);
1455 return ERR_PTR(err);
1456 }
1457
1458 data = mmap(NULL, st.st_size, PROT_READ, MAP_PRIVATE, fd, 0);
1459 err = -errno;
1460 close(fd);
1461
1462 if (data == MAP_FAILED)
1463 return ERR_PTR(err);
1464
1465 btf = btf_new(data, st.st_size, base_btf, true);
1466 if (IS_ERR(btf))
1467 munmap(data, st.st_size);
1468
1469 return btf;
1470 }
1471
1472 static struct btf *btf_parse(const char *path, struct btf *base_btf, struct btf_ext **btf_ext)
1473 {
1474 struct btf *btf;
1475 int err;
1476
1477 if (btf_ext)
1478 *btf_ext = NULL;
1479
1480 btf = btf_parse_raw(path, base_btf);
1481 err = libbpf_get_error(btf);
1482 if (!err)
1483 return btf;
1484 if (err != -EPROTO)
1485 return ERR_PTR(err);
1486 return btf_parse_elf(path, base_btf, btf_ext);
1487 }
1488
1489 struct btf *btf__parse(const char *path, struct btf_ext **btf_ext)
1490 {
1491 return libbpf_ptr(btf_parse(path, NULL, btf_ext));
1492 }
1493
1494 struct btf *btf__parse_split(const char *path, struct btf *base_btf)
1495 {
1496 return libbpf_ptr(btf_parse(path, base_btf, NULL));
1497 }
1498
1499 static void *btf_get_raw_data(const struct btf *btf, __u32 *size, bool swap_endian);
1500
1501 int btf_load_into_kernel(struct btf *btf,
1502 char *log_buf, size_t log_sz, __u32 log_level,
1503 int token_fd)
1504 {
1505 LIBBPF_OPTS(bpf_btf_load_opts, opts);
1506 __u32 buf_sz = 0, raw_size;
1507 char *buf = NULL, *tmp;
1508 void *raw_data;
1509 int err = 0;
1510
1511 if (btf->fd >= 0)
1512 return libbpf_err(-EEXIST);
1513 if (log_sz && !log_buf)
1514 return libbpf_err(-EINVAL);
1515
1516 /* cache native raw data representation */
1517 raw_data = btf_get_raw_data(btf, &raw_size, false);
1518 if (!raw_data) {
1519 err = -ENOMEM;
1520 goto done;
1521 }
1522 btf->raw_size = raw_size;
1523 btf->raw_data = raw_data;
1524
1525 retry_load:
1526 /* if log_level is 0, we won't provide log_buf/log_size to the kernel,
1527 * initially. Only if BTF loading fails, we bump log_level to 1 and
1528 * retry, using either auto-allocated or custom log_buf. This way
1529 * non-NULL custom log_buf provides a buffer just in case, but hopes
1530 * for successful load and no need for log_buf.
1531 */
1532 if (log_level) {
1533 /* if caller didn't provide custom log_buf, we'll keep
1534 * allocating our own progressively bigger buffers for BTF
1535 * verification log
1536 */
1537 if (!log_buf) {
1538 buf_sz = max((__u32)BPF_LOG_BUF_SIZE, buf_sz * 2);
1539 tmp = realloc(buf, buf_sz);
1540 if (!tmp) {
1541 err = -ENOMEM;
1542 goto done;
1543 }
1544 buf = tmp;
1545 buf[0] = '\0';
1546 }
1547
1548 opts.log_buf = log_buf ? log_buf : buf;
1549 opts.log_size = log_buf ? log_sz : buf_sz;
1550 opts.log_level = log_level;
1551 }
1552
1553 opts.token_fd = token_fd;
1554 if (token_fd)
1555 opts.btf_flags |= BPF_F_TOKEN_FD;
1556
1557 btf->fd = bpf_btf_load(raw_data, raw_size, &opts);
1558 if (btf->fd < 0) {
1559 /* time to turn on verbose mode and try again */
1560 if (log_level == 0) {
1561 log_level = 1;
1562 goto retry_load;
1563 }
1564 /* only retry if caller didn't provide custom log_buf, but
1565 * make sure we can never overflow buf_sz
1566 */
1567 if (!log_buf && errno == ENOSPC && buf_sz <= UINT_MAX / 2)
1568 goto retry_load;
1569
1570 err = -errno;
1571 pr_warn("BTF loading error: %s\n", errstr(err));
1572 /* don't print out contents of custom log_buf */
1573 if (!log_buf && buf[0])
1574 pr_warn("-- BEGIN BTF LOAD LOG ---\n%s\n-- END BTF LOAD LOG --\n", buf);
1575 }
1576
1577 done:
1578 free(buf);
1579 return libbpf_err(err);
1580 }
1581
1582 int btf__load_into_kernel(struct btf *btf)
1583 {
1584 return btf_load_into_kernel(btf, NULL, 0, 0, 0);
1585 }
1586
1587 int btf__fd(const struct btf *btf)
1588 {
1589 return btf->fd;
1590 }
1591
1592 void btf__set_fd(struct btf *btf, int fd)
1593 {
1594 btf->fd = fd;
1595 }
1596
1597 static const void *btf_strs_data(const struct btf *btf)
1598 {
1599 return btf->strs_data ? btf->strs_data : strset__data(btf->strs_set);
1600 }
1601
1602 static void *btf_get_raw_data(const struct btf *btf, __u32 *size, bool swap_endian)
1603 {
1604 struct btf_header *hdr = btf->hdr;
1605 struct btf_type *t;
1606 void *data, *p;
1607 __u32 data_sz;
1608 int i;
1609
1610 data = swap_endian ? btf->raw_data_swapped : btf->raw_data;
1611 if (data) {
1612 *size = btf->raw_size;
1613 return data;
1614 }
1615
1616 data_sz = hdr->hdr_len + hdr->type_len + hdr->str_len;
1617 data = calloc(1, data_sz);
1618 if (!data)
1619 return NULL;
1620 p = data;
1621
1622 memcpy(p, hdr, hdr->hdr_len);
1623 if (swap_endian)
1624 btf_bswap_hdr(p);
1625 p += hdr->hdr_len;
1626
1627 memcpy(p, btf->types_data, hdr->type_len);
1628 if (swap_endian) {
1629 for (i = 0; i < btf->nr_types; i++) {
1630 t = p + btf->type_offs[i];
1631 /* btf_bswap_type_rest() relies on native t->info, so
1632 * we swap base type info after we swapped all the
1633 * additional information
1634 */
1635 if (btf_bswap_type_rest(t))
1636 goto err_out;
1637 btf_bswap_type_base(t);
1638 }
1639 }
1640 p += hdr->type_len;
1641
1642 memcpy(p, btf_strs_data(btf), hdr->str_len);
1643 p += hdr->str_len;
1644
1645 *size = data_sz;
1646 return data;
1647 err_out:
1648 free(data);
1649 return NULL;
1650 }
1651
1652 const void *btf__raw_data(const struct btf *btf_ro, __u32 *size)
1653 {
1654 struct btf *btf = (struct btf *)btf_ro;
1655 __u32 data_sz;
1656 void *data;
1657
1658 data = btf_get_raw_data(btf, &data_sz, btf->swapped_endian);
1659 if (!data)
1660 return errno = ENOMEM, NULL;
1661
1662 btf->raw_size = data_sz;
1663 if (btf->swapped_endian)
1664 btf->raw_data_swapped = data;
1665 else
1666 btf->raw_data = data;
1667 *size = data_sz;
1668 return data;
1669 }
1670
1671 __attribute__((alias("btf__raw_data")))
1672 const void *btf__get_raw_data(const struct btf *btf, __u32 *size);
1673
1674 const char *btf__str_by_offset(const struct btf *btf, __u32 offset)
1675 {
1676 if (offset < btf->start_str_off)
1677 return btf__str_by_offset(btf->base_btf, offset);
1678 else if (offset - btf->start_str_off < btf->hdr->str_len)
1679 return btf_strs_data(btf) + (offset - btf->start_str_off);
1680 else
1681 return errno = EINVAL, NULL;
1682 }
1683
1684 const char *btf__name_by_offset(const struct btf *btf, __u32 offset)
1685 {
1686 return btf__str_by_offset(btf, offset);
1687 }
1688
1689 struct btf *btf_get_from_fd(int btf_fd, struct btf *base_btf)
1690 {
1691 struct bpf_btf_info btf_info;
1692 __u32 len = sizeof(btf_info);
1693 __u32 last_size;
1694 struct btf *btf;
1695 void *ptr;
1696 int err;
1697
1698 /* we won't know btf_size until we call bpf_btf_get_info_by_fd(). so
1699 * let's start with a sane default - 4KiB here - and resize it only if
1700 * bpf_btf_get_info_by_fd() needs a bigger buffer.
1701 */
1702 last_size = 4096;
1703 ptr = malloc(last_size);
1704 if (!ptr)
1705 return ERR_PTR(-ENOMEM);
1706
1707 memset(&btf_info, 0, sizeof(btf_info));
1708 btf_info.btf = ptr_to_u64(ptr);
1709 btf_info.btf_size = last_size;
1710 err = bpf_btf_get_info_by_fd(btf_fd, &btf_info, &len);
1711
1712 if (!err && btf_info.btf_size > last_size) {
1713 void *temp_ptr;
1714
1715 last_size = btf_info.btf_size;
1716 temp_ptr = realloc(ptr, last_size);
1717 if (!temp_ptr) {
1718 btf = ERR_PTR(-ENOMEM);
1719 goto exit_free;
1720 }
1721 ptr = temp_ptr;
1722
1723 len = sizeof(btf_info);
1724 memset(&btf_info, 0, sizeof(btf_info));
1725 btf_info.btf = ptr_to_u64(ptr);
1726 btf_info.btf_size = last_size;
1727
1728 err = bpf_btf_get_info_by_fd(btf_fd, &btf_info, &len);
1729 }
1730
1731 if (err || btf_info.btf_size > last_size) {
1732 btf = err ? ERR_PTR(-errno) : ERR_PTR(-E2BIG);
1733 goto exit_free;
1734 }
1735
1736 btf = btf_new(ptr, btf_info.btf_size, base_btf, false);
1737
1738 exit_free:
1739 free(ptr);
1740 return btf;
1741 }
1742
1743 struct btf *btf_load_from_kernel(__u32 id, struct btf *base_btf, int token_fd)
1744 {
1745 struct btf *btf;
1746 int btf_fd;
1747 LIBBPF_OPTS(bpf_get_fd_by_id_opts, opts);
1748
1749 if (token_fd) {
1750 opts.open_flags |= BPF_F_TOKEN_FD;
1751 opts.token_fd = token_fd;
1752 }
1753
1754 btf_fd = bpf_btf_get_fd_by_id_opts(id, &opts);
1755 if (btf_fd < 0)
1756 return libbpf_err_ptr(-errno);
1757
1758 btf = btf_get_from_fd(btf_fd, base_btf);
1759 close(btf_fd);
1760
1761 return libbpf_ptr(btf);
1762 }
1763
1764 struct btf *btf__load_from_kernel_by_id_split(__u32 id, struct btf *base_btf)
1765 {
1766 return btf_load_from_kernel(id, base_btf, 0);
1767 }
1768
1769 struct btf *btf__load_from_kernel_by_id(__u32 id)
1770 {
1771 return btf__load_from_kernel_by_id_split(id, NULL);
1772 }
1773
1774 static void btf_invalidate_raw_data(struct btf *btf)
1775 {
1776 if (btf->raw_data)
1777 btf_free_raw_data(btf);
1778 if (btf->raw_data_swapped) {
1779 free(btf->raw_data_swapped);
1780 btf->raw_data_swapped = NULL;
1781 }
1782 btf->named_start_id = 0;
1783 }
1784
1785 /* Ensure BTF is ready to be modified (by splitting into a three memory
1786 * regions for header, types, and strings). Also invalidate cached
1787 * raw_data, if any.
1788 */
1789 static int btf_ensure_modifiable(struct btf *btf)
1790 {
1791 void *hdr, *types;
1792 struct strset *set = NULL;
1793 int err = -ENOMEM;
1794
1795 if (btf_is_modifiable(btf)) {
1796 /* any BTF modification invalidates raw_data */
1797 btf_invalidate_raw_data(btf);
1798 return 0;
1799 }
1800
1801 /* split raw data into three memory regions */
1802 hdr = malloc(btf->hdr->hdr_len);
1803 types = malloc(btf->hdr->type_len);
1804 if (!hdr || !types)
1805 goto err_out;
1806
1807 memcpy(hdr, btf->hdr, btf->hdr->hdr_len);
1808 memcpy(types, btf->types_data, btf->hdr->type_len);
1809
1810 /* build lookup index for all strings */
1811 set = strset__new(BTF_MAX_STR_OFFSET, btf->strs_data, btf->hdr->str_len);
1812 if (IS_ERR(set)) {
1813 err = PTR_ERR(set);
1814 goto err_out;
1815 }
1816
1817 /* only when everything was successful, update internal state */
1818 btf->hdr = hdr;
1819 btf->types_data = types;
1820 btf->types_data_cap = btf->hdr->type_len;
1821 btf->strs_data = NULL;
1822 btf->strs_set = set;
1823 /* if BTF was created from scratch, all strings are guaranteed to be
1824 * unique and deduplicated
1825 */
1826 if (btf->hdr->str_len == 0)
1827 btf->strs_deduped = true;
1828 if (!btf->base_btf && btf->hdr->str_len == 1)
1829 btf->strs_deduped = true;
1830
1831 /* invalidate raw_data representation */
1832 btf_invalidate_raw_data(btf);
1833
1834 return 0;
1835
1836 err_out:
1837 strset__free(set);
1838 free(hdr);
1839 free(types);
1840 return err;
1841 }
1842
1843 /* Find an offset in BTF string section that corresponds to a given string *s*.
1844 * Returns:
1845 * - >0 offset into string section, if string is found;
1846 * - -ENOENT, if string is not in the string section;
1847 * - <0, on any other error.
1848 */
1849 int btf__find_str(struct btf *btf, const char *s)
1850 {
1851 int off;
1852
1853 if (btf->base_btf) {
1854 off = btf__find_str(btf->base_btf, s);
1855 if (off != -ENOENT)
1856 return off;
1857 }
1858
1859 /* BTF needs to be in a modifiable state to build string lookup index */
1860 if (btf_ensure_modifiable(btf))
1861 return libbpf_err(-ENOMEM);
1862
1863 off = strset__find_str(btf->strs_set, s);
1864 if (off < 0)
1865 return libbpf_err(off);
1866
1867 return btf->start_str_off + off;
1868 }
1869
1870 /* Add a string s to the BTF string section.
1871 * Returns:
1872 * - > 0 offset into string section, on success;
1873 * - < 0, on error.
1874 */
1875 int btf__add_str(struct btf *btf, const char *s)
1876 {
1877 int off;
1878
1879 if (btf->base_btf) {
1880 off = btf__find_str(btf->base_btf, s);
1881 if (off != -ENOENT)
1882 return off;
1883 }
1884
1885 if (btf_ensure_modifiable(btf))
1886 return libbpf_err(-ENOMEM);
1887
1888 off = strset__add_str(btf->strs_set, s);
1889 if (off < 0)
1890 return libbpf_err(off);
1891
1892 btf->hdr->str_len = strset__data_size(btf->strs_set);
1893
1894 return btf->start_str_off + off;
1895 }
1896
1897 static void *btf_add_type_mem(struct btf *btf, size_t add_sz)
1898 {
1899 return libbpf_add_mem(&btf->types_data, &btf->types_data_cap, 1,
1900 btf->hdr->type_len, UINT_MAX, add_sz);
1901 }
1902
1903 static void btf_type_inc_vlen(struct btf_type *t)
1904 {
1905 t->info = btf_type_info(btf_kind(t), btf_vlen(t) + 1, btf_kflag(t));
1906 }
1907
1908 static int btf_commit_type(struct btf *btf, int data_sz)
1909 {
1910 int err;
1911
1912 err = btf_add_type_idx_entry(btf, btf->hdr->type_len);
1913 if (err)
1914 return libbpf_err(err);
1915
1916 btf->hdr->type_len += data_sz;
1917 btf->hdr->str_off += data_sz;
1918 btf->nr_types++;
1919 return btf->start_id + btf->nr_types - 1;
1920 }
1921
1922 struct btf_pipe {
1923 const struct btf *src;
1924 struct btf *dst;
1925 struct hashmap *str_off_map; /* map string offsets from src to dst */
1926 };
1927
1928 static int btf_rewrite_str(struct btf_pipe *p, __u32 *str_off)
1929 {
1930 long mapped_off;
1931 int off, err;
1932
1933 if (!*str_off) /* nothing to do for empty strings */
1934 return 0;
1935
1936 if (p->str_off_map &&
1937 hashmap__find(p->str_off_map, *str_off, &mapped_off)) {
1938 *str_off = mapped_off;
1939 return 0;
1940 }
1941
1942 off = btf__add_str(p->dst, btf__str_by_offset(p->src, *str_off));
1943 if (off < 0)
1944 return off;
1945
1946 /* Remember string mapping from src to dst. It avoids
1947 * performing expensive string comparisons.
1948 */
1949 if (p->str_off_map) {
1950 err = hashmap__append(p->str_off_map, *str_off, off);
1951 if (err)
1952 return err;
1953 }
1954
1955 *str_off = off;
1956 return 0;
1957 }
1958
1959 static int btf_add_type(struct btf_pipe *p, const struct btf_type *src_type)
1960 {
1961 struct btf_field_iter it;
1962 struct btf_type *t;
1963 __u32 *str_off;
1964 int sz, err;
1965
1966 sz = btf_type_size(src_type);
1967 if (sz < 0)
1968 return libbpf_err(sz);
1969
1970 /* deconstruct BTF, if necessary, and invalidate raw_data */
1971 if (btf_ensure_modifiable(p->dst))
1972 return libbpf_err(-ENOMEM);
1973
1974 t = btf_add_type_mem(p->dst, sz);
1975 if (!t)
1976 return libbpf_err(-ENOMEM);
1977
1978 memcpy(t, src_type, sz);
1979
1980 err = btf_field_iter_init(&it, t, BTF_FIELD_ITER_STRS);
1981 if (err)
1982 return libbpf_err(err);
1983
1984 while ((str_off = btf_field_iter_next(&it))) {
1985 err = btf_rewrite_str(p, str_off);
1986 if (err)
1987 return libbpf_err(err);
1988 }
1989
1990 return btf_commit_type(p->dst, sz);
1991 }
1992
1993 int btf__add_type(struct btf *btf, const struct btf *src_btf, const struct btf_type *src_type)
1994 {
1995 struct btf_pipe p = { .src = src_btf, .dst = btf };
1996
1997 return btf_add_type(&p, src_type);
1998 }
1999
2000 static size_t btf_dedup_identity_hash_fn(long key, void *ctx);
2001 static bool btf_dedup_equal_fn(long k1, long k2, void *ctx);
2002
2003 int btf__add_btf(struct btf *btf, const struct btf *src_btf)
2004 {
2005 struct btf_pipe p = { .src = src_btf, .dst = btf };
2006 int data_sz, sz, cnt, i, err, old_strs_len;
2007 __u32 *off;
2008 void *t;
2009
2010 /* appending split BTF isn't supported yet */
2011 if (src_btf->base_btf)
2012 return libbpf_err(-ENOTSUP);
2013
2014 /* deconstruct BTF, if necessary, and invalidate raw_data */
2015 if (btf_ensure_modifiable(btf))
2016 return libbpf_err(-ENOMEM);
2017
2018 /* remember original strings section size if we have to roll back
2019 * partial strings section changes
2020 */
2021 old_strs_len = btf->hdr->str_len;
2022
2023 data_sz = src_btf->hdr->type_len;
2024 cnt = btf__type_cnt(src_btf) - 1;
2025
2026 /* pre-allocate enough memory for new types */
2027 t = btf_add_type_mem(btf, data_sz);
2028 if (!t)
2029 return libbpf_err(-ENOMEM);
2030
2031 /* pre-allocate enough memory for type offset index for new types */
2032 off = btf_add_type_offs_mem(btf, cnt);
2033 if (!off)
2034 return libbpf_err(-ENOMEM);
2035
2036 /* Map the string offsets from src_btf to the offsets from btf to improve performance */
2037 p.str_off_map = hashmap__new(btf_dedup_identity_hash_fn, btf_dedup_equal_fn, NULL);
2038 if (IS_ERR(p.str_off_map))
2039 return libbpf_err(-ENOMEM);
2040
2041 /* bulk copy types data for all types from src_btf */
2042 memcpy(t, src_btf->types_data, data_sz);
2043
2044 for (i = 0; i < cnt; i++) {
2045 struct btf_field_iter it;
2046 __u32 *type_id, *str_off;
2047
2048 sz = btf_type_size(t);
2049 if (sz < 0) {
2050 /* unlikely, has to be corrupted src_btf */
2051 err = sz;
2052 goto err_out;
2053 }
2054
2055 /* fill out type ID to type offset mapping for lookups by type ID */
2056 *off = t - btf->types_data;
2057
2058 /* add, dedup, and remap strings referenced by this BTF type */
2059 err = btf_field_iter_init(&it, t, BTF_FIELD_ITER_STRS);
2060 if (err)
2061 goto err_out;
2062 while ((str_off = btf_field_iter_next(&it))) {
2063 err = btf_rewrite_str(&p, str_off);
2064 if (err)
2065 goto err_out;
2066 }
2067
2068 /* remap all type IDs referenced from this BTF type */
2069 err = btf_field_iter_init(&it, t, BTF_FIELD_ITER_IDS);
2070 if (err)
2071 goto err_out;
2072
2073 while ((type_id = btf_field_iter_next(&it))) {
2074 if (!*type_id) /* nothing to do for VOID references */
2075 continue;
2076
2077 /* we haven't updated btf's type count yet, so
2078 * btf->start_id + btf->nr_types - 1 is the type ID offset we should
2079 * add to all newly added BTF types
2080 */
2081 *type_id += btf->start_id + btf->nr_types - 1;
2082 }
2083
2084 /* go to next type data and type offset index entry */
2085 t += sz;
2086 off++;
2087 }
2088
2089 /* Up until now any of the copied type data was effectively invisible,
2090 * so if we exited early before this point due to error, BTF would be
2091 * effectively unmodified. There would be extra internal memory
2092 * pre-allocated, but it would not be available for querying. But now
2093 * that we've copied and rewritten all the data successfully, we can
2094 * update type count and various internal offsets and sizes to
2095 * "commit" the changes and made them visible to the outside world.
2096 */
2097 btf->hdr->type_len += data_sz;
2098 btf->hdr->str_off += data_sz;
2099 btf->nr_types += cnt;
2100
2101 hashmap__free(p.str_off_map);
2102
2103 /* return type ID of the first added BTF type */
2104 return btf->start_id + btf->nr_types - cnt;
2105 err_out:
2106 /* zero out preallocated memory as if it was just allocated with
2107 * libbpf_add_mem()
2108 */
2109 memset(btf->types_data + btf->hdr->type_len, 0, data_sz);
2110 memset(btf->strs_data + old_strs_len, 0, btf->hdr->str_len - old_strs_len);
2111
2112 /* and now restore original strings section size; types data size
2113 * wasn't modified, so doesn't need restoring, see big comment above
2114 */
2115 btf->hdr->str_len = old_strs_len;
2116
2117 hashmap__free(p.str_off_map);
2118
2119 return libbpf_err(err);
2120 }
2121
2122 /*
2123 * Append new BTF_KIND_INT type with:
2124 * - *name* - non-empty, non-NULL type name;
2125 * - *sz* - power-of-2 (1, 2, 4, ..) size of the type, in bytes;
2126 * - encoding is a combination of BTF_INT_SIGNED, BTF_INT_CHAR, BTF_INT_BOOL.
2127 * Returns:
2128 * - >0, type ID of newly added BTF type;
2129 * - <0, on error.
2130 */
2131 int btf__add_int(struct btf *btf, const char *name, size_t byte_sz, int encoding)
2132 {
2133 struct btf_type *t;
2134 int sz, name_off;
2135
2136 /* non-empty name */
2137 if (str_is_empty(name))
2138 return libbpf_err(-EINVAL);
2139 /* byte_sz must be power of 2 */
2140 if (!byte_sz || (byte_sz & (byte_sz - 1)) || byte_sz > 16)
2141 return libbpf_err(-EINVAL);
2142 if (encoding & ~(BTF_INT_SIGNED | BTF_INT_CHAR | BTF_INT_BOOL))
2143 return libbpf_err(-EINVAL);
2144
2145 /* deconstruct BTF, if necessary, and invalidate raw_data */
2146 if (btf_ensure_modifiable(btf))
2147 return libbpf_err(-ENOMEM);
2148
2149 sz = sizeof(struct btf_type) + sizeof(int);
2150 t = btf_add_type_mem(btf, sz);
2151 if (!t)
2152 return libbpf_err(-ENOMEM);
2153
2154 /* if something goes wrong later, we might end up with an extra string,
2155 * but that shouldn't be a problem, because BTF can't be constructed
2156 * completely anyway and will most probably be just discarded
2157 */
2158 name_off = btf__add_str(btf, name);
2159 if (name_off < 0)
2160 return name_off;
2161
2162 t->name_off = name_off;
2163 t->info = btf_type_info(BTF_KIND_INT, 0, 0);
2164 t->size = byte_sz;
2165 /* set INT info, we don't allow setting legacy bit offset/size */
2166 *(__u32 *)(t + 1) = (encoding << 24) | (byte_sz * 8);
2167
2168 return btf_commit_type(btf, sz);
2169 }
2170
2171 /*
2172 * Append new BTF_KIND_FLOAT type with:
2173 * - *name* - non-empty, non-NULL type name;
2174 * - *sz* - size of the type, in bytes;
2175 * Returns:
2176 * - >0, type ID of newly added BTF type;
2177 * - <0, on error.
2178 */
2179 int btf__add_float(struct btf *btf, const char *name, size_t byte_sz)
2180 {
2181 struct btf_type *t;
2182 int sz, name_off;
2183
2184 /* non-empty name */
2185 if (str_is_empty(name))
2186 return libbpf_err(-EINVAL);
2187
2188 /* byte_sz must be one of the explicitly allowed values */
2189 if (byte_sz != 2 && byte_sz != 4 && byte_sz != 8 && byte_sz != 12 &&
2190 byte_sz != 16)
2191 return libbpf_err(-EINVAL);
2192
2193 if (btf_ensure_modifiable(btf))
2194 return libbpf_err(-ENOMEM);
2195
2196 sz = sizeof(struct btf_type);
2197 t = btf_add_type_mem(btf, sz);
2198 if (!t)
2199 return libbpf_err(-ENOMEM);
2200
2201 name_off = btf__add_str(btf, name);
2202 if (name_off < 0)
2203 return name_off;
2204
2205 t->name_off = name_off;
2206 t->info = btf_type_info(BTF_KIND_FLOAT, 0, 0);
2207 t->size = byte_sz;
2208
2209 return btf_commit_type(btf, sz);
2210 }
2211
2212 /* it's completely legal to append BTF types with type IDs pointing forward to
2213 * types that haven't been appended yet, so we only make sure that id looks
2214 * sane, we can't guarantee that ID will always be valid
2215 */
2216 static int validate_type_id(int id)
2217 {
2218 if (id < 0 || id > BTF_MAX_NR_TYPES)
2219 return -EINVAL;
2220 return 0;
2221 }
2222
2223 /* generic append function for PTR, TYPEDEF, CONST/VOLATILE/RESTRICT */
2224 static int btf_add_ref_kind(struct btf *btf, int kind, const char *name, int ref_type_id, int kflag)
2225 {
2226 struct btf_type *t;
2227 int sz, name_off = 0;
2228
2229 if (validate_type_id(ref_type_id))
2230 return libbpf_err(-EINVAL);
2231
2232 if (btf_ensure_modifiable(btf))
2233 return libbpf_err(-ENOMEM);
2234
2235 sz = sizeof(struct btf_type);
2236 t = btf_add_type_mem(btf, sz);
2237 if (!t)
2238 return libbpf_err(-ENOMEM);
2239
2240 if (!str_is_empty(name)) {
2241 name_off = btf__add_str(btf, name);
2242 if (name_off < 0)
2243 return name_off;
2244 }
2245
2246 t->name_off = name_off;
2247 t->info = btf_type_info(kind, 0, kflag);
2248 t->type = ref_type_id;
2249
2250 return btf_commit_type(btf, sz);
2251 }
2252
2253 /*
2254 * Append new BTF_KIND_PTR type with:
2255 * - *ref_type_id* - referenced type ID, it might not exist yet;
2256 * Returns:
2257 * - >0, type ID of newly added BTF type;
2258 * - <0, on error.
2259 */
2260 int btf__add_ptr(struct btf *btf, int ref_type_id)
2261 {
2262 return btf_add_ref_kind(btf, BTF_KIND_PTR, NULL, ref_type_id, 0);
2263 }
2264
2265 /*
2266 * Append new BTF_KIND_ARRAY type with:
2267 * - *index_type_id* - type ID of the type describing array index;
2268 * - *elem_type_id* - type ID of the type describing array element;
2269 * - *nr_elems* - the size of the array;
2270 * Returns:
2271 * - >0, type ID of newly added BTF type;
2272 * - <0, on error.
2273 */
2274 int btf__add_array(struct btf *btf, int index_type_id, int elem_type_id, __u32 nr_elems)
2275 {
2276 struct btf_type *t;
2277 struct btf_array *a;
2278 int sz;
2279
2280 if (validate_type_id(index_type_id) || validate_type_id(elem_type_id))
2281 return libbpf_err(-EINVAL);
2282
2283 if (btf_ensure_modifiable(btf))
2284 return libbpf_err(-ENOMEM);
2285
2286 sz = sizeof(struct btf_type) + sizeof(struct btf_array);
2287 t = btf_add_type_mem(btf, sz);
2288 if (!t)
2289 return libbpf_err(-ENOMEM);
2290
2291 t->name_off = 0;
2292 t->info = btf_type_info(BTF_KIND_ARRAY, 0, 0);
2293 t->size = 0;
2294
2295 a = btf_array(t);
2296 a->type = elem_type_id;
2297 a->index_type = index_type_id;
2298 a->nelems = nr_elems;
2299
2300 return btf_commit_type(btf, sz);
2301 }
2302
2303 /* generic STRUCT/UNION append function */
2304 static int btf_add_composite(struct btf *btf, int kind, const char *name, __u32 bytes_sz)
2305 {
2306 struct btf_type *t;
2307 int sz, name_off = 0;
2308
2309 if (btf_ensure_modifiable(btf))
2310 return libbpf_err(-ENOMEM);
2311
2312 sz = sizeof(struct btf_type);
2313 t = btf_add_type_mem(btf, sz);
2314 if (!t)
2315 return libbpf_err(-ENOMEM);
2316
2317 if (!str_is_empty(name)) {
2318 name_off = btf__add_str(btf, name);
2319 if (name_off < 0)
2320 return name_off;
2321 }
2322
2323 /* start out with vlen=0 and no kflag; this will be adjusted when
2324 * adding each member
2325 */
2326 t->name_off = name_off;
2327 t->info = btf_type_info(kind, 0, 0);
2328 t->size = bytes_sz;
2329
2330 return btf_commit_type(btf, sz);
2331 }
2332
2333 /*
2334 * Append new BTF_KIND_STRUCT type with:
2335 * - *name* - name of the struct, can be NULL or empty for anonymous structs;
2336 * - *byte_sz* - size of the struct, in bytes;
2337 *
2338 * Struct initially has no fields in it. Fields can be added by
2339 * btf__add_field() right after btf__add_struct() succeeds.
2340 *
2341 * Returns:
2342 * - >0, type ID of newly added BTF type;
2343 * - <0, on error.
2344 */
2345 int btf__add_struct(struct btf *btf, const char *name, __u32 byte_sz)
2346 {
2347 return btf_add_composite(btf, BTF_KIND_STRUCT, name, byte_sz);
2348 }
2349
2350 /*
2351 * Append new BTF_KIND_UNION type with:
2352 * - *name* - name of the union, can be NULL or empty for anonymous union;
2353 * - *byte_sz* - size of the union, in bytes;
2354 *
2355 * Union initially has no fields in it. Fields can be added by
2356 * btf__add_field() right after btf__add_union() succeeds. All fields
2357 * should have *bit_offset* of 0.
2358 *
2359 * Returns:
2360 * - >0, type ID of newly added BTF type;
2361 * - <0, on error.
2362 */
2363 int btf__add_union(struct btf *btf, const char *name, __u32 byte_sz)
2364 {
2365 return btf_add_composite(btf, BTF_KIND_UNION, name, byte_sz);
2366 }
2367
2368 static struct btf_type *btf_last_type(struct btf *btf)
2369 {
2370 return btf_type_by_id(btf, btf__type_cnt(btf) - 1);
2371 }
2372
2373 /*
2374 * Append new field for the current STRUCT/UNION type with:
2375 * - *name* - name of the field, can be NULL or empty for anonymous field;
2376 * - *type_id* - type ID for the type describing field type;
2377 * - *bit_offset* - bit offset of the start of the field within struct/union;
2378 * - *bit_size* - bit size of a bitfield, 0 for non-bitfield fields;
2379 * Returns:
2380 * - 0, on success;
2381 * - <0, on error.
2382 */
2383 int btf__add_field(struct btf *btf, const char *name, int type_id,
2384 __u32 bit_offset, __u32 bit_size)
2385 {
2386 struct btf_type *t;
2387 struct btf_member *m;
2388 bool is_bitfield;
2389 int sz, name_off = 0;
2390
2391 /* last type should be union/struct */
2392 if (btf->nr_types == 0)
2393 return libbpf_err(-EINVAL);
2394 t = btf_last_type(btf);
2395 if (!btf_is_composite(t))
2396 return libbpf_err(-EINVAL);
2397
2398 if (validate_type_id(type_id))
2399 return libbpf_err(-EINVAL);
2400 /* best-effort bit field offset/size enforcement */
2401 is_bitfield = bit_size || (bit_offset % 8 != 0);
2402 if (is_bitfield && (bit_size == 0 || bit_size > 255 || bit_offset > 0xffffff))
2403 return libbpf_err(-EINVAL);
2404
2405 /* only offset 0 is allowed for unions */
2406 if (btf_is_union(t) && bit_offset)
2407 return libbpf_err(-EINVAL);
2408
2409 /* decompose and invalidate raw data */
2410 if (btf_ensure_modifiable(btf))
2411 return libbpf_err(-ENOMEM);
2412
2413 sz = sizeof(struct btf_member);
2414 m = btf_add_type_mem(btf, sz);
2415 if (!m)
2416 return libbpf_err(-ENOMEM);
2417
2418 if (!str_is_empty(name)) {
2419 name_off = btf__add_str(btf, name);
2420 if (name_off < 0)
2421 return name_off;
2422 }
2423
2424 m->name_off = name_off;
2425 m->type = type_id;
2426 m->offset = bit_offset | (bit_size << 24);
2427
2428 /* btf_add_type_mem can invalidate t pointer */
2429 t = btf_last_type(btf);
2430 /* update parent type's vlen and kflag */
2431 t->info = btf_type_info(btf_kind(t), btf_vlen(t) + 1, is_bitfield || btf_kflag(t));
2432
2433 btf->hdr->type_len += sz;
2434 btf->hdr->str_off += sz;
2435 return 0;
2436 }
2437
2438 static int btf_add_enum_common(struct btf *btf, const char *name, __u32 byte_sz,
2439 bool is_signed, __u8 kind)
2440 {
2441 struct btf_type *t;
2442 int sz, name_off = 0;
2443
2444 /* byte_sz must be power of 2 */
2445 if (!byte_sz || (byte_sz & (byte_sz - 1)) || byte_sz > 8)
2446 return libbpf_err(-EINVAL);
2447
2448 if (btf_ensure_modifiable(btf))
2449 return libbpf_err(-ENOMEM);
2450
2451 sz = sizeof(struct btf_type);
2452 t = btf_add_type_mem(btf, sz);
2453 if (!t)
2454 return libbpf_err(-ENOMEM);
2455
2456 if (!str_is_empty(name)) {
2457 name_off = btf__add_str(btf, name);
2458 if (name_off < 0)
2459 return name_off;
2460 }
2461
2462 /* start out with vlen=0; it will be adjusted when adding enum values */
2463 t->name_off = name_off;
2464 t->info = btf_type_info(kind, 0, is_signed);
2465 t->size = byte_sz;
2466
2467 return btf_commit_type(btf, sz);
2468 }
2469
2470 /*
2471 * Append new BTF_KIND_ENUM type with:
2472 * - *name* - name of the enum, can be NULL or empty for anonymous enums;
2473 * - *byte_sz* - size of the enum, in bytes.
2474 *
2475 * Enum initially has no enum values in it (and corresponds to enum forward
2476 * declaration). Enumerator values can be added by btf__add_enum_value()
2477 * immediately after btf__add_enum() succeeds.
2478 *
2479 * Returns:
2480 * - >0, type ID of newly added BTF type;
2481 * - <0, on error.
2482 */
2483 int btf__add_enum(struct btf *btf, const char *name, __u32 byte_sz)
2484 {
2485 /*
2486 * set the signedness to be unsigned, it will change to signed
2487 * if any later enumerator is negative.
2488 */
2489 return btf_add_enum_common(btf, name, byte_sz, false, BTF_KIND_ENUM);
2490 }
2491
2492 /*
2493 * Append new enum value for the current ENUM type with:
2494 * - *name* - name of the enumerator value, can't be NULL or empty;
2495 * - *value* - integer value corresponding to enum value *name*;
2496 * Returns:
2497 * - 0, on success;
2498 * - <0, on error.
2499 */
2500 int btf__add_enum_value(struct btf *btf, const char *name, __s64 value)
2501 {
2502 struct btf_type *t;
2503 struct btf_enum *v;
2504 int sz, name_off;
2505
2506 /* last type should be BTF_KIND_ENUM */
2507 if (btf->nr_types == 0)
2508 return libbpf_err(-EINVAL);
2509 t = btf_last_type(btf);
2510 if (!btf_is_enum(t))
2511 return libbpf_err(-EINVAL);
2512
2513 /* non-empty name */
2514 if (str_is_empty(name))
2515 return libbpf_err(-EINVAL);
2516 if (value < INT_MIN || value > UINT_MAX)
2517 return libbpf_err(-E2BIG);
2518
2519 /* decompose and invalidate raw data */
2520 if (btf_ensure_modifiable(btf))
2521 return libbpf_err(-ENOMEM);
2522
2523 sz = sizeof(struct btf_enum);
2524 v = btf_add_type_mem(btf, sz);
2525 if (!v)
2526 return libbpf_err(-ENOMEM);
2527
2528 name_off = btf__add_str(btf, name);
2529 if (name_off < 0)
2530 return name_off;
2531
2532 v->name_off = name_off;
2533 v->val = value;
2534
2535 /* update parent type's vlen */
2536 t = btf_last_type(btf);
2537 btf_type_inc_vlen(t);
2538
2539 /* if negative value, set signedness to signed */
2540 if (value < 0)
2541 t->info = btf_type_info(btf_kind(t), btf_vlen(t), true);
2542
2543 btf->hdr->type_len += sz;
2544 btf->hdr->str_off += sz;
2545 return 0;
2546 }
2547
2548 /*
2549 * Append new BTF_KIND_ENUM64 type with:
2550 * - *name* - name of the enum, can be NULL or empty for anonymous enums;
2551 * - *byte_sz* - size of the enum, in bytes.
2552 * - *is_signed* - whether the enum values are signed or not;
2553 *
2554 * Enum initially has no enum values in it (and corresponds to enum forward
2555 * declaration). Enumerator values can be added by btf__add_enum64_value()
2556 * immediately after btf__add_enum64() succeeds.
2557 *
2558 * Returns:
2559 * - >0, type ID of newly added BTF type;
2560 * - <0, on error.
2561 */
2562 int btf__add_enum64(struct btf *btf, const char *name, __u32 byte_sz,
2563 bool is_signed)
2564 {
2565 return btf_add_enum_common(btf, name, byte_sz, is_signed,
2566 BTF_KIND_ENUM64);
2567 }
2568
2569 /*
2570 * Append new enum value for the current ENUM64 type with:
2571 * - *name* - name of the enumerator value, can't be NULL or empty;
2572 * - *value* - integer value corresponding to enum value *name*;
2573 * Returns:
2574 * - 0, on success;
2575 * - <0, on error.
2576 */
2577 int btf__add_enum64_value(struct btf *btf, const char *name, __u64 value)
2578 {
2579 struct btf_enum64 *v;
2580 struct btf_type *t;
2581 int sz, name_off;
2582
2583 /* last type should be BTF_KIND_ENUM64 */
2584 if (btf->nr_types == 0)
2585 return libbpf_err(-EINVAL);
2586 t = btf_last_type(btf);
2587 if (!btf_is_enum64(t))
2588 return libbpf_err(-EINVAL);
2589
2590 /* non-empty name */
2591 if (str_is_empty(name))
2592 return libbpf_err(-EINVAL);
2593
2594 /* decompose and invalidate raw data */
2595 if (btf_ensure_modifiable(btf))
2596 return libbpf_err(-ENOMEM);
2597
2598 sz = sizeof(struct btf_enum64);
2599 v = btf_add_type_mem(btf, sz);
2600 if (!v)
2601 return libbpf_err(-ENOMEM);
2602
2603 name_off = btf__add_str(btf, name);
2604 if (name_off < 0)
2605 return name_off;
2606
2607 v->name_off = name_off;
2608 v->val_lo32 = (__u32)value;
2609 v->val_hi32 = value >> 32;
2610
2611 /* update parent type's vlen */
2612 t = btf_last_type(btf);
2613 btf_type_inc_vlen(t);
2614
2615 btf->hdr->type_len += sz;
2616 btf->hdr->str_off += sz;
2617 return 0;
2618 }
2619
2620 /*
2621 * Append new BTF_KIND_FWD type with:
2622 * - *name*, non-empty/non-NULL name;
2623 * - *fwd_kind*, kind of forward declaration, one of BTF_FWD_STRUCT,
2624 * BTF_FWD_UNION, or BTF_FWD_ENUM;
2625 * Returns:
2626 * - >0, type ID of newly added BTF type;
2627 * - <0, on error.
2628 */
2629 int btf__add_fwd(struct btf *btf, const char *name, enum btf_fwd_kind fwd_kind)
2630 {
2631 if (str_is_empty(name))
2632 return libbpf_err(-EINVAL);
2633
2634 switch (fwd_kind) {
2635 case BTF_FWD_STRUCT:
2636 case BTF_FWD_UNION: {
2637 struct btf_type *t;
2638 int id;
2639
2640 id = btf_add_ref_kind(btf, BTF_KIND_FWD, name, 0, 0);
2641 if (id <= 0)
2642 return id;
2643 t = btf_type_by_id(btf, id);
2644 t->info = btf_type_info(BTF_KIND_FWD, 0, fwd_kind == BTF_FWD_UNION);
2645 return id;
2646 }
2647 case BTF_FWD_ENUM:
2648 /* enum forward in BTF currently is just an enum with no enum
2649 * values; we also assume a standard 4-byte size for it
2650 */
2651 return btf__add_enum(btf, name, sizeof(int));
2652 default:
2653 return libbpf_err(-EINVAL);
2654 }
2655 }
2656
2657 /*
2658 * Append new BTF_KING_TYPEDEF type with:
2659 * - *name*, non-empty/non-NULL name;
2660 * - *ref_type_id* - referenced type ID, it might not exist yet;
2661 * Returns:
2662 * - >0, type ID of newly added BTF type;
2663 * - <0, on error.
2664 */
2665 int btf__add_typedef(struct btf *btf, const char *name, int ref_type_id)
2666 {
2667 if (str_is_empty(name))
2668 return libbpf_err(-EINVAL);
2669
2670 return btf_add_ref_kind(btf, BTF_KIND_TYPEDEF, name, ref_type_id, 0);
2671 }
2672
2673 /*
2674 * Append new BTF_KIND_VOLATILE type with:
2675 * - *ref_type_id* - referenced type ID, it might not exist yet;
2676 * Returns:
2677 * - >0, type ID of newly added BTF type;
2678 * - <0, on error.
2679 */
2680 int btf__add_volatile(struct btf *btf, int ref_type_id)
2681 {
2682 return btf_add_ref_kind(btf, BTF_KIND_VOLATILE, NULL, ref_type_id, 0);
2683 }
2684
2685 /*
2686 * Append new BTF_KIND_CONST type with:
2687 * - *ref_type_id* - referenced type ID, it might not exist yet;
2688 * Returns:
2689 * - >0, type ID of newly added BTF type;
2690 * - <0, on error.
2691 */
2692 int btf__add_const(struct btf *btf, int ref_type_id)
2693 {
2694 return btf_add_ref_kind(btf, BTF_KIND_CONST, NULL, ref_type_id, 0);
2695 }
2696
2697 /*
2698 * Append new BTF_KIND_RESTRICT type with:
2699 * - *ref_type_id* - referenced type ID, it might not exist yet;
2700 * Returns:
2701 * - >0, type ID of newly added BTF type;
2702 * - <0, on error.
2703 */
2704 int btf__add_restrict(struct btf *btf, int ref_type_id)
2705 {
2706 return btf_add_ref_kind(btf, BTF_KIND_RESTRICT, NULL, ref_type_id, 0);
2707 }
2708
2709 /*
2710 * Append new BTF_KIND_TYPE_TAG type with:
2711 * - *value*, non-empty/non-NULL tag value;
2712 * - *ref_type_id* - referenced type ID, it might not exist yet;
2713 * Returns:
2714 * - >0, type ID of newly added BTF type;
2715 * - <0, on error.
2716 */
2717 int btf__add_type_tag(struct btf *btf, const char *value, int ref_type_id)
2718 {
2719 if (str_is_empty(value))
2720 return libbpf_err(-EINVAL);
2721
2722 return btf_add_ref_kind(btf, BTF_KIND_TYPE_TAG, value, ref_type_id, 0);
2723 }
2724
2725 /*
2726 * Append new BTF_KIND_TYPE_TAG type with:
2727 * - *value*, non-empty/non-NULL tag value;
2728 * - *ref_type_id* - referenced type ID, it might not exist yet;
2729 * Set info->kflag to 1, indicating this tag is an __attribute__
2730 * Returns:
2731 * - >0, type ID of newly added BTF type;
2732 * - <0, on error.
2733 */
2734 int btf__add_type_attr(struct btf *btf, const char *value, int ref_type_id)
2735 {
2736 if (str_is_empty(value))
2737 return libbpf_err(-EINVAL);
2738
2739 return btf_add_ref_kind(btf, BTF_KIND_TYPE_TAG, value, ref_type_id, 1);
2740 }
2741
2742 /*
2743 * Append new BTF_KIND_FUNC type with:
2744 * - *name*, non-empty/non-NULL name;
2745 * - *proto_type_id* - FUNC_PROTO's type ID, it might not exist yet;
2746 * Returns:
2747 * - >0, type ID of newly added BTF type;
2748 * - <0, on error.
2749 */
2750 int btf__add_func(struct btf *btf, const char *name,
2751 enum btf_func_linkage linkage, int proto_type_id)
2752 {
2753 int id;
2754
2755 if (str_is_empty(name))
2756 return libbpf_err(-EINVAL);
2757 if (linkage != BTF_FUNC_STATIC && linkage != BTF_FUNC_GLOBAL &&
2758 linkage != BTF_FUNC_EXTERN)
2759 return libbpf_err(-EINVAL);
2760
2761 id = btf_add_ref_kind(btf, BTF_KIND_FUNC, name, proto_type_id, 0);
2762 if (id > 0) {
2763 struct btf_type *t = btf_type_by_id(btf, id);
2764
2765 t->info = btf_type_info(BTF_KIND_FUNC, linkage, 0);
2766 }
2767 return libbpf_err(id);
2768 }
2769
2770 /*
2771 * Append new BTF_KIND_FUNC_PROTO with:
2772 * - *ret_type_id* - type ID for return result of a function.
2773 *
2774 * Function prototype initially has no arguments, but they can be added by
2775 * btf__add_func_param() one by one, immediately after
2776 * btf__add_func_proto() succeeded.
2777 *
2778 * Returns:
2779 * - >0, type ID of newly added BTF type;
2780 * - <0, on error.
2781 */
2782 int btf__add_func_proto(struct btf *btf, int ret_type_id)
2783 {
2784 struct btf_type *t;
2785 int sz;
2786
2787 if (validate_type_id(ret_type_id))
2788 return libbpf_err(-EINVAL);
2789
2790 if (btf_ensure_modifiable(btf))
2791 return libbpf_err(-ENOMEM);
2792
2793 sz = sizeof(struct btf_type);
2794 t = btf_add_type_mem(btf, sz);
2795 if (!t)
2796 return libbpf_err(-ENOMEM);
2797
2798 /* start out with vlen=0; this will be adjusted when adding enum
2799 * values, if necessary
2800 */
2801 t->name_off = 0;
2802 t->info = btf_type_info(BTF_KIND_FUNC_PROTO, 0, 0);
2803 t->type = ret_type_id;
2804
2805 return btf_commit_type(btf, sz);
2806 }
2807
2808 /*
2809 * Append new function parameter for current FUNC_PROTO type with:
2810 * - *name* - parameter name, can be NULL or empty;
2811 * - *type_id* - type ID describing the type of the parameter.
2812 * Returns:
2813 * - 0, on success;
2814 * - <0, on error.
2815 */
2816 int btf__add_func_param(struct btf *btf, const char *name, int type_id)
2817 {
2818 struct btf_type *t;
2819 struct btf_param *p;
2820 int sz, name_off = 0;
2821
2822 if (validate_type_id(type_id))
2823 return libbpf_err(-EINVAL);
2824
2825 /* last type should be BTF_KIND_FUNC_PROTO */
2826 if (btf->nr_types == 0)
2827 return libbpf_err(-EINVAL);
2828 t = btf_last_type(btf);
2829 if (!btf_is_func_proto(t))
2830 return libbpf_err(-EINVAL);
2831
2832 /* decompose and invalidate raw data */
2833 if (btf_ensure_modifiable(btf))
2834 return libbpf_err(-ENOMEM);
2835
2836 sz = sizeof(struct btf_param);
2837 p = btf_add_type_mem(btf, sz);
2838 if (!p)
2839 return libbpf_err(-ENOMEM);
2840
2841 if (!str_is_empty(name)) {
2842 name_off = btf__add_str(btf, name);
2843 if (name_off < 0)
2844 return name_off;
2845 }
2846
2847 p->name_off = name_off;
2848 p->type = type_id;
2849
2850 /* update parent type's vlen */
2851 t = btf_last_type(btf);
2852 btf_type_inc_vlen(t);
2853
2854 btf->hdr->type_len += sz;
2855 btf->hdr->str_off += sz;
2856 return 0;
2857 }
2858
2859 /*
2860 * Append new BTF_KIND_VAR type with:
2861 * - *name* - non-empty/non-NULL name;
2862 * - *linkage* - variable linkage, one of BTF_VAR_STATIC,
2863 * BTF_VAR_GLOBAL_ALLOCATED, or BTF_VAR_GLOBAL_EXTERN;
2864 * - *type_id* - type ID of the type describing the type of the variable.
2865 * Returns:
2866 * - >0, type ID of newly added BTF type;
2867 * - <0, on error.
2868 */
2869 int btf__add_var(struct btf *btf, const char *name, int linkage, int type_id)
2870 {
2871 struct btf_type *t;
2872 struct btf_var *v;
2873 int sz, name_off;
2874
2875 /* non-empty name */
2876 if (str_is_empty(name))
2877 return libbpf_err(-EINVAL);
2878 if (linkage != BTF_VAR_STATIC && linkage != BTF_VAR_GLOBAL_ALLOCATED &&
2879 linkage != BTF_VAR_GLOBAL_EXTERN)
2880 return libbpf_err(-EINVAL);
2881 if (validate_type_id(type_id))
2882 return libbpf_err(-EINVAL);
2883
2884 /* deconstruct BTF, if necessary, and invalidate raw_data */
2885 if (btf_ensure_modifiable(btf))
2886 return libbpf_err(-ENOMEM);
2887
2888 sz = sizeof(struct btf_type) + sizeof(struct btf_var);
2889 t = btf_add_type_mem(btf, sz);
2890 if (!t)
2891 return libbpf_err(-ENOMEM);
2892
2893 name_off = btf__add_str(btf, name);
2894 if (name_off < 0)
2895 return name_off;
2896
2897 t->name_off = name_off;
2898 t->info = btf_type_info(BTF_KIND_VAR, 0, 0);
2899 t->type = type_id;
2900
2901 v = btf_var(t);
2902 v->linkage = linkage;
2903
2904 return btf_commit_type(btf, sz);
2905 }
2906
2907 /*
2908 * Append new BTF_KIND_DATASEC type with:
2909 * - *name* - non-empty/non-NULL name;
2910 * - *byte_sz* - data section size, in bytes.
2911 *
2912 * Data section is initially empty. Variables info can be added with
2913 * btf__add_datasec_var_info() calls, after btf__add_datasec() succeeds.
2914 *
2915 * Returns:
2916 * - >0, type ID of newly added BTF type;
2917 * - <0, on error.
2918 */
2919 int btf__add_datasec(struct btf *btf, const char *name, __u32 byte_sz)
2920 {
2921 struct btf_type *t;
2922 int sz, name_off;
2923
2924 /* non-empty name */
2925 if (str_is_empty(name))
2926 return libbpf_err(-EINVAL);
2927
2928 if (btf_ensure_modifiable(btf))
2929 return libbpf_err(-ENOMEM);
2930
2931 sz = sizeof(struct btf_type);
2932 t = btf_add_type_mem(btf, sz);
2933 if (!t)
2934 return libbpf_err(-ENOMEM);
2935
2936 name_off = btf__add_str(btf, name);
2937 if (name_off < 0)
2938 return name_off;
2939
2940 /* start with vlen=0, which will be update as var_secinfos are added */
2941 t->name_off = name_off;
2942 t->info = btf_type_info(BTF_KIND_DATASEC, 0, 0);
2943 t->size = byte_sz;
2944
2945 return btf_commit_type(btf, sz);
2946 }
2947
2948 /*
2949 * Append new data section variable information entry for current DATASEC type:
2950 * - *var_type_id* - type ID, describing type of the variable;
2951 * - *offset* - variable offset within data section, in bytes;
2952 * - *byte_sz* - variable size, in bytes.
2953 *
2954 * Returns:
2955 * - 0, on success;
2956 * - <0, on error.
2957 */
2958 int btf__add_datasec_var_info(struct btf *btf, int var_type_id, __u32 offset, __u32 byte_sz)
2959 {
2960 struct btf_type *t;
2961 struct btf_var_secinfo *v;
2962 int sz;
2963
2964 /* last type should be BTF_KIND_DATASEC */
2965 if (btf->nr_types == 0)
2966 return libbpf_err(-EINVAL);
2967 t = btf_last_type(btf);
2968 if (!btf_is_datasec(t))
2969 return libbpf_err(-EINVAL);
2970
2971 if (validate_type_id(var_type_id))
2972 return libbpf_err(-EINVAL);
2973
2974 /* decompose and invalidate raw data */
2975 if (btf_ensure_modifiable(btf))
2976 return libbpf_err(-ENOMEM);
2977
2978 sz = sizeof(struct btf_var_secinfo);
2979 v = btf_add_type_mem(btf, sz);
2980 if (!v)
2981 return libbpf_err(-ENOMEM);
2982
2983 v->type = var_type_id;
2984 v->offset = offset;
2985 v->size = byte_sz;
2986
2987 /* update parent type's vlen */
2988 t = btf_last_type(btf);
2989 btf_type_inc_vlen(t);
2990
2991 btf->hdr->type_len += sz;
2992 btf->hdr->str_off += sz;
2993 return 0;
2994 }
2995
2996 static int btf_add_decl_tag(struct btf *btf, const char *value, int ref_type_id,
2997 int component_idx, int kflag)
2998 {
2999 struct btf_type *t;
3000 int sz, value_off;
3001
3002 if (str_is_empty(value) || component_idx < -1)
3003 return libbpf_err(-EINVAL);
3004
3005 if (validate_type_id(ref_type_id))
3006 return libbpf_err(-EINVAL);
3007
3008 if (btf_ensure_modifiable(btf))
3009 return libbpf_err(-ENOMEM);
3010
3011 sz = sizeof(struct btf_type) + sizeof(struct btf_decl_tag);
3012 t = btf_add_type_mem(btf, sz);
3013 if (!t)
3014 return libbpf_err(-ENOMEM);
3015
3016 value_off = btf__add_str(btf, value);
3017 if (value_off < 0)
3018 return value_off;
3019
3020 t->name_off = value_off;
3021 t->info = btf_type_info(BTF_KIND_DECL_TAG, 0, kflag);
3022 t->type = ref_type_id;
3023 btf_decl_tag(t)->component_idx = component_idx;
3024
3025 return btf_commit_type(btf, sz);
3026 }
3027
3028 /*
3029 * Append new BTF_KIND_DECL_TAG type with:
3030 * - *value* - non-empty/non-NULL string;
3031 * - *ref_type_id* - referenced type ID, it might not exist yet;
3032 * - *component_idx* - -1 for tagging reference type, otherwise struct/union
3033 * member or function argument index;
3034 * Returns:
3035 * - >0, type ID of newly added BTF type;
3036 * - <0, on error.
3037 */
3038 int btf__add_decl_tag(struct btf *btf, const char *value, int ref_type_id,
3039 int component_idx)
3040 {
3041 return btf_add_decl_tag(btf, value, ref_type_id, component_idx, 0);
3042 }
3043
3044 /*
3045 * Append new BTF_KIND_DECL_TAG type with:
3046 * - *value* - non-empty/non-NULL string;
3047 * - *ref_type_id* - referenced type ID, it might not exist yet;
3048 * - *component_idx* - -1 for tagging reference type, otherwise struct/union
3049 * member or function argument index;
3050 * Set info->kflag to 1, indicating this tag is an __attribute__
3051 * Returns:
3052 * - >0, type ID of newly added BTF type;
3053 * - <0, on error.
3054 */
3055 int btf__add_decl_attr(struct btf *btf, const char *value, int ref_type_id,
3056 int component_idx)
3057 {
3058 return btf_add_decl_tag(btf, value, ref_type_id, component_idx, 1);
3059 }
3060
3061 struct btf_ext_sec_info_param {
3062 __u32 off;
3063 __u32 len;
3064 __u32 min_rec_size;
3065 struct btf_ext_info *ext_info;
3066 const char *desc;
3067 };
3068
3069 /*
3070 * Parse a single info subsection of the BTF.ext info data:
3071 * - validate subsection structure and elements
3072 * - save info subsection start and sizing details in struct btf_ext
3073 * - endian-independent operation, for calling before byte-swapping
3074 */
3075 static int btf_ext_parse_sec_info(struct btf_ext *btf_ext,
3076 struct btf_ext_sec_info_param *ext_sec,
3077 bool is_native)
3078 {
3079 const struct btf_ext_info_sec *sinfo;
3080 struct btf_ext_info *ext_info;
3081 __u32 info_left, record_size;
3082 size_t sec_cnt = 0;
3083 void *info;
3084
3085 if (ext_sec->len == 0)
3086 return 0;
3087
3088 if (ext_sec->off & 0x03) {
3089 pr_debug(".BTF.ext %s section is not aligned to 4 bytes\n",
3090 ext_sec->desc);
3091 return -EINVAL;
3092 }
3093
3094 /* The start of the info sec (including the __u32 record_size). */
3095 info = btf_ext->data + btf_ext->hdr->hdr_len + ext_sec->off;
3096 info_left = ext_sec->len;
3097
3098 if (btf_ext->data + btf_ext->data_size < info + ext_sec->len) {
3099 pr_debug("%s section (off:%u len:%u) is beyond the end of the ELF section .BTF.ext\n",
3100 ext_sec->desc, ext_sec->off, ext_sec->len);
3101 return -EINVAL;
3102 }
3103
3104 /* At least a record size */
3105 if (info_left < sizeof(__u32)) {
3106 pr_debug(".BTF.ext %s record size not found\n", ext_sec->desc);
3107 return -EINVAL;
3108 }
3109
3110 /* The record size needs to meet either the minimum standard or, when
3111 * handling non-native endianness data, the exact standard so as
3112 * to allow safe byte-swapping.
3113 */
3114 record_size = is_native ? *(__u32 *)info : bswap_32(*(__u32 *)info);
3115 if (record_size < ext_sec->min_rec_size ||
3116 (!is_native && record_size != ext_sec->min_rec_size) ||
3117 record_size & 0x03) {
3118 pr_debug("%s section in .BTF.ext has invalid record size %u\n",
3119 ext_sec->desc, record_size);
3120 return -EINVAL;
3121 }
3122
3123 sinfo = info + sizeof(__u32);
3124 info_left -= sizeof(__u32);
3125
3126 /* If no records, return failure now so .BTF.ext won't be used. */
3127 if (!info_left) {
3128 pr_debug("%s section in .BTF.ext has no records\n", ext_sec->desc);
3129 return -EINVAL;
3130 }
3131
3132 while (info_left) {
3133 unsigned int sec_hdrlen = sizeof(struct btf_ext_info_sec);
3134 __u64 total_record_size;
3135 __u32 num_records;
3136
3137 if (info_left < sec_hdrlen) {
3138 pr_debug("%s section header is not found in .BTF.ext\n",
3139 ext_sec->desc);
3140 return -EINVAL;
3141 }
3142
3143 num_records = is_native ? sinfo->num_info : bswap_32(sinfo->num_info);
3144 if (num_records == 0) {
3145 pr_debug("%s section has incorrect num_records in .BTF.ext\n",
3146 ext_sec->desc);
3147 return -EINVAL;
3148 }
3149
3150 total_record_size = sec_hdrlen + (__u64)num_records * record_size;
3151 if (info_left < total_record_size) {
3152 pr_debug("%s section has incorrect num_records in .BTF.ext\n",
3153 ext_sec->desc);
3154 return -EINVAL;
3155 }
3156
3157 info_left -= total_record_size;
3158 sinfo = (void *)sinfo + total_record_size;
3159 sec_cnt++;
3160 }
3161
3162 ext_info = ext_sec->ext_info;
3163 ext_info->len = ext_sec->len - sizeof(__u32);
3164 ext_info->rec_size = record_size;
3165 ext_info->info = info + sizeof(__u32);
3166 ext_info->sec_cnt = sec_cnt;
3167
3168 return 0;
3169 }
3170
3171 /* Parse all info secs in the BTF.ext info data */
3172 static int btf_ext_parse_info(struct btf_ext *btf_ext, bool is_native)
3173 {
3174 struct btf_ext_sec_info_param func_info = {
3175 .off = btf_ext->hdr->func_info_off,
3176 .len = btf_ext->hdr->func_info_len,
3177 .min_rec_size = sizeof(struct bpf_func_info_min),
3178 .ext_info = &btf_ext->func_info,
3179 .desc = "func_info"
3180 };
3181 struct btf_ext_sec_info_param line_info = {
3182 .off = btf_ext->hdr->line_info_off,
3183 .len = btf_ext->hdr->line_info_len,
3184 .min_rec_size = sizeof(struct bpf_line_info_min),
3185 .ext_info = &btf_ext->line_info,
3186 .desc = "line_info",
3187 };
3188 struct btf_ext_sec_info_param core_relo = {
3189 .min_rec_size = sizeof(struct bpf_core_relo),
3190 .ext_info = &btf_ext->core_relo_info,
3191 .desc = "core_relo",
3192 };
3193 int err;
3194
3195 err = btf_ext_parse_sec_info(btf_ext, &func_info, is_native);
3196 if (err)
3197 return err;
3198
3199 err = btf_ext_parse_sec_info(btf_ext, &line_info, is_native);
3200 if (err)
3201 return err;
3202
3203 if (btf_ext->hdr->hdr_len < offsetofend(struct btf_ext_header, core_relo_len))
3204 return 0; /* skip core relos parsing */
3205
3206 core_relo.off = btf_ext->hdr->core_relo_off;
3207 core_relo.len = btf_ext->hdr->core_relo_len;
3208 err = btf_ext_parse_sec_info(btf_ext, &core_relo, is_native);
3209 if (err)
3210 return err;
3211
3212 return 0;
3213 }
3214
3215 /* Swap byte-order of BTF.ext header with any endianness */
3216 static void btf_ext_bswap_hdr(struct btf_ext_header *h)
3217 {
3218 bool is_native = h->magic == BTF_MAGIC;
3219 __u32 hdr_len;
3220
3221 hdr_len = is_native ? h->hdr_len : bswap_32(h->hdr_len);
3222
3223 h->magic = bswap_16(h->magic);
3224 h->hdr_len = bswap_32(h->hdr_len);
3225 h->func_info_off = bswap_32(h->func_info_off);
3226 h->func_info_len = bswap_32(h->func_info_len);
3227 h->line_info_off = bswap_32(h->line_info_off);
3228 h->line_info_len = bswap_32(h->line_info_len);
3229
3230 if (hdr_len < offsetofend(struct btf_ext_header, core_relo_len))
3231 return;
3232
3233 h->core_relo_off = bswap_32(h->core_relo_off);
3234 h->core_relo_len = bswap_32(h->core_relo_len);
3235 }
3236
3237 /* Swap byte-order of generic info subsection */
3238 static void btf_ext_bswap_info_sec(void *info, __u32 len, bool is_native,
3239 info_rec_bswap_fn bswap_fn)
3240 {
3241 struct btf_ext_info_sec *sec;
3242 __u32 info_left, rec_size, *rs;
3243
3244 if (len == 0)
3245 return;
3246
3247 rs = info; /* info record size */
3248 rec_size = is_native ? *rs : bswap_32(*rs);
3249 *rs = bswap_32(*rs);
3250
3251 sec = info + sizeof(__u32); /* info sec #1 */
3252 info_left = len - sizeof(__u32);
3253 while (info_left) {
3254 unsigned int sec_hdrlen = sizeof(struct btf_ext_info_sec);
3255 __u32 i, num_recs;
3256 void *p;
3257
3258 num_recs = is_native ? sec->num_info : bswap_32(sec->num_info);
3259 sec->sec_name_off = bswap_32(sec->sec_name_off);
3260 sec->num_info = bswap_32(sec->num_info);
3261 p = sec->data; /* info rec #1 */
3262 for (i = 0; i < num_recs; i++, p += rec_size)
3263 bswap_fn(p);
3264 sec = p;
3265 info_left -= sec_hdrlen + (__u64)rec_size * num_recs;
3266 }
3267 }
3268
3269 /*
3270 * Swap byte-order of all info data in a BTF.ext section
3271 * - requires BTF.ext hdr in native endianness
3272 */
3273 static void btf_ext_bswap_info(struct btf_ext *btf_ext, void *data)
3274 {
3275 const bool is_native = btf_ext->swapped_endian;
3276 const struct btf_ext_header *h = data;
3277 void *info;
3278
3279 /* Swap func_info subsection byte-order */
3280 info = data + h->hdr_len + h->func_info_off;
3281 btf_ext_bswap_info_sec(info, h->func_info_len, is_native,
3282 (info_rec_bswap_fn)bpf_func_info_bswap);
3283
3284 /* Swap line_info subsection byte-order */
3285 info = data + h->hdr_len + h->line_info_off;
3286 btf_ext_bswap_info_sec(info, h->line_info_len, is_native,
3287 (info_rec_bswap_fn)bpf_line_info_bswap);
3288
3289 /* Swap core_relo subsection byte-order (if present) */
3290 if (h->hdr_len < offsetofend(struct btf_ext_header, core_relo_len))
3291 return;
3292
3293 info = data + h->hdr_len + h->core_relo_off;
3294 btf_ext_bswap_info_sec(info, h->core_relo_len, is_native,
3295 (info_rec_bswap_fn)bpf_core_relo_bswap);
3296 }
3297
3298 /* Parse hdr data and info sections: check and convert to native endianness */
3299 static int btf_ext_parse(struct btf_ext *btf_ext)
3300 {
3301 __u32 hdr_len, data_size = btf_ext->data_size;
3302 struct btf_ext_header *hdr = btf_ext->hdr;
3303 bool swapped_endian = false;
3304 int err;
3305
3306 if (data_size < offsetofend(struct btf_ext_header, hdr_len)) {
3307 pr_debug("BTF.ext header too short\n");
3308 return -EINVAL;
3309 }
3310
3311 hdr_len = hdr->hdr_len;
3312 if (hdr->magic == bswap_16(BTF_MAGIC)) {
3313 swapped_endian = true;
3314 hdr_len = bswap_32(hdr_len);
3315 } else if (hdr->magic != BTF_MAGIC) {
3316 pr_debug("Invalid BTF.ext magic:%x\n", hdr->magic);
3317 return -EINVAL;
3318 }
3319
3320 /* Ensure known version of structs, current BTF_VERSION == 1 */
3321 if (hdr->version != 1) {
3322 pr_debug("Unsupported BTF.ext version:%u\n", hdr->version);
3323 return -ENOTSUP;
3324 }
3325
3326 if (hdr->flags) {
3327 pr_debug("Unsupported BTF.ext flags:%x\n", hdr->flags);
3328 return -ENOTSUP;
3329 }
3330
3331 if (data_size < hdr_len) {
3332 pr_debug("BTF.ext header not found\n");
3333 return -EINVAL;
3334 } else if (data_size == hdr_len) {
3335 pr_debug("BTF.ext has no data\n");
3336 return -EINVAL;
3337 }
3338
3339 /* Verify mandatory hdr info details present */
3340 if (hdr_len < offsetofend(struct btf_ext_header, line_info_len)) {
3341 pr_warn("BTF.ext header missing func_info, line_info\n");
3342 return -EINVAL;
3343 }
3344
3345 /* Keep hdr native byte-order in memory for introspection */
3346 if (swapped_endian)
3347 btf_ext_bswap_hdr(btf_ext->hdr);
3348
3349 /* Validate info subsections and cache key metadata */
3350 err = btf_ext_parse_info(btf_ext, !swapped_endian);
3351 if (err)
3352 return err;
3353
3354 /* Keep infos native byte-order in memory for introspection */
3355 if (swapped_endian)
3356 btf_ext_bswap_info(btf_ext, btf_ext->data);
3357
3358 /*
3359 * Set btf_ext->swapped_endian only after all header and info data has
3360 * been swapped, helping bswap functions determine if their data are
3361 * in native byte-order when called.
3362 */
3363 btf_ext->swapped_endian = swapped_endian;
3364 return 0;
3365 }
3366
3367 void btf_ext__free(struct btf_ext *btf_ext)
3368 {
3369 if (IS_ERR_OR_NULL(btf_ext))
3370 return;
3371 free(btf_ext->func_info.sec_idxs);
3372 free(btf_ext->line_info.sec_idxs);
3373 free(btf_ext->core_relo_info.sec_idxs);
3374 free(btf_ext->data);
3375 free(btf_ext->data_swapped);
3376 free(btf_ext);
3377 }
3378
3379 struct btf_ext *btf_ext__new(const __u8 *data, __u32 size)
3380 {
3381 struct btf_ext *btf_ext;
3382 int err;
3383
3384 btf_ext = calloc(1, sizeof(struct btf_ext));
3385 if (!btf_ext)
3386 return libbpf_err_ptr(-ENOMEM);
3387
3388 btf_ext->data_size = size;
3389 btf_ext->data = malloc(size);
3390 if (!btf_ext->data) {
3391 err = -ENOMEM;
3392 goto done;
3393 }
3394 memcpy(btf_ext->data, data, size);
3395
3396 err = btf_ext_parse(btf_ext);
3397
3398 done:
3399 if (err) {
3400 btf_ext__free(btf_ext);
3401 return libbpf_err_ptr(err);
3402 }
3403
3404 return btf_ext;
3405 }
3406
3407 static void *btf_ext_raw_data(const struct btf_ext *btf_ext_ro, bool swap_endian)
3408 {
3409 struct btf_ext *btf_ext = (struct btf_ext *)btf_ext_ro;
3410 const __u32 data_sz = btf_ext->data_size;
3411 void *data;
3412
3413 /* Return native data (always present) or swapped data if present */
3414 if (!swap_endian)
3415 return btf_ext->data;
3416 else if (btf_ext->data_swapped)
3417 return btf_ext->data_swapped;
3418
3419 /* Recreate missing swapped data, then cache and return */
3420 data = calloc(1, data_sz);
3421 if (!data)
3422 return NULL;
3423 memcpy(data, btf_ext->data, data_sz);
3424
3425 btf_ext_bswap_info(btf_ext, data);
3426 btf_ext_bswap_hdr(data);
3427 btf_ext->data_swapped = data;
3428 return data;
3429 }
3430
3431 const void *btf_ext__raw_data(const struct btf_ext *btf_ext, __u32 *size)
3432 {
3433 void *data;
3434
3435 data = btf_ext_raw_data(btf_ext, btf_ext->swapped_endian);
3436 if (!data)
3437 return errno = ENOMEM, NULL;
3438
3439 *size = btf_ext->data_size;
3440 return data;
3441 }
3442
3443 __attribute__((alias("btf_ext__raw_data")))
3444 const void *btf_ext__get_raw_data(const struct btf_ext *btf_ext, __u32 *size);
3445
3446 enum btf_endianness btf_ext__endianness(const struct btf_ext *btf_ext)
3447 {
3448 if (is_host_big_endian())
3449 return btf_ext->swapped_endian ? BTF_LITTLE_ENDIAN : BTF_BIG_ENDIAN;
3450 else
3451 return btf_ext->swapped_endian ? BTF_BIG_ENDIAN : BTF_LITTLE_ENDIAN;
3452 }
3453
3454 int btf_ext__set_endianness(struct btf_ext *btf_ext, enum btf_endianness endian)
3455 {
3456 if (endian != BTF_LITTLE_ENDIAN && endian != BTF_BIG_ENDIAN)
3457 return libbpf_err(-EINVAL);
3458
3459 btf_ext->swapped_endian = is_host_big_endian() != (endian == BTF_BIG_ENDIAN);
3460
3461 if (!btf_ext->swapped_endian) {
3462 free(btf_ext->data_swapped);
3463 btf_ext->data_swapped = NULL;
3464 }
3465 return 0;
3466 }
3467
3468 struct btf_dedup;
3469
3470 static struct btf_dedup *btf_dedup_new(struct btf *btf, const struct btf_dedup_opts *opts);
3471 static void btf_dedup_free(struct btf_dedup *d);
3472 static int btf_dedup_prep(struct btf_dedup *d);
3473 static int btf_dedup_strings(struct btf_dedup *d);
3474 static int btf_dedup_prim_types(struct btf_dedup *d);
3475 static int btf_dedup_struct_types(struct btf_dedup *d);
3476 static int btf_dedup_ref_types(struct btf_dedup *d);
3477 static int btf_dedup_resolve_fwds(struct btf_dedup *d);
3478 static int btf_dedup_compact_types(struct btf_dedup *d);
3479 static int btf_dedup_remap_types(struct btf_dedup *d);
3480
3481 /*
3482 * Deduplicate BTF types and strings.
3483 *
3484 * BTF dedup algorithm takes as an input `struct btf` representing `.BTF` ELF
3485 * section with all BTF type descriptors and string data. It overwrites that
3486 * memory in-place with deduplicated types and strings without any loss of
3487 * information. If optional `struct btf_ext` representing '.BTF.ext' ELF section
3488 * is provided, all the strings referenced from .BTF.ext section are honored
3489 * and updated to point to the right offsets after deduplication.
3490 *
3491 * If function returns with error, type/string data might be garbled and should
3492 * be discarded.
3493 *
3494 * More verbose and detailed description of both problem btf_dedup is solving,
3495 * as well as solution could be found at:
3496 * https://facebookmicrosites.github.io/bpf/blog/2018/11/14/btf-enhancement.html
3497 *
3498 * Problem description and justification
3499 * =====================================
3500 *
3501 * BTF type information is typically emitted either as a result of conversion
3502 * from DWARF to BTF or directly by compiler. In both cases, each compilation
3503 * unit contains information about a subset of all the types that are used
3504 * in an application. These subsets are frequently overlapping and contain a lot
3505 * of duplicated information when later concatenated together into a single
3506 * binary. This algorithm ensures that each unique type is represented by single
3507 * BTF type descriptor, greatly reducing resulting size of BTF data.
3508 *
3509 * Compilation unit isolation and subsequent duplication of data is not the only
3510 * problem. The same type hierarchy (e.g., struct and all the type that struct
3511 * references) in different compilation units can be represented in BTF to
3512 * various degrees of completeness (or, rather, incompleteness) due to
3513 * struct/union forward declarations.
3514 *
3515 * Let's take a look at an example, that we'll use to better understand the
3516 * problem (and solution). Suppose we have two compilation units, each using
3517 * same `struct S`, but each of them having incomplete type information about
3518 * struct's fields:
3519 *
3520 * // CU #1:
3521 * struct S;
3522 * struct A {
3523 * int a;
3524 * struct A* self;
3525 * struct S* parent;
3526 * };
3527 * struct B;
3528 * struct S {
3529 * struct A* a_ptr;
3530 * struct B* b_ptr;
3531 * };
3532 *
3533 * // CU #2:
3534 * struct S;
3535 * struct A;
3536 * struct B {
3537 * int b;
3538 * struct B* self;
3539 * struct S* parent;
3540 * };
3541 * struct S {
3542 * struct A* a_ptr;
3543 * struct B* b_ptr;
3544 * };
3545 *
3546 * In case of CU #1, BTF data will know only that `struct B` exist (but no
3547 * more), but will know the complete type information about `struct A`. While
3548 * for CU #2, it will know full type information about `struct B`, but will
3549 * only know about forward declaration of `struct A` (in BTF terms, it will
3550 * have `BTF_KIND_FWD` type descriptor with name `B`).
3551 *
3552 * This compilation unit isolation means that it's possible that there is no
3553 * single CU with complete type information describing structs `S`, `A`, and
3554 * `B`. Also, we might get tons of duplicated and redundant type information.
3555 *
3556 * Additional complication we need to keep in mind comes from the fact that
3557 * types, in general, can form graphs containing cycles, not just DAGs.
3558 *
3559 * While algorithm does deduplication, it also merges and resolves type
3560 * information (unless disabled throught `struct btf_opts`), whenever possible.
3561 * E.g., in the example above with two compilation units having partial type
3562 * information for structs `A` and `B`, the output of algorithm will emit
3563 * a single copy of each BTF type that describes structs `A`, `B`, and `S`
3564 * (as well as type information for `int` and pointers), as if they were defined
3565 * in a single compilation unit as:
3566 *
3567 * struct A {
3568 * int a;
3569 * struct A* self;
3570 * struct S* parent;
3571 * };
3572 * struct B {
3573 * int b;
3574 * struct B* self;
3575 * struct S* parent;
3576 * };
3577 * struct S {
3578 * struct A* a_ptr;
3579 * struct B* b_ptr;
3580 * };
3581 *
3582 * Algorithm summary
3583 * =================
3584 *
3585 * Algorithm completes its work in 7 separate passes:
3586 *
3587 * 1. Strings deduplication.
3588 * 2. Primitive types deduplication (int, enum, fwd).
3589 * 3. Struct/union types deduplication.
3590 * 4. Resolve unambiguous forward declarations.
3591 * 5. Reference types deduplication (pointers, typedefs, arrays, funcs, func
3592 * protos, and const/volatile/restrict modifiers).
3593 * 6. Types compaction.
3594 * 7. Types remapping.
3595 *
3596 * Algorithm determines canonical type descriptor, which is a single
3597 * representative type for each truly unique type. This canonical type is the
3598 * one that will go into final deduplicated BTF type information. For
3599 * struct/unions, it is also the type that algorithm will merge additional type
3600 * information into (while resolving FWDs), as it discovers it from data in
3601 * other CUs. Each input BTF type eventually gets either mapped to itself, if
3602 * that type is canonical, or to some other type, if that type is equivalent
3603 * and was chosen as canonical representative. This mapping is stored in
3604 * `btf_dedup->map` array. This map is also used to record STRUCT/UNION that
3605 * FWD type got resolved to.
3606 *
3607 * To facilitate fast discovery of canonical types, we also maintain canonical
3608 * index (`btf_dedup->dedup_table`), which maps type descriptor's signature hash
3609 * (i.e., hashed kind, name, size, fields, etc) into a list of canonical types
3610 * that match that signature. With sufficiently good choice of type signature
3611 * hashing function, we can limit number of canonical types for each unique type
3612 * signature to a very small number, allowing to find canonical type for any
3613 * duplicated type very quickly.
3614 *
3615 * Struct/union deduplication is the most critical part and algorithm for
3616 * deduplicating structs/unions is described in greater details in comments for
3617 * `btf_dedup_is_equiv` function.
3618 */
3619 int btf__dedup(struct btf *btf, const struct btf_dedup_opts *opts)
3620 {
3621 struct btf_dedup *d;
3622 int err;
3623
3624 if (!OPTS_VALID(opts, btf_dedup_opts))
3625 return libbpf_err(-EINVAL);
3626
3627 d = btf_dedup_new(btf, opts);
3628 if (IS_ERR(d)) {
3629 pr_debug("btf_dedup_new failed: %ld\n", PTR_ERR(d));
3630 return libbpf_err(-EINVAL);
3631 }
3632
3633 if (btf_ensure_modifiable(btf)) {
3634 err = -ENOMEM;
3635 goto done;
3636 }
3637
3638 err = btf_dedup_prep(d);
3639 if (err) {
3640 pr_debug("btf_dedup_prep failed: %s\n", errstr(err));
3641 goto done;
3642 }
3643 err = btf_dedup_strings(d);
3644 if (err < 0) {
3645 pr_debug("btf_dedup_strings failed: %s\n", errstr(err));
3646 goto done;
3647 }
3648 err = btf_dedup_prim_types(d);
3649 if (err < 0) {
3650 pr_debug("btf_dedup_prim_types failed: %s\n", errstr(err));
3651 goto done;
3652 }
3653 err = btf_dedup_struct_types(d);
3654 if (err < 0) {
3655 pr_debug("btf_dedup_struct_types failed: %s\n", errstr(err));
3656 goto done;
3657 }
3658 err = btf_dedup_resolve_fwds(d);
3659 if (err < 0) {
3660 pr_debug("btf_dedup_resolve_fwds failed: %s\n", errstr(err));
3661 goto done;
3662 }
3663 err = btf_dedup_ref_types(d);
3664 if (err < 0) {
3665 pr_debug("btf_dedup_ref_types failed: %s\n", errstr(err));
3666 goto done;
3667 }
3668 err = btf_dedup_compact_types(d);
3669 if (err < 0) {
3670 pr_debug("btf_dedup_compact_types failed: %s\n", errstr(err));
3671 goto done;
3672 }
3673 err = btf_dedup_remap_types(d);
3674 if (err < 0) {
3675 pr_debug("btf_dedup_remap_types failed: %s\n", errstr(err));
3676 goto done;
3677 }
3678
3679 done:
3680 btf_dedup_free(d);
3681 return libbpf_err(err);
3682 }
3683
3684 #define BTF_UNPROCESSED_ID ((__u32)-1)
3685 #define BTF_IN_PROGRESS_ID ((__u32)-2)
3686
3687 struct btf_dedup {
3688 /* .BTF section to be deduped in-place */
3689 struct btf *btf;
3690 /*
3691 * Optional .BTF.ext section. When provided, any strings referenced
3692 * from it will be taken into account when deduping strings
3693 */
3694 struct btf_ext *btf_ext;
3695 /*
3696 * This is a map from any type's signature hash to a list of possible
3697 * canonical representative type candidates. Hash collisions are
3698 * ignored, so even types of various kinds can share same list of
3699 * candidates, which is fine because we rely on subsequent
3700 * btf_xxx_equal() checks to authoritatively verify type equality.
3701 */
3702 struct hashmap *dedup_table;
3703 /* Canonical types map */
3704 __u32 *map;
3705 /* Hypothetical mapping, used during type graph equivalence checks */
3706 __u32 *hypot_map;
3707 __u32 *hypot_list;
3708 size_t hypot_cnt;
3709 size_t hypot_cap;
3710 /* Whether hypothetical mapping, if successful, would need to adjust
3711 * already canonicalized types (due to a new forward declaration to
3712 * concrete type resolution). In such case, during split BTF dedup
3713 * candidate type would still be considered as different, because base
3714 * BTF is considered to be immutable.
3715 */
3716 bool hypot_adjust_canon;
3717 /* Various option modifying behavior of algorithm */
3718 struct btf_dedup_opts opts;
3719 /* temporary strings deduplication state */
3720 struct strset *strs_set;
3721 };
3722
3723 static unsigned long hash_combine(unsigned long h, unsigned long value)
3724 {
3725 return h * 31 + value;
3726 }
3727
3728 #define for_each_dedup_cand(d, node, hash) \
3729 hashmap__for_each_key_entry(d->dedup_table, node, hash)
3730
3731 static int btf_dedup_table_add(struct btf_dedup *d, long hash, __u32 type_id)
3732 {
3733 return hashmap__append(d->dedup_table, hash, type_id);
3734 }
3735
3736 static int btf_dedup_hypot_map_add(struct btf_dedup *d,
3737 __u32 from_id, __u32 to_id)
3738 {
3739 if (d->hypot_cnt == d->hypot_cap) {
3740 __u32 *new_list;
3741
3742 d->hypot_cap += max((size_t)16, d->hypot_cap / 2);
3743 new_list = libbpf_reallocarray(d->hypot_list, d->hypot_cap, sizeof(__u32));
3744 if (!new_list)
3745 return -ENOMEM;
3746 d->hypot_list = new_list;
3747 }
3748 d->hypot_list[d->hypot_cnt++] = from_id;
3749 d->hypot_map[from_id] = to_id;
3750 return 0;
3751 }
3752
3753 static void btf_dedup_clear_hypot_map(struct btf_dedup *d)
3754 {
3755 int i;
3756
3757 for (i = 0; i < d->hypot_cnt; i++)
3758 d->hypot_map[d->hypot_list[i]] = BTF_UNPROCESSED_ID;
3759 d->hypot_cnt = 0;
3760 d->hypot_adjust_canon = false;
3761 }
3762
3763 static void btf_dedup_free(struct btf_dedup *d)
3764 {
3765 hashmap__free(d->dedup_table);
3766 d->dedup_table = NULL;
3767
3768 free(d->map);
3769 d->map = NULL;
3770
3771 free(d->hypot_map);
3772 d->hypot_map = NULL;
3773
3774 free(d->hypot_list);
3775 d->hypot_list = NULL;
3776
3777 free(d);
3778 }
3779
3780 static size_t btf_dedup_identity_hash_fn(long key, void *ctx)
3781 {
3782 return key;
3783 }
3784
3785 static size_t btf_dedup_collision_hash_fn(long key, void *ctx)
3786 {
3787 return 0;
3788 }
3789
3790 static bool btf_dedup_equal_fn(long k1, long k2, void *ctx)
3791 {
3792 return k1 == k2;
3793 }
3794
3795 static struct btf_dedup *btf_dedup_new(struct btf *btf, const struct btf_dedup_opts *opts)
3796 {
3797 struct btf_dedup *d = calloc(1, sizeof(struct btf_dedup));
3798 hashmap_hash_fn hash_fn = btf_dedup_identity_hash_fn;
3799 int i, err = 0, type_cnt;
3800
3801 if (!d)
3802 return ERR_PTR(-ENOMEM);
3803
3804 if (OPTS_GET(opts, force_collisions, false))
3805 hash_fn = btf_dedup_collision_hash_fn;
3806
3807 d->btf = btf;
3808 d->btf_ext = OPTS_GET(opts, btf_ext, NULL);
3809
3810 d->dedup_table = hashmap__new(hash_fn, btf_dedup_equal_fn, NULL);
3811 if (IS_ERR(d->dedup_table)) {
3812 err = PTR_ERR(d->dedup_table);
3813 d->dedup_table = NULL;
3814 goto done;
3815 }
3816
3817 type_cnt = btf__type_cnt(btf);
3818 d->map = malloc(sizeof(__u32) * type_cnt);
3819 if (!d->map) {
3820 err = -ENOMEM;
3821 goto done;
3822 }
3823 /* special BTF "void" type is made canonical immediately */
3824 d->map[0] = 0;
3825 for (i = 1; i < type_cnt; i++) {
3826 struct btf_type *t = btf_type_by_id(d->btf, i);
3827
3828 /* VAR and DATASEC are never deduped and are self-canonical */
3829 if (btf_is_var(t) || btf_is_datasec(t))
3830 d->map[i] = i;
3831 else
3832 d->map[i] = BTF_UNPROCESSED_ID;
3833 }
3834
3835 d->hypot_map = malloc(sizeof(__u32) * type_cnt);
3836 if (!d->hypot_map) {
3837 err = -ENOMEM;
3838 goto done;
3839 }
3840 for (i = 0; i < type_cnt; i++)
3841 d->hypot_map[i] = BTF_UNPROCESSED_ID;
3842
3843 done:
3844 if (err) {
3845 btf_dedup_free(d);
3846 return ERR_PTR(err);
3847 }
3848
3849 return d;
3850 }
3851
3852 /*
3853 * Iterate over all possible places in .BTF and .BTF.ext that can reference
3854 * string and pass pointer to it to a provided callback `fn`.
3855 */
3856 static int btf_for_each_str_off(struct btf_dedup *d, str_off_visit_fn fn, void *ctx)
3857 {
3858 int i, r;
3859
3860 for (i = 0; i < d->btf->nr_types; i++) {
3861 struct btf_field_iter it;
3862 struct btf_type *t = btf_type_by_id(d->btf, d->btf->start_id + i);
3863 __u32 *str_off;
3864
3865 r = btf_field_iter_init(&it, t, BTF_FIELD_ITER_STRS);
3866 if (r)
3867 return r;
3868
3869 while ((str_off = btf_field_iter_next(&it))) {
3870 r = fn(str_off, ctx);
3871 if (r)
3872 return r;
3873 }
3874 }
3875
3876 if (!d->btf_ext)
3877 return 0;
3878
3879 r = btf_ext_visit_str_offs(d->btf_ext, fn, ctx);
3880 if (r)
3881 return r;
3882
3883 return 0;
3884 }
3885
3886 static int strs_dedup_remap_str_off(__u32 *str_off_ptr, void *ctx)
3887 {
3888 struct btf_dedup *d = ctx;
3889 __u32 str_off = *str_off_ptr;
3890 const char *s;
3891 int off, err;
3892
3893 /* don't touch empty string or string in main BTF */
3894 if (str_off == 0 || str_off < d->btf->start_str_off)
3895 return 0;
3896
3897 s = btf__str_by_offset(d->btf, str_off);
3898 if (d->btf->base_btf) {
3899 err = btf__find_str(d->btf->base_btf, s);
3900 if (err >= 0) {
3901 *str_off_ptr = err;
3902 return 0;
3903 }
3904 if (err != -ENOENT)
3905 return err;
3906 }
3907
3908 off = strset__add_str(d->strs_set, s);
3909 if (off < 0)
3910 return off;
3911
3912 *str_off_ptr = d->btf->start_str_off + off;
3913 return 0;
3914 }
3915
3916 /*
3917 * Dedup string and filter out those that are not referenced from either .BTF
3918 * or .BTF.ext (if provided) sections.
3919 *
3920 * This is done by building index of all strings in BTF's string section,
3921 * then iterating over all entities that can reference strings (e.g., type
3922 * names, struct field names, .BTF.ext line info, etc) and marking corresponding
3923 * strings as used. After that all used strings are deduped and compacted into
3924 * sequential blob of memory and new offsets are calculated. Then all the string
3925 * references are iterated again and rewritten using new offsets.
3926 */
3927 static int btf_dedup_strings(struct btf_dedup *d)
3928 {
3929 int err;
3930
3931 if (d->btf->strs_deduped)
3932 return 0;
3933
3934 d->strs_set = strset__new(BTF_MAX_STR_OFFSET, NULL, 0);
3935 if (IS_ERR(d->strs_set)) {
3936 err = PTR_ERR(d->strs_set);
3937 goto err_out;
3938 }
3939
3940 if (!d->btf->base_btf) {
3941 /* insert empty string; we won't be looking it up during strings
3942 * dedup, but it's good to have it for generic BTF string lookups
3943 */
3944 err = strset__add_str(d->strs_set, "");
3945 if (err < 0)
3946 goto err_out;
3947 }
3948
3949 /* remap string offsets */
3950 err = btf_for_each_str_off(d, strs_dedup_remap_str_off, d);
3951 if (err)
3952 goto err_out;
3953
3954 /* replace BTF string data and hash with deduped ones */
3955 strset__free(d->btf->strs_set);
3956 d->btf->hdr->str_len = strset__data_size(d->strs_set);
3957 d->btf->strs_set = d->strs_set;
3958 d->strs_set = NULL;
3959 d->btf->strs_deduped = true;
3960 return 0;
3961
3962 err_out:
3963 strset__free(d->strs_set);
3964 d->strs_set = NULL;
3965
3966 return err;
3967 }
3968
3969 /*
3970 * Calculate type signature hash of TYPEDEF, ignoring referenced type IDs,
3971 * as referenced type IDs equivalence is established separately during type
3972 * graph equivalence check algorithm.
3973 */
3974 static long btf_hash_typedef(struct btf_type *t)
3975 {
3976 long h;
3977
3978 h = hash_combine(0, t->name_off);
3979 h = hash_combine(h, t->info);
3980 return h;
3981 }
3982
3983 static long btf_hash_common(struct btf_type *t)
3984 {
3985 long h;
3986
3987 h = hash_combine(0, t->name_off);
3988 h = hash_combine(h, t->info);
3989 h = hash_combine(h, t->size);
3990 return h;
3991 }
3992
3993 static bool btf_equal_common(struct btf_type *t1, struct btf_type *t2)
3994 {
3995 return t1->name_off == t2->name_off &&
3996 t1->info == t2->info &&
3997 t1->size == t2->size;
3998 }
3999
4000 /* Check structural compatibility of two TYPEDEF. */
4001 static bool btf_equal_typedef(struct btf_type *t1, struct btf_type *t2)
4002 {
4003 return t1->name_off == t2->name_off &&
4004 t1->info == t2->info;
4005 }
4006
4007 /* Calculate type signature hash of INT or TAG. */
4008 static long btf_hash_int_decl_tag(struct btf_type *t)
4009 {
4010 __u32 info = *(__u32 *)(t + 1);
4011 long h;
4012
4013 h = btf_hash_common(t);
4014 h = hash_combine(h, info);
4015 return h;
4016 }
4017
4018 /* Check structural equality of two INTs or TAGs. */
4019 static bool btf_equal_int_tag(struct btf_type *t1, struct btf_type *t2)
4020 {
4021 __u32 info1, info2;
4022
4023 if (!btf_equal_common(t1, t2))
4024 return false;
4025 info1 = *(__u32 *)(t1 + 1);
4026 info2 = *(__u32 *)(t2 + 1);
4027 return info1 == info2;
4028 }
4029
4030 /* Calculate type signature hash of ENUM/ENUM64. */
4031 static long btf_hash_enum(struct btf_type *t)
4032 {
4033 long h;
4034
4035 /* don't hash vlen, enum members and size to support enum fwd resolving */
4036 h = hash_combine(0, t->name_off);
4037 return h;
4038 }
4039
4040 static bool btf_equal_enum_members(struct btf_type *t1, struct btf_type *t2)
4041 {
4042 const struct btf_enum *m1, *m2;
4043 __u16 vlen;
4044 int i;
4045
4046 vlen = btf_vlen(t1);
4047 m1 = btf_enum(t1);
4048 m2 = btf_enum(t2);
4049 for (i = 0; i < vlen; i++) {
4050 if (m1->name_off != m2->name_off || m1->val != m2->val)
4051 return false;
4052 m1++;
4053 m2++;
4054 }
4055 return true;
4056 }
4057
4058 static bool btf_equal_enum64_members(struct btf_type *t1, struct btf_type *t2)
4059 {
4060 const struct btf_enum64 *m1, *m2;
4061 __u16 vlen;
4062 int i;
4063
4064 vlen = btf_vlen(t1);
4065 m1 = btf_enum64(t1);
4066 m2 = btf_enum64(t2);
4067 for (i = 0; i < vlen; i++) {
4068 if (m1->name_off != m2->name_off || m1->val_lo32 != m2->val_lo32 ||
4069 m1->val_hi32 != m2->val_hi32)
4070 return false;
4071 m1++;
4072 m2++;
4073 }
4074 return true;
4075 }
4076
4077 /* Check structural equality of two ENUMs or ENUM64s. */
4078 static bool btf_equal_enum(struct btf_type *t1, struct btf_type *t2)
4079 {
4080 if (!btf_equal_common(t1, t2))
4081 return false;
4082
4083 /* t1 & t2 kinds are identical because of btf_equal_common */
4084 if (btf_kind(t1) == BTF_KIND_ENUM)
4085 return btf_equal_enum_members(t1, t2);
4086 else
4087 return btf_equal_enum64_members(t1, t2);
4088 }
4089
4090 static inline bool btf_is_enum_fwd(struct btf_type *t)
4091 {
4092 return btf_is_any_enum(t) && btf_vlen(t) == 0;
4093 }
4094
4095 static bool btf_compat_enum(struct btf_type *t1, struct btf_type *t2)
4096 {
4097 if (!btf_is_enum_fwd(t1) && !btf_is_enum_fwd(t2))
4098 return btf_equal_enum(t1, t2);
4099 /* At this point either t1 or t2 or both are forward declarations, thus:
4100 * - skip comparing vlen because it is zero for forward declarations;
4101 * - skip comparing size to allow enum forward declarations
4102 * to be compatible with enum64 full declarations;
4103 * - skip comparing kind for the same reason.
4104 */
4105 return t1->name_off == t2->name_off &&
4106 btf_is_any_enum(t1) && btf_is_any_enum(t2);
4107 }
4108
4109 /*
4110 * Calculate type signature hash of STRUCT/UNION, ignoring referenced type IDs,
4111 * as referenced type IDs equivalence is established separately during type
4112 * graph equivalence check algorithm.
4113 */
4114 static long btf_hash_struct(struct btf_type *t)
4115 {
4116 const struct btf_member *member = btf_members(t);
4117 __u32 vlen = btf_vlen(t);
4118 long h = btf_hash_common(t);
4119 int i;
4120
4121 for (i = 0; i < vlen; i++) {
4122 h = hash_combine(h, member->name_off);
4123 h = hash_combine(h, member->offset);
4124 /* no hashing of referenced type ID, it can be unresolved yet */
4125 member++;
4126 }
4127 return h;
4128 }
4129
4130 /*
4131 * Check structural compatibility of two STRUCTs/UNIONs, ignoring referenced
4132 * type IDs. This check is performed during type graph equivalence check and
4133 * referenced types equivalence is checked separately.
4134 */
4135 static bool btf_shallow_equal_struct(struct btf_type *t1, struct btf_type *t2)
4136 {
4137 const struct btf_member *m1, *m2;
4138 __u16 vlen;
4139 int i;
4140
4141 if (!btf_equal_common(t1, t2))
4142 return false;
4143
4144 vlen = btf_vlen(t1);
4145 m1 = btf_members(t1);
4146 m2 = btf_members(t2);
4147 for (i = 0; i < vlen; i++) {
4148 if (m1->name_off != m2->name_off || m1->offset != m2->offset)
4149 return false;
4150 m1++;
4151 m2++;
4152 }
4153 return true;
4154 }
4155
4156 /*
4157 * Calculate type signature hash of ARRAY, including referenced type IDs,
4158 * under assumption that they were already resolved to canonical type IDs and
4159 * are not going to change.
4160 */
4161 static long btf_hash_array(struct btf_type *t)
4162 {
4163 const struct btf_array *info = btf_array(t);
4164 long h = btf_hash_common(t);
4165
4166 h = hash_combine(h, info->type);
4167 h = hash_combine(h, info->index_type);
4168 h = hash_combine(h, info->nelems);
4169 return h;
4170 }
4171
4172 /*
4173 * Check exact equality of two ARRAYs, taking into account referenced
4174 * type IDs, under assumption that they were already resolved to canonical
4175 * type IDs and are not going to change.
4176 * This function is called during reference types deduplication to compare
4177 * ARRAY to potential canonical representative.
4178 */
4179 static bool btf_equal_array(struct btf_type *t1, struct btf_type *t2)
4180 {
4181 const struct btf_array *info1, *info2;
4182
4183 if (!btf_equal_common(t1, t2))
4184 return false;
4185
4186 info1 = btf_array(t1);
4187 info2 = btf_array(t2);
4188 return info1->type == info2->type &&
4189 info1->index_type == info2->index_type &&
4190 info1->nelems == info2->nelems;
4191 }
4192
4193 /*
4194 * Check structural compatibility of two ARRAYs, ignoring referenced type
4195 * IDs. This check is performed during type graph equivalence check and
4196 * referenced types equivalence is checked separately.
4197 */
4198 static bool btf_compat_array(struct btf_type *t1, struct btf_type *t2)
4199 {
4200 if (!btf_equal_common(t1, t2))
4201 return false;
4202
4203 return btf_array(t1)->nelems == btf_array(t2)->nelems;
4204 }
4205
4206 /*
4207 * Calculate type signature hash of FUNC_PROTO, including referenced type IDs,
4208 * under assumption that they were already resolved to canonical type IDs and
4209 * are not going to change.
4210 */
4211 static long btf_hash_fnproto(struct btf_type *t)
4212 {
4213 const struct btf_param *member = btf_params(t);
4214 __u16 vlen = btf_vlen(t);
4215 long h = btf_hash_common(t);
4216 int i;
4217
4218 for (i = 0; i < vlen; i++) {
4219 h = hash_combine(h, member->name_off);
4220 h = hash_combine(h, member->type);
4221 member++;
4222 }
4223 return h;
4224 }
4225
4226 /*
4227 * Check exact equality of two FUNC_PROTOs, taking into account referenced
4228 * type IDs, under assumption that they were already resolved to canonical
4229 * type IDs and are not going to change.
4230 * This function is called during reference types deduplication to compare
4231 * FUNC_PROTO to potential canonical representative.
4232 */
4233 static bool btf_equal_fnproto(struct btf_type *t1, struct btf_type *t2)
4234 {
4235 const struct btf_param *m1, *m2;
4236 __u16 vlen;
4237 int i;
4238
4239 if (!btf_equal_common(t1, t2))
4240 return false;
4241
4242 vlen = btf_vlen(t1);
4243 m1 = btf_params(t1);
4244 m2 = btf_params(t2);
4245 for (i = 0; i < vlen; i++) {
4246 if (m1->name_off != m2->name_off || m1->type != m2->type)
4247 return false;
4248 m1++;
4249 m2++;
4250 }
4251 return true;
4252 }
4253
4254 /*
4255 * Check structural compatibility of two FUNC_PROTOs, ignoring referenced type
4256 * IDs. This check is performed during type graph equivalence check and
4257 * referenced types equivalence is checked separately.
4258 */
4259 static bool btf_compat_fnproto(struct btf_type *t1, struct btf_type *t2)
4260 {
4261 const struct btf_param *m1, *m2;
4262 __u16 vlen;
4263 int i;
4264
4265 /* skip return type ID */
4266 if (t1->name_off != t2->name_off || t1->info != t2->info)
4267 return false;
4268
4269 vlen = btf_vlen(t1);
4270 m1 = btf_params(t1);
4271 m2 = btf_params(t2);
4272 for (i = 0; i < vlen; i++) {
4273 if (m1->name_off != m2->name_off)
4274 return false;
4275 m1++;
4276 m2++;
4277 }
4278 return true;
4279 }
4280
4281 /* Prepare split BTF for deduplication by calculating hashes of base BTF's
4282 * types and initializing the rest of the state (canonical type mapping) for
4283 * the fixed base BTF part.
4284 */
4285 static int btf_dedup_prep(struct btf_dedup *d)
4286 {
4287 struct btf_type *t;
4288 int type_id;
4289 long h;
4290
4291 if (!d->btf->base_btf)
4292 return 0;
4293
4294 for (type_id = 1; type_id < d->btf->start_id; type_id++) {
4295 t = btf_type_by_id(d->btf, type_id);
4296
4297 /* all base BTF types are self-canonical by definition */
4298 d->map[type_id] = type_id;
4299
4300 switch (btf_kind(t)) {
4301 case BTF_KIND_VAR:
4302 case BTF_KIND_DATASEC:
4303 /* VAR and DATASEC are never hash/deduplicated */
4304 continue;
4305 case BTF_KIND_CONST:
4306 case BTF_KIND_VOLATILE:
4307 case BTF_KIND_RESTRICT:
4308 case BTF_KIND_PTR:
4309 case BTF_KIND_FWD:
4310 case BTF_KIND_TYPEDEF:
4311 case BTF_KIND_FUNC:
4312 case BTF_KIND_FLOAT:
4313 case BTF_KIND_TYPE_TAG:
4314 h = btf_hash_common(t);
4315 break;
4316 case BTF_KIND_INT:
4317 case BTF_KIND_DECL_TAG:
4318 h = btf_hash_int_decl_tag(t);
4319 break;
4320 case BTF_KIND_ENUM:
4321 case BTF_KIND_ENUM64:
4322 h = btf_hash_enum(t);
4323 break;
4324 case BTF_KIND_STRUCT:
4325 case BTF_KIND_UNION:
4326 h = btf_hash_struct(t);
4327 break;
4328 case BTF_KIND_ARRAY:
4329 h = btf_hash_array(t);
4330 break;
4331 case BTF_KIND_FUNC_PROTO:
4332 h = btf_hash_fnproto(t);
4333 break;
4334 default:
4335 pr_debug("unknown kind %d for type [%d]\n", btf_kind(t), type_id);
4336 return -EINVAL;
4337 }
4338 if (btf_dedup_table_add(d, h, type_id))
4339 return -ENOMEM;
4340 }
4341
4342 return 0;
4343 }
4344
4345 /*
4346 * Deduplicate primitive types, that can't reference other types, by calculating
4347 * their type signature hash and comparing them with any possible canonical
4348 * candidate. If no canonical candidate matches, type itself is marked as
4349 * canonical and is added into `btf_dedup->dedup_table` as another candidate.
4350 */
4351 static int btf_dedup_prim_type(struct btf_dedup *d, __u32 type_id)
4352 {
4353 struct btf_type *t = btf_type_by_id(d->btf, type_id);
4354 struct hashmap_entry *hash_entry;
4355 struct btf_type *cand;
4356 /* if we don't find equivalent type, then we are canonical */
4357 __u32 new_id = type_id;
4358 __u32 cand_id;
4359 long h;
4360
4361 switch (btf_kind(t)) {
4362 case BTF_KIND_CONST:
4363 case BTF_KIND_VOLATILE:
4364 case BTF_KIND_RESTRICT:
4365 case BTF_KIND_PTR:
4366 case BTF_KIND_TYPEDEF:
4367 case BTF_KIND_ARRAY:
4368 case BTF_KIND_STRUCT:
4369 case BTF_KIND_UNION:
4370 case BTF_KIND_FUNC:
4371 case BTF_KIND_FUNC_PROTO:
4372 case BTF_KIND_VAR:
4373 case BTF_KIND_DATASEC:
4374 case BTF_KIND_DECL_TAG:
4375 case BTF_KIND_TYPE_TAG:
4376 return 0;
4377
4378 case BTF_KIND_INT:
4379 h = btf_hash_int_decl_tag(t);
4380 for_each_dedup_cand(d, hash_entry, h) {
4381 cand_id = hash_entry->value;
4382 cand = btf_type_by_id(d->btf, cand_id);
4383 if (btf_equal_int_tag(t, cand)) {
4384 new_id = cand_id;
4385 break;
4386 }
4387 }
4388 break;
4389
4390 case BTF_KIND_ENUM:
4391 case BTF_KIND_ENUM64:
4392 h = btf_hash_enum(t);
4393 for_each_dedup_cand(d, hash_entry, h) {
4394 cand_id = hash_entry->value;
4395 cand = btf_type_by_id(d->btf, cand_id);
4396 if (btf_equal_enum(t, cand)) {
4397 new_id = cand_id;
4398 break;
4399 }
4400 if (btf_compat_enum(t, cand)) {
4401 if (btf_is_enum_fwd(t)) {
4402 /* resolve fwd to full enum */
4403 new_id = cand_id;
4404 break;
4405 }
4406 /* resolve canonical enum fwd to full enum */
4407 d->map[cand_id] = type_id;
4408 }
4409 }
4410 break;
4411
4412 case BTF_KIND_FWD:
4413 case BTF_KIND_FLOAT:
4414 h = btf_hash_common(t);
4415 for_each_dedup_cand(d, hash_entry, h) {
4416 cand_id = hash_entry->value;
4417 cand = btf_type_by_id(d->btf, cand_id);
4418 if (btf_equal_common(t, cand)) {
4419 new_id = cand_id;
4420 break;
4421 }
4422 }
4423 break;
4424
4425 default:
4426 return -EINVAL;
4427 }
4428
4429 d->map[type_id] = new_id;
4430 if (type_id == new_id && btf_dedup_table_add(d, h, type_id))
4431 return -ENOMEM;
4432
4433 return 0;
4434 }
4435
4436 static int btf_dedup_prim_types(struct btf_dedup *d)
4437 {
4438 int i, err;
4439
4440 for (i = 0; i < d->btf->nr_types; i++) {
4441 err = btf_dedup_prim_type(d, d->btf->start_id + i);
4442 if (err)
4443 return err;
4444 }
4445 return 0;
4446 }
4447
4448 /*
4449 * Check whether type is already mapped into canonical one (could be to itself).
4450 */
4451 static inline bool is_type_mapped(struct btf_dedup *d, uint32_t type_id)
4452 {
4453 return d->map[type_id] <= BTF_MAX_NR_TYPES;
4454 }
4455
4456 /*
4457 * Resolve type ID into its canonical type ID, if any; otherwise return original
4458 * type ID. If type is FWD and is resolved into STRUCT/UNION already, follow
4459 * STRUCT/UNION link and resolve it into canonical type ID as well.
4460 */
4461 static inline __u32 resolve_type_id(struct btf_dedup *d, __u32 type_id)
4462 {
4463 while (is_type_mapped(d, type_id) && d->map[type_id] != type_id)
4464 type_id = d->map[type_id];
4465 return type_id;
4466 }
4467
4468 /*
4469 * Resolve FWD to underlying STRUCT/UNION, if any; otherwise return original
4470 * type ID.
4471 */
4472 static uint32_t resolve_fwd_id(struct btf_dedup *d, uint32_t type_id)
4473 {
4474 __u32 orig_type_id = type_id;
4475
4476 if (!btf_is_fwd(btf__type_by_id(d->btf, type_id)))
4477 return type_id;
4478
4479 while (is_type_mapped(d, type_id) && d->map[type_id] != type_id)
4480 type_id = d->map[type_id];
4481
4482 if (!btf_is_fwd(btf__type_by_id(d->btf, type_id)))
4483 return type_id;
4484
4485 return orig_type_id;
4486 }
4487
4488
4489 static inline __u16 btf_fwd_kind(struct btf_type *t)
4490 {
4491 return btf_kflag(t) ? BTF_KIND_UNION : BTF_KIND_STRUCT;
4492 }
4493
4494 static bool btf_dedup_identical_types(struct btf_dedup *d, __u32 id1, __u32 id2, int depth)
4495 {
4496 struct btf_type *t1, *t2;
4497 int k1, k2;
4498 recur:
4499 t1 = btf_type_by_id(d->btf, id1);
4500 t2 = btf_type_by_id(d->btf, id2);
4501 if (depth <= 0) {
4502 pr_debug("Reached depth limit for identical type comparison for '%s'/'%s'\n",
4503 btf__name_by_offset(d->btf, t1->name_off),
4504 btf__name_by_offset(d->btf, t2->name_off));
4505 return false;
4506 }
4507
4508 k1 = btf_kind(t1);
4509 k2 = btf_kind(t2);
4510 if (k1 != k2)
4511 return false;
4512
4513 switch (k1) {
4514 case BTF_KIND_UNKN: /* VOID */
4515 return true;
4516 case BTF_KIND_INT:
4517 return btf_equal_int_tag(t1, t2);
4518 case BTF_KIND_ENUM:
4519 case BTF_KIND_ENUM64:
4520 return btf_compat_enum(t1, t2);
4521 case BTF_KIND_FWD:
4522 case BTF_KIND_FLOAT:
4523 return btf_equal_common(t1, t2);
4524 case BTF_KIND_CONST:
4525 case BTF_KIND_VOLATILE:
4526 case BTF_KIND_RESTRICT:
4527 case BTF_KIND_PTR:
4528 case BTF_KIND_TYPEDEF:
4529 case BTF_KIND_FUNC:
4530 case BTF_KIND_TYPE_TAG:
4531 if (t1->info != t2->info || t1->name_off != t2->name_off)
4532 return false;
4533 id1 = t1->type;
4534 id2 = t2->type;
4535 goto recur;
4536 case BTF_KIND_ARRAY: {
4537 struct btf_array *a1, *a2;
4538
4539 if (!btf_compat_array(t1, t2))
4540 return false;
4541
4542 a1 = btf_array(t1);
4543 a2 = btf_array(t1);
4544
4545 if (a1->index_type != a2->index_type &&
4546 !btf_dedup_identical_types(d, a1->index_type, a2->index_type, depth - 1))
4547 return false;
4548
4549 if (a1->type != a2->type &&
4550 !btf_dedup_identical_types(d, a1->type, a2->type, depth - 1))
4551 return false;
4552
4553 return true;
4554 }
4555 case BTF_KIND_STRUCT:
4556 case BTF_KIND_UNION: {
4557 const struct btf_member *m1, *m2;
4558 int i, n;
4559
4560 if (!btf_shallow_equal_struct(t1, t2))
4561 return false;
4562
4563 m1 = btf_members(t1);
4564 m2 = btf_members(t2);
4565 for (i = 0, n = btf_vlen(t1); i < n; i++, m1++, m2++) {
4566 if (m1->type == m2->type)
4567 continue;
4568 if (!btf_dedup_identical_types(d, m1->type, m2->type, depth - 1)) {
4569 if (t1->name_off) {
4570 pr_debug("%s '%s' size=%d vlen=%d id1[%u] id2[%u] shallow-equal but not identical for field#%d '%s'\n",
4571 k1 == BTF_KIND_STRUCT ? "STRUCT" : "UNION",
4572 btf__name_by_offset(d->btf, t1->name_off),
4573 t1->size, btf_vlen(t1), id1, id2, i,
4574 btf__name_by_offset(d->btf, m1->name_off));
4575 }
4576 return false;
4577 }
4578 }
4579 return true;
4580 }
4581 case BTF_KIND_FUNC_PROTO: {
4582 const struct btf_param *p1, *p2;
4583 int i, n;
4584
4585 if (!btf_compat_fnproto(t1, t2))
4586 return false;
4587
4588 if (t1->type != t2->type &&
4589 !btf_dedup_identical_types(d, t1->type, t2->type, depth - 1))
4590 return false;
4591
4592 p1 = btf_params(t1);
4593 p2 = btf_params(t2);
4594 for (i = 0, n = btf_vlen(t1); i < n; i++, p1++, p2++) {
4595 if (p1->type == p2->type)
4596 continue;
4597 if (!btf_dedup_identical_types(d, p1->type, p2->type, depth - 1))
4598 return false;
4599 }
4600 return true;
4601 }
4602 default:
4603 return false;
4604 }
4605 }
4606
4607
4608 /*
4609 * Check equivalence of BTF type graph formed by candidate struct/union (we'll
4610 * call it "candidate graph" in this description for brevity) to a type graph
4611 * formed by (potential) canonical struct/union ("canonical graph" for brevity
4612 * here, though keep in mind that not all types in canonical graph are
4613 * necessarily canonical representatives themselves, some of them might be
4614 * duplicates or its uniqueness might not have been established yet).
4615 * Returns:
4616 * - >0, if type graphs are equivalent;
4617 * - 0, if not equivalent;
4618 * - <0, on error.
4619 *
4620 * Algorithm performs side-by-side DFS traversal of both type graphs and checks
4621 * equivalence of BTF types at each step. If at any point BTF types in candidate
4622 * and canonical graphs are not compatible structurally, whole graphs are
4623 * incompatible. If types are structurally equivalent (i.e., all information
4624 * except referenced type IDs is exactly the same), a mapping from `canon_id` to
4625 * a `cand_id` is recoded in hypothetical mapping (`btf_dedup->hypot_map`).
4626 * If a type references other types, then those referenced types are checked
4627 * for equivalence recursively.
4628 *
4629 * During DFS traversal, if we find that for current `canon_id` type we
4630 * already have some mapping in hypothetical map, we check for two possible
4631 * situations:
4632 * - `canon_id` is mapped to exactly the same type as `cand_id`. This will
4633 * happen when type graphs have cycles. In this case we assume those two
4634 * types are equivalent.
4635 * - `canon_id` is mapped to different type. This is contradiction in our
4636 * hypothetical mapping, because same graph in canonical graph corresponds
4637 * to two different types in candidate graph, which for equivalent type
4638 * graphs shouldn't happen. This condition terminates equivalence check
4639 * with negative result.
4640 *
4641 * If type graphs traversal exhausts types to check and find no contradiction,
4642 * then type graphs are equivalent.
4643 *
4644 * When checking types for equivalence, there is one special case: FWD types.
4645 * If FWD type resolution is allowed and one of the types (either from canonical
4646 * or candidate graph) is FWD and other is STRUCT/UNION (depending on FWD's kind
4647 * flag) and their names match, hypothetical mapping is updated to point from
4648 * FWD to STRUCT/UNION. If graphs will be determined as equivalent successfully,
4649 * this mapping will be used to record FWD -> STRUCT/UNION mapping permanently.
4650 *
4651 * Technically, this could lead to incorrect FWD to STRUCT/UNION resolution,
4652 * if there are two exactly named (or anonymous) structs/unions that are
4653 * compatible structurally, one of which has FWD field, while other is concrete
4654 * STRUCT/UNION, but according to C sources they are different structs/unions
4655 * that are referencing different types with the same name. This is extremely
4656 * unlikely to happen, but btf_dedup API allows to disable FWD resolution if
4657 * this logic is causing problems.
4658 *
4659 * Doing FWD resolution means that both candidate and/or canonical graphs can
4660 * consists of portions of the graph that come from multiple compilation units.
4661 * This is due to the fact that types within single compilation unit are always
4662 * deduplicated and FWDs are already resolved, if referenced struct/union
4663 * definition is available. So, if we had unresolved FWD and found corresponding
4664 * STRUCT/UNION, they will be from different compilation units. This
4665 * consequently means that when we "link" FWD to corresponding STRUCT/UNION,
4666 * type graph will likely have at least two different BTF types that describe
4667 * same type (e.g., most probably there will be two different BTF types for the
4668 * same 'int' primitive type) and could even have "overlapping" parts of type
4669 * graph that describe same subset of types.
4670 *
4671 * This in turn means that our assumption that each type in canonical graph
4672 * must correspond to exactly one type in candidate graph might not hold
4673 * anymore and will make it harder to detect contradictions using hypothetical
4674 * map. To handle this problem, we allow to follow FWD -> STRUCT/UNION
4675 * resolution only in canonical graph. FWDs in candidate graphs are never
4676 * resolved. To see why it's OK, let's check all possible situations w.r.t. FWDs
4677 * that can occur:
4678 * - Both types in canonical and candidate graphs are FWDs. If they are
4679 * structurally equivalent, then they can either be both resolved to the
4680 * same STRUCT/UNION or not resolved at all. In both cases they are
4681 * equivalent and there is no need to resolve FWD on candidate side.
4682 * - Both types in canonical and candidate graphs are concrete STRUCT/UNION,
4683 * so nothing to resolve as well, algorithm will check equivalence anyway.
4684 * - Type in canonical graph is FWD, while type in candidate is concrete
4685 * STRUCT/UNION. In this case candidate graph comes from single compilation
4686 * unit, so there is exactly one BTF type for each unique C type. After
4687 * resolving FWD into STRUCT/UNION, there might be more than one BTF type
4688 * in canonical graph mapping to single BTF type in candidate graph, but
4689 * because hypothetical mapping maps from canonical to candidate types, it's
4690 * alright, and we still maintain the property of having single `canon_id`
4691 * mapping to single `cand_id` (there could be two different `canon_id`
4692 * mapped to the same `cand_id`, but it's not contradictory).
4693 * - Type in canonical graph is concrete STRUCT/UNION, while type in candidate
4694 * graph is FWD. In this case we are just going to check compatibility of
4695 * STRUCT/UNION and corresponding FWD, and if they are compatible, we'll
4696 * assume that whatever STRUCT/UNION FWD resolves to must be equivalent to
4697 * a concrete STRUCT/UNION from canonical graph. If the rest of type graphs
4698 * turn out equivalent, we'll re-resolve FWD to concrete STRUCT/UNION from
4699 * canonical graph.
4700 */
4701 static int btf_dedup_is_equiv(struct btf_dedup *d, __u32 cand_id,
4702 __u32 canon_id)
4703 {
4704 struct btf_type *cand_type;
4705 struct btf_type *canon_type;
4706 __u32 hypot_type_id;
4707 __u16 cand_kind;
4708 __u16 canon_kind;
4709 int i, eq;
4710
4711 /* if both resolve to the same canonical, they must be equivalent */
4712 if (resolve_type_id(d, cand_id) == resolve_type_id(d, canon_id))
4713 return 1;
4714
4715 canon_id = resolve_fwd_id(d, canon_id);
4716
4717 hypot_type_id = d->hypot_map[canon_id];
4718 if (hypot_type_id <= BTF_MAX_NR_TYPES) {
4719 if (hypot_type_id == cand_id)
4720 return 1;
4721 /* In some cases compiler will generate different DWARF types
4722 * for *identical* array type definitions and use them for
4723 * different fields within the *same* struct. This breaks type
4724 * equivalence check, which makes an assumption that candidate
4725 * types sub-graph has a consistent and deduped-by-compiler
4726 * types within a single CU. And similar situation can happen
4727 * with struct/union sometimes, and event with pointers.
4728 * So accommodate cases like this doing a structural
4729 * comparison recursively, but avoiding being stuck in endless
4730 * loops by limiting the depth up to which we check.
4731 */
4732 if (btf_dedup_identical_types(d, hypot_type_id, cand_id, 16))
4733 return 1;
4734 return 0;
4735 }
4736
4737 if (btf_dedup_hypot_map_add(d, canon_id, cand_id))
4738 return -ENOMEM;
4739
4740 cand_type = btf_type_by_id(d->btf, cand_id);
4741 canon_type = btf_type_by_id(d->btf, canon_id);
4742 cand_kind = btf_kind(cand_type);
4743 canon_kind = btf_kind(canon_type);
4744
4745 if (cand_type->name_off != canon_type->name_off)
4746 return 0;
4747
4748 /* FWD <--> STRUCT/UNION equivalence check, if enabled */
4749 if ((cand_kind == BTF_KIND_FWD || canon_kind == BTF_KIND_FWD)
4750 && cand_kind != canon_kind) {
4751 __u16 real_kind;
4752 __u16 fwd_kind;
4753
4754 if (cand_kind == BTF_KIND_FWD) {
4755 real_kind = canon_kind;
4756 fwd_kind = btf_fwd_kind(cand_type);
4757 } else {
4758 real_kind = cand_kind;
4759 fwd_kind = btf_fwd_kind(canon_type);
4760 /* we'd need to resolve base FWD to STRUCT/UNION */
4761 if (fwd_kind == real_kind && canon_id < d->btf->start_id)
4762 d->hypot_adjust_canon = true;
4763 }
4764 return fwd_kind == real_kind;
4765 }
4766
4767 if (cand_kind != canon_kind)
4768 return 0;
4769
4770 switch (cand_kind) {
4771 case BTF_KIND_INT:
4772 return btf_equal_int_tag(cand_type, canon_type);
4773
4774 case BTF_KIND_ENUM:
4775 case BTF_KIND_ENUM64:
4776 return btf_compat_enum(cand_type, canon_type);
4777
4778 case BTF_KIND_FWD:
4779 case BTF_KIND_FLOAT:
4780 return btf_equal_common(cand_type, canon_type);
4781
4782 case BTF_KIND_CONST:
4783 case BTF_KIND_VOLATILE:
4784 case BTF_KIND_RESTRICT:
4785 case BTF_KIND_PTR:
4786 case BTF_KIND_TYPEDEF:
4787 case BTF_KIND_FUNC:
4788 case BTF_KIND_TYPE_TAG:
4789 if (cand_type->info != canon_type->info)
4790 return 0;
4791 return btf_dedup_is_equiv(d, cand_type->type, canon_type->type);
4792
4793 case BTF_KIND_ARRAY: {
4794 const struct btf_array *cand_arr, *canon_arr;
4795
4796 if (!btf_compat_array(cand_type, canon_type))
4797 return 0;
4798 cand_arr = btf_array(cand_type);
4799 canon_arr = btf_array(canon_type);
4800 eq = btf_dedup_is_equiv(d, cand_arr->index_type, canon_arr->index_type);
4801 if (eq <= 0)
4802 return eq;
4803 return btf_dedup_is_equiv(d, cand_arr->type, canon_arr->type);
4804 }
4805
4806 case BTF_KIND_STRUCT:
4807 case BTF_KIND_UNION: {
4808 const struct btf_member *cand_m, *canon_m;
4809 __u16 vlen;
4810
4811 if (!btf_shallow_equal_struct(cand_type, canon_type))
4812 return 0;
4813 vlen = btf_vlen(cand_type);
4814 cand_m = btf_members(cand_type);
4815 canon_m = btf_members(canon_type);
4816 for (i = 0; i < vlen; i++) {
4817 eq = btf_dedup_is_equiv(d, cand_m->type, canon_m->type);
4818 if (eq <= 0) {
4819 if (cand_type->name_off) {
4820 pr_debug("%s '%s' size=%d vlen=%d cand_id[%u] canon_id[%u] shallow-equal but not equiv for field#%d '%s': %d\n",
4821 cand_kind == BTF_KIND_STRUCT ? "STRUCT" : "UNION",
4822 btf__name_by_offset(d->btf, cand_type->name_off),
4823 cand_type->size, vlen, cand_id, canon_id, i,
4824 btf__name_by_offset(d->btf, cand_m->name_off), eq);
4825 }
4826 return eq;
4827 }
4828 cand_m++;
4829 canon_m++;
4830 }
4831
4832 return 1;
4833 }
4834
4835 case BTF_KIND_FUNC_PROTO: {
4836 const struct btf_param *cand_p, *canon_p;
4837 __u16 vlen;
4838
4839 if (!btf_compat_fnproto(cand_type, canon_type))
4840 return 0;
4841 eq = btf_dedup_is_equiv(d, cand_type->type, canon_type->type);
4842 if (eq <= 0)
4843 return eq;
4844 vlen = btf_vlen(cand_type);
4845 cand_p = btf_params(cand_type);
4846 canon_p = btf_params(canon_type);
4847 for (i = 0; i < vlen; i++) {
4848 eq = btf_dedup_is_equiv(d, cand_p->type, canon_p->type);
4849 if (eq <= 0)
4850 return eq;
4851 cand_p++;
4852 canon_p++;
4853 }
4854 return 1;
4855 }
4856
4857 default:
4858 return -EINVAL;
4859 }
4860 return 0;
4861 }
4862
4863 /*
4864 * Use hypothetical mapping, produced by successful type graph equivalence
4865 * check, to augment existing struct/union canonical mapping, where possible.
4866 *
4867 * If BTF_KIND_FWD resolution is allowed, this mapping is also used to record
4868 * FWD -> STRUCT/UNION correspondence as well. FWD resolution is bidirectional:
4869 * it doesn't matter if FWD type was part of canonical graph or candidate one,
4870 * we are recording the mapping anyway. As opposed to carefulness required
4871 * for struct/union correspondence mapping (described below), for FWD resolution
4872 * it's not important, as by the time that FWD type (reference type) will be
4873 * deduplicated all structs/unions will be deduped already anyway.
4874 *
4875 * Recording STRUCT/UNION mapping is purely a performance optimization and is
4876 * not required for correctness. It needs to be done carefully to ensure that
4877 * struct/union from candidate's type graph is not mapped into corresponding
4878 * struct/union from canonical type graph that itself hasn't been resolved into
4879 * canonical representative. The only guarantee we have is that canonical
4880 * struct/union was determined as canonical and that won't change. But any
4881 * types referenced through that struct/union fields could have been not yet
4882 * resolved, so in case like that it's too early to establish any kind of
4883 * correspondence between structs/unions.
4884 *
4885 * No canonical correspondence is derived for primitive types (they are already
4886 * deduplicated completely already anyway) or reference types (they rely on
4887 * stability of struct/union canonical relationship for equivalence checks).
4888 */
4889 static void btf_dedup_merge_hypot_map(struct btf_dedup *d)
4890 {
4891 __u32 canon_type_id, targ_type_id;
4892 __u16 t_kind, c_kind;
4893 __u32 t_id, c_id;
4894 int i;
4895
4896 for (i = 0; i < d->hypot_cnt; i++) {
4897 canon_type_id = d->hypot_list[i];
4898 targ_type_id = d->hypot_map[canon_type_id];
4899 t_id = resolve_type_id(d, targ_type_id);
4900 c_id = resolve_type_id(d, canon_type_id);
4901 t_kind = btf_kind(btf__type_by_id(d->btf, t_id));
4902 c_kind = btf_kind(btf__type_by_id(d->btf, c_id));
4903 /*
4904 * Resolve FWD into STRUCT/UNION.
4905 * It's ok to resolve FWD into STRUCT/UNION that's not yet
4906 * mapped to canonical representative (as opposed to
4907 * STRUCT/UNION <--> STRUCT/UNION mapping logic below), because
4908 * eventually that struct is going to be mapped and all resolved
4909 * FWDs will automatically resolve to correct canonical
4910 * representative. This will happen before ref type deduping,
4911 * which critically depends on stability of these mapping. This
4912 * stability is not a requirement for STRUCT/UNION equivalence
4913 * checks, though.
4914 */
4915
4916 /* if it's the split BTF case, we still need to point base FWD
4917 * to STRUCT/UNION in a split BTF, because FWDs from split BTF
4918 * will be resolved against base FWD. If we don't point base
4919 * canonical FWD to the resolved STRUCT/UNION, then all the
4920 * FWDs in split BTF won't be correctly resolved to a proper
4921 * STRUCT/UNION.
4922 */
4923 if (t_kind != BTF_KIND_FWD && c_kind == BTF_KIND_FWD)
4924 d->map[c_id] = t_id;
4925
4926 /* if graph equivalence determined that we'd need to adjust
4927 * base canonical types, then we need to only point base FWDs
4928 * to STRUCTs/UNIONs and do no more modifications. For all
4929 * other purposes the type graphs were not equivalent.
4930 */
4931 if (d->hypot_adjust_canon)
4932 continue;
4933
4934 if (t_kind == BTF_KIND_FWD && c_kind != BTF_KIND_FWD)
4935 d->map[t_id] = c_id;
4936
4937 if ((t_kind == BTF_KIND_STRUCT || t_kind == BTF_KIND_UNION) &&
4938 c_kind != BTF_KIND_FWD &&
4939 is_type_mapped(d, c_id) &&
4940 !is_type_mapped(d, t_id)) {
4941 /*
4942 * as a perf optimization, we can map struct/union
4943 * that's part of type graph we just verified for
4944 * equivalence. We can do that for struct/union that has
4945 * canonical representative only, though.
4946 */
4947 d->map[t_id] = c_id;
4948 }
4949 }
4950 }
4951
4952 static inline long btf_hash_by_kind(struct btf_type *t, __u16 kind)
4953 {
4954 if (kind == BTF_KIND_TYPEDEF)
4955 return btf_hash_typedef(t);
4956 else
4957 return btf_hash_struct(t);
4958 }
4959
4960 static inline bool btf_equal_by_kind(struct btf_type *t1, struct btf_type *t2, __u16 kind)
4961 {
4962 if (kind == BTF_KIND_TYPEDEF)
4963 return btf_equal_typedef(t1, t2);
4964 else
4965 return btf_shallow_equal_struct(t1, t2);
4966 }
4967
4968 /*
4969 * Deduplicate struct/union and typedef types.
4970 *
4971 * For each struct/union type its type signature hash is calculated, taking
4972 * into account type's name, size, number, order and names of fields, but
4973 * ignoring type ID's referenced from fields, because they might not be deduped
4974 * completely until after reference types deduplication phase. For each typedef
4975 * type, the hash is computed based on the type’s name and size. This type hash
4976 * is used to iterate over all potential canonical types, sharing same hash.
4977 * For each canonical candidate we check whether type graphs that they form
4978 * (through referenced types in fields and so on) are equivalent using algorithm
4979 * implemented in `btf_dedup_is_equiv`. If such equivalence is found and
4980 * BTF_KIND_FWD resolution is allowed, then hypothetical mapping
4981 * (btf_dedup->hypot_map) produced by aforementioned type graph equivalence
4982 * algorithm is used to record FWD -> STRUCT/UNION mapping. It's also used to
4983 * potentially map other structs/unions to their canonical representatives,
4984 * if such relationship hasn't yet been established. This speeds up algorithm
4985 * by eliminating some of the duplicate work.
4986 *
4987 * If no matching canonical representative was found, struct/union is marked
4988 * as canonical for itself and is added into btf_dedup->dedup_table hash map
4989 * for further look ups.
4990 */
4991 static int btf_dedup_struct_type(struct btf_dedup *d, __u32 type_id)
4992 {
4993 struct btf_type *cand_type, *t;
4994 struct hashmap_entry *hash_entry;
4995 /* if we don't find equivalent type, then we are canonical */
4996 __u32 new_id = type_id;
4997 __u16 kind;
4998 long h;
4999
5000 /* already deduped or is in process of deduping (loop detected) */
5001 if (d->map[type_id] <= BTF_MAX_NR_TYPES)
5002 return 0;
5003
5004 t = btf_type_by_id(d->btf, type_id);
5005 kind = btf_kind(t);
5006
5007 if (kind != BTF_KIND_STRUCT &&
5008 kind != BTF_KIND_UNION &&
5009 kind != BTF_KIND_TYPEDEF)
5010 return 0;
5011
5012 h = btf_hash_by_kind(t, kind);
5013 for_each_dedup_cand(d, hash_entry, h) {
5014 __u32 cand_id = hash_entry->value;
5015 int eq;
5016
5017 /*
5018 * Even though btf_dedup_is_equiv() checks for
5019 * btf_equal_by_kind() internally when checking two
5020 * structs (unions) or typedefs for equivalence, we need to guard here
5021 * from picking matching FWD type as a dedup candidate.
5022 * This can happen due to hash collision. In such case just
5023 * relying on btf_dedup_is_equiv() would lead to potentially
5024 * creating a loop (FWD -> STRUCT and STRUCT -> FWD), because
5025 * FWD and compatible STRUCT/UNION are considered equivalent.
5026 */
5027 cand_type = btf_type_by_id(d->btf, cand_id);
5028 if (!btf_equal_by_kind(t, cand_type, kind))
5029 continue;
5030
5031 btf_dedup_clear_hypot_map(d);
5032 eq = btf_dedup_is_equiv(d, type_id, cand_id);
5033 if (eq < 0)
5034 return eq;
5035 if (!eq)
5036 continue;
5037 btf_dedup_merge_hypot_map(d);
5038 if (d->hypot_adjust_canon) /* not really equivalent */
5039 continue;
5040 new_id = cand_id;
5041 break;
5042 }
5043
5044 d->map[type_id] = new_id;
5045 if (type_id == new_id && btf_dedup_table_add(d, h, type_id))
5046 return -ENOMEM;
5047
5048 return 0;
5049 }
5050
5051 static int btf_dedup_struct_types(struct btf_dedup *d)
5052 {
5053 int i, err;
5054
5055 for (i = 0; i < d->btf->nr_types; i++) {
5056 err = btf_dedup_struct_type(d, d->btf->start_id + i);
5057 if (err)
5058 return err;
5059 }
5060 return 0;
5061 }
5062
5063 /*
5064 * Deduplicate reference type.
5065 *
5066 * Once all primitive, struct/union and typedef types got deduplicated, we can easily
5067 * deduplicate all other (reference) BTF types. This is done in two steps:
5068 *
5069 * 1. Resolve all referenced type IDs into their canonical type IDs. This
5070 * resolution can be done either immediately for primitive, struct/union, and typedef
5071 * types (because they were deduped in previous two phases) or recursively for
5072 * reference types. Recursion will always terminate at either primitive or
5073 * struct/union and typedef types, at which point we can "unwind" chain of reference
5074 * types one by one. There is no danger of encountering cycles in C, as the only way to
5075 * form a type cycle is through struct or union types. Go can form such cycles through
5076 * typedef. Thus, any chain of reference types, even those taking part in a type cycle,
5077 * will inevitably reach a struct/union or typedef type at some point.
5078 *
5079 * 2. Once all referenced type IDs are resolved into canonical ones, BTF type
5080 * becomes "stable", in the sense that no further deduplication will cause
5081 * any changes to it. With that, it's now possible to calculate type's signature
5082 * hash (this time taking into account referenced type IDs) and loop over all
5083 * potential canonical representatives. If no match was found, current type
5084 * will become canonical representative of itself and will be added into
5085 * btf_dedup->dedup_table as another possible canonical representative.
5086 */
5087 static int btf_dedup_ref_type(struct btf_dedup *d, __u32 type_id)
5088 {
5089 struct hashmap_entry *hash_entry;
5090 __u32 new_id = type_id, cand_id;
5091 struct btf_type *t, *cand;
5092 /* if we don't find equivalent type, then we are representative type */
5093 int ref_type_id;
5094 long h;
5095
5096 if (d->map[type_id] == BTF_IN_PROGRESS_ID)
5097 return -ELOOP;
5098 if (d->map[type_id] <= BTF_MAX_NR_TYPES)
5099 return resolve_type_id(d, type_id);
5100
5101 t = btf_type_by_id(d->btf, type_id);
5102 d->map[type_id] = BTF_IN_PROGRESS_ID;
5103
5104 switch (btf_kind(t)) {
5105 case BTF_KIND_CONST:
5106 case BTF_KIND_VOLATILE:
5107 case BTF_KIND_RESTRICT:
5108 case BTF_KIND_PTR:
5109 case BTF_KIND_FUNC:
5110 case BTF_KIND_TYPE_TAG:
5111 ref_type_id = btf_dedup_ref_type(d, t->type);
5112 if (ref_type_id < 0)
5113 return ref_type_id;
5114 t->type = ref_type_id;
5115
5116 h = btf_hash_common(t);
5117 for_each_dedup_cand(d, hash_entry, h) {
5118 cand_id = hash_entry->value;
5119 cand = btf_type_by_id(d->btf, cand_id);
5120 if (btf_equal_common(t, cand)) {
5121 new_id = cand_id;
5122 break;
5123 }
5124 }
5125 break;
5126
5127 case BTF_KIND_DECL_TAG:
5128 ref_type_id = btf_dedup_ref_type(d, t->type);
5129 if (ref_type_id < 0)
5130 return ref_type_id;
5131 t->type = ref_type_id;
5132
5133 h = btf_hash_int_decl_tag(t);
5134 for_each_dedup_cand(d, hash_entry, h) {
5135 cand_id = hash_entry->value;
5136 cand = btf_type_by_id(d->btf, cand_id);
5137 if (btf_equal_int_tag(t, cand)) {
5138 new_id = cand_id;
5139 break;
5140 }
5141 }
5142 break;
5143
5144 case BTF_KIND_ARRAY: {
5145 struct btf_array *info = btf_array(t);
5146
5147 ref_type_id = btf_dedup_ref_type(d, info->type);
5148 if (ref_type_id < 0)
5149 return ref_type_id;
5150 info->type = ref_type_id;
5151
5152 ref_type_id = btf_dedup_ref_type(d, info->index_type);
5153 if (ref_type_id < 0)
5154 return ref_type_id;
5155 info->index_type = ref_type_id;
5156
5157 h = btf_hash_array(t);
5158 for_each_dedup_cand(d, hash_entry, h) {
5159 cand_id = hash_entry->value;
5160 cand = btf_type_by_id(d->btf, cand_id);
5161 if (btf_equal_array(t, cand)) {
5162 new_id = cand_id;
5163 break;
5164 }
5165 }
5166 break;
5167 }
5168
5169 case BTF_KIND_FUNC_PROTO: {
5170 struct btf_param *param;
5171 __u16 vlen;
5172 int i;
5173
5174 ref_type_id = btf_dedup_ref_type(d, t->type);
5175 if (ref_type_id < 0)
5176 return ref_type_id;
5177 t->type = ref_type_id;
5178
5179 vlen = btf_vlen(t);
5180 param = btf_params(t);
5181 for (i = 0; i < vlen; i++) {
5182 ref_type_id = btf_dedup_ref_type(d, param->type);
5183 if (ref_type_id < 0)
5184 return ref_type_id;
5185 param->type = ref_type_id;
5186 param++;
5187 }
5188
5189 h = btf_hash_fnproto(t);
5190 for_each_dedup_cand(d, hash_entry, h) {
5191 cand_id = hash_entry->value;
5192 cand = btf_type_by_id(d->btf, cand_id);
5193 if (btf_equal_fnproto(t, cand)) {
5194 new_id = cand_id;
5195 break;
5196 }
5197 }
5198 break;
5199 }
5200
5201 default:
5202 return -EINVAL;
5203 }
5204
5205 d->map[type_id] = new_id;
5206 if (type_id == new_id && btf_dedup_table_add(d, h, type_id))
5207 return -ENOMEM;
5208
5209 return new_id;
5210 }
5211
5212 static int btf_dedup_ref_types(struct btf_dedup *d)
5213 {
5214 int i, err;
5215
5216 for (i = 0; i < d->btf->nr_types; i++) {
5217 err = btf_dedup_ref_type(d, d->btf->start_id + i);
5218 if (err < 0)
5219 return err;
5220 }
5221 /* we won't need d->dedup_table anymore */
5222 hashmap__free(d->dedup_table);
5223 d->dedup_table = NULL;
5224 return 0;
5225 }
5226
5227 /*
5228 * Collect a map from type names to type ids for all canonical structs
5229 * and unions. If the same name is shared by several canonical types
5230 * use a special value 0 to indicate this fact.
5231 */
5232 static int btf_dedup_fill_unique_names_map(struct btf_dedup *d, struct hashmap *names_map)
5233 {
5234 __u32 nr_types = btf__type_cnt(d->btf);
5235 struct btf_type *t;
5236 __u32 type_id;
5237 __u16 kind;
5238 int err;
5239
5240 /*
5241 * Iterate over base and split module ids in order to get all
5242 * available structs in the map.
5243 */
5244 for (type_id = 1; type_id < nr_types; ++type_id) {
5245 t = btf_type_by_id(d->btf, type_id);
5246 kind = btf_kind(t);
5247
5248 if (kind != BTF_KIND_STRUCT && kind != BTF_KIND_UNION)
5249 continue;
5250
5251 /* Skip non-canonical types */
5252 if (type_id != d->map[type_id])
5253 continue;
5254
5255 err = hashmap__add(names_map, t->name_off, type_id);
5256 if (err == -EEXIST)
5257 err = hashmap__set(names_map, t->name_off, 0, NULL, NULL);
5258
5259 if (err)
5260 return err;
5261 }
5262
5263 return 0;
5264 }
5265
5266 static int btf_dedup_resolve_fwd(struct btf_dedup *d, struct hashmap *names_map, __u32 type_id)
5267 {
5268 struct btf_type *t = btf_type_by_id(d->btf, type_id);
5269 enum btf_fwd_kind fwd_kind = btf_kflag(t);
5270 __u16 cand_kind, kind = btf_kind(t);
5271 struct btf_type *cand_t;
5272 uintptr_t cand_id;
5273
5274 if (kind != BTF_KIND_FWD)
5275 return 0;
5276
5277 /* Skip if this FWD already has a mapping */
5278 if (type_id != d->map[type_id])
5279 return 0;
5280
5281 if (!hashmap__find(names_map, t->name_off, &cand_id))
5282 return 0;
5283
5284 /* Zero is a special value indicating that name is not unique */
5285 if (!cand_id)
5286 return 0;
5287
5288 cand_t = btf_type_by_id(d->btf, cand_id);
5289 cand_kind = btf_kind(cand_t);
5290 if ((cand_kind == BTF_KIND_STRUCT && fwd_kind != BTF_FWD_STRUCT) ||
5291 (cand_kind == BTF_KIND_UNION && fwd_kind != BTF_FWD_UNION))
5292 return 0;
5293
5294 d->map[type_id] = cand_id;
5295
5296 return 0;
5297 }
5298
5299 /*
5300 * Resolve unambiguous forward declarations.
5301 *
5302 * The lion's share of all FWD declarations is resolved during
5303 * `btf_dedup_struct_types` phase when different type graphs are
5304 * compared against each other. However, if in some compilation unit a
5305 * FWD declaration is not a part of a type graph compared against
5306 * another type graph that declaration's canonical type would not be
5307 * changed. Example:
5308 *
5309 * CU #1:
5310 *
5311 * struct foo;
5312 * struct foo *some_global;
5313 *
5314 * CU #2:
5315 *
5316 * struct foo { int u; };
5317 * struct foo *another_global;
5318 *
5319 * After `btf_dedup_struct_types` the BTF looks as follows:
5320 *
5321 * [1] STRUCT 'foo' size=4 vlen=1 ...
5322 * [2] INT 'int' size=4 ...
5323 * [3] PTR '(anon)' type_id=1
5324 * [4] FWD 'foo' fwd_kind=struct
5325 * [5] PTR '(anon)' type_id=4
5326 *
5327 * This pass assumes that such FWD declarations should be mapped to
5328 * structs or unions with identical name in case if the name is not
5329 * ambiguous.
5330 */
5331 static int btf_dedup_resolve_fwds(struct btf_dedup *d)
5332 {
5333 int i, err;
5334 struct hashmap *names_map;
5335
5336 names_map = hashmap__new(btf_dedup_identity_hash_fn, btf_dedup_equal_fn, NULL);
5337 if (IS_ERR(names_map))
5338 return PTR_ERR(names_map);
5339
5340 err = btf_dedup_fill_unique_names_map(d, names_map);
5341 if (err < 0)
5342 goto exit;
5343
5344 for (i = 0; i < d->btf->nr_types; i++) {
5345 err = btf_dedup_resolve_fwd(d, names_map, d->btf->start_id + i);
5346 if (err < 0)
5347 break;
5348 }
5349
5350 exit:
5351 hashmap__free(names_map);
5352 return err;
5353 }
5354
5355 /*
5356 * Compact types.
5357 *
5358 * After we established for each type its corresponding canonical representative
5359 * type, we now can eliminate types that are not canonical and leave only
5360 * canonical ones layed out sequentially in memory by copying them over
5361 * duplicates. During compaction btf_dedup->hypot_map array is reused to store
5362 * a map from original type ID to a new compacted type ID, which will be used
5363 * during next phase to "fix up" type IDs, referenced from struct/union and
5364 * reference types.
5365 */
5366 static int btf_dedup_compact_types(struct btf_dedup *d)
5367 {
5368 __u32 *new_offs;
5369 __u32 next_type_id = d->btf->start_id;
5370 const struct btf_type *t;
5371 void *p;
5372 int i, id, len;
5373
5374 /* we are going to reuse hypot_map to store compaction remapping */
5375 d->hypot_map[0] = 0;
5376 /* base BTF types are not renumbered */
5377 for (id = 1; id < d->btf->start_id; id++)
5378 d->hypot_map[id] = id;
5379 for (i = 0, id = d->btf->start_id; i < d->btf->nr_types; i++, id++)
5380 d->hypot_map[id] = BTF_UNPROCESSED_ID;
5381
5382 p = d->btf->types_data;
5383
5384 for (i = 0, id = d->btf->start_id; i < d->btf->nr_types; i++, id++) {
5385 if (d->map[id] != id)
5386 continue;
5387
5388 t = btf__type_by_id(d->btf, id);
5389 len = btf_type_size(t);
5390 if (len < 0)
5391 return len;
5392
5393 memmove(p, t, len);
5394 d->hypot_map[id] = next_type_id;
5395 d->btf->type_offs[next_type_id - d->btf->start_id] = p - d->btf->types_data;
5396 p += len;
5397 next_type_id++;
5398 }
5399
5400 /* shrink struct btf's internal types index and update btf_header */
5401 d->btf->nr_types = next_type_id - d->btf->start_id;
5402 d->btf->type_offs_cap = d->btf->nr_types;
5403 d->btf->hdr->type_len = p - d->btf->types_data;
5404 new_offs = libbpf_reallocarray(d->btf->type_offs, d->btf->type_offs_cap,
5405 sizeof(*new_offs));
5406 if (d->btf->type_offs_cap && !new_offs)
5407 return -ENOMEM;
5408 d->btf->type_offs = new_offs;
5409 d->btf->hdr->str_off = d->btf->hdr->type_len;
5410 d->btf->raw_size = d->btf->hdr->hdr_len + d->btf->hdr->type_len + d->btf->hdr->str_len;
5411 return 0;
5412 }
5413
5414 /*
5415 * Figure out final (deduplicated and compacted) type ID for provided original
5416 * `type_id` by first resolving it into corresponding canonical type ID and
5417 * then mapping it to a deduplicated type ID, stored in btf_dedup->hypot_map,
5418 * which is populated during compaction phase.
5419 */
5420 static int btf_dedup_remap_type_id(__u32 *type_id, void *ctx)
5421 {
5422 struct btf_dedup *d = ctx;
5423 __u32 resolved_type_id, new_type_id;
5424
5425 resolved_type_id = resolve_type_id(d, *type_id);
5426 new_type_id = d->hypot_map[resolved_type_id];
5427 if (new_type_id > BTF_MAX_NR_TYPES)
5428 return -EINVAL;
5429
5430 *type_id = new_type_id;
5431 return 0;
5432 }
5433
5434 /*
5435 * Remap referenced type IDs into deduped type IDs.
5436 *
5437 * After BTF types are deduplicated and compacted, their final type IDs may
5438 * differ from original ones. The map from original to a corresponding
5439 * deduped type ID is stored in btf_dedup->hypot_map and is populated during
5440 * compaction phase. During remapping phase we are rewriting all type IDs
5441 * referenced from any BTF type (e.g., struct fields, func proto args, etc) to
5442 * their final deduped type IDs.
5443 */
5444 static int btf_dedup_remap_types(struct btf_dedup *d)
5445 {
5446 int i, r;
5447
5448 for (i = 0; i < d->btf->nr_types; i++) {
5449 struct btf_type *t = btf_type_by_id(d->btf, d->btf->start_id + i);
5450 struct btf_field_iter it;
5451 __u32 *type_id;
5452
5453 r = btf_field_iter_init(&it, t, BTF_FIELD_ITER_IDS);
5454 if (r)
5455 return r;
5456
5457 while ((type_id = btf_field_iter_next(&it))) {
5458 __u32 resolved_id, new_id;
5459
5460 resolved_id = resolve_type_id(d, *type_id);
5461 new_id = d->hypot_map[resolved_id];
5462 if (new_id > BTF_MAX_NR_TYPES)
5463 return -EINVAL;
5464
5465 *type_id = new_id;
5466 }
5467 }
5468
5469 if (!d->btf_ext)
5470 return 0;
5471
5472 r = btf_ext_visit_type_ids(d->btf_ext, btf_dedup_remap_type_id, d);
5473 if (r)
5474 return r;
5475
5476 return 0;
5477 }
5478
5479 /*
5480 * Probe few well-known locations for vmlinux kernel image and try to load BTF
5481 * data out of it to use for target BTF.
5482 */
5483 struct btf *btf__load_vmlinux_btf(void)
5484 {
5485 const char *sysfs_btf_path = "/sys/kernel/btf/vmlinux";
5486 /* fall back locations, trying to find vmlinux on disk */
5487 const char *locations[] = {
5488 "/boot/vmlinux-%1$s",
5489 "/lib/modules/%1$s/vmlinux-%1$s",
5490 "/lib/modules/%1$s/build/vmlinux",
5491 "/usr/lib/modules/%1$s/kernel/vmlinux",
5492 "/usr/lib/debug/boot/vmlinux-%1$s",
5493 "/usr/lib/debug/boot/vmlinux-%1$s.debug",
5494 "/usr/lib/debug/lib/modules/%1$s/vmlinux",
5495 };
5496 char path[PATH_MAX + 1];
5497 struct utsname buf;
5498 struct btf *btf;
5499 int i, err;
5500
5501 /* is canonical sysfs location accessible? */
5502 if (faccessat(AT_FDCWD, sysfs_btf_path, F_OK, AT_EACCESS) < 0) {
5503 pr_warn("kernel BTF is missing at '%s', was CONFIG_DEBUG_INFO_BTF enabled?\n",
5504 sysfs_btf_path);
5505 } else {
5506 btf = btf_parse_raw_mmap(sysfs_btf_path, NULL);
5507 if (IS_ERR(btf))
5508 btf = btf__parse(sysfs_btf_path, NULL);
5509
5510 if (!btf) {
5511 err = -errno;
5512 pr_warn("failed to read kernel BTF from '%s': %s\n",
5513 sysfs_btf_path, errstr(err));
5514 return libbpf_err_ptr(err);
5515 }
5516 pr_debug("loaded kernel BTF from '%s'\n", sysfs_btf_path);
5517 return btf;
5518 }
5519
5520 /* try fallback locations */
5521 uname(&buf);
5522 for (i = 0; i < ARRAY_SIZE(locations); i++) {
5523 snprintf(path, PATH_MAX, locations[i], buf.release);
5524
5525 if (faccessat(AT_FDCWD, path, R_OK, AT_EACCESS))
5526 continue;
5527
5528 btf = btf__parse(path, NULL);
5529 err = libbpf_get_error(btf);
5530 pr_debug("loading kernel BTF '%s': %s\n", path, errstr(err));
5531 if (err)
5532 continue;
5533
5534 return btf;
5535 }
5536
5537 pr_warn("failed to find valid kernel BTF\n");
5538 return libbpf_err_ptr(-ESRCH);
5539 }
5540
5541 struct btf *libbpf_find_kernel_btf(void) __attribute__((alias("btf__load_vmlinux_btf")));
5542
5543 struct btf *btf__load_module_btf(const char *module_name, struct btf *vmlinux_btf)
5544 {
5545 char path[80];
5546
5547 snprintf(path, sizeof(path), "/sys/kernel/btf/%s", module_name);
5548 return btf__parse_split(path, vmlinux_btf);
5549 }
5550
5551 int btf_ext_visit_type_ids(struct btf_ext *btf_ext, type_id_visit_fn visit, void *ctx)
5552 {
5553 const struct btf_ext_info *seg;
5554 struct btf_ext_info_sec *sec;
5555 int i, err;
5556
5557 seg = &btf_ext->func_info;
5558 for_each_btf_ext_sec(seg, sec) {
5559 struct bpf_func_info_min *rec;
5560
5561 for_each_btf_ext_rec(seg, sec, i, rec) {
5562 err = visit(&rec->type_id, ctx);
5563 if (err < 0)
5564 return err;
5565 }
5566 }
5567
5568 seg = &btf_ext->core_relo_info;
5569 for_each_btf_ext_sec(seg, sec) {
5570 struct bpf_core_relo *rec;
5571
5572 for_each_btf_ext_rec(seg, sec, i, rec) {
5573 err = visit(&rec->type_id, ctx);
5574 if (err < 0)
5575 return err;
5576 }
5577 }
5578
5579 return 0;
5580 }
5581
5582 int btf_ext_visit_str_offs(struct btf_ext *btf_ext, str_off_visit_fn visit, void *ctx)
5583 {
5584 const struct btf_ext_info *seg;
5585 struct btf_ext_info_sec *sec;
5586 int i, err;
5587
5588 seg = &btf_ext->func_info;
5589 for_each_btf_ext_sec(seg, sec) {
5590 err = visit(&sec->sec_name_off, ctx);
5591 if (err)
5592 return err;
5593 }
5594
5595 seg = &btf_ext->line_info;
5596 for_each_btf_ext_sec(seg, sec) {
5597 struct bpf_line_info_min *rec;
5598
5599 err = visit(&sec->sec_name_off, ctx);
5600 if (err)
5601 return err;
5602
5603 for_each_btf_ext_rec(seg, sec, i, rec) {
5604 err = visit(&rec->file_name_off, ctx);
5605 if (err)
5606 return err;
5607 err = visit(&rec->line_off, ctx);
5608 if (err)
5609 return err;
5610 }
5611 }
5612
5613 seg = &btf_ext->core_relo_info;
5614 for_each_btf_ext_sec(seg, sec) {
5615 struct bpf_core_relo *rec;
5616
5617 err = visit(&sec->sec_name_off, ctx);
5618 if (err)
5619 return err;
5620
5621 for_each_btf_ext_rec(seg, sec, i, rec) {
5622 err = visit(&rec->access_str_off, ctx);
5623 if (err)
5624 return err;
5625 }
5626 }
5627
5628 return 0;
5629 }
5630
5631 struct btf_distill {
5632 struct btf_pipe pipe;
5633 int *id_map;
5634 unsigned int split_start_id;
5635 unsigned int split_start_str;
5636 int diff_id;
5637 };
5638
5639 static int btf_add_distilled_type_ids(struct btf_distill *dist, __u32 i)
5640 {
5641 struct btf_type *split_t = btf_type_by_id(dist->pipe.src, i);
5642 struct btf_field_iter it;
5643 __u32 *id;
5644 int err;
5645
5646 err = btf_field_iter_init(&it, split_t, BTF_FIELD_ITER_IDS);
5647 if (err)
5648 return err;
5649 while ((id = btf_field_iter_next(&it))) {
5650 struct btf_type *base_t;
5651
5652 if (!*id)
5653 continue;
5654 /* split BTF id, not needed */
5655 if (*id >= dist->split_start_id)
5656 continue;
5657 /* already added ? */
5658 if (dist->id_map[*id] > 0)
5659 continue;
5660
5661 /* only a subset of base BTF types should be referenced from
5662 * split BTF; ensure nothing unexpected is referenced.
5663 */
5664 base_t = btf_type_by_id(dist->pipe.src, *id);
5665 switch (btf_kind(base_t)) {
5666 case BTF_KIND_INT:
5667 case BTF_KIND_FLOAT:
5668 case BTF_KIND_FWD:
5669 case BTF_KIND_ARRAY:
5670 case BTF_KIND_STRUCT:
5671 case BTF_KIND_UNION:
5672 case BTF_KIND_TYPEDEF:
5673 case BTF_KIND_ENUM:
5674 case BTF_KIND_ENUM64:
5675 case BTF_KIND_PTR:
5676 case BTF_KIND_CONST:
5677 case BTF_KIND_RESTRICT:
5678 case BTF_KIND_VOLATILE:
5679 case BTF_KIND_FUNC_PROTO:
5680 case BTF_KIND_TYPE_TAG:
5681 dist->id_map[*id] = *id;
5682 break;
5683 default:
5684 pr_warn("unexpected reference to base type[%u] of kind [%u] when creating distilled base BTF.\n",
5685 *id, btf_kind(base_t));
5686 return -EINVAL;
5687 }
5688 /* If a base type is used, ensure types it refers to are
5689 * marked as used also; so for example if we find a PTR to INT
5690 * we need both the PTR and INT.
5691 *
5692 * The only exception is named struct/unions, since distilled
5693 * base BTF composite types have no members.
5694 */
5695 if (btf_is_composite(base_t) && base_t->name_off)
5696 continue;
5697 err = btf_add_distilled_type_ids(dist, *id);
5698 if (err)
5699 return err;
5700 }
5701 return 0;
5702 }
5703
5704 static int btf_add_distilled_types(struct btf_distill *dist)
5705 {
5706 bool adding_to_base = dist->pipe.dst->start_id == 1;
5707 int id = btf__type_cnt(dist->pipe.dst);
5708 struct btf_type *t;
5709 int i, err = 0;
5710
5711
5712 /* Add types for each of the required references to either distilled
5713 * base or split BTF, depending on type characteristics.
5714 */
5715 for (i = 1; i < dist->split_start_id; i++) {
5716 const char *name;
5717 int kind;
5718
5719 if (!dist->id_map[i])
5720 continue;
5721 t = btf_type_by_id(dist->pipe.src, i);
5722 kind = btf_kind(t);
5723 name = btf__name_by_offset(dist->pipe.src, t->name_off);
5724
5725 switch (kind) {
5726 case BTF_KIND_INT:
5727 case BTF_KIND_FLOAT:
5728 case BTF_KIND_FWD:
5729 /* Named int, float, fwd are added to base. */
5730 if (!adding_to_base)
5731 continue;
5732 err = btf_add_type(&dist->pipe, t);
5733 break;
5734 case BTF_KIND_STRUCT:
5735 case BTF_KIND_UNION:
5736 /* Named struct/union are added to base as 0-vlen
5737 * struct/union of same size. Anonymous struct/unions
5738 * are added to split BTF as-is.
5739 */
5740 if (adding_to_base) {
5741 if (!t->name_off)
5742 continue;
5743 err = btf_add_composite(dist->pipe.dst, kind, name, t->size);
5744 } else {
5745 if (t->name_off)
5746 continue;
5747 err = btf_add_type(&dist->pipe, t);
5748 }
5749 break;
5750 case BTF_KIND_ENUM:
5751 case BTF_KIND_ENUM64:
5752 /* Named enum[64]s are added to base as a sized
5753 * enum; relocation will match with appropriately-named
5754 * and sized enum or enum64.
5755 *
5756 * Anonymous enums are added to split BTF as-is.
5757 */
5758 if (adding_to_base) {
5759 if (!t->name_off)
5760 continue;
5761 err = btf__add_enum(dist->pipe.dst, name, t->size);
5762 } else {
5763 if (t->name_off)
5764 continue;
5765 err = btf_add_type(&dist->pipe, t);
5766 }
5767 break;
5768 case BTF_KIND_ARRAY:
5769 case BTF_KIND_TYPEDEF:
5770 case BTF_KIND_PTR:
5771 case BTF_KIND_CONST:
5772 case BTF_KIND_RESTRICT:
5773 case BTF_KIND_VOLATILE:
5774 case BTF_KIND_FUNC_PROTO:
5775 case BTF_KIND_TYPE_TAG:
5776 /* All other types are added to split BTF. */
5777 if (adding_to_base)
5778 continue;
5779 err = btf_add_type(&dist->pipe, t);
5780 break;
5781 default:
5782 pr_warn("unexpected kind when adding base type '%s'[%u] of kind [%u] to distilled base BTF.\n",
5783 name, i, kind);
5784 return -EINVAL;
5785
5786 }
5787 if (err < 0)
5788 break;
5789 dist->id_map[i] = id++;
5790 }
5791 return err;
5792 }
5793
5794 /* Split BTF ids without a mapping will be shifted downwards since distilled
5795 * base BTF is smaller than the original base BTF. For those that have a
5796 * mapping (either to base or updated split BTF), update the id based on
5797 * that mapping.
5798 */
5799 static int btf_update_distilled_type_ids(struct btf_distill *dist, __u32 i)
5800 {
5801 struct btf_type *t = btf_type_by_id(dist->pipe.dst, i);
5802 struct btf_field_iter it;
5803 __u32 *id;
5804 int err;
5805
5806 err = btf_field_iter_init(&it, t, BTF_FIELD_ITER_IDS);
5807 if (err)
5808 return err;
5809 while ((id = btf_field_iter_next(&it))) {
5810 if (dist->id_map[*id])
5811 *id = dist->id_map[*id];
5812 else if (*id >= dist->split_start_id)
5813 *id -= dist->diff_id;
5814 }
5815 return 0;
5816 }
5817
5818 /* Create updated split BTF with distilled base BTF; distilled base BTF
5819 * consists of BTF information required to clarify the types that split
5820 * BTF refers to, omitting unneeded details. Specifically it will contain
5821 * base types and memberless definitions of named structs, unions and enumerated
5822 * types. Associated reference types like pointers, arrays and anonymous
5823 * structs, unions and enumerated types will be added to split BTF.
5824 * Size is recorded for named struct/unions to help guide matching to the
5825 * target base BTF during later relocation.
5826 *
5827 * The only case where structs, unions or enumerated types are fully represented
5828 * is when they are anonymous; in such cases, the anonymous type is added to
5829 * split BTF in full.
5830 *
5831 * We return newly-created split BTF where the split BTF refers to a newly-created
5832 * distilled base BTF. Both must be freed separately by the caller.
5833 */
5834 int btf__distill_base(const struct btf *src_btf, struct btf **new_base_btf,
5835 struct btf **new_split_btf)
5836 {
5837 struct btf *new_base = NULL, *new_split = NULL;
5838 const struct btf *old_base;
5839 unsigned int n = btf__type_cnt(src_btf);
5840 struct btf_distill dist = {};
5841 struct btf_type *t;
5842 int i, err = 0;
5843
5844 /* src BTF must be split BTF. */
5845 old_base = btf__base_btf(src_btf);
5846 if (!new_base_btf || !new_split_btf || !old_base)
5847 return libbpf_err(-EINVAL);
5848
5849 new_base = btf__new_empty();
5850 if (!new_base)
5851 return libbpf_err(-ENOMEM);
5852
5853 btf__set_endianness(new_base, btf__endianness(src_btf));
5854
5855 dist.id_map = calloc(n, sizeof(*dist.id_map));
5856 if (!dist.id_map) {
5857 err = -ENOMEM;
5858 goto done;
5859 }
5860 dist.pipe.src = src_btf;
5861 dist.pipe.dst = new_base;
5862 dist.pipe.str_off_map = hashmap__new(btf_dedup_identity_hash_fn, btf_dedup_equal_fn, NULL);
5863 if (IS_ERR(dist.pipe.str_off_map)) {
5864 err = -ENOMEM;
5865 goto done;
5866 }
5867 dist.split_start_id = btf__type_cnt(old_base);
5868 dist.split_start_str = old_base->hdr->str_len;
5869
5870 /* Pass over src split BTF; generate the list of base BTF type ids it
5871 * references; these will constitute our distilled BTF set to be
5872 * distributed over base and split BTF as appropriate.
5873 */
5874 for (i = src_btf->start_id; i < n; i++) {
5875 err = btf_add_distilled_type_ids(&dist, i);
5876 if (err < 0)
5877 goto done;
5878 }
5879 /* Next add types for each of the required references to base BTF and split BTF
5880 * in turn.
5881 */
5882 err = btf_add_distilled_types(&dist);
5883 if (err < 0)
5884 goto done;
5885
5886 /* Create new split BTF with distilled base BTF as its base; the final
5887 * state is split BTF with distilled base BTF that represents enough
5888 * about its base references to allow it to be relocated with the base
5889 * BTF available.
5890 */
5891 new_split = btf__new_empty_split(new_base);
5892 if (!new_split) {
5893 err = -errno;
5894 goto done;
5895 }
5896 dist.pipe.dst = new_split;
5897 /* First add all split types */
5898 for (i = src_btf->start_id; i < n; i++) {
5899 t = btf_type_by_id(src_btf, i);
5900 err = btf_add_type(&dist.pipe, t);
5901 if (err < 0)
5902 goto done;
5903 }
5904 /* Now add distilled types to split BTF that are not added to base. */
5905 err = btf_add_distilled_types(&dist);
5906 if (err < 0)
5907 goto done;
5908
5909 /* All split BTF ids will be shifted downwards since there are less base
5910 * BTF ids in distilled base BTF.
5911 */
5912 dist.diff_id = dist.split_start_id - btf__type_cnt(new_base);
5913
5914 n = btf__type_cnt(new_split);
5915 /* Now update base/split BTF ids. */
5916 for (i = 1; i < n; i++) {
5917 err = btf_update_distilled_type_ids(&dist, i);
5918 if (err < 0)
5919 break;
5920 }
5921 done:
5922 free(dist.id_map);
5923 hashmap__free(dist.pipe.str_off_map);
5924 if (err) {
5925 btf__free(new_split);
5926 btf__free(new_base);
5927 return libbpf_err(err);
5928 }
5929 *new_base_btf = new_base;
5930 *new_split_btf = new_split;
5931
5932 return 0;
5933 }
5934
5935 const struct btf_header *btf_header(const struct btf *btf)
5936 {
5937 return btf->hdr;
5938 }
5939
5940 void btf_set_base_btf(struct btf *btf, const struct btf *base_btf)
5941 {
5942 btf->base_btf = (struct btf *)base_btf;
5943 btf->start_id = btf__type_cnt(base_btf);
5944 btf->start_str_off = base_btf->hdr->str_len + base_btf->start_str_off;
5945 }
5946
5947 int btf__relocate(struct btf *btf, const struct btf *base_btf)
5948 {
5949 int err = btf_relocate(btf, base_btf, NULL);
5950
5951 if (!err)
5952 btf->owns_base = false;
5953 return libbpf_err(err);
5954 }
5955
5956 struct btf_permute {
5957 struct btf *btf;
5958 __u32 *id_map;
5959 __u32 start_offs;
5960 };
5961
5962 /* Callback function to remap individual type ID references */
5963 static int btf_permute_remap_type_id(__u32 *type_id, void *ctx)
5964 {
5965 struct btf_permute *p = ctx;
5966 __u32 new_id = *type_id;
5967
5968 /* refer to the base BTF or VOID type */
5969 if (new_id < p->btf->start_id)
5970 return 0;
5971
5972 if (new_id >= btf__type_cnt(p->btf))
5973 return -EINVAL;
5974
5975 *type_id = p->id_map[new_id - p->btf->start_id + p->start_offs];
5976 return 0;
5977 }
5978
5979 int btf__permute(struct btf *btf, __u32 *id_map, __u32 id_map_cnt,
5980 const struct btf_permute_opts *opts)
5981 {
5982 struct btf_permute p;
5983 struct btf_ext *btf_ext;
5984 void *nt, *new_types = NULL;
5985 __u32 *order_map = NULL;
5986 int err = 0, i;
5987 __u32 n, id, start_offs = 0;
5988
5989 if (!OPTS_VALID(opts, btf_permute_opts))
5990 return libbpf_err(-EINVAL);
5991
5992 if (btf__base_btf(btf)) {
5993 n = btf->nr_types;
5994 } else {
5995 if (id_map[0] != 0)
5996 return libbpf_err(-EINVAL);
5997 n = btf__type_cnt(btf);
5998 start_offs = 1;
5999 }
6000
6001 if (id_map_cnt != n)
6002 return libbpf_err(-EINVAL);
6003
6004 /* record the sequence of types */
6005 order_map = calloc(id_map_cnt, sizeof(*id_map));
6006 if (!order_map) {
6007 err = -ENOMEM;
6008 goto done;
6009 }
6010
6011 new_types = calloc(btf->hdr->type_len, 1);
6012 if (!new_types) {
6013 err = -ENOMEM;
6014 goto done;
6015 }
6016
6017 if (btf_ensure_modifiable(btf)) {
6018 err = -ENOMEM;
6019 goto done;
6020 }
6021
6022 for (i = start_offs; i < id_map_cnt; i++) {
6023 id = id_map[i];
6024 if (id < btf->start_id || id >= btf__type_cnt(btf)) {
6025 err = -EINVAL;
6026 goto done;
6027 }
6028 id -= btf->start_id - start_offs;
6029 /* cannot be mapped to the same ID */
6030 if (order_map[id]) {
6031 err = -EINVAL;
6032 goto done;
6033 }
6034 order_map[id] = i + btf->start_id - start_offs;
6035 }
6036
6037 p.btf = btf;
6038 p.id_map = id_map;
6039 p.start_offs = start_offs;
6040 nt = new_types;
6041 for (i = start_offs; i < id_map_cnt; i++) {
6042 struct btf_field_iter it;
6043 const struct btf_type *t;
6044 __u32 *type_id;
6045 int type_size;
6046
6047 id = order_map[i];
6048 t = btf__type_by_id(btf, id);
6049 type_size = btf_type_size(t);
6050 memcpy(nt, t, type_size);
6051
6052 /* fix up referenced IDs for BTF */
6053 err = btf_field_iter_init(&it, nt, BTF_FIELD_ITER_IDS);
6054 if (err)
6055 goto done;
6056 while ((type_id = btf_field_iter_next(&it))) {
6057 err = btf_permute_remap_type_id(type_id, &p);
6058 if (err)
6059 goto done;
6060 }
6061
6062 nt += type_size;
6063 }
6064
6065 /* fix up referenced IDs for btf_ext */
6066 btf_ext = OPTS_GET(opts, btf_ext, NULL);
6067 if (btf_ext) {
6068 err = btf_ext_visit_type_ids(btf_ext, btf_permute_remap_type_id, &p);
6069 if (err)
6070 goto done;
6071 }
6072
6073 for (nt = new_types, i = 0; i < id_map_cnt - start_offs; i++) {
6074 btf->type_offs[i] = nt - new_types;
6075 nt += btf_type_size(nt);
6076 }
6077
6078 free(order_map);
6079 free(btf->types_data);
6080 btf->types_data = new_types;
6081 return 0;
6082
6083 done:
6084 free(order_map);
6085 free(new_types);
6086 return libbpf_err(err);
6087 }