| 1 |
// Copyright (c) Meta Platforms, Inc. and affiliates. |
| 2 |
// SPDX-License-Identifier: LGPL-2.1-or-later |
| 3 |
|
| 4 |
#include <byteswap.h> |
| 5 |
#include <elf.h> |
| 6 |
#include <gelf.h> |
| 7 |
#include <limits.h> |
| 8 |
#include <stdlib.h> |
| 9 |
#include <string.h> |
| 10 |
|
| 11 |
#include "binary_search.h" |
| 12 |
#include "cleanup.h" |
| 13 |
#include "debug_info.h" // IWYU pragma: associated |
| 14 |
#include "elf_file.h" |
| 15 |
#include "error.h" |
| 16 |
#include "log.h" |
| 17 |
#include "orc.h" |
| 18 |
#include "platform.h" |
| 19 |
#include "program.h" |
| 20 |
#include "util.h" |
| 21 |
|
| 22 |
DEFINE_VECTOR(uint64_range_vector, struct uint64_range); |
| 23 |
|
| 24 |
#define ORC_HEADER_SIZE 20 |
| 25 |
|
| 26 |
void drgn_module_orc_info_deinit(struct drgn_module *module) |
| 27 |
{ |
| 28 |
free(module->orc.entries); |
| 29 |
free(module->orc.pc_offsets); |
| 30 |
free(module->orc.preferred); |
| 31 |
} |
| 32 |
|
| 33 |
// Getters for "raw" ORC information, i.e., before it is aligned, byte swapped, |
| 34 |
// and normalized to the latest version. |
| 35 |
static inline uint64_t drgn_raw_orc_pc(struct drgn_module *module, |
| 36 |
unsigned int i) |
| 37 |
{ |
| 38 |
int32_t offset; |
| 39 |
memcpy(&offset, &module->orc.pc_offsets[i], sizeof(offset)); |
| 40 |
if (module->orc.bswap) |
| 41 |
offset = bswap_32(offset); |
| 42 |
return module->orc.pc_base + UINT64_C(4) * i + offset; |
| 43 |
} |
| 44 |
|
| 45 |
static bool |
| 46 |
drgn_raw_orc_entry_is_terminator(struct drgn_module *module, unsigned int i) |
| 47 |
{ |
| 48 |
uint16_t flags; |
| 49 |
memcpy(&flags, &module->orc.entries[i].flags, sizeof(flags)); |
| 50 |
if (module->orc.bswap) |
| 51 |
flags = bswap_16(flags); |
| 52 |
if (module->orc.version >= 3) { |
| 53 |
// orc->type == ORC_TYPE_UNDEFINED |
| 54 |
return (flags & 0x700) == 0; |
| 55 |
} else if (module->orc.version == 2) { |
| 56 |
// orc->sp_reg == ORC_REG_UNDEFINED && !orc->end |
| 57 |
return (flags & 0x80f) == 0; |
| 58 |
} else { |
| 59 |
// orc->sp_reg == ORC_REG_UNDEFINED && !orc->end |
| 60 |
return (flags & 0x40f) == 0; |
| 61 |
} |
| 62 |
} |
| 63 |
|
| 64 |
static bool |
| 65 |
drgn_raw_orc_entry_is_preferred(struct drgn_module *module, unsigned int i) |
| 66 |
{ |
| 67 |
uint16_t flags; |
| 68 |
memcpy(&flags, &module->orc.entries[i].flags, sizeof(flags)); |
| 69 |
if (module->orc.bswap) |
| 70 |
flags = bswap_16(flags); |
| 71 |
// ORC_REG_SP_INDIRECT is used for the stack switching pattern used in |
| 72 |
// the Linux kernel's call_on_stack()/call_on_irqstack() macros. See |
| 73 |
// Linux kernel commits 87ccc826bf1c ("x86/unwind/orc: Change |
| 74 |
// REG_SP_INDIRECT"), aafeb14e9da2 ("objtool: Support stack-swizzle"), |
| 75 |
// and a0cfc74d0b00 ("x86/irq: Provide macro for inlining irq stack |
| 76 |
// switching") (in v5.12). These macros switch the stack pointer in |
| 77 |
// inline assembly, resulting in inaccurate DWARF CFI. So, we should use |
| 78 |
// ORC to unwind these instead. |
| 79 |
return (flags & 0xf) == DRGN_ORC_REG_SP_INDIRECT; |
| 80 |
} |
| 81 |
|
| 82 |
static int compare_orc_entries(const void *a, const void *b, void *arg) |
| 83 |
{ |
| 84 |
struct drgn_module *module = arg; |
| 85 |
unsigned int index_a = *(unsigned int *)a; |
| 86 |
unsigned int index_b = *(unsigned int *)b; |
| 87 |
|
| 88 |
uint64_t pc_a = drgn_raw_orc_pc(module, index_a); |
| 89 |
uint64_t pc_b = drgn_raw_orc_pc(module, index_b); |
| 90 |
if (pc_a < pc_b) |
| 91 |
return -1; |
| 92 |
else if (pc_a > pc_b) |
| 93 |
return 1; |
| 94 |
|
| 95 |
/* |
| 96 |
* If two entries have the same PC, then one is probably a "terminator" |
| 97 |
* at the end of a compilation unit. Prefer the real entry. |
| 98 |
*/ |
| 99 |
return (drgn_raw_orc_entry_is_terminator(module, index_b) |
| 100 |
- drgn_raw_orc_entry_is_terminator(module, index_a)); |
| 101 |
} |
| 102 |
|
| 103 |
static unsigned int keep_orc_entry(struct drgn_module *module, |
| 104 |
unsigned int *indices, |
| 105 |
unsigned int num_entries, unsigned int i) |
| 106 |
{ |
| 107 |
|
| 108 |
const struct drgn_orc_entry *entries = module->orc.entries; |
| 109 |
if (num_entries > 0 && |
| 110 |
memcmp(&entries[indices[num_entries - 1]], &entries[indices[i]], |
| 111 |
sizeof(entries[0])) == 0) { |
| 112 |
/* |
| 113 |
* The previous entry is identical to this one, so we can skip |
| 114 |
* this entry (which effectively merges it into the previous |
| 115 |
* one). This usually happens for "terminator" entries. |
| 116 |
*/ |
| 117 |
return num_entries; |
| 118 |
} |
| 119 |
indices[num_entries] = indices[i]; |
| 120 |
return num_entries + 1; |
| 121 |
} |
| 122 |
|
| 123 |
/* |
| 124 |
* The vast majority of ORC entries are redundant with DWARF CFI, and it's a |
| 125 |
* waste to store and binary search those entries. This removes ORC entries that |
| 126 |
* are entirely shadowed by DWARF FDEs. |
| 127 |
* |
| 128 |
* Note that we don't bother checking EH CFI because currently ORC is only used |
| 129 |
* for the Linux kernel on x86-64, which explicitly disables EH data. |
| 130 |
*/ |
| 131 |
static struct drgn_error * |
| 132 |
remove_fdes_from_orc(struct drgn_module *module, unsigned int *indices, |
| 133 |
struct uint64_range_vector *preferred, |
| 134 |
unsigned int *num_entriesp) |
| 135 |
{ |
| 136 |
char *env = getenv("DRGN_PREFER_ORC_UNWINDER"); |
| 137 |
if (env && atoi(env)) { |
| 138 |
struct uint64_range *range = |
| 139 |
uint64_range_vector_append_entry(preferred); |
| 140 |
if (!range) |
| 141 |
return &drgn_enomem; |
| 142 |
range->start = 0; |
| 143 |
range->end = UINT64_MAX; |
| 144 |
return NULL; |
| 145 |
} |
| 146 |
|
| 147 |
if (module->dwarf.debug_frame.num_fdes == 0) |
| 148 |
return NULL; |
| 149 |
|
| 150 |
struct drgn_dwarf_fde *fde = module->dwarf.debug_frame.fdes; |
| 151 |
struct drgn_dwarf_fde *last_fde = |
| 152 |
fde + module->dwarf.debug_frame.num_fdes - 1; |
| 153 |
|
| 154 |
unsigned int num_entries = *num_entriesp; |
| 155 |
unsigned int new_num_entries = 0; |
| 156 |
|
| 157 |
// ORC can be built-in or from the debug file. Because of that, we |
| 158 |
// always store the biased/actual address at orc.pc_base. Since we are |
| 159 |
// comparing to the unbiased addresses in the debug_frame FDEs, we need |
| 160 |
// to subtract the bias from the ORC PC. |
| 161 |
uint64_t start_pc = drgn_raw_orc_pc(module, 0) - module->debug_file_bias; |
| 162 |
uint64_t end_pc; |
| 163 |
for (unsigned int i = 0; i < num_entries; i++, start_pc = end_pc) { |
| 164 |
if (i < num_entries - 1) |
| 165 |
end_pc = drgn_raw_orc_pc(module, i + 1) - module->debug_file_bias; |
| 166 |
else |
| 167 |
end_pc = UINT64_MAX; |
| 168 |
|
| 169 |
if (drgn_raw_orc_entry_is_preferred(module, i)) { |
| 170 |
struct uint64_range *range = |
| 171 |
uint64_range_vector_append_entry(preferred); |
| 172 |
if (!range) |
| 173 |
return &drgn_enomem; |
| 174 |
range->start = start_pc; |
| 175 |
range->end = end_pc; |
| 176 |
new_num_entries = keep_orc_entry(module, indices, |
| 177 |
new_num_entries, i); |
| 178 |
continue; |
| 179 |
} |
| 180 |
|
| 181 |
if (start_pc < fde->initial_location) { |
| 182 |
// The current ORC entry starts before the current FDE |
| 183 |
// (which can only happen if it is the first FDE). Keep |
| 184 |
// it. |
| 185 |
new_num_entries = keep_orc_entry(module, indices, |
| 186 |
new_num_entries, i); |
| 187 |
continue; |
| 188 |
} |
| 189 |
|
| 190 |
/* |
| 191 |
* Find the last FDE that starts at or before the current ORC |
| 192 |
* entry. |
| 193 |
*/ |
| 194 |
while (fde != last_fde && fde[1].initial_location <= start_pc) |
| 195 |
fde++; |
| 196 |
|
| 197 |
/* |
| 198 |
* Check whether the current ORC entry is completely covered by |
| 199 |
* one or more FDEs. |
| 200 |
*/ |
| 201 |
while (end_pc - fde->initial_location > fde->address_range) { |
| 202 |
/* |
| 203 |
* The current FDE doesn't cover the current ORC entry. |
| 204 |
*/ |
| 205 |
if (fde == last_fde |
| 206 |
|| fde[1].initial_location - fde->initial_location |
| 207 |
> fde->address_range) { |
| 208 |
// Either there are no more FDEs or there is a |
| 209 |
// gap between the current FDE and the next FDE |
| 210 |
// that exposes the current ORC entry. Keep it. |
| 211 |
new_num_entries = keep_orc_entry(module, |
| 212 |
indices, |
| 213 |
new_num_entries, |
| 214 |
i); |
| 215 |
break; |
| 216 |
} |
| 217 |
fde++; |
| 218 |
} |
| 219 |
} |
| 220 |
*num_entriesp = new_num_entries; |
| 221 |
return NULL; |
| 222 |
} |
| 223 |
|
| 224 |
static int orc_version_from_header(const void *buffer) |
| 225 |
{ |
| 226 |
// Known version identifiers in .orc_header. These can be generated in |
| 227 |
// the kernel source tree with: |
| 228 |
// sh ./scripts/orc_hash.sh < arch/x86/include/asm/orc_types.h | sed -e 's/^#define ORC_HASH //' -e 's/,/, /g' |
| 229 |
|
| 230 |
// Linux kernel commit 1735858caa4b ("objtool/x86: Reorder ORC register |
| 231 |
// numbering") (in v7.1) |
| 232 |
static const uint8_t orc_hash_7_1[ORC_HEADER_SIZE] = { |
| 233 |
0x13, 0x7c, 0x36, 0xad, 0x76, 0x27, 0xc4, 0xdd, 0x55, 0x93, |
| 234 |
0x21, 0x76, 0xbb, 0x0e, 0xcd, 0xbd, 0x50, 0x41, 0xfc, 0x82, |
| 235 |
}; |
| 236 |
// Linux kernel commit fb799447ae29 ("x86,objtool: Split |
| 237 |
// UNWIND_HINT_EMPTY in two") (in v6.4) |
| 238 |
static const uint8_t orc_hash_6_4[ORC_HEADER_SIZE] = { |
| 239 |
0xfe, 0x5d, 0x32, 0xbf, 0x58, 0x1b, 0xd6, 0x3b, 0x2c, 0xa9, |
| 240 |
0xa5, 0xc6, 0x5b, 0xa5, 0xa6, 0x25, 0xea, 0xb3, 0xfe, 0x24, |
| 241 |
}; |
| 242 |
// Linux kernel commit ffb1b4a41016 ("x86/unwind/orc: Add 'signal' field |
| 243 |
// to ORC metadata") (in v6.3) |
| 244 |
static const uint8_t orc_hash_6_3[ORC_HEADER_SIZE] = { |
| 245 |
0xdb, 0x84, 0xae, 0xd4, 0x10, 0x3b, 0x31, 0xdd, 0x51, 0xe0, |
| 246 |
0x17, 0xf8, 0xf7, 0x97, 0x83, 0xca, 0x98, 0x5c, 0x2c, 0x51, |
| 247 |
}; |
| 248 |
|
| 249 |
if (memcmp(buffer, orc_hash_7_1, ORC_HEADER_SIZE) == 0) |
| 250 |
return 4; |
| 251 |
else if (memcmp(buffer, orc_hash_6_4, ORC_HEADER_SIZE) == 0) |
| 252 |
return 3; |
| 253 |
else if (memcmp(buffer, orc_hash_6_3, ORC_HEADER_SIZE) == 0) |
| 254 |
return 2; |
| 255 |
return -1; |
| 256 |
} |
| 257 |
|
| 258 |
static int orc_version_from_osrelease(struct drgn_program *prog) |
| 259 |
{ |
| 260 |
char *p = (char *)prog->vmcoreinfo.osrelease; |
| 261 |
long major = strtol(p, &p, 10); |
| 262 |
long minor = 0; |
| 263 |
if (*p == '.') |
| 264 |
minor = strtol(p + 1, NULL, 10); |
| 265 |
|
| 266 |
// RHEL & derivatives started shipping backported ORC patches without |
| 267 |
// the ".orc_header" section in 9.6. According to git history this was |
| 268 |
// introduced in kernel version 5.14.0-517.el9. With those changes, the |
| 269 |
// kernel's ORC "format" is indeed v3, but we cannot detect that without |
| 270 |
// the ORC header hash. Add special-case detection for this situation. |
| 271 |
// If it is fixed by including the ".orc_header" section, this fix will |
| 272 |
// not be active, but will still be necessary for the already released |
| 273 |
// kernels. |
| 274 |
char *rhrelease = NULL; |
| 275 |
if (major == 5 && minor == 14 && strstr(p, ".el9") |
| 276 |
&& (rhrelease = strchr(p, '-')) |
| 277 |
&& strtol(rhrelease + 1, NULL, 10) >= 517) |
| 278 |
return 3; |
| 279 |
|
| 280 |
if (major > 6 || (major == 6 && minor >= 4)) |
| 281 |
return 3; |
| 282 |
else if (major == 6 && minor == 3) |
| 283 |
return 2; |
| 284 |
else |
| 285 |
return 1; |
| 286 |
} |
| 287 |
|
| 288 |
static struct drgn_error *drgn_read_orc_sections(struct drgn_module *module) |
| 289 |
{ |
| 290 |
struct drgn_error *err; |
| 291 |
Elf *elf = module->debug_file->elf; |
| 292 |
|
| 293 |
size_t shstrndx; |
| 294 |
if (elf_getshdrstrndx(elf, &shstrndx)) |
| 295 |
return drgn_error_libelf(); |
| 296 |
|
| 297 |
Elf_Scn *orc_unwind_ip_scn = NULL; |
| 298 |
Elf_Scn *orc_unwind_scn = NULL; |
| 299 |
Elf_Scn *orc_header_scn = NULL; |
| 300 |
|
| 301 |
Elf_Scn *scn = NULL; |
| 302 |
while ((scn = elf_nextscn(elf, scn))) { |
| 303 |
GElf_Shdr shdr_mem, *shdr = gelf_getshdr(scn, &shdr_mem); |
| 304 |
if (!shdr) |
| 305 |
return drgn_error_libelf(); |
| 306 |
|
| 307 |
if (shdr->sh_type != SHT_PROGBITS) |
| 308 |
continue; |
| 309 |
|
| 310 |
const char *scnname = elf_strptr(elf, shstrndx, shdr->sh_name); |
| 311 |
if (!scnname) |
| 312 |
return drgn_error_libelf(); |
| 313 |
|
| 314 |
if (!orc_unwind_ip_scn |
| 315 |
&& strcmp(scnname, ".orc_unwind_ip") == 0) { |
| 316 |
orc_unwind_ip_scn = scn; |
| 317 |
module->orc.pc_base = shdr->sh_addr; |
| 318 |
} else if (!orc_unwind_scn |
| 319 |
&& strcmp(scnname, ".orc_unwind") == 0) { |
| 320 |
orc_unwind_scn = scn; |
| 321 |
} else if (!orc_header_scn |
| 322 |
&& strcmp(scnname, ".orc_header") == 0) { |
| 323 |
orc_header_scn = scn; |
| 324 |
} |
| 325 |
} |
| 326 |
|
| 327 |
if (!orc_unwind_ip_scn || !orc_unwind_scn) { |
| 328 |
module->orc.num_entries = 0; |
| 329 |
return NULL; |
| 330 |
} |
| 331 |
|
| 332 |
err = drgn_elf_file_apply_relocations(module->debug_file); |
| 333 |
if (err) |
| 334 |
return err; |
| 335 |
|
| 336 |
// Since Linux kernel b9f174c811e3 ("x86/unwind/orc: Add ELF section |
| 337 |
// with ORC version identifier") (in v6.4), which was also backported to |
| 338 |
// Linux 6.3.10, vmlinux and kernel modules have a .orc_header ELF |
| 339 |
// section containing a 20-byte hash identifying the ORC version. |
| 340 |
// |
| 341 |
// Because there are 6.3 and 6.4 kernels without .orc_header, we have to |
| 342 |
// fall back to checking the kernel version. |
| 343 |
if (orc_header_scn) { |
| 344 |
Elf_Data *orc_header; |
| 345 |
err = read_elf_section(orc_header_scn, &orc_header); |
| 346 |
if (err) |
| 347 |
return err; |
| 348 |
module->orc.version = -1; |
| 349 |
if (orc_header->d_size == ORC_HEADER_SIZE) |
| 350 |
module->orc.version = orc_version_from_header(orc_header->d_buf); |
| 351 |
if (module->orc.version < 0) { |
| 352 |
return drgn_error_create(DRGN_ERROR_OTHER, |
| 353 |
"unrecognized .orc_header"); |
| 354 |
} |
| 355 |
} else { |
| 356 |
module->orc.version = orc_version_from_osrelease(module->prog); |
| 357 |
} |
| 358 |
|
| 359 |
Elf_Data *orc_unwind_ip, *orc_unwind; |
| 360 |
err = read_elf_section(orc_unwind_ip_scn, &orc_unwind_ip); |
| 361 |
if (err) |
| 362 |
return err; |
| 363 |
err = read_elf_section(orc_unwind_scn, &orc_unwind); |
| 364 |
if (err) |
| 365 |
return err; |
| 366 |
|
| 367 |
size_t num_entries = orc_unwind_ip->d_size / sizeof(int32_t); |
| 368 |
if (num_entries > UINT_MAX) { |
| 369 |
return drgn_error_create(DRGN_ERROR_OTHER, |
| 370 |
".orc_unwind_ip is too large"); |
| 371 |
} |
| 372 |
module->orc.num_entries = num_entries; |
| 373 |
|
| 374 |
if (orc_unwind_ip->d_size % sizeof(int32_t) != 0 || |
| 375 |
orc_unwind->d_size % sizeof(struct drgn_orc_entry) != 0 || |
| 376 |
orc_unwind->d_size / sizeof(struct drgn_orc_entry) |
| 377 |
!= module->orc.num_entries) { |
| 378 |
return drgn_error_create(DRGN_ERROR_OTHER, |
| 379 |
".orc_unwind_ip and/or .orc_unwind has invalid size"); |
| 380 |
} |
| 381 |
|
| 382 |
module->orc.pc_offsets = orc_unwind_ip->d_buf; |
| 383 |
module->orc.entries = orc_unwind->d_buf; |
| 384 |
|
| 385 |
return NULL; |
| 386 |
} |
| 387 |
|
| 388 |
static struct drgn_error * |
| 389 |
copy_builtin_orc_buffers(struct drgn_module *module, uint64_t num_entries, |
| 390 |
uint64_t unwind, uint64_t unwind_ip, uint64_t header) |
| 391 |
{ |
| 392 |
uint8_t header_data[ORC_HEADER_SIZE]; |
| 393 |
|
| 394 |
struct drgn_error *err; |
| 395 |
|
| 396 |
if (header) { |
| 397 |
err = drgn_program_read_memory(module->prog, header_data, |
| 398 |
header, sizeof(header_data), |
| 399 |
false); |
| 400 |
|
| 401 |
if (err) |
| 402 |
return err; |
| 403 |
|
| 404 |
module->orc.version = orc_version_from_header(header_data); |
| 405 |
if (module->orc.version < 0) |
| 406 |
return drgn_error_create(DRGN_ERROR_OTHER, |
| 407 |
"unrecognized .orc_header"); |
| 408 |
} else { |
| 409 |
module->orc.version = orc_version_from_osrelease(module->prog); |
| 410 |
} |
| 411 |
|
| 412 |
_cleanup_free_ int32_t *pc_offsets = malloc_array(num_entries, |
| 413 |
sizeof(pc_offsets[0])); |
| 414 |
if (!pc_offsets) |
| 415 |
return &drgn_enomem; |
| 416 |
err = drgn_program_read_memory(module->prog, pc_offsets, unwind_ip, |
| 417 |
num_entries * sizeof(pc_offsets[0]), false); |
| 418 |
if (err) |
| 419 |
return err; |
| 420 |
|
| 421 |
_cleanup_free_ struct drgn_orc_entry *entries = |
| 422 |
malloc_array(num_entries, sizeof(entries[0])); |
| 423 |
if (!entries) |
| 424 |
return &drgn_enomem; |
| 425 |
err = drgn_program_read_memory(module->prog, entries, unwind, |
| 426 |
num_entries * sizeof(entries[0]), false); |
| 427 |
if (err) |
| 428 |
return err; |
| 429 |
|
| 430 |
module->orc.entries = no_cleanup_ptr(entries); |
| 431 |
module->orc.pc_offsets = no_cleanup_ptr(pc_offsets); |
| 432 |
module->orc.num_entries = num_entries; |
| 433 |
module->orc.pc_base = unwind_ip; |
| 434 |
drgn_log_debug(module->prog, "Loaded built-in ORC (v%d) for module %s", |
| 435 |
module->orc.version, module->name); |
| 436 |
return NULL; |
| 437 |
} |
| 438 |
|
| 439 |
static struct drgn_error *drgn_read_vmlinux_orc(struct drgn_module *module) |
| 440 |
{ |
| 441 |
struct drgn_error *err; |
| 442 |
struct drgn_symbol *sym; |
| 443 |
|
| 444 |
uint64_t unwind_ip_start, unwind_ip_end; |
| 445 |
uint64_t unwind_start, unwind_end; |
| 446 |
uint64_t header_start = 0, header_end = 0; |
| 447 |
|
| 448 |
#define get_symbol(name, var, optional) \ |
| 449 |
err = drgn_program_find_symbol_by_name(module->prog, name, &sym); \ |
| 450 |
if (!err) { \ |
| 451 |
var = sym->address; \ |
| 452 |
drgn_symbol_destroy(sym); \ |
| 453 |
sym = NULL; \ |
| 454 |
} else if (optional && drgn_error_catch(&err, DRGN_ERROR_LOOKUP)) { \ |
| 455 |
sym = NULL; \ |
| 456 |
} else { \ |
| 457 |
drgn_error_catch(&err, DRGN_ERROR_LOOKUP); \ |
| 458 |
return err; \ |
| 459 |
} |
| 460 |
|
| 461 |
get_symbol("__start_orc_unwind_ip", unwind_ip_start, false); |
| 462 |
get_symbol("__stop_orc_unwind_ip", unwind_ip_end, false); |
| 463 |
get_symbol("__start_orc_unwind", unwind_start, false); |
| 464 |
get_symbol("__stop_orc_unwind", unwind_end, false); |
| 465 |
get_symbol("__start_orc_header", header_start, true); |
| 466 |
get_symbol("__stop_orc_header", header_end, true); |
| 467 |
#undef get_symbol |
| 468 |
|
| 469 |
if ((unwind_ip_end - unwind_ip_start) % sizeof(int32_t)) |
| 470 |
return drgn_error_create(DRGN_ERROR_OTHER, "invalid built-in orc_unwind_ip range"); |
| 471 |
uint64_t num_entries = (unwind_ip_end - unwind_ip_start) / sizeof(int32_t); |
| 472 |
if (num_entries > UINT_MAX) |
| 473 |
return drgn_error_create(DRGN_ERROR_OTHER, |
| 474 |
"built-in orc_unwind_ip range is too large"); |
| 475 |
|
| 476 |
if ((unwind_end - unwind_start) % sizeof(struct drgn_orc_entry) |
| 477 |
|| (unwind_end - unwind_start) / sizeof(struct drgn_orc_entry) != num_entries) |
| 478 |
return drgn_error_create(DRGN_ERROR_OTHER, "invalid built-in orc_unwind range"); |
| 479 |
|
| 480 |
if (header_start && header_end && header_end - header_start != ORC_HEADER_SIZE) |
| 481 |
return drgn_error_create(DRGN_ERROR_OTHER, "invalid built-in orc_header size"); |
| 482 |
|
| 483 |
return copy_builtin_orc_buffers(module, num_entries, unwind_start, |
| 484 |
unwind_ip_start, header_start); |
| 485 |
} |
| 486 |
|
| 487 |
static struct drgn_error *drgn_read_builtin_orc(struct drgn_module *module) |
| 488 |
{ |
| 489 |
if (!(module->prog->flags & DRGN_PROGRAM_IS_LINUX_KERNEL)) |
| 490 |
return NULL; |
| 491 |
if (module->kind == DRGN_MODULE_MAIN) |
| 492 |
return drgn_read_vmlinux_orc(module); |
| 493 |
else if (module->kind != DRGN_MODULE_RELOCATABLE) |
| 494 |
return NULL; |
| 495 |
else if (module->object.kind == DRGN_OBJECT_ABSENT) |
| 496 |
return NULL; |
| 497 |
|
| 498 |
// num_entries is implied by the size of the arrays. We can get the |
| 499 |
// array addresses from the section address info, but not their size. So |
| 500 |
// we need to find num_orcs by reading it out of the arch-specific |
| 501 |
// module info. |
| 502 |
DRGN_OBJECT(tmp, module->prog); |
| 503 |
struct drgn_error *err; |
| 504 |
|
| 505 |
err = drgn_object_dereference(&tmp, &module->object); |
| 506 |
if (err) |
| 507 |
return err; |
| 508 |
|
| 509 |
err = drgn_object_member(&tmp, &tmp, "arch"); |
| 510 |
if (err) |
| 511 |
return err; |
| 512 |
|
| 513 |
err = drgn_object_member(&tmp, &tmp, "num_orcs"); |
| 514 |
// If the kernel does not support ORC (e.g. it is too old), this will be |
| 515 |
// the first lookup error we encounter. Catch it and don't return any |
| 516 |
// error. |
| 517 |
if (drgn_error_catch(&err, DRGN_ERROR_LOOKUP) || err) |
| 518 |
return err; |
| 519 |
|
| 520 |
uint64_t num_entries; |
| 521 |
err = drgn_object_read_unsigned(&tmp, &num_entries); |
| 522 |
if (err) |
| 523 |
return err; |
| 524 |
|
| 525 |
// We'll still use the section addresses for everything else, because |
| 526 |
// the orc_header is only present there, and it should be a bit faster |
| 527 |
// to read data which we already parsed, rather than going back to read |
| 528 |
// it from program memory. |
| 529 |
uint64_t orc_unwind; |
| 530 |
uint64_t orc_unwind_ip; |
| 531 |
uint64_t orc_header = 0; |
| 532 |
err = drgn_module_get_section_address(module, ".orc_unwind", &orc_unwind); |
| 533 |
if (err) |
| 534 |
return err; |
| 535 |
err = drgn_module_get_section_address(module, ".orc_unwind_ip", &orc_unwind_ip); |
| 536 |
if (err) |
| 537 |
return err; |
| 538 |
err = drgn_module_get_section_address(module, ".orc_header", &orc_header); |
| 539 |
drgn_error_catch(&err, DRGN_ERROR_LOOKUP); |
| 540 |
if (err) |
| 541 |
return err; |
| 542 |
|
| 543 |
return copy_builtin_orc_buffers(module, num_entries, orc_unwind, |
| 544 |
orc_unwind_ip, orc_header); |
| 545 |
} |
| 546 |
|
| 547 |
static inline void drgn_module_clear_orc(struct drgn_module **modulep) |
| 548 |
{ |
| 549 |
if (*modulep) { |
| 550 |
(*modulep)->orc.pc_offsets = NULL; |
| 551 |
(*modulep)->orc.entries = NULL; |
| 552 |
} |
| 553 |
} |
| 554 |
|
| 555 |
struct drgn_error *drgn_module_parse_orc(struct drgn_module *module, |
| 556 |
bool use_builtin) |
| 557 |
{ |
| 558 |
struct drgn_error *err; |
| 559 |
|
| 560 |
if (module->prog->platform.arch->arch != DRGN_ARCH_X86_64) |
| 561 |
return NULL; |
| 562 |
|
| 563 |
// When loading from the debug_file, pc_offsets and entries point to the |
| 564 |
// Elf_Data buffers until the end of this function, when copies are made. |
| 565 |
// When loading built-in ORC, we own both buffers. In either case, we |
| 566 |
// don't want these pointers freed by drgn_module_orc_info_deinit() if |
| 567 |
// we fail early. So we use this to reset the pointers to NULL on |
| 568 |
// failure. |
| 569 |
_cleanup_(drgn_module_clear_orc) struct drgn_module *clear = module; |
| 570 |
|
| 571 |
// For the built-in ORC, we use these to clean up the memory allocated |
| 572 |
// in drgn_read_builtin_orc(). This happens even on success, because the |
| 573 |
// buffers are copied at the end of this function. |
| 574 |
_cleanup_free_ void *cleanup_pc_offsets = NULL; |
| 575 |
_cleanup_free_ void *cleanup_entries = NULL; |
| 576 |
|
| 577 |
if (use_builtin) { |
| 578 |
err = drgn_read_builtin_orc(module); |
| 579 |
cleanup_pc_offsets = module->orc.pc_offsets; |
| 580 |
cleanup_entries = module->orc.entries; |
| 581 |
} else { |
| 582 |
err = drgn_read_orc_sections(module); |
| 583 |
module->orc.pc_base += module->debug_file_bias; |
| 584 |
} |
| 585 |
if (err || !module->orc.num_entries) |
| 586 |
return err; |
| 587 |
|
| 588 |
// We may need to byte swap ORC entries. Rather than checking the |
| 589 |
// debug_file's platform, use the program's platform (since they are the |
| 590 |
// same) because it's possible there is no debug_file (e.g. for builtin |
| 591 |
// ORC). |
| 592 |
bool bswap; |
| 593 |
err = drgn_program_bswap(module->prog, &bswap); |
| 594 |
if (err) |
| 595 |
return err; |
| 596 |
|
| 597 |
unsigned int num_entries = module->orc.num_entries; |
| 598 |
_cleanup_free_ unsigned int *indices = |
| 599 |
malloc_array(num_entries, sizeof(indices[0])); |
| 600 |
if (!indices) |
| 601 |
return &drgn_enomem; |
| 602 |
for (unsigned int i = 0; i < num_entries; i++) |
| 603 |
indices[i] = i; |
| 604 |
|
| 605 |
/* |
| 606 |
* Sort the ORC entries for binary search. Since Linux kernel commit |
| 607 |
* f14bf6a350df ("x86/unwind/orc: Remove boot-time ORC unwind tables |
| 608 |
* sorting") (in v5.6), this is already sorted for vmlinux, so only sort |
| 609 |
* it if necessary. |
| 610 |
*/ |
| 611 |
for (unsigned int i = 1; i < num_entries; i++) { |
| 612 |
if (compare_orc_entries(&indices[i - 1], &indices[i], module) > 0) { |
| 613 |
qsort_arg(indices, num_entries, sizeof(indices[0]), |
| 614 |
compare_orc_entries, module); |
| 615 |
break; |
| 616 |
} |
| 617 |
} |
| 618 |
|
| 619 |
VECTOR(uint64_range_vector, preferred); |
| 620 |
|
| 621 |
err = remove_fdes_from_orc(module, indices, &preferred, &num_entries); |
| 622 |
if (err) |
| 623 |
return err; |
| 624 |
|
| 625 |
_cleanup_free_ int32_t *pc_offsets = |
| 626 |
malloc_array(num_entries, sizeof(pc_offsets[0])); |
| 627 |
if (!pc_offsets) |
| 628 |
return &drgn_enomem; |
| 629 |
_cleanup_free_ struct drgn_orc_entry *entries = |
| 630 |
malloc_array(num_entries, sizeof(entries[0])); |
| 631 |
if (!entries) |
| 632 |
return &drgn_enomem; |
| 633 |
const int32_t *orig_offsets = module->orc.pc_offsets; |
| 634 |
const struct drgn_orc_entry *orig_entries = module->orc.entries; |
| 635 |
const int version = module->orc.version; |
| 636 |
for (unsigned int i = 0; i < num_entries; i++) { |
| 637 |
unsigned int index = indices[i]; |
| 638 |
int32_t offset; |
| 639 |
memcpy(&offset, &orig_offsets[index], sizeof(offset)); |
| 640 |
memcpy(&entries[i], &orig_entries[index], sizeof(entries[i])); |
| 641 |
if (bswap) { |
| 642 |
offset = bswap_32(offset); |
| 643 |
entries[i].sp_offset = bswap_16(entries[i].sp_offset); |
| 644 |
entries[i].bp_offset = bswap_16(entries[i].bp_offset); |
| 645 |
entries[i].flags = bswap_16(entries[i].flags); |
| 646 |
} |
| 647 |
// "Upgrade" the format to version 4. See struct |
| 648 |
// drgn_orc_entry::flags and the DRGN_ORC_REG_* enum. |
| 649 |
if (version <= 3) { |
| 650 |
// Versions 1 and 2 need type/signal flag upgrades to |
| 651 |
// the version 3 layout first. |
| 652 |
if (version == 2) { |
| 653 |
// There are no UNDEFINED or END_OF_STACK types |
| 654 |
// in versions 1 and 2. Instead, sp_reg == |
| 655 |
// ORC_REG_UNDEFINED && !end is equivalent to |
| 656 |
// UNDEFINED, and sp_reg == ORC_REG_UNDEFINED && |
| 657 |
// end is equivalent to END_OF_STACK. |
| 658 |
int type; |
| 659 |
if ((entries[i].flags & 0x80f) == 0) |
| 660 |
type = DRGN_ORC_TYPE_UNDEFINED << 8; |
| 661 |
else if ((entries[i].flags & 0x80f) == 0x800) |
| 662 |
type = DRGN_ORC_TYPE_END_OF_STACK << 8; |
| 663 |
else |
| 664 |
type = (entries[i].flags & 0x300) + 0x200; |
| 665 |
int signal = (entries[i].flags & 0x400) << 1; |
| 666 |
entries[i].flags = ((entries[i].flags & 0xff) |
| 667 |
| type |
| 668 |
| signal); |
| 669 |
} else if (version == 1) { |
| 670 |
int type; |
| 671 |
if ((entries[i].flags & 0x40f) == 0) |
| 672 |
type = DRGN_ORC_TYPE_UNDEFINED << 8; |
| 673 |
else if ((entries[i].flags & 0x40f) == 0x400) |
| 674 |
type = DRGN_ORC_TYPE_END_OF_STACK << 8; |
| 675 |
else |
| 676 |
type = (entries[i].flags & 0x300) + 0x200; |
| 677 |
// There is no signal flag in version 1. |
| 678 |
// Instead, ORC_TYPE_REGS and |
| 679 |
// ORC_TYPE_REGS_PARTIAL imply the signal flag, |
| 680 |
// and ORC_TYPE_CALL does not. |
| 681 |
int signal = (entries[i].flags & 0x300) > 0 ? 0x800 : 0; |
| 682 |
entries[i].flags = ((entries[i].flags & 0xff) |
| 683 |
| type |
| 684 |
| signal); |
| 685 |
} |
| 686 |
// Remap register numbers from the pre-v7.1 numbering to |
| 687 |
// the current numbering. |
| 688 |
static const uint8_t reg_map[] = { |
| 689 |
[0] = DRGN_ORC_REG_UNDEFINED, |
| 690 |
[1] = DRGN_ORC_REG_PREV_SP, |
| 691 |
[2] = DRGN_ORC_REG_DX, |
| 692 |
[3] = DRGN_ORC_REG_DI, |
| 693 |
[4] = DRGN_ORC_REG_BP, |
| 694 |
[5] = DRGN_ORC_REG_SP, |
| 695 |
[6] = DRGN_ORC_REG_R10, |
| 696 |
[7] = DRGN_ORC_REG_R13, |
| 697 |
[8] = DRGN_ORC_REG_BP_INDIRECT, |
| 698 |
[9] = DRGN_ORC_REG_SP_INDIRECT, |
| 699 |
[10] = DRGN_ORC_REG_AX, |
| 700 |
}; |
| 701 |
int sp_reg = entries[i].flags & 0xf; |
| 702 |
int bp_reg = (entries[i].flags >> 4) & 0xf; |
| 703 |
if (sp_reg < sizeof(reg_map)) |
| 704 |
sp_reg = reg_map[sp_reg]; |
| 705 |
if (bp_reg < sizeof(reg_map)) |
| 706 |
bp_reg = reg_map[bp_reg]; |
| 707 |
entries[i].flags = ((entries[i].flags & 0xff00) |
| 708 |
| (bp_reg << 4) |
| 709 |
| sp_reg); |
| 710 |
} |
| 711 |
pc_offsets[i] = UINT64_C(4) * index + offset - UINT64_C(4) * i; |
| 712 |
} |
| 713 |
|
| 714 |
uint64_range_vector_shrink_to_fit(&preferred); |
| 715 |
uint64_range_vector_steal(&preferred, &module->orc.preferred, |
| 716 |
&module->orc.num_preferred); |
| 717 |
module->orc.pc_offsets = no_cleanup_ptr(pc_offsets); |
| 718 |
module->orc.entries = no_cleanup_ptr(entries); |
| 719 |
module->orc.num_entries = num_entries; |
| 720 |
module->orc.bswap = bswap; |
| 721 |
clear = NULL; |
| 722 |
return NULL; |
| 723 |
} |
| 724 |
|
| 725 |
bool drgn_module_should_prefer_orc_cfi(struct drgn_module *module, uint64_t pc) |
| 726 |
{ |
| 727 |
uint64_t unbiased_pc = pc - module->debug_file_bias; |
| 728 |
#define less_than_uint64_range_start(a, b) (*(a) < (b)->start) |
| 729 |
size_t i = binary_search_gt(module->orc.preferred, |
| 730 |
module->orc.num_preferred, &unbiased_pc, |
| 731 |
less_than_uint64_range_start); |
| 732 |
#undef less_than_uint64_range_start |
| 733 |
return i > 0 && module->orc.preferred[i - 1].end > unbiased_pc; |
| 734 |
} |
| 735 |
|
| 736 |
static inline uint64_t drgn_orc_pc(struct drgn_module *module, unsigned int i) |
| 737 |
{ |
| 738 |
return module->orc.pc_base + UINT64_C(4) * i + module->orc.pc_offsets[i]; |
| 739 |
} |
| 740 |
|
| 741 |
struct drgn_error * |
| 742 |
drgn_module_find_orc_cfi(struct drgn_module *module, uint64_t pc, |
| 743 |
struct drgn_cfi_row **row_ret, bool *interrupted_ret, |
| 744 |
drgn_register_number *ret_addr_regno_ret) |
| 745 |
{ |
| 746 |
#define less_than_orc_pc(a, b) \ |
| 747 |
(*(a) < drgn_orc_pc(module, (b) - module->orc.pc_offsets)) |
| 748 |
size_t i = binary_search_gt(module->orc.pc_offsets, |
| 749 |
module->orc.num_entries, &pc, |
| 750 |
less_than_orc_pc); |
| 751 |
#undef less_than_orc_pc |
| 752 |
// We can tell when the program counter is below the minimum program |
| 753 |
// counter included in the ORC data, but we don't know the maximum. The |
| 754 |
// last entry seems to always be a terminator, so it doesn't matter. All |
| 755 |
// addresses beyond the max will fall into the last entry. |
| 756 |
if (i == 0) |
| 757 |
return &drgn_not_found; |
| 758 |
return drgn_orc_to_cfi_x86_64(&module->orc.entries[i - 1], row_ret, |
| 759 |
interrupted_ret, ret_addr_regno_ret); |
| 760 |
} |