Debian Codegraph @

compiler-rt/lib/hwasan/scripts/hwasan_symbolize

raw
1 #!/usr/bin/env python3
2 #===- lib/hwasan/scripts/hwasan_symbolize ----------------------------------===#
3 #
4 # Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
5 # See https:#llvm.org/LICENSE.txt for license information.
6 # SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
7 #
8 #===------------------------------------------------------------------------===#
9 #
10 # HWAddressSanitizer offline symbolization script.
11 #
12 #===------------------------------------------------------------------------===#
13
14 from __future__ import print_function
15 from __future__ import unicode_literals
16
17 import argparse
18 import glob
19 import hashlib
20 import html
21 import json
22 import mmap
23 import os
24 import re
25 import struct
26 import subprocess
27 import sys
28
29 if sys.version_info.major < 3:
30 # Simulate Python 3.x behaviour of defaulting to UTF-8 for print. This is
31 # important in case any symbols are non-ASCII.
32 import codecs
33 sys.stdout = codecs.getwriter("utf-8")(sys.stdout)
34
35 # Below, a parser for a subset of ELF. It only supports 64 bit, little-endian,
36 # and only parses what is necessary to find the build ids. It uses a memoryview
37 # into an mmap to avoid copying.
38 Ehdr_size = 64
39 e_shnum_offset = 60
40 e_shoff_offset = 40
41 e_shstrndx_offset = 62
42 Shdr_size = 64
43 sh_name_offset = 0
44 sh_type_offset = 4
45 sh_offset_offset = 24
46 sh_size_offset = 32
47 SHT_NOTE = 7
48
49 Nhdr_size = 12
50 NT_GNU_BUILD_ID = 3
51
52 def align_up(size, alignment):
53 return (size + alignment - 1) & ~(alignment - 1)
54
55 def handle_Nhdr(mv, sh_size):
56 offset = 0
57 while offset < sh_size:
58 n_namesz, n_descsz, n_type = struct.unpack_from('<III', buffer=mv,
59 offset=offset)
60 if (n_type == NT_GNU_BUILD_ID and n_namesz == 4 and
61 mv[offset + Nhdr_size: offset + Nhdr_size + 4] == b"GNU\x00"):
62 value = mv[offset + Nhdr_size + 4: offset + Nhdr_size + 4 + n_descsz]
63 return value.hex()
64 offset += Nhdr_size + align_up(n_namesz, 4) + align_up(n_descsz, 4)
65 return None
66
67 def handle_shstrtab(mv, e_shoff):
68 e_shstrndx, = struct.unpack_from('<H', buffer=mv, offset=e_shstrndx_offset)
69
70 start_shstrndx = e_shoff + e_shstrndx * Shdr_size
71 shstrndx_sh = mv[start_shstrndx: start_shstrndx + Shdr_size]
72 _, shstrndx_sh_offset, shstrndx_sh_size = handle_Shdr(shstrndx_sh)
73 return mv[shstrndx_sh_offset:shstrndx_sh_offset + shstrndx_sh_size]
74
75 def read_string(mv):
76 name = ""
77 for byte in mv:
78 char = chr(byte)
79 if char == '\x00':
80 break
81 name += char
82 return name
83
84 def unpack_sh_type(mv):
85 sh_type, = struct.unpack_from('<I', buffer=mv, offset=sh_type_offset)
86 return sh_type
87
88 def handle_Shdr(mv):
89 name_offset, = struct.unpack_from('<I', buffer=mv, offset=sh_name_offset)
90 sh_offset, = struct.unpack_from('<Q', buffer=mv, offset=sh_offset_offset)
91 sh_size, = struct.unpack_from('<Q', buffer=mv, offset=sh_size_offset)
92 return name_offset, sh_offset, sh_size
93
94 def handle_elf(mv):
95 # \x02 is ELFCLASS64, \x01 is ELFDATA2LSB. HWASan currently only works on
96 # 64-bit little endian platforms (x86_64 and ARM64). If this changes, we will
97 # have to extend the parsing code.
98 if mv[:6] != b'\x7fELF\x02\x01':
99 return None
100 found_symbols = False
101 bid = None
102 e_shnum, = struct.unpack_from('<H', buffer=mv, offset=e_shnum_offset)
103 e_shoff, = struct.unpack_from('<Q', buffer=mv, offset=e_shoff_offset)
104
105 # Section where all the section header names are stored.
106 shstr = handle_shstrtab(mv, e_shoff)
107
108 for i in range(0, e_shnum):
109 start = e_shoff + i * Shdr_size
110 sh = mv[start: start + Shdr_size]
111 sh_name_offset, sh_offset, sh_size = handle_Shdr(sh)
112 sh_name = read_string(shstr[sh_name_offset:])
113 sh_type = unpack_sh_type(sh)
114
115 if sh_name == ".debug_info":
116 found_symbols = True
117 if sh_type == SHT_NOTE:
118 if sh_offset is None:
119 continue
120 note_hdr = mv[sh_offset: sh_offset + sh_size]
121 result = handle_Nhdr(note_hdr, sh_size)
122 if result is not None:
123 bid = result
124
125 if found_symbols:
126 return bid
127 else:
128 return None
129
130 def read_elf(filename):
131 with open(filename, "r") as fd:
132 if os.fstat(fd.fileno()).st_size < Ehdr_size:
133 return None
134 with mmap.mmap(fd.fileno(), 0, access=mmap.ACCESS_READ) as m:
135 with memoryview(m) as mv:
136 return handle_elf(mv)
137
138 class Symbolizer:
139 def __init__(self, path, binary_prefixes, paths_to_cut):
140 self.__pipe = None
141 self.__path = path
142 self.__binary_prefixes = binary_prefixes
143 self.__paths_to_cut = paths_to_cut
144 self.__log = False
145 self.__warnings = set()
146 self.__index = {}
147 self.__link_prefixes = []
148 self.__html = False
149 self.__last_access_address = None
150 self.__last_access_tag = None
151 self.__tag_dump = []
152 self.__tag_dump_match_idx = None
153 self.__matched_stack_uas = False
154 self.__offsets = []
155
156 def enable_html(self, enable):
157 self.__html = enable
158
159 def enable_logging(self, enable):
160 self.__log = enable
161
162 def maybe_escape(self, text):
163 if self.__html:
164 # We need to manually use &nbsp; for leading spaces, html.escape does
165 # not do that, and HTML ignores them.
166 spaces = 0
167 for i, c in enumerate(text):
168 spaces = i
169 if c != ' ':
170 break
171 text = text[spaces:]
172 return spaces * '&nbsp;' + html.escape(text)
173 return text
174
175 def print(self, line, escape=True):
176 if escape:
177 line = self.maybe_escape(line)
178 if self.__html:
179 line += '<br/>'
180 print(line)
181
182 def read_linkify(self, filename):
183 with open(filename, 'r') as fd:
184 data = json.load(fd)
185 self.__link_prefixes = [(e["prefix"], e["link"]) for e in data]
186
187 def __open_pipe(self):
188 if not self.__pipe:
189 opt = {}
190 if sys.version_info.major > 2:
191 opt['encoding'] = 'utf-8'
192 self.__pipe = subprocess.Popen([self.__path, "--inlining", "--functions"],
193 stdin=subprocess.PIPE, stdout=subprocess.PIPE,
194 **opt)
195
196 class __EOF(Exception):
197 pass
198
199 def __write(self, s):
200 print(s, file=self.__pipe.stdin)
201 self.__pipe.stdin.flush()
202 if self.__log:
203 print("#>> |%s|" % (s,), file=sys.stderr)
204
205 def __read(self):
206 s = self.__pipe.stdout.readline().rstrip()
207 if self.__log:
208 print("# << |%s|" % (s,), file=sys.stderr)
209 if s == '':
210 raise Symbolizer.__EOF
211 return s
212
213 def __process_source_path(self, file_name):
214 for path_to_cut in self.__paths_to_cut:
215 file_name = re.sub(".*" + path_to_cut, "", file_name)
216 file_name = re.sub(".*hwasan_[a-z_]*.(cc|h):[0-9]*", "[hwasan_rtl]", file_name)
217 file_name = re.sub(".*asan_[a-z_]*.(cc|h):[0-9]*", "[asan_rtl]", file_name)
218 file_name = re.sub(".*crtstuff.c:0", "???:0", file_name)
219 return file_name
220
221 def __process_binary_name(self, name, buildid):
222 if name.startswith('/'):
223 name = name[1:]
224 if buildid is not None and buildid in self.__index:
225 return self.__index[buildid]
226
227 for p in self.__binary_prefixes:
228 full_path = os.path.join(p, name)
229 if os.path.exists(full_path):
230 return full_path
231 apex_prefix = "apex/com.android."
232 if name.startswith(apex_prefix):
233 full_path = os.path.join(p, "apex/com.google.android." + name[len(apex_prefix):])
234 if os.path.exists(full_path):
235 return full_path
236 # Try stripping extra path components as the last resort.
237 for p in self.__binary_prefixes:
238 full_path = os.path.join(p, os.path.basename(name))
239 if os.path.exists(full_path):
240 return full_path
241 if name not in self.__warnings:
242 print("Could not find symbols for {} (Build ID: {})".format(name, buildid), file=sys.stderr)
243 self.__warnings.add(name)
244 return None
245
246 def iter_locals(self, binary, addr, buildid):
247 self.__open_pipe()
248 p = self.__pipe
249 binary = self.__process_binary_name(binary, buildid)
250 if not binary:
251 return
252 self.__write("FRAME %s %s" % (binary, addr))
253 try:
254 while True:
255 function_name = self.__read()
256 local_name = self.__read()
257 file_line = self.__read()
258 extra = self.__read().split()
259
260 file_line = self.__process_source_path(file_line)
261 offset = None if extra[0] == '??' else int(extra[0])
262 size = None if extra[1] == '??' else int(extra[1])
263 tag_offset = None if extra[2] == '??' else int(extra[2])
264 yield (function_name, file_line, local_name, offset, size, tag_offset)
265 except Symbolizer.__EOF:
266 pass
267
268 def iter_call_stack(self, binary, buildid, addr):
269 self.__open_pipe()
270 p = self.__pipe
271 binary = self.__process_binary_name(binary, buildid)
272 if not binary:
273 return
274 self.__write("CODE %s %s" % (binary, addr))
275 try:
276 while True:
277 function_name = self.__read()
278 file_line = self.__read()
279 file_line = self.__process_source_path(file_line)
280 yield (function_name, file_line)
281 except Symbolizer.__EOF:
282 pass
283
284 def maybe_linkify(self, file_line):
285 if not self.__html or not self.__link_prefixes:
286 return file_line
287 filename, line_col = file_line.split(':', 1)
288 if not line_col:
289 line = '0' # simplify the link generation
290 else:
291 line = line_col.split(':')[0]
292 longest_prefix = max((
293 (prefix, link) for prefix, link in self.__link_prefixes
294 if filename.startswith(prefix)),
295 key=lambda x: len(x[0]), default=None)
296 if longest_prefix is None:
297 return file_line
298 else:
299 prefix, link = longest_prefix
300 return '<a href="{}">{}</a>'.format(
301 html.escape(link.format(file=filename[len(prefix):], line=line,
302 file_line=file_line, prefix=prefix)), file_line)
303
304 def build_index(self):
305 for p in self.__binary_prefixes:
306 for dname, _, fnames in os.walk(p):
307 for fn in fnames:
308 filename = os.path.join(dname, fn)
309 try:
310 bid = read_elf(filename)
311 except FileNotFoundError:
312 continue
313 except Exception as e:
314 print("Failed to parse {}: {}".format(filename, e), file=sys.stderr)
315 continue
316 if bid is None:
317 continue
318
319 if bid in self.__index:
320 index_filename = self.__index[bid]
321
322 if os.path.samefile(index_filename, filename):
323 continue
324
325 with open(filename, "rb") as f:
326 file_hash = hashlib.file_digest(f, "sha256")
327
328 with open(index_filename, "rb") as f:
329 index_file_hash = hashlib.file_digest(f, "sha256")
330
331 if index_file_hash.digest() != file_hash.digest():
332 print("Build ID collision! Files share the same BuildId ({}) but their contents differ. Files {} and {} ".format(bid, filename, index_filename), file=sys.stderr)
333 else:
334 self.__index[bid] = filename
335
336 def symbolize_line(self, line):
337 #0 0x7f6e35cf2e45 (/blah/foo.so+0x11fe45) (BuildId: 4abce4cd41ea5c2f34753297b7e774d9)
338 match = re.match(r'^(.*?)#([0-9]+)( *)(0x[0-9a-f]*) *\((.*)\+(0x[0-9a-f]+)\)'
339 r'(?:\s*\(BuildId: ([0-9a-f]+)\))?', line, re.UNICODE)
340 if match:
341 frameno = match.group(2)
342 binary = match.group(5)
343 addr = int(match.group(6), 16)
344 buildid = match.group(7)
345
346 frames = list(self.iter_call_stack(binary, buildid, addr))
347
348 if len(frames) > 0:
349 self.print(
350 self.maybe_escape(
351 "%s#%s%s%s in " % (match.group(1), match.group(2), match.group(3),
352 frames[0][0])
353 ) + self.maybe_linkify(frames[0][1]),
354 escape=False)
355 for i in range(1, len(frames)):
356 space1 = ' ' * match.end(1)
357 space2 = ' ' * (match.start(4) - match.end(1) - 2)
358 self.print(
359 self.maybe_escape("%s->%s%s in " % (space1, space2, frames[i][0]))
360 + self.maybe_linkify(frames[i][1]), escape=False)
361 else:
362 self.print(line.rstrip())
363 else:
364 self.print(line.rstrip())
365
366 def save_access_address(self, line):
367 match = re.match(r'^(.*?)HWAddressSanitizer: tag-mismatch on address (0x[0-9a-f]+) ', line, re.UNICODE)
368 if match:
369 self.__last_access_address = int(match.group(2), 16)
370 match = re.match(r'^(.*?) of size [0-9]+ at 0x[0-9a-f]* tags: ([0-9a-f]+)/[0-9a-f]+(\([0-9a-f]+\))? \(ptr/mem\)', line, re.UNICODE)
371 if match:
372 self.__last_access_tag = int(match.group(2), 16)
373
374 def process_tag_dump_line(self, line, ignore_tags=False):
375 m = re.match(r'.*?(0x[0-9a-f]+):' + r'([ ]*[\[ ][0-9a-f][0-9a-f]\]?)' * 16, line)
376 if m is None:
377 return False
378 addr = m.group(1)
379 tags = m.group(*range(2, 18))
380 fault = [i for i, x in enumerate(tags) if '[' in x]
381 if fault:
382 self.__tag_dump_match_idx = len(self.__tag_dump) + fault[0]
383 self.__tag_dump.extend(int(x.strip(' [').rstrip('] '), 16) for x in tags)
384 return True
385
386 def finish_tag_dump(self):
387 if self.__matched_stack_uas or self.__tag_dump_match_idx is None:
388 return
389 for offset, size, local in sorted(self.__offsets, key=lambda x: abs(x[0])):
390 idx = self.__tag_dump_match_idx - offset // 16
391 if idx < 0 or idx > len(self.__tag_dump):
392 continue
393 if self.__tag_dump[idx] == self.__last_access_tag:
394 self.print('')
395 self.print('Potentially referenced stack object:')
396 if offset > 0:
397 self.print(' %d bytes after a variable "%s" in stack frame of function "%s"' % (offset - size, local[2], local[0]))
398 if offset < 0:
399 self.print(' %d bytes before a variable "%s" in stack frame of function "%s"' % (-offset, local[2], local[0]))
400 self.print(' at %s' % (local[1],))
401
402 def process_stack_history(self, line, ignore_tags=False):
403 if self.__last_access_address is None or self.__last_access_tag is None:
404 return
405 if re.match(r'Previously allocated frames:', line, re.UNICODE):
406 return True
407 pc_mask = (1 << 48) - 1
408 fp_mask = (1 << 20) - 1
409 # record_addr:0x1234ABCD record:0x1234ABCD (/path/to/binary+0x1234ABCD) (BuildId: 4abce4cd41ea5c2f34753297b7e774d9)
410 match = re.match(r'^(.*?)record_addr:(0x[0-9a-f]+) +record:(0x[0-9a-f]+) +\((.*)\+(0x[0-9a-f]+)\)'
411 r'(?:\s*\(BuildId: ([0-9a-f]+)\))?', line, re.UNICODE)
412 if match:
413 record_addr = int(match.group(2), 16)
414 record = int(match.group(3), 16)
415 binary = match.group(4)
416 addr = int(match.group(5), 16)
417 buildid = match.group(6)
418 base_tag = (record_addr >> 3) & 0xFF
419 fp = (record >> 48) << 4
420 pc = record & pc_mask
421
422 for local in self.iter_locals(binary, addr, buildid):
423 frame_offset = local[3]
424 size = local[4]
425 if frame_offset is None or size is None:
426 continue
427 obj_offset = (self.__last_access_address & fp_mask) - ((fp & fp_mask) + frame_offset)
428 tag_offset = local[5]
429 if not ignore_tags and (tag_offset is None or base_tag ^ tag_offset != self.__last_access_tag):
430 continue
431 if obj_offset < 0 or obj_offset >= size:
432 self.__offsets.append((obj_offset, size, local))
433 continue
434 self.print('')
435 self.print('Potentially referenced stack object:')
436 self.print(' %d bytes inside a variable "%s" in stack frame of function "%s"' % (obj_offset, local[2], local[0]))
437 self.print(' at %s' % (local[1],))
438 self.__matched_stack_uas = True
439 return True
440 return False
441
442 def extract_version(s):
443 idx = s.rfind('-')
444 if idx == -1:
445 return 0
446 x = float(s[idx + 1:])
447 return x
448
449 def main():
450 parser = argparse.ArgumentParser()
451 parser.add_argument('-d', action='store_true')
452 parser.add_argument('-v', action='store_true')
453 parser.add_argument('--ignore-tags', action='store_true')
454 parser.add_argument('--symbols', action='append')
455 parser.add_argument('--source', action='append')
456 parser.add_argument('--index', action='store_true')
457 parser.add_argument('--symbolizer')
458 parser.add_argument('--linkify', type=str)
459 parser.add_argument('--html', action='store_true')
460 parser.add_argument('args', nargs=argparse.REMAINDER)
461 args = parser.parse_args()
462
463 # Unstripped binaries location.
464 binary_prefixes = args.symbols or []
465 if not binary_prefixes:
466 if 'ANDROID_PRODUCT_OUT' in os.environ:
467 product_out = os.path.join(os.environ['ANDROID_PRODUCT_OUT'], 'symbols')
468 binary_prefixes.append(product_out)
469 binary_prefixes.append('/')
470
471 for p in binary_prefixes:
472 if not os.path.isdir(p):
473 print("Symbols path does not exist or is not a directory:", p, file=sys.stderr)
474 sys.exit(1)
475
476 # Source location.
477 paths_to_cut = args.source or []
478 if not paths_to_cut:
479 paths_to_cut.append(os.getcwd() + '/')
480 if 'ANDROID_BUILD_TOP' in os.environ:
481 paths_to_cut.append(os.environ['ANDROID_BUILD_TOP'] + '/')
482
483 # llvm-symbolizer binary.
484 # 1. --symbolizer flag
485 # 2. environment variable
486 # 3. unsuffixed binary in the current directory
487 # 4. if inside Android platform, prebuilt binary at a known path
488 # 5. first "llvm-symbolizer", then "llvm-symbolizer-$VER" with the
489 # highest available version in $PATH
490 symbolizer_path = args.symbolizer
491 if not symbolizer_path:
492 if 'LLVM_SYMBOLIZER_PATH' in os.environ:
493 symbolizer_path = os.environ['LLVM_SYMBOLIZER_PATH']
494 elif 'HWASAN_SYMBOLIZER_PATH' in os.environ:
495 symbolizer_path = os.environ['HWASAN_SYMBOLIZER_PATH']
496
497 if not symbolizer_path:
498 s = os.path.join(os.path.dirname(sys.argv[0]), 'llvm-symbolizer')
499 if os.path.exists(s):
500 symbolizer_path = s
501
502 if not symbolizer_path:
503 if 'ANDROID_BUILD_TOP' in os.environ:
504 s = os.path.join(os.environ['ANDROID_BUILD_TOP'], 'prebuilts/clang/host/linux-x86/llvm-binutils-stable/llvm-symbolizer')
505 if os.path.exists(s):
506 symbolizer_path = s
507
508 if not symbolizer_path:
509 for path in os.environ["PATH"].split(os.pathsep):
510 p = os.path.join(path, 'llvm-symbolizer')
511 if os.path.exists(p):
512 symbolizer_path = p
513 break
514
515 if not symbolizer_path:
516 for path in os.environ["PATH"].split(os.pathsep):
517 candidates = glob.glob(os.path.join(path, 'llvm-symbolizer-*'))
518 if len(candidates) > 0:
519 candidates.sort(key = extract_version, reverse = True)
520 symbolizer_path = candidates[0]
521 break
522
523 if not os.path.exists(symbolizer_path):
524 print("Symbolizer path does not exist:", symbolizer_path, file=sys.stderr)
525 sys.exit(1)
526
527 if args.v:
528 print("Looking for symbols in:")
529 for s in binary_prefixes:
530 print(" %s" % (s,))
531 print("Stripping source path prefixes:")
532 for s in paths_to_cut:
533 print(" %s" % (s,))
534 print("Using llvm-symbolizer binary in:\n %s" % (symbolizer_path,))
535 print()
536
537 symbolizer = Symbolizer(symbolizer_path, binary_prefixes, paths_to_cut)
538 symbolizer.enable_html(args.html)
539 symbolizer.enable_logging(args.d)
540 if args.index:
541 symbolizer.build_index()
542
543 if args.linkify:
544 if not args.html:
545 print('Need --html to --linkify', file=sys.stderr)
546 sys.exit(1)
547 symbolizer.read_linkify(args.linkify)
548
549 tag_dump = False
550 for line in sys.stdin:
551 if sys.version_info.major < 3:
552 line = line.decode('utf-8')
553 if tag_dump:
554 tag_dump = symbolizer.process_tag_dump_line(line)
555 if tag_dump:
556 continue
557 symbolizer.finish_tag_dump()
558 if 'Memory tags around the buggy address' in line:
559 tag_dump = True
560
561 symbolizer.save_access_address(line)
562 if symbolizer.process_stack_history(line, ignore_tags=args.ignore_tags):
563 continue
564 symbolizer.symbolize_line(line)
565
566
567 if __name__ == '__main__':
568 main()