| 1 |
#!/usr/bin/env python |
| 2 |
# Merge or print the coverage data collected by asan's coverage. |
| 3 |
# Input files are sequences of 4-byte integers. |
| 4 |
# We need to merge these integers into a set and then |
| 5 |
# either print them (as hex) or dump them into another file. |
| 6 |
import array |
| 7 |
import bisect |
| 8 |
import glob |
| 9 |
import os.path |
| 10 |
import struct |
| 11 |
import subprocess |
| 12 |
import sys |
| 13 |
|
| 14 |
prog_name = "" |
| 15 |
|
| 16 |
def Usage(): |
| 17 |
sys.stderr.write( |
| 18 |
"Usage: \n" + \ |
| 19 |
" " + prog_name + " merge FILE [FILE...] > OUTPUT\n" \ |
| 20 |
" " + prog_name + " print FILE [FILE...]\n" \ |
| 21 |
" " + prog_name + " unpack FILE [FILE...]\n" \ |
| 22 |
" " + prog_name + " rawunpack FILE [FILE ...]\n" \ |
| 23 |
" " + prog_name + " missing BINARY < LIST_OF_PCS\n" \ |
| 24 |
"\n") |
| 25 |
exit(1) |
| 26 |
|
| 27 |
def CheckBits(bits): |
| 28 |
if bits != 32 and bits != 64: |
| 29 |
raise Exception("Wrong bitness: %d" % bits) |
| 30 |
|
| 31 |
def TypeCodeForBits(bits): |
| 32 |
CheckBits(bits) |
| 33 |
return 'L' if bits == 64 else 'I' |
| 34 |
|
| 35 |
def TypeCodeForStruct(bits): |
| 36 |
CheckBits(bits) |
| 37 |
return 'Q' if bits == 64 else 'I' |
| 38 |
|
| 39 |
kMagic32SecondHalf = 0xFFFFFF32; |
| 40 |
kMagic64SecondHalf = 0xFFFFFF64; |
| 41 |
kMagicFirstHalf = 0xC0BFFFFF; |
| 42 |
|
| 43 |
def MagicForBits(bits): |
| 44 |
CheckBits(bits) |
| 45 |
if sys.byteorder == 'little': |
| 46 |
return [kMagic64SecondHalf if bits == 64 else kMagic32SecondHalf, kMagicFirstHalf] |
| 47 |
else: |
| 48 |
return [kMagicFirstHalf, kMagic64SecondHalf if bits == 64 else kMagic32SecondHalf] |
| 49 |
|
| 50 |
def ReadMagicAndReturnBitness(f, path): |
| 51 |
magic_bytes = f.read(8) |
| 52 |
magic_words = struct.unpack('II', magic_bytes); |
| 53 |
bits = 0 |
| 54 |
idx = 1 if sys.byteorder == 'little' else 0 |
| 55 |
if magic_words[idx] == kMagicFirstHalf: |
| 56 |
if magic_words[1-idx] == kMagic64SecondHalf: |
| 57 |
bits = 64 |
| 58 |
elif magic_words[1-idx] == kMagic32SecondHalf: |
| 59 |
bits = 32 |
| 60 |
if bits == 0: |
| 61 |
raise Exception('Bad magic word in %s' % path) |
| 62 |
return bits |
| 63 |
|
| 64 |
def ReadOneFile(path): |
| 65 |
with open(path, mode="rb") as f: |
| 66 |
f.seek(0, 2) |
| 67 |
size = f.tell() |
| 68 |
f.seek(0, 0) |
| 69 |
if size < 8: |
| 70 |
raise Exception('File %s is short (< 8 bytes)' % path) |
| 71 |
bits = ReadMagicAndReturnBitness(f, path) |
| 72 |
size -= 8 |
| 73 |
w = size * 8 // bits |
| 74 |
s = struct.unpack_from(TypeCodeForStruct(bits) * (w), f.read(size)) |
| 75 |
sys.stderr.write( |
| 76 |
"%s: read %d %d-bit PCs from %s\n" % (prog_name, w, bits, path)) |
| 77 |
return s |
| 78 |
|
| 79 |
def Merge(files): |
| 80 |
s = set() |
| 81 |
for f in files: |
| 82 |
s = s.union(set(ReadOneFile(f))) |
| 83 |
sys.stderr.write( |
| 84 |
"%s: %d files merged; %d PCs total\n" % (prog_name, len(files), len(s)) |
| 85 |
) |
| 86 |
return sorted(s) |
| 87 |
|
| 88 |
def PrintFiles(files): |
| 89 |
if len(files) > 1: |
| 90 |
s = Merge(files) |
| 91 |
else: # If there is just on file, print the PCs in order. |
| 92 |
s = ReadOneFile(files[0]) |
| 93 |
sys.stderr.write("%s: 1 file merged; %d PCs total\n" % (prog_name, len(s))) |
| 94 |
for i in s: |
| 95 |
print("0x%x" % i) |
| 96 |
|
| 97 |
def MergeAndPrint(files): |
| 98 |
if sys.stdout.isatty(): |
| 99 |
Usage() |
| 100 |
s = Merge(files) |
| 101 |
bits = 32 |
| 102 |
if max(s) > 0xFFFFFFFF: |
| 103 |
bits = 64 |
| 104 |
stdout_buf = getattr(sys.stdout, 'buffer', sys.stdout) |
| 105 |
array.array('I', MagicForBits(bits)).tofile(stdout_buf) |
| 106 |
a = struct.pack(TypeCodeForStruct(bits) * len(s), *s) |
| 107 |
stdout_buf.write(a) |
| 108 |
|
| 109 |
|
| 110 |
def UnpackOneFile(path): |
| 111 |
with open(path, mode="rb") as f: |
| 112 |
sys.stderr.write("%s: unpacking %s\n" % (prog_name, path)) |
| 113 |
while True: |
| 114 |
header = f.read(12) |
| 115 |
if not header: return |
| 116 |
if len(header) < 12: |
| 117 |
break |
| 118 |
pid, module_length, blob_size = struct.unpack('iII', header) |
| 119 |
module = f.read(module_length).decode('utf-8') |
| 120 |
blob = f.read(blob_size) |
| 121 |
assert(len(module) == module_length) |
| 122 |
assert(len(blob) == blob_size) |
| 123 |
extracted_file = "%s.%d.sancov" % (module, pid) |
| 124 |
sys.stderr.write("%s: extracting %s\n" % (prog_name, extracted_file)) |
| 125 |
# The packed file may contain multiple blobs for the same pid/module |
| 126 |
# pair. Append to the end of the file instead of overwriting. |
| 127 |
with open(extracted_file, 'ab') as f2: |
| 128 |
f2.write(blob) |
| 129 |
# fail |
| 130 |
raise Exception('Error reading file %s' % path) |
| 131 |
|
| 132 |
|
| 133 |
def Unpack(files): |
| 134 |
for f in files: |
| 135 |
UnpackOneFile(f) |
| 136 |
|
| 137 |
def UnpackOneRawFile(path, map_path): |
| 138 |
mem_map = [] |
| 139 |
with open(map_path, mode="rt") as f_map: |
| 140 |
sys.stderr.write("%s: reading map %s\n" % (prog_name, map_path)) |
| 141 |
bits = int(f_map.readline()) |
| 142 |
if bits != 32 and bits != 64: |
| 143 |
raise Exception('Wrong bits size in the map') |
| 144 |
for line in f_map: |
| 145 |
parts = line.rstrip().split() |
| 146 |
mem_map.append((int(parts[0], 16), |
| 147 |
int(parts[1], 16), |
| 148 |
int(parts[2], 16), |
| 149 |
' '.join(parts[3:]))) |
| 150 |
mem_map.sort(key=lambda m : m[0]) |
| 151 |
mem_map_keys = [m[0] for m in mem_map] |
| 152 |
|
| 153 |
with open(path, mode="rb") as f: |
| 154 |
sys.stderr.write("%s: unpacking %s\n" % (prog_name, path)) |
| 155 |
|
| 156 |
f.seek(0, 2) |
| 157 |
size = f.tell() |
| 158 |
f.seek(0, 0) |
| 159 |
pcs = struct.unpack_from(TypeCodeForStruct(bits) * (size * 8 // bits), f.read(size)) |
| 160 |
mem_map_pcs = [[] for i in range(0, len(mem_map))] |
| 161 |
|
| 162 |
for pc in pcs: |
| 163 |
if pc == 0: continue |
| 164 |
map_idx = bisect.bisect(mem_map_keys, pc) - 1 |
| 165 |
(start, end, base, module_path) = mem_map[map_idx] |
| 166 |
assert pc >= start |
| 167 |
if pc >= end: |
| 168 |
sys.stderr.write("warning: %s: pc %x outside of any known mapping\n" % (prog_name, pc)) |
| 169 |
continue |
| 170 |
mem_map_pcs[map_idx].append(pc - base) |
| 171 |
|
| 172 |
for ((start, end, base, module_path), pc_list) in zip(mem_map, mem_map_pcs): |
| 173 |
if len(pc_list) == 0: continue |
| 174 |
assert path.endswith('.sancov.raw') |
| 175 |
dst_path = module_path + '.' + os.path.basename(path)[:-4] |
| 176 |
sys.stderr.write("%s: writing %d PCs to %s\n" % (prog_name, len(pc_list), dst_path)) |
| 177 |
sorted_pc_list = sorted(pc_list) |
| 178 |
pc_buffer = struct.pack(TypeCodeForStruct(bits) * len(pc_list), *sorted_pc_list) |
| 179 |
with open(dst_path, 'ab+') as f2: |
| 180 |
array.array('I', MagicForBits(bits)).tofile(f2) |
| 181 |
f2.seek(0, 2) |
| 182 |
f2.write(pc_buffer) |
| 183 |
|
| 184 |
def RawUnpack(files): |
| 185 |
for f in files: |
| 186 |
if not f.endswith('.sancov.raw'): |
| 187 |
raise Exception('Unexpected raw file name %s' % f) |
| 188 |
f_map = f[:-3] + 'map' |
| 189 |
UnpackOneRawFile(f, f_map) |
| 190 |
|
| 191 |
def GetInstrumentedPCs(binary): |
| 192 |
# This looks scary, but all it does is extract all offsets where we call: |
| 193 |
# - __sanitizer_cov() or __sanitizer_cov_with_check(), |
| 194 |
# - with call or callq, |
| 195 |
# - directly or via PLT. |
| 196 |
cmd = r"objdump --no-show-raw-insn -d %s | " \ |
| 197 |
r"grep '^\s\+[0-9a-f]\+:\s\+call\(q\|\)\s\+\(0x\|\)[0-9a-f]\+ <__sanitizer_cov\(_with_check\|\|_trace_pc_guard\)\(@plt\|\)>' | " \ |
| 198 |
r"grep -o '^\s\+[0-9a-f]\+'" % binary |
| 199 |
lines = subprocess.check_output(cmd, stdin=subprocess.PIPE, shell=True).splitlines() |
| 200 |
# The PCs we get from objdump are off by 4 bytes, as they point to the |
| 201 |
# beginning of the callq instruction. Empirically this is true on x86 and |
| 202 |
# x86_64. |
| 203 |
return set(int(line.strip(), 16) + 4 for line in lines) |
| 204 |
|
| 205 |
def PrintMissing(binary): |
| 206 |
if not os.path.isfile(binary): |
| 207 |
raise Exception('File not found: %s' % binary) |
| 208 |
instrumented = GetInstrumentedPCs(binary) |
| 209 |
sys.stderr.write("%s: found %d instrumented PCs in %s\n" % (prog_name, |
| 210 |
len(instrumented), |
| 211 |
binary)) |
| 212 |
covered = set(int(line, 16) for line in sys.stdin) |
| 213 |
sys.stderr.write("%s: read %d PCs from stdin\n" % (prog_name, len(covered))) |
| 214 |
missing = instrumented - covered |
| 215 |
sys.stderr.write("%s: %d PCs missing from coverage\n" % (prog_name, len(missing))) |
| 216 |
if (len(missing) > len(instrumented) - len(covered)): |
| 217 |
sys.stderr.write( |
| 218 |
"%s: WARNING: stdin contains PCs not found in binary\n" % prog_name |
| 219 |
) |
| 220 |
for pc in sorted(missing): |
| 221 |
print("0x%x" % pc) |
| 222 |
|
| 223 |
if __name__ == '__main__': |
| 224 |
prog_name = sys.argv[0] |
| 225 |
if len(sys.argv) <= 2: |
| 226 |
Usage(); |
| 227 |
|
| 228 |
if sys.argv[1] == "missing": |
| 229 |
if len(sys.argv) != 3: |
| 230 |
Usage() |
| 231 |
PrintMissing(sys.argv[2]) |
| 232 |
exit(0) |
| 233 |
|
| 234 |
file_list = [] |
| 235 |
for f in sys.argv[2:]: |
| 236 |
file_list += glob.glob(f) |
| 237 |
if not file_list: |
| 238 |
Usage() |
| 239 |
|
| 240 |
if sys.argv[1] == "print": |
| 241 |
PrintFiles(file_list) |
| 242 |
elif sys.argv[1] == "merge": |
| 243 |
MergeAndPrint(file_list) |
| 244 |
elif sys.argv[1] == "unpack": |
| 245 |
Unpack(file_list) |
| 246 |
elif sys.argv[1] == "rawunpack": |
| 247 |
RawUnpack(file_list) |
| 248 |
else: |
| 249 |
Usage() |