| 1 |
# DExTer : Debugging Experience Tester |
| 2 |
# ~~~~~~ ~ ~~ ~ ~~ |
| 3 |
# |
| 4 |
# Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. |
| 5 |
# See https://llvm.org/LICENSE.txt for license information. |
| 6 |
# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception |
| 7 |
"""Test tool.""" |
| 8 |
|
| 9 |
import math |
| 10 |
import os |
| 11 |
import csv |
| 12 |
import pickle |
| 13 |
import shutil |
| 14 |
import platform |
| 15 |
|
| 16 |
from dex.command.ParseCommand import get_command_infos |
| 17 |
from dex.debugger.Debuggers import run_debugger_subprocess |
| 18 |
from dex.debugger.DebuggerControllers.DefaultController import DefaultController |
| 19 |
from dex.debugger.DebuggerControllers.ConditionalController import ConditionalController |
| 20 |
from dex.dextIR.DextIR import DextIR |
| 21 |
from dex.heuristic import Heuristic |
| 22 |
from dex.tools import TestToolBase |
| 23 |
from dex.utils.Exceptions import DebuggerException |
| 24 |
from dex.utils.Exceptions import BuildScriptException, HeuristicException |
| 25 |
from dex.utils.PrettyOutputBase import Stream |
| 26 |
from dex.utils.ReturnCode import ReturnCode |
| 27 |
|
| 28 |
|
| 29 |
class TestCase(object): |
| 30 |
def __init__(self, context, name, heuristic, error): |
| 31 |
self.context = context |
| 32 |
self.name = name |
| 33 |
self.heuristic = heuristic |
| 34 |
self.error = error |
| 35 |
|
| 36 |
@property |
| 37 |
def penalty(self): |
| 38 |
try: |
| 39 |
return self.heuristic.penalty |
| 40 |
except AttributeError: |
| 41 |
return float("nan") |
| 42 |
|
| 43 |
@property |
| 44 |
def max_penalty(self): |
| 45 |
try: |
| 46 |
return self.heuristic.max_penalty |
| 47 |
except AttributeError: |
| 48 |
return float("nan") |
| 49 |
|
| 50 |
@property |
| 51 |
def score(self): |
| 52 |
try: |
| 53 |
return self.heuristic.score |
| 54 |
except AttributeError: |
| 55 |
return float("nan") |
| 56 |
|
| 57 |
def __str__(self): |
| 58 |
if self.error and self.context.options.verbose: |
| 59 |
verbose_error = str(self.error) |
| 60 |
else: |
| 61 |
verbose_error = "" |
| 62 |
|
| 63 |
if self.error: |
| 64 |
script_error = ( |
| 65 |
" : {}".format(self.error.script_error.splitlines()[0]) |
| 66 |
if getattr(self.error, "script_error", None) |
| 67 |
else "" |
| 68 |
) |
| 69 |
|
| 70 |
error = " [{}{}]".format(str(self.error).splitlines()[0], script_error) |
| 71 |
else: |
| 72 |
error = "" |
| 73 |
|
| 74 |
try: |
| 75 |
summary = self.heuristic.summary_string |
| 76 |
except AttributeError: |
| 77 |
summary = "<r>nan/nan (nan)</>" |
| 78 |
return "{}: {}{}\n{}".format(self.name, summary, error, verbose_error) |
| 79 |
|
| 80 |
|
| 81 |
class Tool(TestToolBase): |
| 82 |
"""Run the specified DExTer test(s) with the specified compiler and linker |
| 83 |
options and produce a dextIR file as well as printing out the debugging |
| 84 |
experience score calculated by the DExTer heuristic. |
| 85 |
""" |
| 86 |
|
| 87 |
def __init__(self, *args, **kwargs): |
| 88 |
super(Tool, self).__init__(*args, **kwargs) |
| 89 |
self._test_cases = [] |
| 90 |
|
| 91 |
@property |
| 92 |
def name(self): |
| 93 |
return "DExTer test" |
| 94 |
|
| 95 |
def add_tool_arguments(self, parser, defaults): |
| 96 |
parser.add_argument( |
| 97 |
"--fail-lt", |
| 98 |
type=float, |
| 99 |
default=0.0, # By default TEST always succeeds. |
| 100 |
help="exit with status FAIL(2) if the test result" |
| 101 |
" is less than this value.", |
| 102 |
metavar="<float>", |
| 103 |
) |
| 104 |
parser.add_argument( |
| 105 |
"--calculate-average", |
| 106 |
action="store_true", |
| 107 |
help="calculate the average score of every test run", |
| 108 |
) |
| 109 |
super(Tool, self).add_tool_arguments(parser, defaults) |
| 110 |
|
| 111 |
def _init_debugger_controller(self): |
| 112 |
step_collection = DextIR( |
| 113 |
executable_path=self.context.options.executable, |
| 114 |
source_paths=self.context.options.source_files, |
| 115 |
dexter_version=self.context.version, |
| 116 |
) |
| 117 |
|
| 118 |
step_collection.commands, new_source_files = get_command_infos( |
| 119 |
self.context.options.test_files, self.context.options.source_root_dir |
| 120 |
) |
| 121 |
|
| 122 |
self.context.options.source_files.extend(list(new_source_files)) |
| 123 |
|
| 124 |
cond_controller_cmds = ["DexLimitSteps", "DexStepFunction", "DexContinue"] |
| 125 |
if any(c in step_collection.commands for c in cond_controller_cmds): |
| 126 |
debugger_controller = ConditionalController(self.context, step_collection) |
| 127 |
else: |
| 128 |
debugger_controller = DefaultController(self.context, step_collection) |
| 129 |
|
| 130 |
return debugger_controller |
| 131 |
|
| 132 |
def _get_steps(self): |
| 133 |
"""Generate a list of debugger steps from a test case.""" |
| 134 |
debugger_controller = self._init_debugger_controller() |
| 135 |
debugger_controller = run_debugger_subprocess( |
| 136 |
debugger_controller, self.context.working_directory.path |
| 137 |
) |
| 138 |
steps = debugger_controller.step_collection |
| 139 |
return steps |
| 140 |
|
| 141 |
def _get_results_basename(self, test_name): |
| 142 |
def splitall(x): |
| 143 |
while len(x) > 0: |
| 144 |
x, y = os.path.split(x) |
| 145 |
yield y |
| 146 |
|
| 147 |
all_components = reversed([x for x in splitall(test_name)]) |
| 148 |
return "_".join(all_components) |
| 149 |
|
| 150 |
def _get_results_path(self, test_name): |
| 151 |
"""Returns the path to the test results directory for the test denoted |
| 152 |
by test_name. |
| 153 |
""" |
| 154 |
assert self.context.options.results_directory is not None |
| 155 |
return os.path.join( |
| 156 |
self.context.options.results_directory, |
| 157 |
self._get_results_basename(test_name), |
| 158 |
) |
| 159 |
|
| 160 |
def _get_results_text_path(self, test_name): |
| 161 |
"""Returns path results .txt file for test denoted by test_name.""" |
| 162 |
test_results_path = self._get_results_path(test_name) |
| 163 |
return "{}.txt".format(test_results_path) |
| 164 |
|
| 165 |
def _get_results_pickle_path(self, test_name): |
| 166 |
"""Returns path results .dextIR file for test denoted by test_name.""" |
| 167 |
test_results_path = self._get_results_path(test_name) |
| 168 |
return "{}.dextIR".format(test_results_path) |
| 169 |
|
| 170 |
def _record_steps(self, test_name, steps): |
| 171 |
"""Write out the set of steps out to the test's .txt and .json |
| 172 |
results file if a results directory has been specified. |
| 173 |
""" |
| 174 |
if self.context.options.results_directory: |
| 175 |
output_text_path = self._get_results_text_path(test_name) |
| 176 |
with open(output_text_path, "w") as fp: |
| 177 |
self.context.o.auto(str(steps), stream=Stream(fp)) |
| 178 |
|
| 179 |
output_dextIR_path = self._get_results_pickle_path(test_name) |
| 180 |
with open(output_dextIR_path, "wb") as fp: |
| 181 |
pickle.dump(steps, fp, protocol=pickle.HIGHEST_PROTOCOL) |
| 182 |
|
| 183 |
def _record_score(self, test_name, heuristic): |
| 184 |
"""Write out the test's heuristic score to the results .txt file |
| 185 |
if a results directory has been specified. |
| 186 |
""" |
| 187 |
if self.context.options.results_directory: |
| 188 |
output_text_path = self._get_results_text_path(test_name) |
| 189 |
with open(output_text_path, "a") as fp: |
| 190 |
self.context.o.auto(heuristic.verbose_output, stream=Stream(fp)) |
| 191 |
|
| 192 |
def _record_test_and_display(self, test_case): |
| 193 |
"""Output test case to o stream and record test case internally for |
| 194 |
handling later. |
| 195 |
""" |
| 196 |
self.context.o.auto(test_case) |
| 197 |
self._test_cases.append(test_case) |
| 198 |
|
| 199 |
def _record_failed_test(self, test_name, exception): |
| 200 |
"""Instantiate a failed test case with failure exception and |
| 201 |
store internally. |
| 202 |
""" |
| 203 |
test_case = TestCase(self.context, test_name, None, exception) |
| 204 |
self._record_test_and_display(test_case) |
| 205 |
|
| 206 |
def _record_successful_test(self, test_name, steps, heuristic): |
| 207 |
"""Instantiate a successful test run, store test for handling later. |
| 208 |
Display verbose output for test case if required. |
| 209 |
""" |
| 210 |
test_case = TestCase(self.context, test_name, heuristic, None) |
| 211 |
self._record_test_and_display(test_case) |
| 212 |
if self.context.options.verbose: |
| 213 |
self.context.o.auto("\n{}\n".format(steps)) |
| 214 |
self.context.o.auto(heuristic.verbose_output) |
| 215 |
|
| 216 |
def _run_test(self, test_name): |
| 217 |
"""Attempt to run test files specified in options.source_files. Store |
| 218 |
result internally in self._test_cases. |
| 219 |
""" |
| 220 |
try: |
| 221 |
if self.context.options.binary: |
| 222 |
if platform.system() == 'Darwin' and os.path.exists(self.context.options.binary + '.dSYM'): |
| 223 |
# On Darwin, the debug info is in the .dSYM which might not be found by lldb, copy it into the tmp working directory |
| 224 |
shutil.copytree(self.context.options.binary + '.dSYM', self.context.options.executable + '.dSYM') |
| 225 |
# Copy user's binary into the tmp working directory. |
| 226 |
shutil.copy( |
| 227 |
self.context.options.binary, self.context.options.executable |
| 228 |
) |
| 229 |
steps = self._get_steps() |
| 230 |
self._record_steps(test_name, steps) |
| 231 |
heuristic_score = Heuristic(self.context, steps) |
| 232 |
self._record_score(test_name, heuristic_score) |
| 233 |
except (BuildScriptException, DebuggerException, HeuristicException) as e: |
| 234 |
self._record_failed_test(test_name, e) |
| 235 |
return |
| 236 |
|
| 237 |
self._record_successful_test(test_name, steps, heuristic_score) |
| 238 |
return |
| 239 |
|
| 240 |
def _handle_results(self) -> ReturnCode: |
| 241 |
return_code = ReturnCode.OK |
| 242 |
options = self.context.options |
| 243 |
|
| 244 |
if not options.verbose: |
| 245 |
self.context.o.auto("\n") |
| 246 |
|
| 247 |
if options.calculate_average: |
| 248 |
# Calculate and print the average score |
| 249 |
score_sum = 0.0 |
| 250 |
num_tests = 0 |
| 251 |
for test_case in self._test_cases: |
| 252 |
score = test_case.score |
| 253 |
if not test_case.error and not math.isnan(score): |
| 254 |
score_sum += test_case.score |
| 255 |
num_tests += 1 |
| 256 |
|
| 257 |
if num_tests != 0: |
| 258 |
print("@avg: ({:.4f})".format(score_sum / num_tests)) |
| 259 |
|
| 260 |
has_failed = lambda test: test.score < options.fail_lt or test.error |
| 261 |
if any(map(has_failed, self._test_cases)): |
| 262 |
return_code = ReturnCode.FAIL |
| 263 |
|
| 264 |
if options.results_directory: |
| 265 |
summary_path = os.path.join(options.results_directory, "summary.csv") |
| 266 |
with open(summary_path, mode="w", newline="") as fp: |
| 267 |
writer = csv.writer(fp, delimiter=",") |
| 268 |
writer.writerow(["Test Case", "Score", "Error"]) |
| 269 |
|
| 270 |
for test_case in self._test_cases: |
| 271 |
writer.writerow( |
| 272 |
[ |
| 273 |
test_case.name, |
| 274 |
"{:.4f}".format(test_case.score), |
| 275 |
test_case.error, |
| 276 |
] |
| 277 |
) |
| 278 |
|
| 279 |
return return_code |