Debian Codegraph @

cross-project-tests/debuginfo-tests/dexter/dex/tools/test/Tool.py

raw
1 # DExTer : Debugging Experience Tester
2 # ~~~~~~ ~ ~~ ~ ~~
3 #
4 # Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
5 # See https://llvm.org/LICENSE.txt for license information.
6 # SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
7 """Test tool."""
8
9 import math
10 import os
11 import csv
12 import pickle
13 import shutil
14 import platform
15
16 from dex.command.ParseCommand import get_command_infos
17 from dex.debugger.Debuggers import run_debugger_subprocess
18 from dex.debugger.DebuggerControllers.DefaultController import DefaultController
19 from dex.debugger.DebuggerControllers.ConditionalController import ConditionalController
20 from dex.dextIR.DextIR import DextIR
21 from dex.heuristic import Heuristic
22 from dex.tools import TestToolBase
23 from dex.utils.Exceptions import DebuggerException
24 from dex.utils.Exceptions import BuildScriptException, HeuristicException
25 from dex.utils.PrettyOutputBase import Stream
26 from dex.utils.ReturnCode import ReturnCode
27
28
29 class TestCase(object):
30 def __init__(self, context, name, heuristic, error):
31 self.context = context
32 self.name = name
33 self.heuristic = heuristic
34 self.error = error
35
36 @property
37 def penalty(self):
38 try:
39 return self.heuristic.penalty
40 except AttributeError:
41 return float("nan")
42
43 @property
44 def max_penalty(self):
45 try:
46 return self.heuristic.max_penalty
47 except AttributeError:
48 return float("nan")
49
50 @property
51 def score(self):
52 try:
53 return self.heuristic.score
54 except AttributeError:
55 return float("nan")
56
57 def __str__(self):
58 if self.error and self.context.options.verbose:
59 verbose_error = str(self.error)
60 else:
61 verbose_error = ""
62
63 if self.error:
64 script_error = (
65 " : {}".format(self.error.script_error.splitlines()[0])
66 if getattr(self.error, "script_error", None)
67 else ""
68 )
69
70 error = " [{}{}]".format(str(self.error).splitlines()[0], script_error)
71 else:
72 error = ""
73
74 try:
75 summary = self.heuristic.summary_string
76 except AttributeError:
77 summary = "<r>nan/nan (nan)</>"
78 return "{}: {}{}\n{}".format(self.name, summary, error, verbose_error)
79
80
81 class Tool(TestToolBase):
82 """Run the specified DExTer test(s) with the specified compiler and linker
83 options and produce a dextIR file as well as printing out the debugging
84 experience score calculated by the DExTer heuristic.
85 """
86
87 def __init__(self, *args, **kwargs):
88 super(Tool, self).__init__(*args, **kwargs)
89 self._test_cases = []
90
91 @property
92 def name(self):
93 return "DExTer test"
94
95 def add_tool_arguments(self, parser, defaults):
96 parser.add_argument(
97 "--fail-lt",
98 type=float,
99 default=0.0, # By default TEST always succeeds.
100 help="exit with status FAIL(2) if the test result"
101 " is less than this value.",
102 metavar="<float>",
103 )
104 parser.add_argument(
105 "--calculate-average",
106 action="store_true",
107 help="calculate the average score of every test run",
108 )
109 super(Tool, self).add_tool_arguments(parser, defaults)
110
111 def _init_debugger_controller(self):
112 step_collection = DextIR(
113 executable_path=self.context.options.executable,
114 source_paths=self.context.options.source_files,
115 dexter_version=self.context.version,
116 )
117
118 step_collection.commands, new_source_files = get_command_infos(
119 self.context.options.test_files, self.context.options.source_root_dir
120 )
121
122 self.context.options.source_files.extend(list(new_source_files))
123
124 cond_controller_cmds = ["DexLimitSteps", "DexStepFunction", "DexContinue"]
125 if any(c in step_collection.commands for c in cond_controller_cmds):
126 debugger_controller = ConditionalController(self.context, step_collection)
127 else:
128 debugger_controller = DefaultController(self.context, step_collection)
129
130 return debugger_controller
131
132 def _get_steps(self):
133 """Generate a list of debugger steps from a test case."""
134 debugger_controller = self._init_debugger_controller()
135 debugger_controller = run_debugger_subprocess(
136 debugger_controller, self.context.working_directory.path
137 )
138 steps = debugger_controller.step_collection
139 return steps
140
141 def _get_results_basename(self, test_name):
142 def splitall(x):
143 while len(x) > 0:
144 x, y = os.path.split(x)
145 yield y
146
147 all_components = reversed([x for x in splitall(test_name)])
148 return "_".join(all_components)
149
150 def _get_results_path(self, test_name):
151 """Returns the path to the test results directory for the test denoted
152 by test_name.
153 """
154 assert self.context.options.results_directory is not None
155 return os.path.join(
156 self.context.options.results_directory,
157 self._get_results_basename(test_name),
158 )
159
160 def _get_results_text_path(self, test_name):
161 """Returns path results .txt file for test denoted by test_name."""
162 test_results_path = self._get_results_path(test_name)
163 return "{}.txt".format(test_results_path)
164
165 def _get_results_pickle_path(self, test_name):
166 """Returns path results .dextIR file for test denoted by test_name."""
167 test_results_path = self._get_results_path(test_name)
168 return "{}.dextIR".format(test_results_path)
169
170 def _record_steps(self, test_name, steps):
171 """Write out the set of steps out to the test's .txt and .json
172 results file if a results directory has been specified.
173 """
174 if self.context.options.results_directory:
175 output_text_path = self._get_results_text_path(test_name)
176 with open(output_text_path, "w") as fp:
177 self.context.o.auto(str(steps), stream=Stream(fp))
178
179 output_dextIR_path = self._get_results_pickle_path(test_name)
180 with open(output_dextIR_path, "wb") as fp:
181 pickle.dump(steps, fp, protocol=pickle.HIGHEST_PROTOCOL)
182
183 def _record_score(self, test_name, heuristic):
184 """Write out the test's heuristic score to the results .txt file
185 if a results directory has been specified.
186 """
187 if self.context.options.results_directory:
188 output_text_path = self._get_results_text_path(test_name)
189 with open(output_text_path, "a") as fp:
190 self.context.o.auto(heuristic.verbose_output, stream=Stream(fp))
191
192 def _record_test_and_display(self, test_case):
193 """Output test case to o stream and record test case internally for
194 handling later.
195 """
196 self.context.o.auto(test_case)
197 self._test_cases.append(test_case)
198
199 def _record_failed_test(self, test_name, exception):
200 """Instantiate a failed test case with failure exception and
201 store internally.
202 """
203 test_case = TestCase(self.context, test_name, None, exception)
204 self._record_test_and_display(test_case)
205
206 def _record_successful_test(self, test_name, steps, heuristic):
207 """Instantiate a successful test run, store test for handling later.
208 Display verbose output for test case if required.
209 """
210 test_case = TestCase(self.context, test_name, heuristic, None)
211 self._record_test_and_display(test_case)
212 if self.context.options.verbose:
213 self.context.o.auto("\n{}\n".format(steps))
214 self.context.o.auto(heuristic.verbose_output)
215
216 def _run_test(self, test_name):
217 """Attempt to run test files specified in options.source_files. Store
218 result internally in self._test_cases.
219 """
220 try:
221 if self.context.options.binary:
222 if platform.system() == 'Darwin' and os.path.exists(self.context.options.binary + '.dSYM'):
223 # On Darwin, the debug info is in the .dSYM which might not be found by lldb, copy it into the tmp working directory
224 shutil.copytree(self.context.options.binary + '.dSYM', self.context.options.executable + '.dSYM')
225 # Copy user's binary into the tmp working directory.
226 shutil.copy(
227 self.context.options.binary, self.context.options.executable
228 )
229 steps = self._get_steps()
230 self._record_steps(test_name, steps)
231 heuristic_score = Heuristic(self.context, steps)
232 self._record_score(test_name, heuristic_score)
233 except (BuildScriptException, DebuggerException, HeuristicException) as e:
234 self._record_failed_test(test_name, e)
235 return
236
237 self._record_successful_test(test_name, steps, heuristic_score)
238 return
239
240 def _handle_results(self) -> ReturnCode:
241 return_code = ReturnCode.OK
242 options = self.context.options
243
244 if not options.verbose:
245 self.context.o.auto("\n")
246
247 if options.calculate_average:
248 # Calculate and print the average score
249 score_sum = 0.0
250 num_tests = 0
251 for test_case in self._test_cases:
252 score = test_case.score
253 if not test_case.error and not math.isnan(score):
254 score_sum += test_case.score
255 num_tests += 1
256
257 if num_tests != 0:
258 print("@avg: ({:.4f})".format(score_sum / num_tests))
259
260 has_failed = lambda test: test.score < options.fail_lt or test.error
261 if any(map(has_failed, self._test_cases)):
262 return_code = ReturnCode.FAIL
263
264 if options.results_directory:
265 summary_path = os.path.join(options.results_directory, "summary.csv")
266 with open(summary_path, mode="w", newline="") as fp:
267 writer = csv.writer(fp, delimiter=",")
268 writer.writerow(["Test Case", "Score", "Error"])
269
270 for test_case in self._test_cases:
271 writer.writerow(
272 [
273 test_case.name,
274 "{:.4f}".format(test_case.score),
275 test_case.error,
276 ]
277 )
278
279 return return_code