Debian Codegraph @

Lib/test/libregrtest/results.py

raw
1 import sys
2 import trace
3 from _colorize import get_colors # type: ignore[import-not-found]
4 from typing import TYPE_CHECKING
5
6 from .runtests import RunTests
7 from .result import State, TestResult, TestStats, Location
8 from .utils import (
9 StrPath, TestName, TestTuple, TestList, FilterDict,
10 printlist, count, format_duration)
11
12 if TYPE_CHECKING:
13 from xml.etree.ElementTree import Element
14
15
16 # Python uses exit code 1 when an exception is not caught
17 # argparse.ArgumentParser.error() uses exit code 2
18 EXITCODE_BAD_TEST = 2
19 EXITCODE_ENV_CHANGED = 3
20 EXITCODE_NO_TESTS_RAN = 4
21 EXITCODE_RERUN_FAIL = 5
22 EXITCODE_INTERRUPTED = 130 # 128 + signal.SIGINT=2
23
24
25 class TestResults:
26 def __init__(self) -> None:
27 self.bad: TestList = []
28 self.good: TestList = []
29 self.rerun_bad: TestList = []
30 self.skipped: TestList = []
31 self.resource_denied: TestList = []
32 self.env_changed: TestList = []
33 self.run_no_tests: TestList = []
34 self.rerun: TestList = []
35 self.rerun_results: list[TestResult] = []
36
37 self.interrupted: bool = False
38 self.worker_bug: bool = False
39 self.test_times: list[tuple[float, TestName]] = []
40 self.stats = TestStats()
41 # used by --junit-xml
42 self.testsuite_xml: list['Element'] = []
43 # used by -T with -j
44 self.covered_lines: set[Location] = set()
45
46 def is_all_good(self) -> bool:
47 return (not self.bad
48 and not self.skipped
49 and not self.interrupted
50 and not self.worker_bug)
51
52 def get_executed(self) -> set[TestName]:
53 return (set(self.good) | set(self.bad) | set(self.skipped)
54 | set(self.resource_denied) | set(self.env_changed)
55 | set(self.run_no_tests))
56
57 def no_tests_run(self) -> bool:
58 return not any((self.good, self.bad, self.skipped, self.interrupted,
59 self.env_changed))
60
61 def get_state(self, fail_env_changed: bool) -> str:
62 state = []
63 ansi = get_colors()
64 green = ansi.GREEN
65 red = ansi.BOLD_RED
66 reset = ansi.RESET
67 yellow = ansi.YELLOW
68 if self.bad:
69 state.append(f"{red}FAILURE{reset}")
70 elif fail_env_changed and self.env_changed:
71 state.append(f"{yellow}ENV CHANGED{reset}")
72 elif self.no_tests_run():
73 state.append(f"{yellow}NO TESTS RAN{reset}")
74
75 if self.interrupted:
76 state.append(f"{yellow}INTERRUPTED{reset}")
77 if self.worker_bug:
78 state.append(f"{red}WORKER BUG{reset}")
79 if not state:
80 state.append(f"{green}SUCCESS{reset}")
81
82 return ', '.join(state)
83
84 def get_exitcode(self, fail_env_changed: bool, fail_rerun: bool) -> int:
85 exitcode = 0
86 if self.bad:
87 exitcode = EXITCODE_BAD_TEST
88 elif self.interrupted:
89 exitcode = EXITCODE_INTERRUPTED
90 elif fail_env_changed and self.env_changed:
91 exitcode = EXITCODE_ENV_CHANGED
92 elif self.no_tests_run():
93 exitcode = EXITCODE_NO_TESTS_RAN
94 elif fail_rerun and self.rerun:
95 exitcode = EXITCODE_RERUN_FAIL
96 elif self.worker_bug:
97 exitcode = EXITCODE_BAD_TEST
98 return exitcode
99
100 def accumulate_result(self, result: TestResult, runtests: RunTests) -> None:
101 test_name = result.test_name
102 rerun = runtests.rerun
103 fail_env_changed = runtests.fail_env_changed
104
105 match result.state:
106 case State.PASSED:
107 self.good.append(test_name)
108 case State.ENV_CHANGED:
109 self.env_changed.append(test_name)
110 self.rerun_results.append(result)
111 case State.SKIPPED:
112 self.skipped.append(test_name)
113 case State.RESOURCE_DENIED:
114 self.resource_denied.append(test_name)
115 case State.INTERRUPTED:
116 self.interrupted = True
117 case State.DID_NOT_RUN:
118 self.run_no_tests.append(test_name)
119 case _:
120 if result.is_failed(fail_env_changed):
121 self.bad.append(test_name)
122 self.rerun_results.append(result)
123 else:
124 raise ValueError(f"invalid test state: {result.state!r}")
125
126 if result.state == State.WORKER_BUG:
127 self.worker_bug = True
128
129 if result.has_meaningful_duration() and not rerun:
130 if result.duration is None:
131 raise ValueError("result.duration is None")
132 self.test_times.append((result.duration, test_name))
133 if result.stats is not None:
134 self.stats.accumulate(result.stats)
135 if rerun:
136 self.rerun.append(test_name)
137 if result.covered_lines:
138 # we don't care about trace counts so we don't have to sum them up
139 self.covered_lines.update(result.covered_lines)
140 xml_data = result.xml_data
141 if xml_data:
142 self.add_junit(xml_data)
143
144 def get_coverage_results(self) -> trace.CoverageResults:
145 counts = {loc: 1 for loc in self.covered_lines}
146 return trace.CoverageResults(counts=counts)
147
148 def need_rerun(self) -> bool:
149 return bool(self.rerun_results)
150
151 def prepare_rerun(self, *, clear: bool = True) -> tuple[TestTuple, FilterDict]:
152 tests: TestList = []
153 match_tests_dict = {}
154 for result in self.rerun_results:
155 tests.append(result.test_name)
156
157 match_tests = result.get_rerun_match_tests()
158 # ignore empty match list
159 if match_tests:
160 match_tests_dict[result.test_name] = match_tests
161
162 if clear:
163 # Clear previously failed tests
164 self.rerun_bad.extend(self.bad)
165 self.bad.clear()
166 self.env_changed.clear()
167 self.rerun_results.clear()
168
169 return (tuple(tests), match_tests_dict)
170
171 def add_junit(self, xml_data: list[str]) -> None:
172 import xml.etree.ElementTree as ET
173 for e in xml_data:
174 try:
175 self.testsuite_xml.append(ET.fromstring(e))
176 except ET.ParseError:
177 print(xml_data, file=sys.__stderr__)
178 raise
179
180 def write_junit(self, filename: StrPath) -> None:
181 if not self.testsuite_xml:
182 # Don't create empty XML file
183 return
184
185 import xml.etree.ElementTree as ET
186 root = ET.Element("testsuites")
187
188 # Manually count the totals for the overall summary
189 totals = {'tests': 0, 'errors': 0, 'failures': 0}
190 for suite in self.testsuite_xml:
191 root.append(suite)
192 for k in totals:
193 try:
194 totals[k] += int(suite.get(k, 0))
195 except ValueError:
196 pass
197
198 for k, v in totals.items():
199 root.set(k, str(v))
200
201 with open(filename, 'wb') as f:
202 for s in ET.tostringlist(root):
203 f.write(s)
204
205 def display_result(self, tests: TestTuple, quiet: bool, print_slowest: bool) -> None:
206 ansi = get_colors()
207 green = ansi.GREEN
208 red = ansi.BOLD_RED
209 reset = ansi.RESET
210 yellow = ansi.YELLOW
211
212 if print_slowest:
213 self.test_times.sort(reverse=True)
214 print()
215 print(f"{yellow}10 slowest tests:{reset}")
216 for test_time, test in self.test_times[:10]:
217 print(f"- {test}: {format_duration(test_time)}")
218
219 all_tests = []
220 omitted = set(tests) - self.get_executed()
221
222 # less important
223 all_tests.append(
224 (sorted(omitted), "test", f"{yellow}{{}} omitted:{reset}")
225 )
226 if not quiet:
227 all_tests.append(
228 (self.skipped, "test", f"{yellow}{{}} skipped:{reset}")
229 )
230 all_tests.append(
231 (
232 self.resource_denied,
233 "test",
234 f"{yellow}{{}} skipped (resource denied):{reset}",
235 )
236 )
237 all_tests.append(
238 (self.run_no_tests, "test", f"{yellow}{{}} run no tests:{reset}")
239 )
240
241 # more important
242 all_tests.append(
243 (
244 self.env_changed,
245 "test",
246 f"{yellow}{{}} altered the execution environment (env changed):{reset}",
247 )
248 )
249 all_tests.append((self.rerun, "re-run test", f"{yellow}{{}}:{reset}"))
250 all_tests.append((self.bad, "test", f"{red}{{}} failed:{reset}"))
251
252 for tests_list, count_text, title_format in all_tests:
253 if tests_list:
254 print()
255 count_text = count(len(tests_list), count_text)
256 print(title_format.format(count_text))
257 printlist(tests_list)
258
259 if self.good and not quiet:
260 print()
261 text = count(len(self.good), "test")
262 text = f"{green}{text} OK.{reset}"
263 if self.is_all_good() and len(self.good) > 1:
264 text = f"All {text}"
265 print(text)
266
267 if self.interrupted:
268 print()
269 print(f"{yellow}Test suite interrupted by signal SIGINT.{reset}")
270
271 def display_summary(self, first_runtests: RunTests, filtered: bool) -> None:
272 # Total tests
273 ansi = get_colors()
274 red, reset, yellow = ansi.RED, ansi.RESET, ansi.YELLOW
275
276 stats = self.stats
277 text = f'run={stats.tests_run:,}'
278 if filtered:
279 text = f"{text} (filtered)"
280 report = [text]
281 if stats.failures:
282 report.append(f'{red}failures={stats.failures:,}{reset}')
283 if stats.skipped:
284 report.append(f'{yellow}skipped={stats.skipped:,}{reset}')
285 print(f"Total tests: {' '.join(report)}")
286
287 # Total test files
288 all_tests = [self.good, self.bad, self.rerun,
289 self.skipped,
290 self.env_changed, self.run_no_tests]
291 run = sum(map(len, all_tests))
292 text = f'run={run}'
293 if not first_runtests.forever:
294 ntest = len(first_runtests.tests)
295 text = f"{text}/{ntest}"
296 if filtered:
297 text = f"{text} (filtered)"
298 report = [text]
299 for name, tests, color in (
300 ('failed', self.bad, red),
301 ('env_changed', self.env_changed, yellow),
302 ('skipped', self.skipped, yellow),
303 ('resource_denied', self.resource_denied, yellow),
304 ('rerun', self.rerun, yellow),
305 ('run_no_tests', self.run_no_tests, yellow),
306 ):
307 if tests:
308 report.append(f'{color}{name}={len(tests)}{reset}')
309 print(f"Total test files: {' '.join(report)}")