| 1 |
import sys |
| 2 |
import trace |
| 3 |
from _colorize import get_colors # type: ignore[import-not-found] |
| 4 |
from typing import TYPE_CHECKING |
| 5 |
|
| 6 |
from .runtests import RunTests |
| 7 |
from .result import State, TestResult, TestStats, Location |
| 8 |
from .utils import ( |
| 9 |
StrPath, TestName, TestTuple, TestList, FilterDict, |
| 10 |
printlist, count, format_duration) |
| 11 |
|
| 12 |
if TYPE_CHECKING: |
| 13 |
from xml.etree.ElementTree import Element |
| 14 |
|
| 15 |
|
| 16 |
# Python uses exit code 1 when an exception is not caught |
| 17 |
# argparse.ArgumentParser.error() uses exit code 2 |
| 18 |
EXITCODE_BAD_TEST = 2 |
| 19 |
EXITCODE_ENV_CHANGED = 3 |
| 20 |
EXITCODE_NO_TESTS_RAN = 4 |
| 21 |
EXITCODE_RERUN_FAIL = 5 |
| 22 |
EXITCODE_INTERRUPTED = 130 # 128 + signal.SIGINT=2 |
| 23 |
|
| 24 |
|
| 25 |
class TestResults: |
| 26 |
def __init__(self) -> None: |
| 27 |
self.bad: TestList = [] |
| 28 |
self.good: TestList = [] |
| 29 |
self.rerun_bad: TestList = [] |
| 30 |
self.skipped: TestList = [] |
| 31 |
self.resource_denied: TestList = [] |
| 32 |
self.env_changed: TestList = [] |
| 33 |
self.run_no_tests: TestList = [] |
| 34 |
self.rerun: TestList = [] |
| 35 |
self.rerun_results: list[TestResult] = [] |
| 36 |
|
| 37 |
self.interrupted: bool = False |
| 38 |
self.worker_bug: bool = False |
| 39 |
self.test_times: list[tuple[float, TestName]] = [] |
| 40 |
self.stats = TestStats() |
| 41 |
# used by --junit-xml |
| 42 |
self.testsuite_xml: list['Element'] = [] |
| 43 |
# used by -T with -j |
| 44 |
self.covered_lines: set[Location] = set() |
| 45 |
|
| 46 |
def is_all_good(self) -> bool: |
| 47 |
return (not self.bad |
| 48 |
and not self.skipped |
| 49 |
and not self.interrupted |
| 50 |
and not self.worker_bug) |
| 51 |
|
| 52 |
def get_executed(self) -> set[TestName]: |
| 53 |
return (set(self.good) | set(self.bad) | set(self.skipped) |
| 54 |
| set(self.resource_denied) | set(self.env_changed) |
| 55 |
| set(self.run_no_tests)) |
| 56 |
|
| 57 |
def no_tests_run(self) -> bool: |
| 58 |
return not any((self.good, self.bad, self.skipped, self.interrupted, |
| 59 |
self.env_changed)) |
| 60 |
|
| 61 |
def get_state(self, fail_env_changed: bool) -> str: |
| 62 |
state = [] |
| 63 |
ansi = get_colors() |
| 64 |
green = ansi.GREEN |
| 65 |
red = ansi.BOLD_RED |
| 66 |
reset = ansi.RESET |
| 67 |
yellow = ansi.YELLOW |
| 68 |
if self.bad: |
| 69 |
state.append(f"{red}FAILURE{reset}") |
| 70 |
elif fail_env_changed and self.env_changed: |
| 71 |
state.append(f"{yellow}ENV CHANGED{reset}") |
| 72 |
elif self.no_tests_run(): |
| 73 |
state.append(f"{yellow}NO TESTS RAN{reset}") |
| 74 |
|
| 75 |
if self.interrupted: |
| 76 |
state.append(f"{yellow}INTERRUPTED{reset}") |
| 77 |
if self.worker_bug: |
| 78 |
state.append(f"{red}WORKER BUG{reset}") |
| 79 |
if not state: |
| 80 |
state.append(f"{green}SUCCESS{reset}") |
| 81 |
|
| 82 |
return ', '.join(state) |
| 83 |
|
| 84 |
def get_exitcode(self, fail_env_changed: bool, fail_rerun: bool) -> int: |
| 85 |
exitcode = 0 |
| 86 |
if self.bad: |
| 87 |
exitcode = EXITCODE_BAD_TEST |
| 88 |
elif self.interrupted: |
| 89 |
exitcode = EXITCODE_INTERRUPTED |
| 90 |
elif fail_env_changed and self.env_changed: |
| 91 |
exitcode = EXITCODE_ENV_CHANGED |
| 92 |
elif self.no_tests_run(): |
| 93 |
exitcode = EXITCODE_NO_TESTS_RAN |
| 94 |
elif fail_rerun and self.rerun: |
| 95 |
exitcode = EXITCODE_RERUN_FAIL |
| 96 |
elif self.worker_bug: |
| 97 |
exitcode = EXITCODE_BAD_TEST |
| 98 |
return exitcode |
| 99 |
|
| 100 |
def accumulate_result(self, result: TestResult, runtests: RunTests) -> None: |
| 101 |
test_name = result.test_name |
| 102 |
rerun = runtests.rerun |
| 103 |
fail_env_changed = runtests.fail_env_changed |
| 104 |
|
| 105 |
match result.state: |
| 106 |
case State.PASSED: |
| 107 |
self.good.append(test_name) |
| 108 |
case State.ENV_CHANGED: |
| 109 |
self.env_changed.append(test_name) |
| 110 |
self.rerun_results.append(result) |
| 111 |
case State.SKIPPED: |
| 112 |
self.skipped.append(test_name) |
| 113 |
case State.RESOURCE_DENIED: |
| 114 |
self.resource_denied.append(test_name) |
| 115 |
case State.INTERRUPTED: |
| 116 |
self.interrupted = True |
| 117 |
case State.DID_NOT_RUN: |
| 118 |
self.run_no_tests.append(test_name) |
| 119 |
case _: |
| 120 |
if result.is_failed(fail_env_changed): |
| 121 |
self.bad.append(test_name) |
| 122 |
self.rerun_results.append(result) |
| 123 |
else: |
| 124 |
raise ValueError(f"invalid test state: {result.state!r}") |
| 125 |
|
| 126 |
if result.state == State.WORKER_BUG: |
| 127 |
self.worker_bug = True |
| 128 |
|
| 129 |
if result.has_meaningful_duration() and not rerun: |
| 130 |
if result.duration is None: |
| 131 |
raise ValueError("result.duration is None") |
| 132 |
self.test_times.append((result.duration, test_name)) |
| 133 |
if result.stats is not None: |
| 134 |
self.stats.accumulate(result.stats) |
| 135 |
if rerun: |
| 136 |
self.rerun.append(test_name) |
| 137 |
if result.covered_lines: |
| 138 |
# we don't care about trace counts so we don't have to sum them up |
| 139 |
self.covered_lines.update(result.covered_lines) |
| 140 |
xml_data = result.xml_data |
| 141 |
if xml_data: |
| 142 |
self.add_junit(xml_data) |
| 143 |
|
| 144 |
def get_coverage_results(self) -> trace.CoverageResults: |
| 145 |
counts = {loc: 1 for loc in self.covered_lines} |
| 146 |
return trace.CoverageResults(counts=counts) |
| 147 |
|
| 148 |
def need_rerun(self) -> bool: |
| 149 |
return bool(self.rerun_results) |
| 150 |
|
| 151 |
def prepare_rerun(self, *, clear: bool = True) -> tuple[TestTuple, FilterDict]: |
| 152 |
tests: TestList = [] |
| 153 |
match_tests_dict = {} |
| 154 |
for result in self.rerun_results: |
| 155 |
tests.append(result.test_name) |
| 156 |
|
| 157 |
match_tests = result.get_rerun_match_tests() |
| 158 |
# ignore empty match list |
| 159 |
if match_tests: |
| 160 |
match_tests_dict[result.test_name] = match_tests |
| 161 |
|
| 162 |
if clear: |
| 163 |
# Clear previously failed tests |
| 164 |
self.rerun_bad.extend(self.bad) |
| 165 |
self.bad.clear() |
| 166 |
self.env_changed.clear() |
| 167 |
self.rerun_results.clear() |
| 168 |
|
| 169 |
return (tuple(tests), match_tests_dict) |
| 170 |
|
| 171 |
def add_junit(self, xml_data: list[str]) -> None: |
| 172 |
import xml.etree.ElementTree as ET |
| 173 |
for e in xml_data: |
| 174 |
try: |
| 175 |
self.testsuite_xml.append(ET.fromstring(e)) |
| 176 |
except ET.ParseError: |
| 177 |
print(xml_data, file=sys.__stderr__) |
| 178 |
raise |
| 179 |
|
| 180 |
def write_junit(self, filename: StrPath) -> None: |
| 181 |
if not self.testsuite_xml: |
| 182 |
# Don't create empty XML file |
| 183 |
return |
| 184 |
|
| 185 |
import xml.etree.ElementTree as ET |
| 186 |
root = ET.Element("testsuites") |
| 187 |
|
| 188 |
# Manually count the totals for the overall summary |
| 189 |
totals = {'tests': 0, 'errors': 0, 'failures': 0} |
| 190 |
for suite in self.testsuite_xml: |
| 191 |
root.append(suite) |
| 192 |
for k in totals: |
| 193 |
try: |
| 194 |
totals[k] += int(suite.get(k, 0)) |
| 195 |
except ValueError: |
| 196 |
pass |
| 197 |
|
| 198 |
for k, v in totals.items(): |
| 199 |
root.set(k, str(v)) |
| 200 |
|
| 201 |
with open(filename, 'wb') as f: |
| 202 |
for s in ET.tostringlist(root): |
| 203 |
f.write(s) |
| 204 |
|
| 205 |
def display_result(self, tests: TestTuple, quiet: bool, print_slowest: bool) -> None: |
| 206 |
ansi = get_colors() |
| 207 |
green = ansi.GREEN |
| 208 |
red = ansi.BOLD_RED |
| 209 |
reset = ansi.RESET |
| 210 |
yellow = ansi.YELLOW |
| 211 |
|
| 212 |
if print_slowest: |
| 213 |
self.test_times.sort(reverse=True) |
| 214 |
print() |
| 215 |
print(f"{yellow}10 slowest tests:{reset}") |
| 216 |
for test_time, test in self.test_times[:10]: |
| 217 |
print(f"- {test}: {format_duration(test_time)}") |
| 218 |
|
| 219 |
all_tests = [] |
| 220 |
omitted = set(tests) - self.get_executed() |
| 221 |
|
| 222 |
# less important |
| 223 |
all_tests.append( |
| 224 |
(sorted(omitted), "test", f"{yellow}{{}} omitted:{reset}") |
| 225 |
) |
| 226 |
if not quiet: |
| 227 |
all_tests.append( |
| 228 |
(self.skipped, "test", f"{yellow}{{}} skipped:{reset}") |
| 229 |
) |
| 230 |
all_tests.append( |
| 231 |
( |
| 232 |
self.resource_denied, |
| 233 |
"test", |
| 234 |
f"{yellow}{{}} skipped (resource denied):{reset}", |
| 235 |
) |
| 236 |
) |
| 237 |
all_tests.append( |
| 238 |
(self.run_no_tests, "test", f"{yellow}{{}} run no tests:{reset}") |
| 239 |
) |
| 240 |
|
| 241 |
# more important |
| 242 |
all_tests.append( |
| 243 |
( |
| 244 |
self.env_changed, |
| 245 |
"test", |
| 246 |
f"{yellow}{{}} altered the execution environment (env changed):{reset}", |
| 247 |
) |
| 248 |
) |
| 249 |
all_tests.append((self.rerun, "re-run test", f"{yellow}{{}}:{reset}")) |
| 250 |
all_tests.append((self.bad, "test", f"{red}{{}} failed:{reset}")) |
| 251 |
|
| 252 |
for tests_list, count_text, title_format in all_tests: |
| 253 |
if tests_list: |
| 254 |
print() |
| 255 |
count_text = count(len(tests_list), count_text) |
| 256 |
print(title_format.format(count_text)) |
| 257 |
printlist(tests_list) |
| 258 |
|
| 259 |
if self.good and not quiet: |
| 260 |
print() |
| 261 |
text = count(len(self.good), "test") |
| 262 |
text = f"{green}{text} OK.{reset}" |
| 263 |
if self.is_all_good() and len(self.good) > 1: |
| 264 |
text = f"All {text}" |
| 265 |
print(text) |
| 266 |
|
| 267 |
if self.interrupted: |
| 268 |
print() |
| 269 |
print(f"{yellow}Test suite interrupted by signal SIGINT.{reset}") |
| 270 |
|
| 271 |
def display_summary(self, first_runtests: RunTests, filtered: bool) -> None: |
| 272 |
# Total tests |
| 273 |
ansi = get_colors() |
| 274 |
red, reset, yellow = ansi.RED, ansi.RESET, ansi.YELLOW |
| 275 |
|
| 276 |
stats = self.stats |
| 277 |
text = f'run={stats.tests_run:,}' |
| 278 |
if filtered: |
| 279 |
text = f"{text} (filtered)" |
| 280 |
report = [text] |
| 281 |
if stats.failures: |
| 282 |
report.append(f'{red}failures={stats.failures:,}{reset}') |
| 283 |
if stats.skipped: |
| 284 |
report.append(f'{yellow}skipped={stats.skipped:,}{reset}') |
| 285 |
print(f"Total tests: {' '.join(report)}") |
| 286 |
|
| 287 |
# Total test files |
| 288 |
all_tests = [self.good, self.bad, self.rerun, |
| 289 |
self.skipped, |
| 290 |
self.env_changed, self.run_no_tests] |
| 291 |
run = sum(map(len, all_tests)) |
| 292 |
text = f'run={run}' |
| 293 |
if not first_runtests.forever: |
| 294 |
ntest = len(first_runtests.tests) |
| 295 |
text = f"{text}/{ntest}" |
| 296 |
if filtered: |
| 297 |
text = f"{text} (filtered)" |
| 298 |
report = [text] |
| 299 |
for name, tests, color in ( |
| 300 |
('failed', self.bad, red), |
| 301 |
('env_changed', self.env_changed, yellow), |
| 302 |
('skipped', self.skipped, yellow), |
| 303 |
('resource_denied', self.resource_denied, yellow), |
| 304 |
('rerun', self.rerun, yellow), |
| 305 |
('run_no_tests', self.run_no_tests, yellow), |
| 306 |
): |
| 307 |
if tests: |
| 308 |
report.append(f'{color}{name}={len(tests)}{reset}') |
| 309 |
print(f"Total test files: {' '.join(report)}") |