| 1 |
import faulthandler |
| 2 |
import gc |
| 3 |
import importlib |
| 4 |
import io |
| 5 |
import sys |
| 6 |
import time |
| 7 |
import traceback |
| 8 |
import unittest |
| 9 |
|
| 10 |
from _colorize import get_colors # type: ignore[import-not-found] |
| 11 |
from test import support |
| 12 |
from test.support import threading_helper |
| 13 |
|
| 14 |
from .filter import match_test |
| 15 |
from .result import State, TestResult, TestStats |
| 16 |
from .runtests import RunTests |
| 17 |
from .save_env import saved_test_environment |
| 18 |
from .setup import setup_tests |
| 19 |
from .testresult import get_test_runner |
| 20 |
from .parallel_case import ParallelTestCase |
| 21 |
from .utils import ( |
| 22 |
TestName, |
| 23 |
clear_caches, remove_testfn, abs_module_name, print_warning) |
| 24 |
|
| 25 |
|
| 26 |
# Minimum duration of a test to display its duration or to mention that |
| 27 |
# the test is running in background |
| 28 |
PROGRESS_MIN_TIME = 30.0 # seconds |
| 29 |
|
| 30 |
|
| 31 |
def run_unittest(test_mod, runtests: RunTests): |
| 32 |
loader = unittest.TestLoader() |
| 33 |
tests = loader.loadTestsFromModule(test_mod) |
| 34 |
|
| 35 |
for error in loader.errors: |
| 36 |
print(error, file=sys.stderr) |
| 37 |
if loader.errors: |
| 38 |
raise Exception("errors while loading tests") |
| 39 |
_filter_suite(tests, match_test) |
| 40 |
if runtests.parallel_threads: |
| 41 |
_parallelize_tests(tests, runtests.parallel_threads) |
| 42 |
return _run_suite(tests) |
| 43 |
|
| 44 |
def _filter_suite(suite, pred): |
| 45 |
"""Recursively filter test cases in a suite based on a predicate.""" |
| 46 |
newtests = [] |
| 47 |
for test in suite._tests: |
| 48 |
if isinstance(test, unittest.TestSuite): |
| 49 |
_filter_suite(test, pred) |
| 50 |
newtests.append(test) |
| 51 |
else: |
| 52 |
if pred(test): |
| 53 |
newtests.append(test) |
| 54 |
suite._tests = newtests |
| 55 |
|
| 56 |
def _parallelize_tests(suite, parallel_threads: int): |
| 57 |
def is_thread_unsafe(test): |
| 58 |
test_method = getattr(test, test._testMethodName) |
| 59 |
instance = test_method.__self__ |
| 60 |
return (getattr(test_method, "__unittest_thread_unsafe__", False) or |
| 61 |
getattr(instance, "__unittest_thread_unsafe__", False)) |
| 62 |
|
| 63 |
newtests: list[object] = [] |
| 64 |
for test in suite._tests: |
| 65 |
if isinstance(test, unittest.TestSuite): |
| 66 |
_parallelize_tests(test, parallel_threads) |
| 67 |
newtests.append(test) |
| 68 |
continue |
| 69 |
|
| 70 |
if is_thread_unsafe(test): |
| 71 |
# Don't parallelize thread-unsafe tests |
| 72 |
newtests.append(test) |
| 73 |
continue |
| 74 |
|
| 75 |
newtests.append(ParallelTestCase(test, parallel_threads)) |
| 76 |
suite._tests = newtests |
| 77 |
|
| 78 |
def _run_suite(suite): |
| 79 |
"""Run tests from a unittest.TestSuite-derived class.""" |
| 80 |
runner = get_test_runner(sys.stdout, |
| 81 |
verbosity=support.verbose, |
| 82 |
capture_output=(support.junit_xml_list is not None)) |
| 83 |
|
| 84 |
result = runner.run(suite) |
| 85 |
|
| 86 |
if support.junit_xml_list is not None: |
| 87 |
import xml.etree.ElementTree as ET |
| 88 |
xml_elem = result.get_xml_element() |
| 89 |
xml_str = ET.tostring(xml_elem).decode('ascii') |
| 90 |
support.junit_xml_list.append(xml_str) |
| 91 |
|
| 92 |
if not result.testsRun and not result.skipped and not result.errors: |
| 93 |
raise support.TestDidNotRun |
| 94 |
if not result.wasSuccessful(): |
| 95 |
stats = TestStats.from_unittest(result) |
| 96 |
if len(result.errors) == 1 and not result.failures: |
| 97 |
err = result.errors[0][1] |
| 98 |
elif len(result.failures) == 1 and not result.errors: |
| 99 |
err = result.failures[0][1] |
| 100 |
else: |
| 101 |
err = "multiple errors occurred" |
| 102 |
if not support.verbose: err += "; run in verbose mode for details" |
| 103 |
errors = [(str(tc), exc_str) for tc, exc_str in result.errors] |
| 104 |
failures = [(str(tc), exc_str) for tc, exc_str in result.failures] |
| 105 |
raise support.TestFailedWithDetails(err, errors, failures, stats=stats) |
| 106 |
return result |
| 107 |
|
| 108 |
|
| 109 |
def regrtest_runner(result: TestResult, test_func, runtests: RunTests) -> None: |
| 110 |
# Run test_func(), collect statistics, and detect reference and memory |
| 111 |
# leaks. |
| 112 |
if runtests.hunt_refleak: |
| 113 |
from .refleak import runtest_refleak |
| 114 |
refleak, test_result = runtest_refleak(result.test_name, test_func, |
| 115 |
runtests.hunt_refleak, |
| 116 |
runtests.quiet) |
| 117 |
else: |
| 118 |
test_result = test_func() |
| 119 |
refleak = False |
| 120 |
|
| 121 |
if refleak: |
| 122 |
result.state = State.REFLEAK |
| 123 |
|
| 124 |
stats: TestStats | None |
| 125 |
|
| 126 |
match test_result: |
| 127 |
case TestStats(): |
| 128 |
stats = test_result |
| 129 |
case unittest.TestResult(): |
| 130 |
stats = TestStats.from_unittest(test_result) |
| 131 |
case None: |
| 132 |
print_warning(f"{result.test_name} test runner returned None: {test_func}") |
| 133 |
stats = None |
| 134 |
case _: |
| 135 |
# Don't import doctest at top level since only few tests return |
| 136 |
# a doctest.TestResult instance. |
| 137 |
import doctest |
| 138 |
if isinstance(test_result, doctest.TestResults): |
| 139 |
stats = TestStats.from_doctest(test_result) |
| 140 |
else: |
| 141 |
print_warning(f"Unknown test result type: {type(test_result)}") |
| 142 |
stats = None |
| 143 |
|
| 144 |
result.stats = stats |
| 145 |
|
| 146 |
|
| 147 |
# Storage of uncollectable GC objects (gc.garbage) |
| 148 |
GC_GARBAGE = [] |
| 149 |
|
| 150 |
|
| 151 |
def _load_run_test(result: TestResult, runtests: RunTests) -> None: |
| 152 |
# Load the test module and run the tests. |
| 153 |
test_name = result.test_name |
| 154 |
module_name = abs_module_name(test_name, runtests.test_dir) |
| 155 |
test_mod = importlib.import_module(module_name) |
| 156 |
|
| 157 |
if hasattr(test_mod, "test_main"): |
| 158 |
# https://github.com/python/cpython/issues/89392 |
| 159 |
raise Exception(f"Module {test_name} defines test_main() which " |
| 160 |
f"is no longer supported by regrtest") |
| 161 |
def test_func(): |
| 162 |
return run_unittest(test_mod, runtests) |
| 163 |
|
| 164 |
try: |
| 165 |
regrtest_runner(result, test_func, runtests) |
| 166 |
finally: |
| 167 |
# First kill any dangling references to open files etc. |
| 168 |
# This can also issue some ResourceWarnings which would otherwise get |
| 169 |
# triggered during the following test run, and possibly produce |
| 170 |
# failures. |
| 171 |
support.gc_collect() |
| 172 |
|
| 173 |
remove_testfn(test_name, runtests.verbose) |
| 174 |
|
| 175 |
if gc.garbage: |
| 176 |
support.environment_altered = True |
| 177 |
print_warning(f"{test_name} created {len(gc.garbage)} " |
| 178 |
f"uncollectable object(s)") |
| 179 |
|
| 180 |
# move the uncollectable objects somewhere, |
| 181 |
# so we don't see them again |
| 182 |
GC_GARBAGE.extend(gc.garbage) |
| 183 |
gc.garbage.clear() |
| 184 |
|
| 185 |
support.reap_children() |
| 186 |
|
| 187 |
|
| 188 |
def _runtest_env_changed_exc(result: TestResult, runtests: RunTests, |
| 189 |
display_failure: bool = True) -> None: |
| 190 |
# Handle exceptions, detect environment changes. |
| 191 |
stdout = get_colors(file=sys.stdout) |
| 192 |
stderr = get_colors(file=sys.stderr) |
| 193 |
|
| 194 |
# Reset the environment_altered flag to detect if a test altered |
| 195 |
# the environment |
| 196 |
support.environment_altered = False |
| 197 |
|
| 198 |
pgo = runtests.pgo |
| 199 |
if pgo: |
| 200 |
display_failure = False |
| 201 |
quiet = runtests.quiet |
| 202 |
|
| 203 |
test_name = result.test_name |
| 204 |
try: |
| 205 |
clear_caches() |
| 206 |
support.gc_collect() |
| 207 |
|
| 208 |
with saved_test_environment(test_name, |
| 209 |
runtests.verbose, quiet, pgo=pgo): |
| 210 |
_load_run_test(result, runtests) |
| 211 |
except support.ResourceDenied as exc: |
| 212 |
if not quiet and not pgo: |
| 213 |
print( |
| 214 |
f"{stdout.YELLOW}{test_name} skipped -- {exc}{stdout.RESET}", |
| 215 |
flush=True, |
| 216 |
) |
| 217 |
result.state = State.RESOURCE_DENIED |
| 218 |
return |
| 219 |
except unittest.SkipTest as exc: |
| 220 |
if not quiet and not pgo: |
| 221 |
print( |
| 222 |
f"{stdout.YELLOW}{test_name} skipped -- {exc}{stdout.RESET}", |
| 223 |
flush=True, |
| 224 |
) |
| 225 |
result.state = State.SKIPPED |
| 226 |
return |
| 227 |
except support.TestFailedWithDetails as exc: |
| 228 |
msg = f"{stderr.RED}test {test_name} failed{stderr.RESET}" |
| 229 |
if display_failure: |
| 230 |
msg = f"{stderr.RED}{msg} -- {exc}{stderr.RESET}" |
| 231 |
print(msg, file=sys.stderr, flush=True) |
| 232 |
result.state = State.FAILED |
| 233 |
result.errors = exc.errors |
| 234 |
result.failures = exc.failures |
| 235 |
result.stats = exc.stats |
| 236 |
return |
| 237 |
except support.TestFailed as exc: |
| 238 |
msg = f"{stderr.RED}test {test_name} failed{stderr.RESET}" |
| 239 |
if display_failure: |
| 240 |
msg = f"{stderr.RED}{msg} -- {exc}{stderr.RESET}" |
| 241 |
print(msg, file=sys.stderr, flush=True) |
| 242 |
result.state = State.FAILED |
| 243 |
result.stats = exc.stats |
| 244 |
return |
| 245 |
except support.TestDidNotRun: |
| 246 |
result.state = State.DID_NOT_RUN |
| 247 |
return |
| 248 |
except KeyboardInterrupt: |
| 249 |
print() |
| 250 |
result.state = State.INTERRUPTED |
| 251 |
return |
| 252 |
except: |
| 253 |
if not pgo: |
| 254 |
msg = traceback.format_exc() |
| 255 |
print( |
| 256 |
f"{stderr.RED}test {test_name} crashed -- {msg}{stderr.RESET}", |
| 257 |
file=sys.stderr, |
| 258 |
flush=True, |
| 259 |
) |
| 260 |
result.state = State.UNCAUGHT_EXC |
| 261 |
return |
| 262 |
|
| 263 |
if support.environment_altered: |
| 264 |
result.set_env_changed() |
| 265 |
# Don't override the state if it was already set (REFLEAK or ENV_CHANGED) |
| 266 |
if result.state is None: |
| 267 |
result.state = State.PASSED |
| 268 |
|
| 269 |
|
| 270 |
def _runtest(result: TestResult, runtests: RunTests) -> None: |
| 271 |
# Capture stdout and stderr, set faulthandler timeout, |
| 272 |
# and create JUnit XML report. |
| 273 |
verbose = runtests.verbose |
| 274 |
output_on_failure = runtests.output_on_failure |
| 275 |
timeout = runtests.timeout |
| 276 |
|
| 277 |
if timeout is not None and threading_helper.can_start_thread: |
| 278 |
use_timeout = True |
| 279 |
faulthandler.dump_traceback_later(timeout, exit=True) |
| 280 |
else: |
| 281 |
use_timeout = False |
| 282 |
|
| 283 |
try: |
| 284 |
setup_tests(runtests) |
| 285 |
|
| 286 |
if output_on_failure or runtests.pgo: |
| 287 |
support.verbose = True |
| 288 |
|
| 289 |
stream = io.StringIO() |
| 290 |
orig_stdout = sys.stdout |
| 291 |
orig_stderr = sys.stderr |
| 292 |
print_warning = support.print_warning |
| 293 |
orig_print_warnings_stderr = print_warning.orig_stderr |
| 294 |
|
| 295 |
output = None |
| 296 |
try: |
| 297 |
sys.stdout = stream |
| 298 |
sys.stderr = stream |
| 299 |
# print_warning() writes into the temporary stream to preserve |
| 300 |
# messages order. If support.environment_altered becomes true, |
| 301 |
# warnings will be written to sys.stderr below. |
| 302 |
print_warning.orig_stderr = stream |
| 303 |
|
| 304 |
_runtest_env_changed_exc(result, runtests, display_failure=False) |
| 305 |
# Ignore output if the test passed successfully |
| 306 |
if result.state != State.PASSED: |
| 307 |
output = stream.getvalue() |
| 308 |
finally: |
| 309 |
sys.stdout = orig_stdout |
| 310 |
sys.stderr = orig_stderr |
| 311 |
print_warning.orig_stderr = orig_print_warnings_stderr |
| 312 |
|
| 313 |
if output is not None: |
| 314 |
sys.stderr.write(output) |
| 315 |
sys.stderr.flush() |
| 316 |
else: |
| 317 |
# Tell tests to be moderately quiet |
| 318 |
support.verbose = verbose |
| 319 |
_runtest_env_changed_exc(result, runtests, |
| 320 |
display_failure=not verbose) |
| 321 |
|
| 322 |
xml_list = support.junit_xml_list |
| 323 |
if xml_list: |
| 324 |
result.xml_data = xml_list |
| 325 |
finally: |
| 326 |
if use_timeout: |
| 327 |
faulthandler.cancel_dump_traceback_later() |
| 328 |
support.junit_xml_list = None |
| 329 |
|
| 330 |
|
| 331 |
def run_single_test(test_name: TestName, runtests: RunTests) -> TestResult: |
| 332 |
"""Run a single test. |
| 333 |
|
| 334 |
test_name -- the name of the test |
| 335 |
|
| 336 |
Returns a TestResult. |
| 337 |
|
| 338 |
If runtests.use_junit, xml_data is a list containing each generated |
| 339 |
testsuite element. |
| 340 |
""" |
| 341 |
ansi = get_colors(file=sys.stderr) |
| 342 |
red, reset, yellow = ansi.BOLD_RED, ansi.RESET, ansi.YELLOW |
| 343 |
|
| 344 |
start_time = time.perf_counter() |
| 345 |
result = TestResult(test_name) |
| 346 |
pgo = runtests.pgo |
| 347 |
try: |
| 348 |
_runtest(result, runtests) |
| 349 |
except: |
| 350 |
if not pgo: |
| 351 |
msg = traceback.format_exc() |
| 352 |
print(f"{red}test {test_name} crashed -- {msg}{reset}", |
| 353 |
file=sys.stderr, flush=True) |
| 354 |
result.state = State.UNCAUGHT_EXC |
| 355 |
|
| 356 |
sys.stdout.flush() |
| 357 |
sys.stderr.flush() |
| 358 |
|
| 359 |
result.duration = time.perf_counter() - start_time |
| 360 |
return result |