Debian Codegraph @

Lib/test/libregrtest/single.py

raw
1 import faulthandler
2 import gc
3 import importlib
4 import io
5 import sys
6 import time
7 import traceback
8 import unittest
9
10 from _colorize import get_colors # type: ignore[import-not-found]
11 from test import support
12 from test.support import threading_helper
13
14 from .filter import match_test
15 from .result import State, TestResult, TestStats
16 from .runtests import RunTests
17 from .save_env import saved_test_environment
18 from .setup import setup_tests
19 from .testresult import get_test_runner
20 from .parallel_case import ParallelTestCase
21 from .utils import (
22 TestName,
23 clear_caches, remove_testfn, abs_module_name, print_warning)
24
25
26 # Minimum duration of a test to display its duration or to mention that
27 # the test is running in background
28 PROGRESS_MIN_TIME = 30.0 # seconds
29
30
31 def run_unittest(test_mod, runtests: RunTests):
32 loader = unittest.TestLoader()
33 tests = loader.loadTestsFromModule(test_mod)
34
35 for error in loader.errors:
36 print(error, file=sys.stderr)
37 if loader.errors:
38 raise Exception("errors while loading tests")
39 _filter_suite(tests, match_test)
40 if runtests.parallel_threads:
41 _parallelize_tests(tests, runtests.parallel_threads)
42 return _run_suite(tests)
43
44 def _filter_suite(suite, pred):
45 """Recursively filter test cases in a suite based on a predicate."""
46 newtests = []
47 for test in suite._tests:
48 if isinstance(test, unittest.TestSuite):
49 _filter_suite(test, pred)
50 newtests.append(test)
51 else:
52 if pred(test):
53 newtests.append(test)
54 suite._tests = newtests
55
56 def _parallelize_tests(suite, parallel_threads: int):
57 def is_thread_unsafe(test):
58 test_method = getattr(test, test._testMethodName)
59 instance = test_method.__self__
60 return (getattr(test_method, "__unittest_thread_unsafe__", False) or
61 getattr(instance, "__unittest_thread_unsafe__", False))
62
63 newtests: list[object] = []
64 for test in suite._tests:
65 if isinstance(test, unittest.TestSuite):
66 _parallelize_tests(test, parallel_threads)
67 newtests.append(test)
68 continue
69
70 if is_thread_unsafe(test):
71 # Don't parallelize thread-unsafe tests
72 newtests.append(test)
73 continue
74
75 newtests.append(ParallelTestCase(test, parallel_threads))
76 suite._tests = newtests
77
78 def _run_suite(suite):
79 """Run tests from a unittest.TestSuite-derived class."""
80 runner = get_test_runner(sys.stdout,
81 verbosity=support.verbose,
82 capture_output=(support.junit_xml_list is not None))
83
84 result = runner.run(suite)
85
86 if support.junit_xml_list is not None:
87 import xml.etree.ElementTree as ET
88 xml_elem = result.get_xml_element()
89 xml_str = ET.tostring(xml_elem).decode('ascii')
90 support.junit_xml_list.append(xml_str)
91
92 if not result.testsRun and not result.skipped and not result.errors:
93 raise support.TestDidNotRun
94 if not result.wasSuccessful():
95 stats = TestStats.from_unittest(result)
96 if len(result.errors) == 1 and not result.failures:
97 err = result.errors[0][1]
98 elif len(result.failures) == 1 and not result.errors:
99 err = result.failures[0][1]
100 else:
101 err = "multiple errors occurred"
102 if not support.verbose: err += "; run in verbose mode for details"
103 errors = [(str(tc), exc_str) for tc, exc_str in result.errors]
104 failures = [(str(tc), exc_str) for tc, exc_str in result.failures]
105 raise support.TestFailedWithDetails(err, errors, failures, stats=stats)
106 return result
107
108
109 def regrtest_runner(result: TestResult, test_func, runtests: RunTests) -> None:
110 # Run test_func(), collect statistics, and detect reference and memory
111 # leaks.
112 if runtests.hunt_refleak:
113 from .refleak import runtest_refleak
114 refleak, test_result = runtest_refleak(result.test_name, test_func,
115 runtests.hunt_refleak,
116 runtests.quiet)
117 else:
118 test_result = test_func()
119 refleak = False
120
121 if refleak:
122 result.state = State.REFLEAK
123
124 stats: TestStats | None
125
126 match test_result:
127 case TestStats():
128 stats = test_result
129 case unittest.TestResult():
130 stats = TestStats.from_unittest(test_result)
131 case None:
132 print_warning(f"{result.test_name} test runner returned None: {test_func}")
133 stats = None
134 case _:
135 # Don't import doctest at top level since only few tests return
136 # a doctest.TestResult instance.
137 import doctest
138 if isinstance(test_result, doctest.TestResults):
139 stats = TestStats.from_doctest(test_result)
140 else:
141 print_warning(f"Unknown test result type: {type(test_result)}")
142 stats = None
143
144 result.stats = stats
145
146
147 # Storage of uncollectable GC objects (gc.garbage)
148 GC_GARBAGE = []
149
150
151 def _load_run_test(result: TestResult, runtests: RunTests) -> None:
152 # Load the test module and run the tests.
153 test_name = result.test_name
154 module_name = abs_module_name(test_name, runtests.test_dir)
155 test_mod = importlib.import_module(module_name)
156
157 if hasattr(test_mod, "test_main"):
158 # https://github.com/python/cpython/issues/89392
159 raise Exception(f"Module {test_name} defines test_main() which "
160 f"is no longer supported by regrtest")
161 def test_func():
162 return run_unittest(test_mod, runtests)
163
164 try:
165 regrtest_runner(result, test_func, runtests)
166 finally:
167 # First kill any dangling references to open files etc.
168 # This can also issue some ResourceWarnings which would otherwise get
169 # triggered during the following test run, and possibly produce
170 # failures.
171 support.gc_collect()
172
173 remove_testfn(test_name, runtests.verbose)
174
175 if gc.garbage:
176 support.environment_altered = True
177 print_warning(f"{test_name} created {len(gc.garbage)} "
178 f"uncollectable object(s)")
179
180 # move the uncollectable objects somewhere,
181 # so we don't see them again
182 GC_GARBAGE.extend(gc.garbage)
183 gc.garbage.clear()
184
185 support.reap_children()
186
187
188 def _runtest_env_changed_exc(result: TestResult, runtests: RunTests,
189 display_failure: bool = True) -> None:
190 # Handle exceptions, detect environment changes.
191 stdout = get_colors(file=sys.stdout)
192 stderr = get_colors(file=sys.stderr)
193
194 # Reset the environment_altered flag to detect if a test altered
195 # the environment
196 support.environment_altered = False
197
198 pgo = runtests.pgo
199 if pgo:
200 display_failure = False
201 quiet = runtests.quiet
202
203 test_name = result.test_name
204 try:
205 clear_caches()
206 support.gc_collect()
207
208 with saved_test_environment(test_name,
209 runtests.verbose, quiet, pgo=pgo):
210 _load_run_test(result, runtests)
211 except support.ResourceDenied as exc:
212 if not quiet and not pgo:
213 print(
214 f"{stdout.YELLOW}{test_name} skipped -- {exc}{stdout.RESET}",
215 flush=True,
216 )
217 result.state = State.RESOURCE_DENIED
218 return
219 except unittest.SkipTest as exc:
220 if not quiet and not pgo:
221 print(
222 f"{stdout.YELLOW}{test_name} skipped -- {exc}{stdout.RESET}",
223 flush=True,
224 )
225 result.state = State.SKIPPED
226 return
227 except support.TestFailedWithDetails as exc:
228 msg = f"{stderr.RED}test {test_name} failed{stderr.RESET}"
229 if display_failure:
230 msg = f"{stderr.RED}{msg} -- {exc}{stderr.RESET}"
231 print(msg, file=sys.stderr, flush=True)
232 result.state = State.FAILED
233 result.errors = exc.errors
234 result.failures = exc.failures
235 result.stats = exc.stats
236 return
237 except support.TestFailed as exc:
238 msg = f"{stderr.RED}test {test_name} failed{stderr.RESET}"
239 if display_failure:
240 msg = f"{stderr.RED}{msg} -- {exc}{stderr.RESET}"
241 print(msg, file=sys.stderr, flush=True)
242 result.state = State.FAILED
243 result.stats = exc.stats
244 return
245 except support.TestDidNotRun:
246 result.state = State.DID_NOT_RUN
247 return
248 except KeyboardInterrupt:
249 print()
250 result.state = State.INTERRUPTED
251 return
252 except:
253 if not pgo:
254 msg = traceback.format_exc()
255 print(
256 f"{stderr.RED}test {test_name} crashed -- {msg}{stderr.RESET}",
257 file=sys.stderr,
258 flush=True,
259 )
260 result.state = State.UNCAUGHT_EXC
261 return
262
263 if support.environment_altered:
264 result.set_env_changed()
265 # Don't override the state if it was already set (REFLEAK or ENV_CHANGED)
266 if result.state is None:
267 result.state = State.PASSED
268
269
270 def _runtest(result: TestResult, runtests: RunTests) -> None:
271 # Capture stdout and stderr, set faulthandler timeout,
272 # and create JUnit XML report.
273 verbose = runtests.verbose
274 output_on_failure = runtests.output_on_failure
275 timeout = runtests.timeout
276
277 if timeout is not None and threading_helper.can_start_thread:
278 use_timeout = True
279 faulthandler.dump_traceback_later(timeout, exit=True)
280 else:
281 use_timeout = False
282
283 try:
284 setup_tests(runtests)
285
286 if output_on_failure or runtests.pgo:
287 support.verbose = True
288
289 stream = io.StringIO()
290 orig_stdout = sys.stdout
291 orig_stderr = sys.stderr
292 print_warning = support.print_warning
293 orig_print_warnings_stderr = print_warning.orig_stderr
294
295 output = None
296 try:
297 sys.stdout = stream
298 sys.stderr = stream
299 # print_warning() writes into the temporary stream to preserve
300 # messages order. If support.environment_altered becomes true,
301 # warnings will be written to sys.stderr below.
302 print_warning.orig_stderr = stream
303
304 _runtest_env_changed_exc(result, runtests, display_failure=False)
305 # Ignore output if the test passed successfully
306 if result.state != State.PASSED:
307 output = stream.getvalue()
308 finally:
309 sys.stdout = orig_stdout
310 sys.stderr = orig_stderr
311 print_warning.orig_stderr = orig_print_warnings_stderr
312
313 if output is not None:
314 sys.stderr.write(output)
315 sys.stderr.flush()
316 else:
317 # Tell tests to be moderately quiet
318 support.verbose = verbose
319 _runtest_env_changed_exc(result, runtests,
320 display_failure=not verbose)
321
322 xml_list = support.junit_xml_list
323 if xml_list:
324 result.xml_data = xml_list
325 finally:
326 if use_timeout:
327 faulthandler.cancel_dump_traceback_later()
328 support.junit_xml_list = None
329
330
331 def run_single_test(test_name: TestName, runtests: RunTests) -> TestResult:
332 """Run a single test.
333
334 test_name -- the name of the test
335
336 Returns a TestResult.
337
338 If runtests.use_junit, xml_data is a list containing each generated
339 testsuite element.
340 """
341 ansi = get_colors(file=sys.stderr)
342 red, reset, yellow = ansi.BOLD_RED, ansi.RESET, ansi.YELLOW
343
344 start_time = time.perf_counter()
345 result = TestResult(test_name)
346 pgo = runtests.pgo
347 try:
348 _runtest(result, runtests)
349 except:
350 if not pgo:
351 msg = traceback.format_exc()
352 print(f"{red}test {test_name} crashed -- {msg}{reset}",
353 file=sys.stderr, flush=True)
354 result.state = State.UNCAUGHT_EXC
355
356 sys.stdout.flush()
357 sys.stderr.flush()
358
359 result.duration = time.perf_counter() - start_time
360 return result