| 1 |
import argparse |
| 2 |
import ctypes |
| 3 |
import faulthandler |
| 4 |
import hashlib |
| 5 |
import io |
| 6 |
import itertools |
| 7 |
import logging |
| 8 |
import multiprocessing |
| 9 |
import os |
| 10 |
import pickle |
| 11 |
import random |
| 12 |
import sys |
| 13 |
import textwrap |
| 14 |
import unittest |
| 15 |
import unittest.suite |
| 16 |
from collections import defaultdict |
| 17 |
from contextlib import contextmanager |
| 18 |
from importlib import import_module |
| 19 |
from io import StringIO |
| 20 |
|
| 21 |
import sqlparse |
| 22 |
|
| 23 |
import django |
| 24 |
from django.core.management import call_command |
| 25 |
from django.db import connections |
| 26 |
from django.test import SimpleTestCase, TestCase |
| 27 |
from django.test.utils import NullTimeKeeper, TimeKeeper, iter_test_cases |
| 28 |
from django.test.utils import setup_databases as _setup_databases |
| 29 |
from django.test.utils import setup_test_environment |
| 30 |
from django.test.utils import teardown_databases as _teardown_databases |
| 31 |
from django.test.utils import teardown_test_environment |
| 32 |
from django.utils.datastructures import OrderedSet |
| 33 |
from django.utils.version import PY312, PY313 |
| 34 |
|
| 35 |
try: |
| 36 |
import ipdb as pdb |
| 37 |
except ImportError: |
| 38 |
import pdb |
| 39 |
|
| 40 |
try: |
| 41 |
import tblib.pickling_support |
| 42 |
except ImportError: |
| 43 |
tblib = None |
| 44 |
|
| 45 |
|
| 46 |
class DebugSQLTextTestResult(unittest.TextTestResult): |
| 47 |
def __init__(self, stream, descriptions, verbosity): |
| 48 |
self.logger = logging.getLogger("django.db.backends") |
| 49 |
self.logger.setLevel(logging.DEBUG) |
| 50 |
self.debug_sql_stream = None |
| 51 |
super().__init__(stream, descriptions, verbosity) |
| 52 |
|
| 53 |
def startTest(self, test): |
| 54 |
self.debug_sql_stream = StringIO() |
| 55 |
self.handler = logging.StreamHandler(self.debug_sql_stream) |
| 56 |
self.logger.addHandler(self.handler) |
| 57 |
super().startTest(test) |
| 58 |
|
| 59 |
def stopTest(self, test): |
| 60 |
super().stopTest(test) |
| 61 |
self.logger.removeHandler(self.handler) |
| 62 |
if self.showAll: |
| 63 |
self.debug_sql_stream.seek(0) |
| 64 |
self.stream.write(self.debug_sql_stream.read()) |
| 65 |
self.stream.writeln(self.separator2) |
| 66 |
|
| 67 |
def addError(self, test, err): |
| 68 |
super().addError(test, err) |
| 69 |
if self.debug_sql_stream is None: |
| 70 |
# Error before tests e.g. in setUpTestData(). |
| 71 |
sql = "" |
| 72 |
else: |
| 73 |
self.debug_sql_stream.seek(0) |
| 74 |
sql = self.debug_sql_stream.read() |
| 75 |
self.errors[-1] = self.errors[-1] + (sql,) |
| 76 |
|
| 77 |
def addFailure(self, test, err): |
| 78 |
super().addFailure(test, err) |
| 79 |
self.debug_sql_stream.seek(0) |
| 80 |
self.failures[-1] = self.failures[-1] + (self.debug_sql_stream.read(),) |
| 81 |
|
| 82 |
def addSubTest(self, test, subtest, err): |
| 83 |
super().addSubTest(test, subtest, err) |
| 84 |
if err is not None: |
| 85 |
self.debug_sql_stream.seek(0) |
| 86 |
errors = ( |
| 87 |
self.failures |
| 88 |
if issubclass(err[0], test.failureException) |
| 89 |
else self.errors |
| 90 |
) |
| 91 |
errors[-1] = errors[-1] + (self.debug_sql_stream.read(),) |
| 92 |
|
| 93 |
def printErrorList(self, flavour, errors): |
| 94 |
for test, err, sql_debug in errors: |
| 95 |
self.stream.writeln(self.separator1) |
| 96 |
self.stream.writeln("%s: %s" % (flavour, self.getDescription(test))) |
| 97 |
self.stream.writeln(self.separator2) |
| 98 |
self.stream.writeln(err) |
| 99 |
self.stream.writeln(self.separator2) |
| 100 |
self.stream.writeln( |
| 101 |
sqlparse.format(sql_debug, reindent=True, keyword_case="upper") |
| 102 |
) |
| 103 |
|
| 104 |
|
| 105 |
class PDBDebugResult(unittest.TextTestResult): |
| 106 |
""" |
| 107 |
Custom result class that triggers a PDB session when an error or failure |
| 108 |
occurs. |
| 109 |
""" |
| 110 |
|
| 111 |
def addError(self, test, err): |
| 112 |
super().addError(test, err) |
| 113 |
self.debug(err) |
| 114 |
|
| 115 |
def addFailure(self, test, err): |
| 116 |
super().addFailure(test, err) |
| 117 |
self.debug(err) |
| 118 |
|
| 119 |
def addSubTest(self, test, subtest, err): |
| 120 |
if err is not None: |
| 121 |
self.debug(err) |
| 122 |
super().addSubTest(test, subtest, err) |
| 123 |
|
| 124 |
def debug(self, error): |
| 125 |
self._restoreStdout() |
| 126 |
self.buffer = False |
| 127 |
exc_type, exc_value, traceback = error |
| 128 |
print("\nOpening PDB: %r" % exc_value) |
| 129 |
if PY313: |
| 130 |
pdb.post_mortem(exc_value) |
| 131 |
else: |
| 132 |
pdb.post_mortem(traceback) |
| 133 |
|
| 134 |
|
| 135 |
class DummyList: |
| 136 |
""" |
| 137 |
Dummy list class for faking storage of results in unittest.TestResult. |
| 138 |
""" |
| 139 |
|
| 140 |
__slots__ = () |
| 141 |
|
| 142 |
def append(self, item): |
| 143 |
pass |
| 144 |
|
| 145 |
|
| 146 |
class RemoteTestResult(unittest.TestResult): |
| 147 |
""" |
| 148 |
Extend unittest.TestResult to record events in the child processes so they |
| 149 |
can be replayed in the parent process. Events include things like which |
| 150 |
tests succeeded or failed. |
| 151 |
""" |
| 152 |
|
| 153 |
def __init__(self, *args, **kwargs): |
| 154 |
super().__init__(*args, **kwargs) |
| 155 |
# Fake storage of results to reduce memory usage. These are used by the |
| 156 |
# unittest default methods, but here 'events' is used instead. |
| 157 |
dummy_list = DummyList() |
| 158 |
self.failures = dummy_list |
| 159 |
self.errors = dummy_list |
| 160 |
self.skipped = dummy_list |
| 161 |
self.expectedFailures = dummy_list |
| 162 |
self.unexpectedSuccesses = dummy_list |
| 163 |
|
| 164 |
if tblib is not None: |
| 165 |
tblib.pickling_support.install() |
| 166 |
self.events = [] |
| 167 |
|
| 168 |
def __getstate__(self): |
| 169 |
# Make this class picklable by removing the file-like buffer |
| 170 |
# attributes. This is possible since they aren't used after unpickling |
| 171 |
# after being sent to ParallelTestSuite. |
| 172 |
state = self.__dict__.copy() |
| 173 |
state.pop("_stdout_buffer", None) |
| 174 |
state.pop("_stderr_buffer", None) |
| 175 |
state.pop("_original_stdout", None) |
| 176 |
state.pop("_original_stderr", None) |
| 177 |
return state |
| 178 |
|
| 179 |
@property |
| 180 |
def test_index(self): |
| 181 |
return self.testsRun - 1 |
| 182 |
|
| 183 |
def _confirm_picklable(self, obj): |
| 184 |
""" |
| 185 |
Confirm that obj can be pickled and unpickled as multiprocessing will |
| 186 |
need to pickle the exception in the child process and unpickle it in |
| 187 |
the parent process. Let the exception rise, if not. |
| 188 |
""" |
| 189 |
pickle.loads(pickle.dumps(obj)) |
| 190 |
|
| 191 |
def _print_unpicklable_subtest(self, test, subtest, pickle_exc): |
| 192 |
print(""" |
| 193 |
Subtest failed: |
| 194 |
|
| 195 |
test: {} |
| 196 |
subtest: {} |
| 197 |
|
| 198 |
Unfortunately, the subtest that failed cannot be pickled, so the parallel |
| 199 |
test runner cannot handle it cleanly. Here is the pickling error: |
| 200 |
|
| 201 |
> {} |
| 202 |
|
| 203 |
You should re-run this test with --parallel=1 to reproduce the failure |
| 204 |
with a cleaner failure message. |
| 205 |
""".format(test, subtest, pickle_exc)) |
| 206 |
|
| 207 |
def check_picklable(self, test, err): |
| 208 |
# Ensure that sys.exc_info() tuples are picklable. This displays a |
| 209 |
# clear multiprocessing.pool.RemoteTraceback generated in the child |
| 210 |
# process instead of a multiprocessing.pool.MaybeEncodingError, making |
| 211 |
# the root cause easier to figure out for users who aren't familiar |
| 212 |
# with the multiprocessing module. Since we're in a forked process, |
| 213 |
# our best chance to communicate with them is to print to stdout. |
| 214 |
try: |
| 215 |
self._confirm_picklable(err) |
| 216 |
except Exception as exc: |
| 217 |
original_exc_txt = repr(err[1]) |
| 218 |
original_exc_txt = textwrap.fill( |
| 219 |
original_exc_txt, 75, initial_indent=" ", subsequent_indent=" " |
| 220 |
) |
| 221 |
pickle_exc_txt = repr(exc) |
| 222 |
pickle_exc_txt = textwrap.fill( |
| 223 |
pickle_exc_txt, 75, initial_indent=" ", subsequent_indent=" " |
| 224 |
) |
| 225 |
if tblib is None: |
| 226 |
print(""" |
| 227 |
|
| 228 |
{} failed: |
| 229 |
|
| 230 |
{} |
| 231 |
|
| 232 |
Unfortunately, tracebacks cannot be pickled, making it impossible for the |
| 233 |
parallel test runner to handle this exception cleanly. |
| 234 |
|
| 235 |
In order to see the traceback, you should install tblib: |
| 236 |
|
| 237 |
python -m pip install tblib |
| 238 |
""".format(test, original_exc_txt)) |
| 239 |
else: |
| 240 |
print(""" |
| 241 |
|
| 242 |
{} failed: |
| 243 |
|
| 244 |
{} |
| 245 |
|
| 246 |
Unfortunately, the exception it raised cannot be pickled, making it impossible |
| 247 |
for the parallel test runner to handle it cleanly. |
| 248 |
|
| 249 |
Here's the error encountered while trying to pickle the exception: |
| 250 |
|
| 251 |
{} |
| 252 |
|
| 253 |
You should re-run this test with the --parallel=1 option to reproduce the |
| 254 |
failure and get a correct traceback. |
| 255 |
""".format(test, original_exc_txt, pickle_exc_txt)) |
| 256 |
raise |
| 257 |
|
| 258 |
def check_subtest_picklable(self, test, subtest): |
| 259 |
try: |
| 260 |
self._confirm_picklable(subtest) |
| 261 |
except Exception as exc: |
| 262 |
self._print_unpicklable_subtest(test, subtest, exc) |
| 263 |
raise |
| 264 |
|
| 265 |
def startTestRun(self): |
| 266 |
super().startTestRun() |
| 267 |
self.events.append(("startTestRun",)) |
| 268 |
|
| 269 |
def stopTestRun(self): |
| 270 |
super().stopTestRun() |
| 271 |
self.events.append(("stopTestRun",)) |
| 272 |
|
| 273 |
def startTest(self, test): |
| 274 |
super().startTest(test) |
| 275 |
self.events.append(("startTest", self.test_index)) |
| 276 |
|
| 277 |
def stopTest(self, test): |
| 278 |
super().stopTest(test) |
| 279 |
self.events.append(("stopTest", self.test_index)) |
| 280 |
|
| 281 |
def addDuration(self, test, elapsed): |
| 282 |
super().addDuration(test, elapsed) |
| 283 |
self.events.append(("addDuration", self.test_index, elapsed)) |
| 284 |
|
| 285 |
def addError(self, test, err): |
| 286 |
self.check_picklable(test, err) |
| 287 |
|
| 288 |
event_occurred_before_first_test = self.test_index == -1 |
| 289 |
if event_occurred_before_first_test and isinstance( |
| 290 |
test, unittest.suite._ErrorHolder |
| 291 |
): |
| 292 |
self.events.append(("addError", self.test_index, test.id(), err)) |
| 293 |
else: |
| 294 |
self.events.append(("addError", self.test_index, err)) |
| 295 |
|
| 296 |
super().addError(test, err) |
| 297 |
|
| 298 |
def addFailure(self, test, err): |
| 299 |
self.check_picklable(test, err) |
| 300 |
self.events.append(("addFailure", self.test_index, err)) |
| 301 |
super().addFailure(test, err) |
| 302 |
|
| 303 |
def addSubTest(self, test, subtest, err): |
| 304 |
# Follow Python's implementation of unittest.TestResult.addSubTest() by |
| 305 |
# not doing anything when a subtest is successful. |
| 306 |
if err is not None: |
| 307 |
# Call check_picklable() before check_subtest_picklable() since |
| 308 |
# check_picklable() performs the tblib check. |
| 309 |
self.check_picklable(test, err) |
| 310 |
self.check_subtest_picklable(test, subtest) |
| 311 |
self.events.append(("addSubTest", self.test_index, subtest, err)) |
| 312 |
super().addSubTest(test, subtest, err) |
| 313 |
|
| 314 |
def addSuccess(self, test): |
| 315 |
self.events.append(("addSuccess", self.test_index)) |
| 316 |
super().addSuccess(test) |
| 317 |
|
| 318 |
def addSkip(self, test, reason): |
| 319 |
self.events.append(("addSkip", self.test_index, reason)) |
| 320 |
super().addSkip(test, reason) |
| 321 |
|
| 322 |
def addExpectedFailure(self, test, err): |
| 323 |
# If tblib isn't installed, pickling the traceback will always fail. |
| 324 |
# However we don't want tblib to be required for running the tests |
| 325 |
# when they pass or fail as expected. Drop the traceback when an |
| 326 |
# expected failure occurs. |
| 327 |
if tblib is None: |
| 328 |
err = err[0], err[1], None |
| 329 |
self.check_picklable(test, err) |
| 330 |
self.events.append(("addExpectedFailure", self.test_index, err)) |
| 331 |
super().addExpectedFailure(test, err) |
| 332 |
|
| 333 |
def addUnexpectedSuccess(self, test): |
| 334 |
self.events.append(("addUnexpectedSuccess", self.test_index)) |
| 335 |
super().addUnexpectedSuccess(test) |
| 336 |
|
| 337 |
def wasSuccessful(self): |
| 338 |
"""Tells whether or not this result was a success.""" |
| 339 |
failure_types = {"addError", "addFailure", "addSubTest", "addUnexpectedSuccess"} |
| 340 |
return all(e[0] not in failure_types for e in self.events) |
| 341 |
|
| 342 |
def _exc_info_to_string(self, err, test): |
| 343 |
# Make this method no-op. It only powers the default unittest behavior |
| 344 |
# for recording errors, but this class pickles errors into 'events' |
| 345 |
# instead. |
| 346 |
return "" |
| 347 |
|
| 348 |
|
| 349 |
class RemoteTestRunner: |
| 350 |
""" |
| 351 |
Run tests and record everything but don't display anything. |
| 352 |
|
| 353 |
The implementation matches the unpythonic coding style of unittest2. |
| 354 |
""" |
| 355 |
|
| 356 |
resultclass = RemoteTestResult |
| 357 |
|
| 358 |
def __init__(self, failfast=False, resultclass=None, buffer=False): |
| 359 |
self.failfast = failfast |
| 360 |
self.buffer = buffer |
| 361 |
if resultclass is not None: |
| 362 |
self.resultclass = resultclass |
| 363 |
|
| 364 |
def run(self, test): |
| 365 |
result = self.resultclass() |
| 366 |
unittest.registerResult(result) |
| 367 |
result.failfast = self.failfast |
| 368 |
result.buffer = self.buffer |
| 369 |
test(result) |
| 370 |
return result |
| 371 |
|
| 372 |
|
| 373 |
def get_max_test_processes(): |
| 374 |
""" |
| 375 |
The maximum number of test processes when using the --parallel option. |
| 376 |
""" |
| 377 |
# The current implementation of the parallel test runner requires |
| 378 |
# multiprocessing to start subprocesses with fork() or spawn(). |
| 379 |
if multiprocessing.get_start_method() not in {"fork", "spawn"}: |
| 380 |
return 1 |
| 381 |
try: |
| 382 |
return int(os.environ["DJANGO_TEST_PROCESSES"]) |
| 383 |
except KeyError: |
| 384 |
return multiprocessing.cpu_count() |
| 385 |
|
| 386 |
|
| 387 |
def parallel_type(value): |
| 388 |
"""Parse value passed to the --parallel option.""" |
| 389 |
if value == "auto": |
| 390 |
return value |
| 391 |
try: |
| 392 |
return int(value) |
| 393 |
except ValueError: |
| 394 |
raise argparse.ArgumentTypeError( |
| 395 |
f"{value!r} is not an integer or the string 'auto'" |
| 396 |
) |
| 397 |
|
| 398 |
|
| 399 |
_worker_id = 0 |
| 400 |
|
| 401 |
|
| 402 |
def _init_worker( |
| 403 |
counter, |
| 404 |
initial_settings=None, |
| 405 |
serialized_contents=None, |
| 406 |
process_setup=None, |
| 407 |
process_setup_args=None, |
| 408 |
debug_mode=None, |
| 409 |
used_aliases=None, |
| 410 |
): |
| 411 |
""" |
| 412 |
Switch to databases dedicated to this worker. |
| 413 |
|
| 414 |
This helper lives at module-level because of the multiprocessing module's |
| 415 |
requirements. |
| 416 |
""" |
| 417 |
|
| 418 |
global _worker_id |
| 419 |
|
| 420 |
with counter.get_lock(): |
| 421 |
counter.value += 1 |
| 422 |
_worker_id = counter.value |
| 423 |
|
| 424 |
start_method = multiprocessing.get_start_method() |
| 425 |
|
| 426 |
if start_method == "spawn": |
| 427 |
if process_setup and callable(process_setup): |
| 428 |
if process_setup_args is None: |
| 429 |
process_setup_args = () |
| 430 |
process_setup(*process_setup_args) |
| 431 |
django.setup() |
| 432 |
setup_test_environment(debug=debug_mode) |
| 433 |
|
| 434 |
db_aliases = used_aliases if used_aliases is not None else connections |
| 435 |
for alias in db_aliases: |
| 436 |
connection = connections[alias] |
| 437 |
if start_method == "spawn": |
| 438 |
# Restore initial settings in spawned processes. |
| 439 |
connection.settings_dict.update(initial_settings[alias]) |
| 440 |
if value := serialized_contents.get(alias): |
| 441 |
connection._test_serialized_contents = value |
| 442 |
connection.creation.setup_worker_connection(_worker_id) |
| 443 |
|
| 444 |
|
| 445 |
def _run_subsuite(args): |
| 446 |
""" |
| 447 |
Run a suite of tests with a RemoteTestRunner and return a RemoteTestResult. |
| 448 |
|
| 449 |
This helper lives at module-level and its arguments are wrapped in a tuple |
| 450 |
because of the multiprocessing module's requirements. |
| 451 |
""" |
| 452 |
runner_class, subsuite_index, subsuite, failfast, buffer = args |
| 453 |
runner = runner_class(failfast=failfast, buffer=buffer) |
| 454 |
result = runner.run(subsuite) |
| 455 |
return subsuite_index, result.events |
| 456 |
|
| 457 |
|
| 458 |
def _process_setup_stub(*args): |
| 459 |
"""Stub method to simplify run() implementation.""" |
| 460 |
pass |
| 461 |
|
| 462 |
|
| 463 |
class ParallelTestSuite(unittest.TestSuite): |
| 464 |
""" |
| 465 |
Run a series of tests in parallel in several processes. |
| 466 |
|
| 467 |
While the unittest module's documentation implies that orchestrating the |
| 468 |
execution of tests is the responsibility of the test runner, in practice, |
| 469 |
it appears that TestRunner classes are more concerned with formatting and |
| 470 |
displaying test results. |
| 471 |
|
| 472 |
Since there are fewer use cases for customizing TestSuite than TestRunner, |
| 473 |
implementing parallelization at the level of the TestSuite improves |
| 474 |
interoperability with existing custom test runners. A single instance of a |
| 475 |
test runner can still collect results from all tests without being aware |
| 476 |
that they have been run in parallel. |
| 477 |
""" |
| 478 |
|
| 479 |
# In case someone wants to modify these in a subclass. |
| 480 |
init_worker = _init_worker |
| 481 |
process_setup = _process_setup_stub |
| 482 |
process_setup_args = () |
| 483 |
run_subsuite = _run_subsuite |
| 484 |
runner_class = RemoteTestRunner |
| 485 |
|
| 486 |
def __init__( |
| 487 |
self, subsuites, processes, failfast=False, debug_mode=False, buffer=False |
| 488 |
): |
| 489 |
self.subsuites = subsuites |
| 490 |
self.processes = processes |
| 491 |
self.failfast = failfast |
| 492 |
self.debug_mode = debug_mode |
| 493 |
self.buffer = buffer |
| 494 |
self.initial_settings = None |
| 495 |
self.serialized_contents = None |
| 496 |
self.used_aliases = None |
| 497 |
super().__init__() |
| 498 |
|
| 499 |
def run(self, result): |
| 500 |
""" |
| 501 |
Distribute TestCases across workers. |
| 502 |
|
| 503 |
Return an identifier of each TestCase with its result in order to use |
| 504 |
imap_unordered to show results as soon as they're available. |
| 505 |
|
| 506 |
To minimize pickling errors when getting results from workers: |
| 507 |
|
| 508 |
- pass back numeric indexes in self.subsuites instead of tests |
| 509 |
- make tracebacks picklable with tblib, if available |
| 510 |
|
| 511 |
Even with tblib, errors may still occur for dynamically created |
| 512 |
exception classes which cannot be unpickled. |
| 513 |
""" |
| 514 |
self.initialize_suite() |
| 515 |
counter = multiprocessing.Value(ctypes.c_int, 0) |
| 516 |
pool = multiprocessing.Pool( |
| 517 |
processes=self.processes, |
| 518 |
initializer=self.init_worker.__func__, |
| 519 |
initargs=[ |
| 520 |
counter, |
| 521 |
self.initial_settings, |
| 522 |
self.serialized_contents, |
| 523 |
self.process_setup.__func__, |
| 524 |
self.process_setup_args, |
| 525 |
self.debug_mode, |
| 526 |
self.used_aliases, |
| 527 |
], |
| 528 |
) |
| 529 |
args = [ |
| 530 |
(self.runner_class, index, subsuite, self.failfast, self.buffer) |
| 531 |
for index, subsuite in enumerate(self.subsuites) |
| 532 |
] |
| 533 |
test_results = pool.imap_unordered(self.run_subsuite.__func__, args) |
| 534 |
|
| 535 |
while True: |
| 536 |
if result.shouldStop: |
| 537 |
pool.terminate() |
| 538 |
break |
| 539 |
|
| 540 |
try: |
| 541 |
subsuite_index, events = test_results.next(timeout=0.1) |
| 542 |
except multiprocessing.TimeoutError: |
| 543 |
continue |
| 544 |
except StopIteration: |
| 545 |
pool.close() |
| 546 |
break |
| 547 |
|
| 548 |
tests = list(self.subsuites[subsuite_index]) |
| 549 |
for event in events: |
| 550 |
self.handle_event(result, tests, event) |
| 551 |
|
| 552 |
pool.join() |
| 553 |
|
| 554 |
return result |
| 555 |
|
| 556 |
def handle_event(self, result, tests, event): |
| 557 |
event_name = event[0] |
| 558 |
handler = getattr(result, event_name, None) |
| 559 |
if handler is None: |
| 560 |
return |
| 561 |
test_index = event[1] |
| 562 |
event_occurred_before_first_test = test_index == -1 |
| 563 |
if ( |
| 564 |
event_name == "addError" |
| 565 |
and event_occurred_before_first_test |
| 566 |
and len(event) >= 4 |
| 567 |
): |
| 568 |
test_id = event[2] |
| 569 |
test = unittest.suite._ErrorHolder(test_id) |
| 570 |
args = event[3:] |
| 571 |
else: |
| 572 |
test = tests[test_index] |
| 573 |
args = event[2:] |
| 574 |
handler(test, *args) |
| 575 |
|
| 576 |
def __iter__(self): |
| 577 |
return iter(self.subsuites) |
| 578 |
|
| 579 |
def initialize_suite(self): |
| 580 |
if multiprocessing.get_start_method() == "spawn": |
| 581 |
self.initial_settings = { |
| 582 |
alias: connections[alias].settings_dict for alias in connections |
| 583 |
} |
| 584 |
self.serialized_contents = { |
| 585 |
alias: connections[alias]._test_serialized_contents |
| 586 |
for alias in connections |
| 587 |
if alias in self.serialized_aliases |
| 588 |
} |
| 589 |
|
| 590 |
|
| 591 |
class Shuffler: |
| 592 |
""" |
| 593 |
This class implements shuffling with a special consistency property. |
| 594 |
Consistency means that, for a given seed and key function, if two sets of |
| 595 |
items are shuffled, the resulting order will agree on the intersection of |
| 596 |
the two sets. For example, if items are removed from an original set, the |
| 597 |
shuffled order for the new set will be the shuffled order of the original |
| 598 |
set restricted to the smaller set. |
| 599 |
""" |
| 600 |
|
| 601 |
# This doesn't need to be cryptographically strong, so use what's fastest. |
| 602 |
hash_algorithm = "md5" |
| 603 |
|
| 604 |
@classmethod |
| 605 |
def _hash_text(cls, text): |
| 606 |
h = hashlib.new(cls.hash_algorithm, usedforsecurity=False) |
| 607 |
h.update(text.encode("utf-8")) |
| 608 |
return h.hexdigest() |
| 609 |
|
| 610 |
def __init__(self, seed=None): |
| 611 |
if seed is None: |
| 612 |
# Limit seeds to 10 digits for simpler output. |
| 613 |
seed = random.randint(0, 10**10 - 1) |
| 614 |
seed_source = "generated" |
| 615 |
else: |
| 616 |
seed_source = "given" |
| 617 |
self.seed = seed |
| 618 |
self.seed_source = seed_source |
| 619 |
|
| 620 |
@property |
| 621 |
def seed_display(self): |
| 622 |
return f"{self.seed!r} ({self.seed_source})" |
| 623 |
|
| 624 |
def _hash_item(self, item, key): |
| 625 |
text = "{}{}".format(self.seed, key(item)) |
| 626 |
return self._hash_text(text) |
| 627 |
|
| 628 |
def shuffle(self, items, key): |
| 629 |
""" |
| 630 |
Return a new list of the items in a shuffled order. |
| 631 |
|
| 632 |
The `key` is a function that accepts an item in `items` and returns |
| 633 |
a string unique for that item that can be viewed as a string id. The |
| 634 |
order of the return value is deterministic. It depends on the seed |
| 635 |
and key function but not on the original order. |
| 636 |
""" |
| 637 |
hashes = {} |
| 638 |
for item in items: |
| 639 |
hashed = self._hash_item(item, key) |
| 640 |
if hashed in hashes: |
| 641 |
msg = "item {!r} has same hash {!r} as item {!r}".format( |
| 642 |
item, |
| 643 |
hashed, |
| 644 |
hashes[hashed], |
| 645 |
) |
| 646 |
raise RuntimeError(msg) |
| 647 |
hashes[hashed] = item |
| 648 |
return [hashes[hashed] for hashed in sorted(hashes)] |
| 649 |
|
| 650 |
|
| 651 |
class DiscoverRunner: |
| 652 |
"""A Django test runner that uses unittest2 test discovery.""" |
| 653 |
|
| 654 |
test_suite = unittest.TestSuite |
| 655 |
parallel_test_suite = ParallelTestSuite |
| 656 |
test_runner = unittest.TextTestRunner |
| 657 |
test_loader = unittest.defaultTestLoader |
| 658 |
reorder_by = (TestCase, SimpleTestCase) |
| 659 |
|
| 660 |
def __init__( |
| 661 |
self, |
| 662 |
pattern=None, |
| 663 |
top_level=None, |
| 664 |
verbosity=1, |
| 665 |
interactive=True, |
| 666 |
failfast=False, |
| 667 |
keepdb=False, |
| 668 |
reverse=False, |
| 669 |
debug_mode=False, |
| 670 |
debug_sql=False, |
| 671 |
parallel=0, |
| 672 |
tags=None, |
| 673 |
exclude_tags=None, |
| 674 |
test_name_patterns=None, |
| 675 |
pdb=False, |
| 676 |
buffer=False, |
| 677 |
enable_faulthandler=True, |
| 678 |
timing=False, |
| 679 |
shuffle=False, |
| 680 |
logger=None, |
| 681 |
durations=None, |
| 682 |
**kwargs, |
| 683 |
): |
| 684 |
self.pattern = pattern |
| 685 |
self.top_level = top_level |
| 686 |
self.verbosity = verbosity |
| 687 |
self.interactive = interactive |
| 688 |
self.failfast = failfast |
| 689 |
self.keepdb = keepdb |
| 690 |
self.reverse = reverse |
| 691 |
self.debug_mode = debug_mode |
| 692 |
self.debug_sql = debug_sql |
| 693 |
self.parallel = parallel |
| 694 |
self.tags = set(tags or []) |
| 695 |
self.exclude_tags = set(exclude_tags or []) |
| 696 |
if not faulthandler.is_enabled() and enable_faulthandler: |
| 697 |
try: |
| 698 |
faulthandler.enable(file=sys.stderr.fileno()) |
| 699 |
except (AttributeError, io.UnsupportedOperation): |
| 700 |
faulthandler.enable(file=sys.__stderr__.fileno()) |
| 701 |
self.pdb = pdb |
| 702 |
if self.pdb and self.parallel > 1: |
| 703 |
raise ValueError( |
| 704 |
"You cannot use --pdb with parallel tests; pass --parallel=1 to use it." |
| 705 |
) |
| 706 |
self.buffer = buffer |
| 707 |
self.test_name_patterns = None |
| 708 |
self.time_keeper = TimeKeeper() if timing else NullTimeKeeper() |
| 709 |
if test_name_patterns: |
| 710 |
# unittest does not export the _convert_select_pattern function |
| 711 |
# that converts command-line arguments to patterns. |
| 712 |
self.test_name_patterns = { |
| 713 |
pattern if "*" in pattern else "*%s*" % pattern |
| 714 |
for pattern in test_name_patterns |
| 715 |
} |
| 716 |
self.shuffle = shuffle |
| 717 |
self._shuffler = None |
| 718 |
self.logger = logger |
| 719 |
self.durations = durations |
| 720 |
|
| 721 |
@classmethod |
| 722 |
def add_arguments(cls, parser): |
| 723 |
parser.add_argument( |
| 724 |
"--failfast", |
| 725 |
action="store_true", |
| 726 |
help="Stops the test suite after the first failure.", |
| 727 |
) |
| 728 |
parser.add_argument( |
| 729 |
"-t", |
| 730 |
"--top-level-directory", |
| 731 |
dest="top_level", |
| 732 |
help="Top level of project for unittest discovery.", |
| 733 |
) |
| 734 |
parser.add_argument( |
| 735 |
"-p", |
| 736 |
"--pattern", |
| 737 |
default="test*.py", |
| 738 |
help="The test matching pattern. Defaults to test*.py.", |
| 739 |
) |
| 740 |
parser.add_argument( |
| 741 |
"--keepdb", action="store_true", help="Preserves the test DB between runs." |
| 742 |
) |
| 743 |
parser.add_argument( |
| 744 |
"--shuffle", |
| 745 |
nargs="?", |
| 746 |
default=False, |
| 747 |
type=int, |
| 748 |
metavar="SEED", |
| 749 |
help="Shuffles test case order.", |
| 750 |
) |
| 751 |
parser.add_argument( |
| 752 |
"-r", |
| 753 |
"--reverse", |
| 754 |
action="store_true", |
| 755 |
help="Reverses test case order.", |
| 756 |
) |
| 757 |
parser.add_argument( |
| 758 |
"--debug-mode", |
| 759 |
action="store_true", |
| 760 |
help="Sets settings.DEBUG to True.", |
| 761 |
) |
| 762 |
parser.add_argument( |
| 763 |
"-d", |
| 764 |
"--debug-sql", |
| 765 |
action="store_true", |
| 766 |
help="Prints logged SQL queries on failure.", |
| 767 |
) |
| 768 |
parser.add_argument( |
| 769 |
"--parallel", |
| 770 |
nargs="?", |
| 771 |
const="auto", |
| 772 |
default=0, |
| 773 |
type=parallel_type, |
| 774 |
metavar="N", |
| 775 |
help=( |
| 776 |
"Run tests using up to N parallel processes. Use the value " |
| 777 |
'"auto" to run one test process for each processor core.' |
| 778 |
), |
| 779 |
) |
| 780 |
parser.add_argument( |
| 781 |
"--tag", |
| 782 |
action="append", |
| 783 |
dest="tags", |
| 784 |
help="Run only tests with the specified tag. Can be used multiple times.", |
| 785 |
) |
| 786 |
parser.add_argument( |
| 787 |
"--exclude-tag", |
| 788 |
action="append", |
| 789 |
dest="exclude_tags", |
| 790 |
help="Do not run tests with the specified tag. Can be used multiple times.", |
| 791 |
) |
| 792 |
parser.add_argument( |
| 793 |
"--pdb", |
| 794 |
action="store_true", |
| 795 |
help="Runs a debugger (pdb, or ipdb if installed) on error or failure.", |
| 796 |
) |
| 797 |
parser.add_argument( |
| 798 |
"-b", |
| 799 |
"--buffer", |
| 800 |
action="store_true", |
| 801 |
help="Discard output from passing tests.", |
| 802 |
) |
| 803 |
parser.add_argument( |
| 804 |
"--no-faulthandler", |
| 805 |
action="store_false", |
| 806 |
dest="enable_faulthandler", |
| 807 |
help="Disables the Python faulthandler module during tests.", |
| 808 |
) |
| 809 |
parser.add_argument( |
| 810 |
"--timing", |
| 811 |
action="store_true", |
| 812 |
help=("Output timings, including database set up and total run time."), |
| 813 |
) |
| 814 |
parser.add_argument( |
| 815 |
"-k", |
| 816 |
action="append", |
| 817 |
dest="test_name_patterns", |
| 818 |
help=( |
| 819 |
"Only run test methods and classes that match the pattern " |
| 820 |
"or substring. Can be used multiple times. Same as " |
| 821 |
"unittest -k option." |
| 822 |
), |
| 823 |
) |
| 824 |
if PY312: |
| 825 |
parser.add_argument( |
| 826 |
"--durations", |
| 827 |
dest="durations", |
| 828 |
type=int, |
| 829 |
default=None, |
| 830 |
metavar="N", |
| 831 |
help="Show the N slowest test cases (N=0 for all).", |
| 832 |
) |
| 833 |
|
| 834 |
@property |
| 835 |
def shuffle_seed(self): |
| 836 |
if self._shuffler is None: |
| 837 |
return None |
| 838 |
return self._shuffler.seed |
| 839 |
|
| 840 |
def log(self, msg, level=None): |
| 841 |
""" |
| 842 |
Log the message at the given logging level (the default is INFO). |
| 843 |
|
| 844 |
If a logger isn't set, the message is instead printed to the console, |
| 845 |
respecting the configured verbosity. A verbosity of 0 prints no output, |
| 846 |
a verbosity of 1 prints INFO and above, and a verbosity of 2 or higher |
| 847 |
prints all levels. |
| 848 |
""" |
| 849 |
if level is None: |
| 850 |
level = logging.INFO |
| 851 |
if self.logger is None: |
| 852 |
if self.verbosity <= 0 or (self.verbosity == 1 and level < logging.INFO): |
| 853 |
return |
| 854 |
print(msg) |
| 855 |
else: |
| 856 |
self.logger.log(level, msg) |
| 857 |
|
| 858 |
def setup_test_environment(self, **kwargs): |
| 859 |
setup_test_environment(debug=self.debug_mode) |
| 860 |
unittest.installHandler() |
| 861 |
|
| 862 |
def setup_shuffler(self): |
| 863 |
if self.shuffle is False: |
| 864 |
return |
| 865 |
shuffler = Shuffler(seed=self.shuffle) |
| 866 |
self.log(f"Using shuffle seed: {shuffler.seed_display}") |
| 867 |
self._shuffler = shuffler |
| 868 |
|
| 869 |
@contextmanager |
| 870 |
def load_with_patterns(self): |
| 871 |
original_test_name_patterns = self.test_loader.testNamePatterns |
| 872 |
self.test_loader.testNamePatterns = self.test_name_patterns |
| 873 |
try: |
| 874 |
yield |
| 875 |
finally: |
| 876 |
# Restore the original patterns. |
| 877 |
self.test_loader.testNamePatterns = original_test_name_patterns |
| 878 |
|
| 879 |
def load_tests_for_label(self, label, discover_kwargs): |
| 880 |
label_as_path = os.path.abspath(label) |
| 881 |
tests = None |
| 882 |
|
| 883 |
# If a module, or "module.ClassName[.method_name]", just run those. |
| 884 |
if not os.path.exists(label_as_path): |
| 885 |
with self.load_with_patterns(): |
| 886 |
tests = self.test_loader.loadTestsFromName(label) |
| 887 |
if tests.countTestCases(): |
| 888 |
return tests |
| 889 |
# Try discovery if "label" is a package or directory. |
| 890 |
is_importable, is_package = try_importing(label) |
| 891 |
if is_importable: |
| 892 |
if not is_package: |
| 893 |
return tests |
| 894 |
elif not os.path.isdir(label_as_path): |
| 895 |
if os.path.exists(label_as_path): |
| 896 |
assert tests is None |
| 897 |
raise RuntimeError( |
| 898 |
f"One of the test labels is a path to a file: {label!r}, " |
| 899 |
f"which is not supported. Use a dotted module name or " |
| 900 |
f"path to a directory instead." |
| 901 |
) |
| 902 |
return tests |
| 903 |
|
| 904 |
kwargs = discover_kwargs.copy() |
| 905 |
if os.path.isdir(label_as_path) and not self.top_level: |
| 906 |
kwargs["top_level_dir"] = find_top_level(label_as_path) |
| 907 |
|
| 908 |
with self.load_with_patterns(): |
| 909 |
tests = self.test_loader.discover(start_dir=label, **kwargs) |
| 910 |
|
| 911 |
# Make unittest forget the top-level dir it calculated from this run, |
| 912 |
# to support running tests from two different top-levels. |
| 913 |
self.test_loader._top_level_dir = None |
| 914 |
return tests |
| 915 |
|
| 916 |
def build_suite(self, test_labels=None, **kwargs): |
| 917 |
test_labels = test_labels or ["."] |
| 918 |
|
| 919 |
discover_kwargs = {} |
| 920 |
if self.pattern is not None: |
| 921 |
discover_kwargs["pattern"] = self.pattern |
| 922 |
if self.top_level is not None: |
| 923 |
discover_kwargs["top_level_dir"] = self.top_level |
| 924 |
self.setup_shuffler() |
| 925 |
|
| 926 |
all_tests = [] |
| 927 |
for label in test_labels: |
| 928 |
tests = self.load_tests_for_label(label, discover_kwargs) |
| 929 |
all_tests.extend(iter_test_cases(tests)) |
| 930 |
|
| 931 |
if self.tags or self.exclude_tags: |
| 932 |
if self.tags: |
| 933 |
self.log( |
| 934 |
"Including test tag(s): %s." % ", ".join(sorted(self.tags)), |
| 935 |
level=logging.DEBUG, |
| 936 |
) |
| 937 |
if self.exclude_tags: |
| 938 |
self.log( |
| 939 |
"Excluding test tag(s): %s." % ", ".join(sorted(self.exclude_tags)), |
| 940 |
level=logging.DEBUG, |
| 941 |
) |
| 942 |
all_tests = filter_tests_by_tags(all_tests, self.tags, self.exclude_tags) |
| 943 |
|
| 944 |
# Put the failures detected at load time first for quicker feedback. |
| 945 |
# _FailedTest objects include things like test modules that couldn't be |
| 946 |
# found or that couldn't be loaded due to syntax errors. |
| 947 |
test_types = (unittest.loader._FailedTest, *self.reorder_by) |
| 948 |
all_tests = list( |
| 949 |
reorder_tests( |
| 950 |
all_tests, |
| 951 |
test_types, |
| 952 |
shuffler=self._shuffler, |
| 953 |
reverse=self.reverse, |
| 954 |
) |
| 955 |
) |
| 956 |
self.log("Found %d test(s)." % len(all_tests)) |
| 957 |
suite = self.test_suite(all_tests) |
| 958 |
|
| 959 |
if self.parallel > 1: |
| 960 |
subsuites = partition_suite_by_case(suite) |
| 961 |
# Since tests are distributed across processes on a per-TestCase |
| 962 |
# basis, there's no need for more processes than TestCases. |
| 963 |
processes = min(self.parallel, len(subsuites)) |
| 964 |
# Update also "parallel" because it's used to determine the number |
| 965 |
# of test databases. |
| 966 |
self.parallel = processes |
| 967 |
if processes > 1: |
| 968 |
suite = self.parallel_test_suite( |
| 969 |
subsuites, |
| 970 |
processes, |
| 971 |
self.failfast, |
| 972 |
self.debug_mode, |
| 973 |
self.buffer, |
| 974 |
) |
| 975 |
return suite |
| 976 |
|
| 977 |
def setup_databases(self, **kwargs): |
| 978 |
return _setup_databases( |
| 979 |
self.verbosity, |
| 980 |
self.interactive, |
| 981 |
time_keeper=self.time_keeper, |
| 982 |
keepdb=self.keepdb, |
| 983 |
debug_sql=self.debug_sql, |
| 984 |
parallel=self.parallel, |
| 985 |
**kwargs, |
| 986 |
) |
| 987 |
|
| 988 |
def get_resultclass(self): |
| 989 |
if self.debug_sql: |
| 990 |
return DebugSQLTextTestResult |
| 991 |
elif self.pdb: |
| 992 |
return PDBDebugResult |
| 993 |
|
| 994 |
def get_test_runner_kwargs(self): |
| 995 |
kwargs = { |
| 996 |
"failfast": self.failfast, |
| 997 |
"resultclass": self.get_resultclass(), |
| 998 |
"verbosity": self.verbosity, |
| 999 |
"buffer": self.buffer, |
| 1000 |
} |
| 1001 |
if PY312: |
| 1002 |
kwargs["durations"] = self.durations |
| 1003 |
return kwargs |
| 1004 |
|
| 1005 |
def run_checks(self, databases): |
| 1006 |
# Checks are run after database creation since some checks require |
| 1007 |
# database access. |
| 1008 |
call_command("check", verbosity=self.verbosity, databases=databases) |
| 1009 |
|
| 1010 |
def run_suite(self, suite, **kwargs): |
| 1011 |
kwargs = self.get_test_runner_kwargs() |
| 1012 |
runner = self.test_runner(**kwargs) |
| 1013 |
try: |
| 1014 |
return runner.run(suite) |
| 1015 |
finally: |
| 1016 |
if self._shuffler is not None: |
| 1017 |
seed_display = self._shuffler.seed_display |
| 1018 |
self.log(f"Used shuffle seed: {seed_display}") |
| 1019 |
|
| 1020 |
def teardown_databases(self, old_config, **kwargs): |
| 1021 |
"""Destroy all the non-mirror databases.""" |
| 1022 |
_teardown_databases( |
| 1023 |
old_config, |
| 1024 |
verbosity=self.verbosity, |
| 1025 |
parallel=self.parallel, |
| 1026 |
keepdb=self.keepdb, |
| 1027 |
) |
| 1028 |
|
| 1029 |
def teardown_test_environment(self, **kwargs): |
| 1030 |
unittest.removeHandler() |
| 1031 |
teardown_test_environment() |
| 1032 |
|
| 1033 |
def suite_result(self, suite, result, **kwargs): |
| 1034 |
return ( |
| 1035 |
len(result.failures) + len(result.errors) + len(result.unexpectedSuccesses) |
| 1036 |
) |
| 1037 |
|
| 1038 |
def _get_databases(self, suite): |
| 1039 |
databases = {} |
| 1040 |
for test in iter_test_cases(suite): |
| 1041 |
test_databases = getattr(test, "databases", None) |
| 1042 |
if test_databases == "__all__": |
| 1043 |
test_databases = connections |
| 1044 |
if test_databases: |
| 1045 |
serialized_rollback = getattr(test, "serialized_rollback", False) |
| 1046 |
databases.update( |
| 1047 |
(alias, serialized_rollback or databases.get(alias, False)) |
| 1048 |
for alias in test_databases |
| 1049 |
) |
| 1050 |
return databases |
| 1051 |
|
| 1052 |
def get_databases(self, suite): |
| 1053 |
databases = self._get_databases(suite) |
| 1054 |
unused_databases = [alias for alias in connections if alias not in databases] |
| 1055 |
if unused_databases: |
| 1056 |
self.log( |
| 1057 |
"Skipping setup of unused database(s): %s." |
| 1058 |
% ", ".join(sorted(unused_databases)), |
| 1059 |
level=logging.DEBUG, |
| 1060 |
) |
| 1061 |
return databases |
| 1062 |
|
| 1063 |
def run_tests(self, test_labels, **kwargs): |
| 1064 |
""" |
| 1065 |
Run the unit tests for all the test labels in the provided list. |
| 1066 |
|
| 1067 |
Test labels should be dotted Python paths to test modules, test |
| 1068 |
classes, or test methods. |
| 1069 |
|
| 1070 |
Return the number of tests that failed. |
| 1071 |
""" |
| 1072 |
self.setup_test_environment() |
| 1073 |
suite = self.build_suite(test_labels) |
| 1074 |
databases = self.get_databases(suite) |
| 1075 |
suite.serialized_aliases = set( |
| 1076 |
alias for alias, serialize in databases.items() if serialize |
| 1077 |
) |
| 1078 |
suite.used_aliases = set(databases) |
| 1079 |
with self.time_keeper.timed("Total database setup"): |
| 1080 |
old_config = self.setup_databases( |
| 1081 |
aliases=databases, |
| 1082 |
serialized_aliases=suite.serialized_aliases, |
| 1083 |
) |
| 1084 |
run_failed = False |
| 1085 |
try: |
| 1086 |
self.run_checks(databases) |
| 1087 |
result = self.run_suite(suite) |
| 1088 |
except Exception: |
| 1089 |
run_failed = True |
| 1090 |
raise |
| 1091 |
finally: |
| 1092 |
try: |
| 1093 |
with self.time_keeper.timed("Total database teardown"): |
| 1094 |
self.teardown_databases(old_config) |
| 1095 |
self.teardown_test_environment() |
| 1096 |
except Exception: |
| 1097 |
# Silence teardown exceptions if an exception was raised during |
| 1098 |
# runs to avoid shadowing it. |
| 1099 |
if not run_failed: |
| 1100 |
raise |
| 1101 |
self.time_keeper.print_results() |
| 1102 |
return self.suite_result(suite, result) |
| 1103 |
|
| 1104 |
|
| 1105 |
def try_importing(label): |
| 1106 |
""" |
| 1107 |
Try importing a test label, and return (is_importable, is_package). |
| 1108 |
|
| 1109 |
Relative labels like "." and ".." are seen as directories. |
| 1110 |
""" |
| 1111 |
try: |
| 1112 |
mod = import_module(label) |
| 1113 |
except (ImportError, TypeError): |
| 1114 |
return (False, False) |
| 1115 |
|
| 1116 |
return (True, hasattr(mod, "__path__")) |
| 1117 |
|
| 1118 |
|
| 1119 |
def find_top_level(top_level): |
| 1120 |
# Try to be a bit smarter than unittest about finding the default top-level |
| 1121 |
# for a given directory path, to avoid breaking relative imports. |
| 1122 |
# (Unittest's default is to set top-level equal to the path, which means |
| 1123 |
# relative imports will result in "Attempted relative import in |
| 1124 |
# non-package."). |
| 1125 |
|
| 1126 |
# We'd be happy to skip this and require dotted module paths (which don't |
| 1127 |
# cause this problem) instead of file paths (which do), but in the case of |
| 1128 |
# a directory in the cwd, which would be equally valid if considered as a |
| 1129 |
# top-level module or as a directory path, unittest unfortunately prefers |
| 1130 |
# the latter. |
| 1131 |
while True: |
| 1132 |
init_py = os.path.join(top_level, "__init__.py") |
| 1133 |
if not os.path.exists(init_py): |
| 1134 |
break |
| 1135 |
try_next = os.path.dirname(top_level) |
| 1136 |
if try_next == top_level: |
| 1137 |
# __init__.py all the way down? give up. |
| 1138 |
break |
| 1139 |
top_level = try_next |
| 1140 |
return top_level |
| 1141 |
|
| 1142 |
|
| 1143 |
def _class_shuffle_key(cls): |
| 1144 |
return f"{cls.__module__}.{cls.__qualname__}" |
| 1145 |
|
| 1146 |
|
| 1147 |
def shuffle_tests(tests, shuffler): |
| 1148 |
""" |
| 1149 |
Return an iterator over the given tests in a shuffled order, keeping tests |
| 1150 |
next to other tests of their class. |
| 1151 |
|
| 1152 |
`tests` should be an iterable of tests. |
| 1153 |
""" |
| 1154 |
tests_by_type = {} |
| 1155 |
for _, class_tests in itertools.groupby(tests, type): |
| 1156 |
class_tests = list(class_tests) |
| 1157 |
test_type = type(class_tests[0]) |
| 1158 |
class_tests = shuffler.shuffle(class_tests, key=lambda test: test.id()) |
| 1159 |
tests_by_type[test_type] = class_tests |
| 1160 |
|
| 1161 |
classes = shuffler.shuffle(tests_by_type, key=_class_shuffle_key) |
| 1162 |
|
| 1163 |
return itertools.chain(*(tests_by_type[cls] for cls in classes)) |
| 1164 |
|
| 1165 |
|
| 1166 |
def reorder_test_bin(tests, shuffler=None, reverse=False): |
| 1167 |
""" |
| 1168 |
Return an iterator that reorders the given tests, keeping tests next to |
| 1169 |
other tests of their class. |
| 1170 |
|
| 1171 |
`tests` should be an iterable of tests that supports reversed(). |
| 1172 |
""" |
| 1173 |
if shuffler is None: |
| 1174 |
if reverse: |
| 1175 |
return reversed(tests) |
| 1176 |
# The function must return an iterator. |
| 1177 |
return iter(tests) |
| 1178 |
|
| 1179 |
tests = shuffle_tests(tests, shuffler) |
| 1180 |
if not reverse: |
| 1181 |
return tests |
| 1182 |
# Arguments to reversed() must be reversible. |
| 1183 |
return reversed(list(tests)) |
| 1184 |
|
| 1185 |
|
| 1186 |
def reorder_tests(tests, classes, reverse=False, shuffler=None): |
| 1187 |
""" |
| 1188 |
Reorder an iterable of tests, grouping by the given TestCase classes. |
| 1189 |
|
| 1190 |
This function also removes any duplicates and reorders so that tests of the |
| 1191 |
same type are consecutive. |
| 1192 |
|
| 1193 |
The result is returned as an iterator. `classes` is a sequence of types. |
| 1194 |
Tests that are instances of `classes[0]` are grouped first, followed by |
| 1195 |
instances of `classes[1]`, etc. Tests that are not instances of any of the |
| 1196 |
classes are grouped last. |
| 1197 |
|
| 1198 |
If `reverse` is True, the tests within each `classes` group are reversed, |
| 1199 |
but without reversing the order of `classes` itself. |
| 1200 |
|
| 1201 |
The `shuffler` argument is an optional instance of this module's `Shuffler` |
| 1202 |
class. If provided, tests will be shuffled within each `classes` group, but |
| 1203 |
keeping tests with other tests of their TestCase class. Reversing is |
| 1204 |
applied after shuffling to allow reversing the same random order. |
| 1205 |
""" |
| 1206 |
# Each bin maps TestCase class to OrderedSet of tests. This permits tests |
| 1207 |
# to be grouped by TestCase class even if provided non-consecutively. |
| 1208 |
bins = [defaultdict(OrderedSet) for i in range(len(classes) + 1)] |
| 1209 |
*class_bins, last_bin = bins |
| 1210 |
|
| 1211 |
for test in tests: |
| 1212 |
for test_bin, test_class in zip(class_bins, classes): |
| 1213 |
if isinstance(test, test_class): |
| 1214 |
break |
| 1215 |
else: |
| 1216 |
test_bin = last_bin |
| 1217 |
test_bin[type(test)].add(test) |
| 1218 |
|
| 1219 |
for test_bin in bins: |
| 1220 |
# Call list() since reorder_test_bin()'s input must support reversed(). |
| 1221 |
tests = list(itertools.chain.from_iterable(test_bin.values())) |
| 1222 |
yield from reorder_test_bin(tests, shuffler=shuffler, reverse=reverse) |
| 1223 |
|
| 1224 |
|
| 1225 |
def partition_suite_by_case(suite): |
| 1226 |
"""Partition a test suite by TestCase, preserving the order of tests.""" |
| 1227 |
suite_class = type(suite) |
| 1228 |
all_tests = iter_test_cases(suite) |
| 1229 |
return [suite_class(tests) for _, tests in itertools.groupby(all_tests, type)] |
| 1230 |
|
| 1231 |
|
| 1232 |
def test_match_tags(test, tags, exclude_tags): |
| 1233 |
if isinstance(test, unittest.loader._FailedTest): |
| 1234 |
# Tests that couldn't load always match to prevent tests from falsely |
| 1235 |
# passing due e.g. to syntax errors. |
| 1236 |
return True |
| 1237 |
test_tags = set(getattr(test, "tags", [])) |
| 1238 |
test_fn_name = getattr(test, "_testMethodName", str(test)) |
| 1239 |
if hasattr(test, test_fn_name): |
| 1240 |
test_fn = getattr(test, test_fn_name) |
| 1241 |
test_fn_tags = list(getattr(test_fn, "tags", [])) |
| 1242 |
test_tags = test_tags.union(test_fn_tags) |
| 1243 |
if tags and test_tags.isdisjoint(tags): |
| 1244 |
return False |
| 1245 |
return test_tags.isdisjoint(exclude_tags) |
| 1246 |
|
| 1247 |
|
| 1248 |
def filter_tests_by_tags(tests, tags, exclude_tags): |
| 1249 |
"""Return the matching tests as an iterator.""" |
| 1250 |
return (test for test in tests if test_match_tags(test, tags, exclude_tags)) |