Debian Codegraph @

django/test/runner.py

raw
1 import argparse
2 import ctypes
3 import faulthandler
4 import hashlib
5 import io
6 import itertools
7 import logging
8 import multiprocessing
9 import os
10 import pickle
11 import random
12 import sys
13 import textwrap
14 import unittest
15 import unittest.suite
16 from collections import defaultdict
17 from contextlib import contextmanager
18 from importlib import import_module
19 from io import StringIO
20
21 import sqlparse
22
23 import django
24 from django.core.management import call_command
25 from django.db import connections
26 from django.test import SimpleTestCase, TestCase
27 from django.test.utils import NullTimeKeeper, TimeKeeper, iter_test_cases
28 from django.test.utils import setup_databases as _setup_databases
29 from django.test.utils import setup_test_environment
30 from django.test.utils import teardown_databases as _teardown_databases
31 from django.test.utils import teardown_test_environment
32 from django.utils.datastructures import OrderedSet
33 from django.utils.version import PY312, PY313
34
35 try:
36 import ipdb as pdb
37 except ImportError:
38 import pdb
39
40 try:
41 import tblib.pickling_support
42 except ImportError:
43 tblib = None
44
45
46 class DebugSQLTextTestResult(unittest.TextTestResult):
47 def __init__(self, stream, descriptions, verbosity):
48 self.logger = logging.getLogger("django.db.backends")
49 self.logger.setLevel(logging.DEBUG)
50 self.debug_sql_stream = None
51 super().__init__(stream, descriptions, verbosity)
52
53 def startTest(self, test):
54 self.debug_sql_stream = StringIO()
55 self.handler = logging.StreamHandler(self.debug_sql_stream)
56 self.logger.addHandler(self.handler)
57 super().startTest(test)
58
59 def stopTest(self, test):
60 super().stopTest(test)
61 self.logger.removeHandler(self.handler)
62 if self.showAll:
63 self.debug_sql_stream.seek(0)
64 self.stream.write(self.debug_sql_stream.read())
65 self.stream.writeln(self.separator2)
66
67 def addError(self, test, err):
68 super().addError(test, err)
69 if self.debug_sql_stream is None:
70 # Error before tests e.g. in setUpTestData().
71 sql = ""
72 else:
73 self.debug_sql_stream.seek(0)
74 sql = self.debug_sql_stream.read()
75 self.errors[-1] = self.errors[-1] + (sql,)
76
77 def addFailure(self, test, err):
78 super().addFailure(test, err)
79 self.debug_sql_stream.seek(0)
80 self.failures[-1] = self.failures[-1] + (self.debug_sql_stream.read(),)
81
82 def addSubTest(self, test, subtest, err):
83 super().addSubTest(test, subtest, err)
84 if err is not None:
85 self.debug_sql_stream.seek(0)
86 errors = (
87 self.failures
88 if issubclass(err[0], test.failureException)
89 else self.errors
90 )
91 errors[-1] = errors[-1] + (self.debug_sql_stream.read(),)
92
93 def printErrorList(self, flavour, errors):
94 for test, err, sql_debug in errors:
95 self.stream.writeln(self.separator1)
96 self.stream.writeln("%s: %s" % (flavour, self.getDescription(test)))
97 self.stream.writeln(self.separator2)
98 self.stream.writeln(err)
99 self.stream.writeln(self.separator2)
100 self.stream.writeln(
101 sqlparse.format(sql_debug, reindent=True, keyword_case="upper")
102 )
103
104
105 class PDBDebugResult(unittest.TextTestResult):
106 """
107 Custom result class that triggers a PDB session when an error or failure
108 occurs.
109 """
110
111 def addError(self, test, err):
112 super().addError(test, err)
113 self.debug(err)
114
115 def addFailure(self, test, err):
116 super().addFailure(test, err)
117 self.debug(err)
118
119 def addSubTest(self, test, subtest, err):
120 if err is not None:
121 self.debug(err)
122 super().addSubTest(test, subtest, err)
123
124 def debug(self, error):
125 self._restoreStdout()
126 self.buffer = False
127 exc_type, exc_value, traceback = error
128 print("\nOpening PDB: %r" % exc_value)
129 if PY313:
130 pdb.post_mortem(exc_value)
131 else:
132 pdb.post_mortem(traceback)
133
134
135 class DummyList:
136 """
137 Dummy list class for faking storage of results in unittest.TestResult.
138 """
139
140 __slots__ = ()
141
142 def append(self, item):
143 pass
144
145
146 class RemoteTestResult(unittest.TestResult):
147 """
148 Extend unittest.TestResult to record events in the child processes so they
149 can be replayed in the parent process. Events include things like which
150 tests succeeded or failed.
151 """
152
153 def __init__(self, *args, **kwargs):
154 super().__init__(*args, **kwargs)
155 # Fake storage of results to reduce memory usage. These are used by the
156 # unittest default methods, but here 'events' is used instead.
157 dummy_list = DummyList()
158 self.failures = dummy_list
159 self.errors = dummy_list
160 self.skipped = dummy_list
161 self.expectedFailures = dummy_list
162 self.unexpectedSuccesses = dummy_list
163
164 if tblib is not None:
165 tblib.pickling_support.install()
166 self.events = []
167
168 def __getstate__(self):
169 # Make this class picklable by removing the file-like buffer
170 # attributes. This is possible since they aren't used after unpickling
171 # after being sent to ParallelTestSuite.
172 state = self.__dict__.copy()
173 state.pop("_stdout_buffer", None)
174 state.pop("_stderr_buffer", None)
175 state.pop("_original_stdout", None)
176 state.pop("_original_stderr", None)
177 return state
178
179 @property
180 def test_index(self):
181 return self.testsRun - 1
182
183 def _confirm_picklable(self, obj):
184 """
185 Confirm that obj can be pickled and unpickled as multiprocessing will
186 need to pickle the exception in the child process and unpickle it in
187 the parent process. Let the exception rise, if not.
188 """
189 pickle.loads(pickle.dumps(obj))
190
191 def _print_unpicklable_subtest(self, test, subtest, pickle_exc):
192 print("""
193 Subtest failed:
194
195 test: {}
196 subtest: {}
197
198 Unfortunately, the subtest that failed cannot be pickled, so the parallel
199 test runner cannot handle it cleanly. Here is the pickling error:
200
201 > {}
202
203 You should re-run this test with --parallel=1 to reproduce the failure
204 with a cleaner failure message.
205 """.format(test, subtest, pickle_exc))
206
207 def check_picklable(self, test, err):
208 # Ensure that sys.exc_info() tuples are picklable. This displays a
209 # clear multiprocessing.pool.RemoteTraceback generated in the child
210 # process instead of a multiprocessing.pool.MaybeEncodingError, making
211 # the root cause easier to figure out for users who aren't familiar
212 # with the multiprocessing module. Since we're in a forked process,
213 # our best chance to communicate with them is to print to stdout.
214 try:
215 self._confirm_picklable(err)
216 except Exception as exc:
217 original_exc_txt = repr(err[1])
218 original_exc_txt = textwrap.fill(
219 original_exc_txt, 75, initial_indent=" ", subsequent_indent=" "
220 )
221 pickle_exc_txt = repr(exc)
222 pickle_exc_txt = textwrap.fill(
223 pickle_exc_txt, 75, initial_indent=" ", subsequent_indent=" "
224 )
225 if tblib is None:
226 print("""
227
228 {} failed:
229
230 {}
231
232 Unfortunately, tracebacks cannot be pickled, making it impossible for the
233 parallel test runner to handle this exception cleanly.
234
235 In order to see the traceback, you should install tblib:
236
237 python -m pip install tblib
238 """.format(test, original_exc_txt))
239 else:
240 print("""
241
242 {} failed:
243
244 {}
245
246 Unfortunately, the exception it raised cannot be pickled, making it impossible
247 for the parallel test runner to handle it cleanly.
248
249 Here's the error encountered while trying to pickle the exception:
250
251 {}
252
253 You should re-run this test with the --parallel=1 option to reproduce the
254 failure and get a correct traceback.
255 """.format(test, original_exc_txt, pickle_exc_txt))
256 raise
257
258 def check_subtest_picklable(self, test, subtest):
259 try:
260 self._confirm_picklable(subtest)
261 except Exception as exc:
262 self._print_unpicklable_subtest(test, subtest, exc)
263 raise
264
265 def startTestRun(self):
266 super().startTestRun()
267 self.events.append(("startTestRun",))
268
269 def stopTestRun(self):
270 super().stopTestRun()
271 self.events.append(("stopTestRun",))
272
273 def startTest(self, test):
274 super().startTest(test)
275 self.events.append(("startTest", self.test_index))
276
277 def stopTest(self, test):
278 super().stopTest(test)
279 self.events.append(("stopTest", self.test_index))
280
281 def addDuration(self, test, elapsed):
282 super().addDuration(test, elapsed)
283 self.events.append(("addDuration", self.test_index, elapsed))
284
285 def addError(self, test, err):
286 self.check_picklable(test, err)
287
288 event_occurred_before_first_test = self.test_index == -1
289 if event_occurred_before_first_test and isinstance(
290 test, unittest.suite._ErrorHolder
291 ):
292 self.events.append(("addError", self.test_index, test.id(), err))
293 else:
294 self.events.append(("addError", self.test_index, err))
295
296 super().addError(test, err)
297
298 def addFailure(self, test, err):
299 self.check_picklable(test, err)
300 self.events.append(("addFailure", self.test_index, err))
301 super().addFailure(test, err)
302
303 def addSubTest(self, test, subtest, err):
304 # Follow Python's implementation of unittest.TestResult.addSubTest() by
305 # not doing anything when a subtest is successful.
306 if err is not None:
307 # Call check_picklable() before check_subtest_picklable() since
308 # check_picklable() performs the tblib check.
309 self.check_picklable(test, err)
310 self.check_subtest_picklable(test, subtest)
311 self.events.append(("addSubTest", self.test_index, subtest, err))
312 super().addSubTest(test, subtest, err)
313
314 def addSuccess(self, test):
315 self.events.append(("addSuccess", self.test_index))
316 super().addSuccess(test)
317
318 def addSkip(self, test, reason):
319 self.events.append(("addSkip", self.test_index, reason))
320 super().addSkip(test, reason)
321
322 def addExpectedFailure(self, test, err):
323 # If tblib isn't installed, pickling the traceback will always fail.
324 # However we don't want tblib to be required for running the tests
325 # when they pass or fail as expected. Drop the traceback when an
326 # expected failure occurs.
327 if tblib is None:
328 err = err[0], err[1], None
329 self.check_picklable(test, err)
330 self.events.append(("addExpectedFailure", self.test_index, err))
331 super().addExpectedFailure(test, err)
332
333 def addUnexpectedSuccess(self, test):
334 self.events.append(("addUnexpectedSuccess", self.test_index))
335 super().addUnexpectedSuccess(test)
336
337 def wasSuccessful(self):
338 """Tells whether or not this result was a success."""
339 failure_types = {"addError", "addFailure", "addSubTest", "addUnexpectedSuccess"}
340 return all(e[0] not in failure_types for e in self.events)
341
342 def _exc_info_to_string(self, err, test):
343 # Make this method no-op. It only powers the default unittest behavior
344 # for recording errors, but this class pickles errors into 'events'
345 # instead.
346 return ""
347
348
349 class RemoteTestRunner:
350 """
351 Run tests and record everything but don't display anything.
352
353 The implementation matches the unpythonic coding style of unittest2.
354 """
355
356 resultclass = RemoteTestResult
357
358 def __init__(self, failfast=False, resultclass=None, buffer=False):
359 self.failfast = failfast
360 self.buffer = buffer
361 if resultclass is not None:
362 self.resultclass = resultclass
363
364 def run(self, test):
365 result = self.resultclass()
366 unittest.registerResult(result)
367 result.failfast = self.failfast
368 result.buffer = self.buffer
369 test(result)
370 return result
371
372
373 def get_max_test_processes():
374 """
375 The maximum number of test processes when using the --parallel option.
376 """
377 # The current implementation of the parallel test runner requires
378 # multiprocessing to start subprocesses with fork() or spawn().
379 if multiprocessing.get_start_method() not in {"fork", "spawn"}:
380 return 1
381 try:
382 return int(os.environ["DJANGO_TEST_PROCESSES"])
383 except KeyError:
384 return multiprocessing.cpu_count()
385
386
387 def parallel_type(value):
388 """Parse value passed to the --parallel option."""
389 if value == "auto":
390 return value
391 try:
392 return int(value)
393 except ValueError:
394 raise argparse.ArgumentTypeError(
395 f"{value!r} is not an integer or the string 'auto'"
396 )
397
398
399 _worker_id = 0
400
401
402 def _init_worker(
403 counter,
404 initial_settings=None,
405 serialized_contents=None,
406 process_setup=None,
407 process_setup_args=None,
408 debug_mode=None,
409 used_aliases=None,
410 ):
411 """
412 Switch to databases dedicated to this worker.
413
414 This helper lives at module-level because of the multiprocessing module's
415 requirements.
416 """
417
418 global _worker_id
419
420 with counter.get_lock():
421 counter.value += 1
422 _worker_id = counter.value
423
424 start_method = multiprocessing.get_start_method()
425
426 if start_method == "spawn":
427 if process_setup and callable(process_setup):
428 if process_setup_args is None:
429 process_setup_args = ()
430 process_setup(*process_setup_args)
431 django.setup()
432 setup_test_environment(debug=debug_mode)
433
434 db_aliases = used_aliases if used_aliases is not None else connections
435 for alias in db_aliases:
436 connection = connections[alias]
437 if start_method == "spawn":
438 # Restore initial settings in spawned processes.
439 connection.settings_dict.update(initial_settings[alias])
440 if value := serialized_contents.get(alias):
441 connection._test_serialized_contents = value
442 connection.creation.setup_worker_connection(_worker_id)
443
444
445 def _run_subsuite(args):
446 """
447 Run a suite of tests with a RemoteTestRunner and return a RemoteTestResult.
448
449 This helper lives at module-level and its arguments are wrapped in a tuple
450 because of the multiprocessing module's requirements.
451 """
452 runner_class, subsuite_index, subsuite, failfast, buffer = args
453 runner = runner_class(failfast=failfast, buffer=buffer)
454 result = runner.run(subsuite)
455 return subsuite_index, result.events
456
457
458 def _process_setup_stub(*args):
459 """Stub method to simplify run() implementation."""
460 pass
461
462
463 class ParallelTestSuite(unittest.TestSuite):
464 """
465 Run a series of tests in parallel in several processes.
466
467 While the unittest module's documentation implies that orchestrating the
468 execution of tests is the responsibility of the test runner, in practice,
469 it appears that TestRunner classes are more concerned with formatting and
470 displaying test results.
471
472 Since there are fewer use cases for customizing TestSuite than TestRunner,
473 implementing parallelization at the level of the TestSuite improves
474 interoperability with existing custom test runners. A single instance of a
475 test runner can still collect results from all tests without being aware
476 that they have been run in parallel.
477 """
478
479 # In case someone wants to modify these in a subclass.
480 init_worker = _init_worker
481 process_setup = _process_setup_stub
482 process_setup_args = ()
483 run_subsuite = _run_subsuite
484 runner_class = RemoteTestRunner
485
486 def __init__(
487 self, subsuites, processes, failfast=False, debug_mode=False, buffer=False
488 ):
489 self.subsuites = subsuites
490 self.processes = processes
491 self.failfast = failfast
492 self.debug_mode = debug_mode
493 self.buffer = buffer
494 self.initial_settings = None
495 self.serialized_contents = None
496 self.used_aliases = None
497 super().__init__()
498
499 def run(self, result):
500 """
501 Distribute TestCases across workers.
502
503 Return an identifier of each TestCase with its result in order to use
504 imap_unordered to show results as soon as they're available.
505
506 To minimize pickling errors when getting results from workers:
507
508 - pass back numeric indexes in self.subsuites instead of tests
509 - make tracebacks picklable with tblib, if available
510
511 Even with tblib, errors may still occur for dynamically created
512 exception classes which cannot be unpickled.
513 """
514 self.initialize_suite()
515 counter = multiprocessing.Value(ctypes.c_int, 0)
516 pool = multiprocessing.Pool(
517 processes=self.processes,
518 initializer=self.init_worker.__func__,
519 initargs=[
520 counter,
521 self.initial_settings,
522 self.serialized_contents,
523 self.process_setup.__func__,
524 self.process_setup_args,
525 self.debug_mode,
526 self.used_aliases,
527 ],
528 )
529 args = [
530 (self.runner_class, index, subsuite, self.failfast, self.buffer)
531 for index, subsuite in enumerate(self.subsuites)
532 ]
533 test_results = pool.imap_unordered(self.run_subsuite.__func__, args)
534
535 while True:
536 if result.shouldStop:
537 pool.terminate()
538 break
539
540 try:
541 subsuite_index, events = test_results.next(timeout=0.1)
542 except multiprocessing.TimeoutError:
543 continue
544 except StopIteration:
545 pool.close()
546 break
547
548 tests = list(self.subsuites[subsuite_index])
549 for event in events:
550 self.handle_event(result, tests, event)
551
552 pool.join()
553
554 return result
555
556 def handle_event(self, result, tests, event):
557 event_name = event[0]
558 handler = getattr(result, event_name, None)
559 if handler is None:
560 return
561 test_index = event[1]
562 event_occurred_before_first_test = test_index == -1
563 if (
564 event_name == "addError"
565 and event_occurred_before_first_test
566 and len(event) >= 4
567 ):
568 test_id = event[2]
569 test = unittest.suite._ErrorHolder(test_id)
570 args = event[3:]
571 else:
572 test = tests[test_index]
573 args = event[2:]
574 handler(test, *args)
575
576 def __iter__(self):
577 return iter(self.subsuites)
578
579 def initialize_suite(self):
580 if multiprocessing.get_start_method() == "spawn":
581 self.initial_settings = {
582 alias: connections[alias].settings_dict for alias in connections
583 }
584 self.serialized_contents = {
585 alias: connections[alias]._test_serialized_contents
586 for alias in connections
587 if alias in self.serialized_aliases
588 }
589
590
591 class Shuffler:
592 """
593 This class implements shuffling with a special consistency property.
594 Consistency means that, for a given seed and key function, if two sets of
595 items are shuffled, the resulting order will agree on the intersection of
596 the two sets. For example, if items are removed from an original set, the
597 shuffled order for the new set will be the shuffled order of the original
598 set restricted to the smaller set.
599 """
600
601 # This doesn't need to be cryptographically strong, so use what's fastest.
602 hash_algorithm = "md5"
603
604 @classmethod
605 def _hash_text(cls, text):
606 h = hashlib.new(cls.hash_algorithm, usedforsecurity=False)
607 h.update(text.encode("utf-8"))
608 return h.hexdigest()
609
610 def __init__(self, seed=None):
611 if seed is None:
612 # Limit seeds to 10 digits for simpler output.
613 seed = random.randint(0, 10**10 - 1)
614 seed_source = "generated"
615 else:
616 seed_source = "given"
617 self.seed = seed
618 self.seed_source = seed_source
619
620 @property
621 def seed_display(self):
622 return f"{self.seed!r} ({self.seed_source})"
623
624 def _hash_item(self, item, key):
625 text = "{}{}".format(self.seed, key(item))
626 return self._hash_text(text)
627
628 def shuffle(self, items, key):
629 """
630 Return a new list of the items in a shuffled order.
631
632 The `key` is a function that accepts an item in `items` and returns
633 a string unique for that item that can be viewed as a string id. The
634 order of the return value is deterministic. It depends on the seed
635 and key function but not on the original order.
636 """
637 hashes = {}
638 for item in items:
639 hashed = self._hash_item(item, key)
640 if hashed in hashes:
641 msg = "item {!r} has same hash {!r} as item {!r}".format(
642 item,
643 hashed,
644 hashes[hashed],
645 )
646 raise RuntimeError(msg)
647 hashes[hashed] = item
648 return [hashes[hashed] for hashed in sorted(hashes)]
649
650
651 class DiscoverRunner:
652 """A Django test runner that uses unittest2 test discovery."""
653
654 test_suite = unittest.TestSuite
655 parallel_test_suite = ParallelTestSuite
656 test_runner = unittest.TextTestRunner
657 test_loader = unittest.defaultTestLoader
658 reorder_by = (TestCase, SimpleTestCase)
659
660 def __init__(
661 self,
662 pattern=None,
663 top_level=None,
664 verbosity=1,
665 interactive=True,
666 failfast=False,
667 keepdb=False,
668 reverse=False,
669 debug_mode=False,
670 debug_sql=False,
671 parallel=0,
672 tags=None,
673 exclude_tags=None,
674 test_name_patterns=None,
675 pdb=False,
676 buffer=False,
677 enable_faulthandler=True,
678 timing=False,
679 shuffle=False,
680 logger=None,
681 durations=None,
682 **kwargs,
683 ):
684 self.pattern = pattern
685 self.top_level = top_level
686 self.verbosity = verbosity
687 self.interactive = interactive
688 self.failfast = failfast
689 self.keepdb = keepdb
690 self.reverse = reverse
691 self.debug_mode = debug_mode
692 self.debug_sql = debug_sql
693 self.parallel = parallel
694 self.tags = set(tags or [])
695 self.exclude_tags = set(exclude_tags or [])
696 if not faulthandler.is_enabled() and enable_faulthandler:
697 try:
698 faulthandler.enable(file=sys.stderr.fileno())
699 except (AttributeError, io.UnsupportedOperation):
700 faulthandler.enable(file=sys.__stderr__.fileno())
701 self.pdb = pdb
702 if self.pdb and self.parallel > 1:
703 raise ValueError(
704 "You cannot use --pdb with parallel tests; pass --parallel=1 to use it."
705 )
706 self.buffer = buffer
707 self.test_name_patterns = None
708 self.time_keeper = TimeKeeper() if timing else NullTimeKeeper()
709 if test_name_patterns:
710 # unittest does not export the _convert_select_pattern function
711 # that converts command-line arguments to patterns.
712 self.test_name_patterns = {
713 pattern if "*" in pattern else "*%s*" % pattern
714 for pattern in test_name_patterns
715 }
716 self.shuffle = shuffle
717 self._shuffler = None
718 self.logger = logger
719 self.durations = durations
720
721 @classmethod
722 def add_arguments(cls, parser):
723 parser.add_argument(
724 "--failfast",
725 action="store_true",
726 help="Stops the test suite after the first failure.",
727 )
728 parser.add_argument(
729 "-t",
730 "--top-level-directory",
731 dest="top_level",
732 help="Top level of project for unittest discovery.",
733 )
734 parser.add_argument(
735 "-p",
736 "--pattern",
737 default="test*.py",
738 help="The test matching pattern. Defaults to test*.py.",
739 )
740 parser.add_argument(
741 "--keepdb", action="store_true", help="Preserves the test DB between runs."
742 )
743 parser.add_argument(
744 "--shuffle",
745 nargs="?",
746 default=False,
747 type=int,
748 metavar="SEED",
749 help="Shuffles test case order.",
750 )
751 parser.add_argument(
752 "-r",
753 "--reverse",
754 action="store_true",
755 help="Reverses test case order.",
756 )
757 parser.add_argument(
758 "--debug-mode",
759 action="store_true",
760 help="Sets settings.DEBUG to True.",
761 )
762 parser.add_argument(
763 "-d",
764 "--debug-sql",
765 action="store_true",
766 help="Prints logged SQL queries on failure.",
767 )
768 parser.add_argument(
769 "--parallel",
770 nargs="?",
771 const="auto",
772 default=0,
773 type=parallel_type,
774 metavar="N",
775 help=(
776 "Run tests using up to N parallel processes. Use the value "
777 '"auto" to run one test process for each processor core.'
778 ),
779 )
780 parser.add_argument(
781 "--tag",
782 action="append",
783 dest="tags",
784 help="Run only tests with the specified tag. Can be used multiple times.",
785 )
786 parser.add_argument(
787 "--exclude-tag",
788 action="append",
789 dest="exclude_tags",
790 help="Do not run tests with the specified tag. Can be used multiple times.",
791 )
792 parser.add_argument(
793 "--pdb",
794 action="store_true",
795 help="Runs a debugger (pdb, or ipdb if installed) on error or failure.",
796 )
797 parser.add_argument(
798 "-b",
799 "--buffer",
800 action="store_true",
801 help="Discard output from passing tests.",
802 )
803 parser.add_argument(
804 "--no-faulthandler",
805 action="store_false",
806 dest="enable_faulthandler",
807 help="Disables the Python faulthandler module during tests.",
808 )
809 parser.add_argument(
810 "--timing",
811 action="store_true",
812 help=("Output timings, including database set up and total run time."),
813 )
814 parser.add_argument(
815 "-k",
816 action="append",
817 dest="test_name_patterns",
818 help=(
819 "Only run test methods and classes that match the pattern "
820 "or substring. Can be used multiple times. Same as "
821 "unittest -k option."
822 ),
823 )
824 if PY312:
825 parser.add_argument(
826 "--durations",
827 dest="durations",
828 type=int,
829 default=None,
830 metavar="N",
831 help="Show the N slowest test cases (N=0 for all).",
832 )
833
834 @property
835 def shuffle_seed(self):
836 if self._shuffler is None:
837 return None
838 return self._shuffler.seed
839
840 def log(self, msg, level=None):
841 """
842 Log the message at the given logging level (the default is INFO).
843
844 If a logger isn't set, the message is instead printed to the console,
845 respecting the configured verbosity. A verbosity of 0 prints no output,
846 a verbosity of 1 prints INFO and above, and a verbosity of 2 or higher
847 prints all levels.
848 """
849 if level is None:
850 level = logging.INFO
851 if self.logger is None:
852 if self.verbosity <= 0 or (self.verbosity == 1 and level < logging.INFO):
853 return
854 print(msg)
855 else:
856 self.logger.log(level, msg)
857
858 def setup_test_environment(self, **kwargs):
859 setup_test_environment(debug=self.debug_mode)
860 unittest.installHandler()
861
862 def setup_shuffler(self):
863 if self.shuffle is False:
864 return
865 shuffler = Shuffler(seed=self.shuffle)
866 self.log(f"Using shuffle seed: {shuffler.seed_display}")
867 self._shuffler = shuffler
868
869 @contextmanager
870 def load_with_patterns(self):
871 original_test_name_patterns = self.test_loader.testNamePatterns
872 self.test_loader.testNamePatterns = self.test_name_patterns
873 try:
874 yield
875 finally:
876 # Restore the original patterns.
877 self.test_loader.testNamePatterns = original_test_name_patterns
878
879 def load_tests_for_label(self, label, discover_kwargs):
880 label_as_path = os.path.abspath(label)
881 tests = None
882
883 # If a module, or "module.ClassName[.method_name]", just run those.
884 if not os.path.exists(label_as_path):
885 with self.load_with_patterns():
886 tests = self.test_loader.loadTestsFromName(label)
887 if tests.countTestCases():
888 return tests
889 # Try discovery if "label" is a package or directory.
890 is_importable, is_package = try_importing(label)
891 if is_importable:
892 if not is_package:
893 return tests
894 elif not os.path.isdir(label_as_path):
895 if os.path.exists(label_as_path):
896 assert tests is None
897 raise RuntimeError(
898 f"One of the test labels is a path to a file: {label!r}, "
899 f"which is not supported. Use a dotted module name or "
900 f"path to a directory instead."
901 )
902 return tests
903
904 kwargs = discover_kwargs.copy()
905 if os.path.isdir(label_as_path) and not self.top_level:
906 kwargs["top_level_dir"] = find_top_level(label_as_path)
907
908 with self.load_with_patterns():
909 tests = self.test_loader.discover(start_dir=label, **kwargs)
910
911 # Make unittest forget the top-level dir it calculated from this run,
912 # to support running tests from two different top-levels.
913 self.test_loader._top_level_dir = None
914 return tests
915
916 def build_suite(self, test_labels=None, **kwargs):
917 test_labels = test_labels or ["."]
918
919 discover_kwargs = {}
920 if self.pattern is not None:
921 discover_kwargs["pattern"] = self.pattern
922 if self.top_level is not None:
923 discover_kwargs["top_level_dir"] = self.top_level
924 self.setup_shuffler()
925
926 all_tests = []
927 for label in test_labels:
928 tests = self.load_tests_for_label(label, discover_kwargs)
929 all_tests.extend(iter_test_cases(tests))
930
931 if self.tags or self.exclude_tags:
932 if self.tags:
933 self.log(
934 "Including test tag(s): %s." % ", ".join(sorted(self.tags)),
935 level=logging.DEBUG,
936 )
937 if self.exclude_tags:
938 self.log(
939 "Excluding test tag(s): %s." % ", ".join(sorted(self.exclude_tags)),
940 level=logging.DEBUG,
941 )
942 all_tests = filter_tests_by_tags(all_tests, self.tags, self.exclude_tags)
943
944 # Put the failures detected at load time first for quicker feedback.
945 # _FailedTest objects include things like test modules that couldn't be
946 # found or that couldn't be loaded due to syntax errors.
947 test_types = (unittest.loader._FailedTest, *self.reorder_by)
948 all_tests = list(
949 reorder_tests(
950 all_tests,
951 test_types,
952 shuffler=self._shuffler,
953 reverse=self.reverse,
954 )
955 )
956 self.log("Found %d test(s)." % len(all_tests))
957 suite = self.test_suite(all_tests)
958
959 if self.parallel > 1:
960 subsuites = partition_suite_by_case(suite)
961 # Since tests are distributed across processes on a per-TestCase
962 # basis, there's no need for more processes than TestCases.
963 processes = min(self.parallel, len(subsuites))
964 # Update also "parallel" because it's used to determine the number
965 # of test databases.
966 self.parallel = processes
967 if processes > 1:
968 suite = self.parallel_test_suite(
969 subsuites,
970 processes,
971 self.failfast,
972 self.debug_mode,
973 self.buffer,
974 )
975 return suite
976
977 def setup_databases(self, **kwargs):
978 return _setup_databases(
979 self.verbosity,
980 self.interactive,
981 time_keeper=self.time_keeper,
982 keepdb=self.keepdb,
983 debug_sql=self.debug_sql,
984 parallel=self.parallel,
985 **kwargs,
986 )
987
988 def get_resultclass(self):
989 if self.debug_sql:
990 return DebugSQLTextTestResult
991 elif self.pdb:
992 return PDBDebugResult
993
994 def get_test_runner_kwargs(self):
995 kwargs = {
996 "failfast": self.failfast,
997 "resultclass": self.get_resultclass(),
998 "verbosity": self.verbosity,
999 "buffer": self.buffer,
1000 }
1001 if PY312:
1002 kwargs["durations"] = self.durations
1003 return kwargs
1004
1005 def run_checks(self, databases):
1006 # Checks are run after database creation since some checks require
1007 # database access.
1008 call_command("check", verbosity=self.verbosity, databases=databases)
1009
1010 def run_suite(self, suite, **kwargs):
1011 kwargs = self.get_test_runner_kwargs()
1012 runner = self.test_runner(**kwargs)
1013 try:
1014 return runner.run(suite)
1015 finally:
1016 if self._shuffler is not None:
1017 seed_display = self._shuffler.seed_display
1018 self.log(f"Used shuffle seed: {seed_display}")
1019
1020 def teardown_databases(self, old_config, **kwargs):
1021 """Destroy all the non-mirror databases."""
1022 _teardown_databases(
1023 old_config,
1024 verbosity=self.verbosity,
1025 parallel=self.parallel,
1026 keepdb=self.keepdb,
1027 )
1028
1029 def teardown_test_environment(self, **kwargs):
1030 unittest.removeHandler()
1031 teardown_test_environment()
1032
1033 def suite_result(self, suite, result, **kwargs):
1034 return (
1035 len(result.failures) + len(result.errors) + len(result.unexpectedSuccesses)
1036 )
1037
1038 def _get_databases(self, suite):
1039 databases = {}
1040 for test in iter_test_cases(suite):
1041 test_databases = getattr(test, "databases", None)
1042 if test_databases == "__all__":
1043 test_databases = connections
1044 if test_databases:
1045 serialized_rollback = getattr(test, "serialized_rollback", False)
1046 databases.update(
1047 (alias, serialized_rollback or databases.get(alias, False))
1048 for alias in test_databases
1049 )
1050 return databases
1051
1052 def get_databases(self, suite):
1053 databases = self._get_databases(suite)
1054 unused_databases = [alias for alias in connections if alias not in databases]
1055 if unused_databases:
1056 self.log(
1057 "Skipping setup of unused database(s): %s."
1058 % ", ".join(sorted(unused_databases)),
1059 level=logging.DEBUG,
1060 )
1061 return databases
1062
1063 def run_tests(self, test_labels, **kwargs):
1064 """
1065 Run the unit tests for all the test labels in the provided list.
1066
1067 Test labels should be dotted Python paths to test modules, test
1068 classes, or test methods.
1069
1070 Return the number of tests that failed.
1071 """
1072 self.setup_test_environment()
1073 suite = self.build_suite(test_labels)
1074 databases = self.get_databases(suite)
1075 suite.serialized_aliases = set(
1076 alias for alias, serialize in databases.items() if serialize
1077 )
1078 suite.used_aliases = set(databases)
1079 with self.time_keeper.timed("Total database setup"):
1080 old_config = self.setup_databases(
1081 aliases=databases,
1082 serialized_aliases=suite.serialized_aliases,
1083 )
1084 run_failed = False
1085 try:
1086 self.run_checks(databases)
1087 result = self.run_suite(suite)
1088 except Exception:
1089 run_failed = True
1090 raise
1091 finally:
1092 try:
1093 with self.time_keeper.timed("Total database teardown"):
1094 self.teardown_databases(old_config)
1095 self.teardown_test_environment()
1096 except Exception:
1097 # Silence teardown exceptions if an exception was raised during
1098 # runs to avoid shadowing it.
1099 if not run_failed:
1100 raise
1101 self.time_keeper.print_results()
1102 return self.suite_result(suite, result)
1103
1104
1105 def try_importing(label):
1106 """
1107 Try importing a test label, and return (is_importable, is_package).
1108
1109 Relative labels like "." and ".." are seen as directories.
1110 """
1111 try:
1112 mod = import_module(label)
1113 except (ImportError, TypeError):
1114 return (False, False)
1115
1116 return (True, hasattr(mod, "__path__"))
1117
1118
1119 def find_top_level(top_level):
1120 # Try to be a bit smarter than unittest about finding the default top-level
1121 # for a given directory path, to avoid breaking relative imports.
1122 # (Unittest's default is to set top-level equal to the path, which means
1123 # relative imports will result in "Attempted relative import in
1124 # non-package.").
1125
1126 # We'd be happy to skip this and require dotted module paths (which don't
1127 # cause this problem) instead of file paths (which do), but in the case of
1128 # a directory in the cwd, which would be equally valid if considered as a
1129 # top-level module or as a directory path, unittest unfortunately prefers
1130 # the latter.
1131 while True:
1132 init_py = os.path.join(top_level, "__init__.py")
1133 if not os.path.exists(init_py):
1134 break
1135 try_next = os.path.dirname(top_level)
1136 if try_next == top_level:
1137 # __init__.py all the way down? give up.
1138 break
1139 top_level = try_next
1140 return top_level
1141
1142
1143 def _class_shuffle_key(cls):
1144 return f"{cls.__module__}.{cls.__qualname__}"
1145
1146
1147 def shuffle_tests(tests, shuffler):
1148 """
1149 Return an iterator over the given tests in a shuffled order, keeping tests
1150 next to other tests of their class.
1151
1152 `tests` should be an iterable of tests.
1153 """
1154 tests_by_type = {}
1155 for _, class_tests in itertools.groupby(tests, type):
1156 class_tests = list(class_tests)
1157 test_type = type(class_tests[0])
1158 class_tests = shuffler.shuffle(class_tests, key=lambda test: test.id())
1159 tests_by_type[test_type] = class_tests
1160
1161 classes = shuffler.shuffle(tests_by_type, key=_class_shuffle_key)
1162
1163 return itertools.chain(*(tests_by_type[cls] for cls in classes))
1164
1165
1166 def reorder_test_bin(tests, shuffler=None, reverse=False):
1167 """
1168 Return an iterator that reorders the given tests, keeping tests next to
1169 other tests of their class.
1170
1171 `tests` should be an iterable of tests that supports reversed().
1172 """
1173 if shuffler is None:
1174 if reverse:
1175 return reversed(tests)
1176 # The function must return an iterator.
1177 return iter(tests)
1178
1179 tests = shuffle_tests(tests, shuffler)
1180 if not reverse:
1181 return tests
1182 # Arguments to reversed() must be reversible.
1183 return reversed(list(tests))
1184
1185
1186 def reorder_tests(tests, classes, reverse=False, shuffler=None):
1187 """
1188 Reorder an iterable of tests, grouping by the given TestCase classes.
1189
1190 This function also removes any duplicates and reorders so that tests of the
1191 same type are consecutive.
1192
1193 The result is returned as an iterator. `classes` is a sequence of types.
1194 Tests that are instances of `classes[0]` are grouped first, followed by
1195 instances of `classes[1]`, etc. Tests that are not instances of any of the
1196 classes are grouped last.
1197
1198 If `reverse` is True, the tests within each `classes` group are reversed,
1199 but without reversing the order of `classes` itself.
1200
1201 The `shuffler` argument is an optional instance of this module's `Shuffler`
1202 class. If provided, tests will be shuffled within each `classes` group, but
1203 keeping tests with other tests of their TestCase class. Reversing is
1204 applied after shuffling to allow reversing the same random order.
1205 """
1206 # Each bin maps TestCase class to OrderedSet of tests. This permits tests
1207 # to be grouped by TestCase class even if provided non-consecutively.
1208 bins = [defaultdict(OrderedSet) for i in range(len(classes) + 1)]
1209 *class_bins, last_bin = bins
1210
1211 for test in tests:
1212 for test_bin, test_class in zip(class_bins, classes):
1213 if isinstance(test, test_class):
1214 break
1215 else:
1216 test_bin = last_bin
1217 test_bin[type(test)].add(test)
1218
1219 for test_bin in bins:
1220 # Call list() since reorder_test_bin()'s input must support reversed().
1221 tests = list(itertools.chain.from_iterable(test_bin.values()))
1222 yield from reorder_test_bin(tests, shuffler=shuffler, reverse=reverse)
1223
1224
1225 def partition_suite_by_case(suite):
1226 """Partition a test suite by TestCase, preserving the order of tests."""
1227 suite_class = type(suite)
1228 all_tests = iter_test_cases(suite)
1229 return [suite_class(tests) for _, tests in itertools.groupby(all_tests, type)]
1230
1231
1232 def test_match_tags(test, tags, exclude_tags):
1233 if isinstance(test, unittest.loader._FailedTest):
1234 # Tests that couldn't load always match to prevent tests from falsely
1235 # passing due e.g. to syntax errors.
1236 return True
1237 test_tags = set(getattr(test, "tags", []))
1238 test_fn_name = getattr(test, "_testMethodName", str(test))
1239 if hasattr(test, test_fn_name):
1240 test_fn = getattr(test, test_fn_name)
1241 test_fn_tags = list(getattr(test_fn, "tags", []))
1242 test_tags = test_tags.union(test_fn_tags)
1243 if tags and test_tags.isdisjoint(tags):
1244 return False
1245 return test_tags.isdisjoint(exclude_tags)
1246
1247
1248 def filter_tests_by_tags(tests, tags, exclude_tags):
1249 """Return the matching tests as an iterator."""
1250 return (test for test in tests if test_match_tags(test, tags, exclude_tags))