Debian Codegraph @

Lib/test/test_regrtest.py

raw
1 """
2 Tests of regrtest.py.
3
4 Note: test_regrtest cannot be run twice in parallel.
5 """
6
7 import _colorize
8 import contextlib
9 import dataclasses
10 import glob
11 import io
12 import locale
13 import os.path
14 import platform
15 import random
16 import re
17 import shlex
18 import signal
19 import subprocess
20 import sys
21 import sysconfig
22 import tempfile
23 import textwrap
24 import unittest
25 import unittest.mock
26 from xml.etree import ElementTree
27
28 from test import support
29 from test.support import import_helper
30 from test.support import os_helper
31 from test.libregrtest import cmdline
32 from test.libregrtest import main
33 from test.libregrtest import setup
34 from test.libregrtest import utils
35 from test.libregrtest.filter import get_match_tests, set_match_tests, match_test
36 from test.libregrtest.result import TestStats
37 from test.libregrtest.utils import normalize_test_name
38
39 if not support.has_subprocess_support:
40 raise unittest.SkipTest("test module requires subprocess")
41
42 ROOT_DIR = os.path.join(os.path.dirname(__file__), '..', '..')
43 ROOT_DIR = os.path.abspath(os.path.normpath(ROOT_DIR))
44 LOG_PREFIX = r'[0-9]+:[0-9]+:[0-9]+ (?:load avg: [0-9]+\.[0-9]{2} )?'
45 RESULT_REGEX = (
46 'passed',
47 'failed',
48 'skipped',
49 'interrupted',
50 'env changed',
51 'timed out',
52 'ran no tests',
53 'worker non-zero exit code',
54 )
55 RESULT_REGEX = fr'(?:{"|".join(RESULT_REGEX)})'
56
57 EXITCODE_BAD_TEST = 2
58 EXITCODE_ENV_CHANGED = 3
59 EXITCODE_NO_TESTS_RAN = 4
60 EXITCODE_RERUN_FAIL = 5
61 EXITCODE_INTERRUPTED = 130
62
63 TEST_INTERRUPTED = textwrap.dedent("""
64 from signal import SIGINT, raise_signal
65 try:
66 raise_signal(SIGINT)
67 except ImportError:
68 import os
69 os.kill(os.getpid(), SIGINT)
70 """)
71
72
73 class ParseArgsTestCase(unittest.TestCase):
74 """
75 Test regrtest's argument parsing, function _parse_args().
76 """
77
78 @staticmethod
79 def parse_args(args):
80 return cmdline._parse_args(args)
81
82 def checkError(self, args, msg):
83 with support.captured_stderr() as err, self.assertRaises(SystemExit):
84 self.parse_args(args)
85 self.assertIn(msg, err.getvalue())
86
87 def test_help(self):
88 for opt in '-h', '--help':
89 with self.subTest(opt=opt):
90 with support.captured_stdout() as out, \
91 self.assertRaises(SystemExit):
92 self.parse_args([opt])
93 self.assertIn('Run Python regression tests.', out.getvalue())
94
95 def test_timeout(self):
96 ns = self.parse_args(['--timeout', '4.2'])
97 self.assertEqual(ns.timeout, 4.2)
98
99 # negative, zero and empty string are treated as "no timeout"
100 for value in ('-1', '0', ''):
101 with self.subTest(value=value):
102 ns = self.parse_args([f'--timeout={value}'])
103 self.assertEqual(ns.timeout, None)
104
105 self.checkError(['--timeout'], 'expected one argument')
106 self.checkError(['--timeout', 'foo'], 'invalid timeout value:')
107
108 def test_wait(self):
109 ns = self.parse_args(['--wait'])
110 self.assertTrue(ns.wait)
111
112 def test_start(self):
113 for opt in '-S', '--start':
114 with self.subTest(opt=opt):
115 ns = self.parse_args([opt, 'foo'])
116 self.assertEqual(ns.start, 'foo')
117 self.checkError([opt], 'expected one argument')
118
119 def test_verbose(self):
120 ns = self.parse_args(['-v'])
121 self.assertEqual(ns.verbose, 1)
122 ns = self.parse_args(['-vvv'])
123 self.assertEqual(ns.verbose, 3)
124 ns = self.parse_args(['--verbose'])
125 self.assertEqual(ns.verbose, 1)
126 ns = self.parse_args(['--verbose'] * 3)
127 self.assertEqual(ns.verbose, 3)
128 ns = self.parse_args([])
129 self.assertEqual(ns.verbose, 0)
130
131 def test_rerun(self):
132 for opt in '-w', '--rerun', '--verbose2':
133 with self.subTest(opt=opt):
134 ns = self.parse_args([opt])
135 self.assertTrue(ns.rerun)
136
137 def test_verbose3(self):
138 for opt in '-W', '--verbose3':
139 with self.subTest(opt=opt):
140 ns = self.parse_args([opt])
141 self.assertTrue(ns.verbose3)
142
143 def test_quiet(self):
144 for opt in '-q', '--quiet':
145 with self.subTest(opt=opt):
146 ns = self.parse_args([opt])
147 self.assertTrue(ns.quiet)
148 self.assertEqual(ns.verbose, 0)
149
150 def test_slowest(self):
151 for opt in '-o', '--slowest':
152 with self.subTest(opt=opt):
153 ns = self.parse_args([opt])
154 self.assertTrue(ns.print_slow)
155
156 def test_header(self):
157 ns = self.parse_args(['--header'])
158 self.assertTrue(ns.header)
159
160 ns = self.parse_args(['--verbose'])
161 self.assertTrue(ns.header)
162
163 def test_randomize(self):
164 for opt in ('-r', '--randomize'):
165 with self.subTest(opt=opt):
166 ns = self.parse_args([opt])
167 self.assertTrue(ns.randomize)
168
169 with os_helper.EnvironmentVarGuard() as env:
170 # with SOURCE_DATE_EPOCH
171 env['SOURCE_DATE_EPOCH'] = '1697839080'
172 ns = self.parse_args(['--randomize'])
173 regrtest = main.Regrtest(ns)
174 self.assertFalse(regrtest.randomize)
175 self.assertIsInstance(regrtest.random_seed, str)
176 self.assertEqual(regrtest.random_seed, '1697839080')
177
178 # without SOURCE_DATE_EPOCH
179 del env['SOURCE_DATE_EPOCH']
180 ns = self.parse_args(['--randomize'])
181 regrtest = main.Regrtest(ns)
182 self.assertTrue(regrtest.randomize)
183 self.assertIsInstance(regrtest.random_seed, int)
184
185 def test_no_randomize(self):
186 ns = self.parse_args([])
187 self.assertIs(ns.randomize, False)
188
189 ns = self.parse_args(["--randomize"])
190 self.assertIs(ns.randomize, True)
191
192 ns = self.parse_args(["--no-randomize"])
193 self.assertIs(ns.randomize, False)
194
195 ns = self.parse_args(["--randomize", "--no-randomize"])
196 self.assertIs(ns.randomize, False)
197
198 ns = self.parse_args(["--no-randomize", "--randomize"])
199 self.assertIs(ns.randomize, False)
200
201 def test_randseed(self):
202 ns = self.parse_args(['--randseed', '12345'])
203 self.assertEqual(ns.random_seed, 12345)
204 self.assertTrue(ns.randomize)
205 self.checkError(['--randseed'], 'expected one argument')
206 self.checkError(['--randseed', 'foo'], 'invalid int value')
207
208 ns = self.parse_args(['--randseed', '12345', '--no-randomize'])
209 self.assertEqual(ns.random_seed, 12345)
210 self.assertFalse(ns.randomize)
211
212 def test_fromfile(self):
213 for opt in '-f', '--fromfile':
214 with self.subTest(opt=opt):
215 ns = self.parse_args([opt, 'foo'])
216 self.assertEqual(ns.fromfile, 'foo')
217 self.checkError([opt], 'expected one argument')
218 self.checkError([opt, 'foo', '-s'], "don't go together")
219
220 def test_exclude(self):
221 for opt in '-x', '--exclude':
222 with self.subTest(opt=opt):
223 ns = self.parse_args([opt])
224 self.assertTrue(ns.exclude)
225
226 def test_single(self):
227 for opt in '-s', '--single':
228 with self.subTest(opt=opt):
229 ns = self.parse_args([opt])
230 self.assertTrue(ns.single)
231 self.checkError([opt, '-f', 'foo'], "don't go together")
232
233 def test_match(self):
234 for opt in '-m', '--match':
235 with self.subTest(opt=opt):
236 ns = self.parse_args([opt, 'pattern'])
237 self.assertEqual(ns.match_tests, [('pattern', True)])
238 self.checkError([opt], 'expected one argument')
239
240 for opt in '-i', '--ignore':
241 with self.subTest(opt=opt):
242 ns = self.parse_args([opt, 'pattern'])
243 self.assertEqual(ns.match_tests, [('pattern', False)])
244 self.checkError([opt], 'expected one argument')
245
246 ns = self.parse_args(['-m', 'pattern1', '-m', 'pattern2'])
247 self.assertEqual(ns.match_tests, [('pattern1', True), ('pattern2', True)])
248
249 ns = self.parse_args(['-m', 'pattern1', '-i', 'pattern2'])
250 self.assertEqual(ns.match_tests, [('pattern1', True), ('pattern2', False)])
251
252 ns = self.parse_args(['-i', 'pattern1', '-m', 'pattern2'])
253 self.assertEqual(ns.match_tests, [('pattern1', False), ('pattern2', True)])
254
255 self.addCleanup(os_helper.unlink, os_helper.TESTFN)
256 with open(os_helper.TESTFN, "w") as fp:
257 print('matchfile1', file=fp)
258 print('matchfile2', file=fp)
259
260 filename = os.path.abspath(os_helper.TESTFN)
261 ns = self.parse_args(['-m', 'match', '--matchfile', filename])
262 self.assertEqual(ns.match_tests,
263 [('match', True), ('matchfile1', True), ('matchfile2', True)])
264
265 ns = self.parse_args(['-i', 'match', '--ignorefile', filename])
266 self.assertEqual(ns.match_tests,
267 [('match', False), ('matchfile1', False), ('matchfile2', False)])
268
269 def test_failfast(self):
270 for opt in '-G', '--failfast':
271 with self.subTest(opt=opt):
272 ns = self.parse_args([opt, '-v'])
273 self.assertTrue(ns.failfast)
274 ns = self.parse_args([opt, '-W'])
275 self.assertTrue(ns.failfast)
276 self.checkError([opt], '-G/--failfast needs either -v or -W')
277
278 def test_use(self):
279 for opt in '-u', '--use':
280 with self.subTest(opt=opt):
281 ns = self.parse_args([opt, 'gui,network'])
282 self.assertEqual(ns.use_resources, {'gui': None, 'network': None})
283 ns = self.parse_args([opt, 'gui', opt, 'network'])
284 self.assertEqual(ns.use_resources, {'gui': None, 'network': None})
285
286 ns = self.parse_args([opt, 'gui,none,network'])
287 self.assertEqual(ns.use_resources, {'network': None})
288 ns = self.parse_args([opt, 'gui', opt, 'none', opt, 'network'])
289 self.assertEqual(ns.use_resources, {'network': None})
290
291 expected = dict.fromkeys(cmdline.ALL_RESOURCES)
292 del expected['gui']
293 ns = self.parse_args([opt, 'all,-gui'])
294 self.assertEqual(ns.use_resources, expected)
295
296 self.checkError([opt], 'expected one argument')
297 self.checkError([opt, 'foo'], 'invalid resource')
298
299 # all + a resource not part of "all"
300 expected = dict.fromkeys(cmdline.ALL_RESOURCES)
301 expected['tzdata'] = None
302 ns = self.parse_args([opt, 'all,tzdata'])
303 self.assertEqual(ns.use_resources, expected)
304 ns = self.parse_args([opt, 'all', opt, 'tzdata'])
305 self.assertEqual(ns.use_resources, expected)
306
307 # test another resource which is not part of "all"
308 ns = self.parse_args([opt, 'extralargefile'])
309 self.assertEqual(ns.use_resources, {'extralargefile': None})
310
311 # test resource with value
312 ns = self.parse_args([opt, 'xpickle=2.7'])
313 self.assertEqual(ns.use_resources, {'xpickle': '2.7'})
314 ns = self.parse_args([opt, 'xpickle=2.7,xpickle=3.3'])
315 self.assertEqual(ns.use_resources, {'xpickle': '3.3'})
316 ns = self.parse_args([opt, 'xpickle=2.7,none'])
317 self.assertEqual(ns.use_resources, {})
318 ns = self.parse_args([opt, 'xpickle=2.7,-xpickle'])
319 self.assertEqual(ns.use_resources, {})
320
321 expected = dict.fromkeys(cmdline.ALL_RESOURCES)
322 expected['xpickle'] = '2.7'
323 ns = self.parse_args([opt, 'all,xpickle=2.7'])
324 self.assertEqual(ns.use_resources, expected)
325 ns = self.parse_args([opt, 'all', opt, 'xpickle=2.7'])
326 self.assertEqual(ns.use_resources, expected)
327
328 # test invalid resources with value
329 self.checkError([opt, 'all=0'], 'invalid resource: all=0')
330 self.checkError([opt, 'none=0'], 'invalid resource: none=0')
331 self.checkError([opt, 'all,-gui=0'], 'invalid resource: -gui=0')
332
333 def test_memlimit(self):
334 for opt in '-M', '--memlimit':
335 with self.subTest(opt=opt):
336 ns = self.parse_args([opt, '4G'])
337 self.assertEqual(ns.memlimit, '4G')
338 self.checkError([opt], 'expected one argument')
339
340 def test_testdir(self):
341 ns = self.parse_args(['--testdir', 'foo'])
342 self.assertEqual(ns.testdir, os.path.join(os_helper.SAVEDCWD, 'foo'))
343 self.checkError(['--testdir'], 'expected one argument')
344
345 def test_runleaks(self):
346 for opt in '-L', '--runleaks':
347 with self.subTest(opt=opt):
348 ns = self.parse_args([opt])
349 self.assertTrue(ns.runleaks)
350
351 def test_huntrleaks(self):
352 for opt in '-R', '--huntrleaks':
353 with self.subTest(opt=opt):
354 ns = self.parse_args([opt, ':'])
355 self.assertEqual(ns.huntrleaks, (5, 4, 'reflog.txt'))
356 ns = self.parse_args([opt, '6:'])
357 self.assertEqual(ns.huntrleaks, (6, 4, 'reflog.txt'))
358 ns = self.parse_args([opt, ':3'])
359 self.assertEqual(ns.huntrleaks, (5, 3, 'reflog.txt'))
360 ns = self.parse_args([opt, '6:3:leaks.log'])
361 self.assertEqual(ns.huntrleaks, (6, 3, 'leaks.log'))
362 self.checkError([opt], 'expected one argument')
363 self.checkError([opt, '6'],
364 'needs 2 or 3 colon-separated arguments')
365 self.checkError([opt, 'foo:'], 'invalid huntrleaks value')
366 self.checkError([opt, '6:foo'], 'invalid huntrleaks value')
367
368 def test_multiprocess(self):
369 for opt in '-j', '--multiprocess':
370 with self.subTest(opt=opt):
371 ns = self.parse_args([opt, '2'])
372 self.assertEqual(ns.use_mp, 2)
373 self.checkError([opt], 'expected one argument')
374 self.checkError([opt, 'foo'], 'invalid int value')
375
376 def test_coverage_sequential(self):
377 for opt in '-T', '--coverage':
378 with self.subTest(opt=opt):
379 with support.captured_stderr() as stderr:
380 ns = self.parse_args([opt])
381 self.assertTrue(ns.trace)
382 self.assertIn(
383 "collecting coverage without -j is imprecise",
384 stderr.getvalue(),
385 )
386
387 @unittest.skipUnless(support.Py_DEBUG, 'need a debug build')
388 def test_coverage_mp(self):
389 for opt in '-T', '--coverage':
390 with self.subTest(opt=opt):
391 ns = self.parse_args([opt, '-j1'])
392 self.assertTrue(ns.trace)
393
394 def test_coverdir(self):
395 for opt in '-D', '--coverdir':
396 with self.subTest(opt=opt):
397 ns = self.parse_args([opt, 'foo'])
398 self.assertEqual(ns.coverdir,
399 os.path.join(os_helper.SAVEDCWD, 'foo'))
400 self.checkError([opt], 'expected one argument')
401
402 def test_nocoverdir(self):
403 for opt in '-N', '--nocoverdir':
404 with self.subTest(opt=opt):
405 ns = self.parse_args([opt])
406 self.assertIsNone(ns.coverdir)
407
408 def test_threshold(self):
409 for opt in '-t', '--threshold':
410 with self.subTest(opt=opt):
411 ns = self.parse_args([opt, '1000'])
412 self.assertEqual(ns.threshold, 1000)
413 self.checkError([opt], 'expected one argument')
414 self.checkError([opt, 'foo'], 'invalid int value')
415
416 def test_nowindows(self):
417 for opt in '-n', '--nowindows':
418 with self.subTest(opt=opt):
419 with contextlib.redirect_stderr(io.StringIO()) as stderr:
420 ns = self.parse_args([opt])
421 self.assertTrue(ns.nowindows)
422 err = stderr.getvalue()
423 self.assertIn('the --nowindows (-n) option is deprecated', err)
424
425 def test_forever(self):
426 for opt in '-F', '--forever':
427 with self.subTest(opt=opt):
428 ns = self.parse_args([opt])
429 self.assertTrue(ns.forever)
430
431 def test_unrecognized_argument(self):
432 self.checkError(['--xxx'], 'usage:')
433
434 def test_long_option__partial(self):
435 ns = self.parse_args(['--qui'])
436 self.assertTrue(ns.quiet)
437 self.assertEqual(ns.verbose, 0)
438
439 def test_two_options(self):
440 ns = self.parse_args(['--quiet', '--exclude'])
441 self.assertTrue(ns.quiet)
442 self.assertEqual(ns.verbose, 0)
443 self.assertTrue(ns.exclude)
444
445 def test_option_with_empty_string_value(self):
446 ns = self.parse_args(['--start', ''])
447 self.assertEqual(ns.start, '')
448
449 def test_arg(self):
450 ns = self.parse_args(['foo'])
451 self.assertEqual(ns.args, ['foo'])
452
453 def test_option_and_arg(self):
454 ns = self.parse_args(['--quiet', 'foo'])
455 self.assertTrue(ns.quiet)
456 self.assertEqual(ns.verbose, 0)
457 self.assertEqual(ns.args, ['foo'])
458
459 def test_arg_option_arg(self):
460 ns = self.parse_args(['test_unaryop', '-v', 'test_binop'])
461 self.assertEqual(ns.verbose, 1)
462 self.assertEqual(ns.args, ['test_unaryop', 'test_binop'])
463
464 def test_unknown_option(self):
465 self.checkError(['--unknown-option'],
466 'unrecognized arguments: --unknown-option')
467
468 def create_regrtest(self, args):
469 ns = cmdline._parse_args(args)
470
471 # Check Regrtest attributes which are more reliable than Namespace
472 # which has an unclear API
473 with os_helper.EnvironmentVarGuard() as env:
474 # Ignore SOURCE_DATE_EPOCH env var if it's set
475 del env['SOURCE_DATE_EPOCH']
476
477 regrtest = main.Regrtest(ns)
478
479 return regrtest
480
481 def check_ci_mode(self, args, use_resources,
482 *, rerun=True, randomize=True, output_on_failure=True):
483 regrtest = self.create_regrtest(args)
484 self.assertEqual(regrtest.num_workers, -1)
485 self.assertEqual(regrtest.want_rerun, rerun)
486 self.assertEqual(regrtest.fail_rerun, False)
487 self.assertEqual(regrtest.randomize, randomize)
488 self.assertIsInstance(regrtest.random_seed, int)
489 self.assertTrue(regrtest.fail_env_changed)
490 self.assertTrue(regrtest.print_slowest)
491 self.assertEqual(regrtest.output_on_failure, output_on_failure)
492 self.assertEqual(regrtest.use_resources, use_resources)
493 return regrtest
494
495 def test_fast_ci(self):
496 args = ['--fast-ci']
497 use_resources = dict.fromkeys(cmdline.ALL_RESOURCES)
498 del use_resources['cpu']
499 regrtest = self.check_ci_mode(args, use_resources)
500 self.assertEqual(regrtest.timeout, 10 * 60)
501
502 def test_fast_ci_python_cmd(self):
503 args = ['--fast-ci', '--python', 'python -X dev']
504 use_resources = dict.fromkeys(cmdline.ALL_RESOURCES)
505 del use_resources['cpu']
506 regrtest = self.check_ci_mode(args, use_resources, rerun=False)
507 self.assertEqual(regrtest.timeout, 10 * 60)
508 self.assertEqual(regrtest.python_cmd, ('python', '-X', 'dev'))
509
510 def test_fast_ci_resource(self):
511 # it should be possible to override resources individually
512 args = ['--fast-ci', '-u-network']
513 use_resources = dict.fromkeys(cmdline.ALL_RESOURCES)
514 del use_resources['cpu']
515 del use_resources['network']
516 self.check_ci_mode(args, use_resources)
517
518 def test_fast_ci_verbose(self):
519 args = ['--fast-ci', '--verbose']
520 use_resources = dict.fromkeys(cmdline.ALL_RESOURCES)
521 del use_resources['cpu']
522 regrtest = self.check_ci_mode(args, use_resources,
523 output_on_failure=False)
524 self.assertEqual(regrtest.verbose, True)
525
526 def test_slow_ci(self):
527 args = ['--slow-ci']
528 use_resources = dict.fromkeys(cmdline.ALL_RESOURCES)
529 regrtest = self.check_ci_mode(args, use_resources)
530 self.assertEqual(regrtest.timeout, 20 * 60)
531
532 def test_ci_no_randomize(self):
533 use_resources = dict.fromkeys(cmdline.ALL_RESOURCES)
534 self.check_ci_mode(
535 ["--slow-ci", "--no-randomize"], use_resources, randomize=False
536 )
537 del use_resources['cpu']
538 self.check_ci_mode(
539 ["--fast-ci", "--no-randomize"], use_resources, randomize=False
540 )
541
542 def test_dont_add_python_opts(self):
543 args = ['--dont-add-python-opts']
544 ns = cmdline._parse_args(args)
545 self.assertFalse(ns._add_python_opts)
546
547 def test_bisect(self):
548 args = ['--bisect']
549 regrtest = self.create_regrtest(args)
550 self.assertTrue(regrtest.want_bisect)
551
552 def test_verbose3_huntrleaks(self):
553 args = ['-R', '3:10', '--verbose3']
554 with support.captured_stderr():
555 regrtest = self.create_regrtest(args)
556 self.assertIsNotNone(regrtest.hunt_refleak)
557 self.assertEqual(regrtest.hunt_refleak.warmups, 3)
558 self.assertEqual(regrtest.hunt_refleak.runs, 10)
559 self.assertFalse(regrtest.output_on_failure)
560
561 def test_single_process(self):
562 args = ['-j2', '--single-process']
563 with support.captured_stderr():
564 regrtest = self.create_regrtest(args)
565 self.assertEqual(regrtest.num_workers, 0)
566 self.assertTrue(regrtest.single_process)
567
568 args = ['--fast-ci', '--single-process']
569 with support.captured_stderr():
570 regrtest = self.create_regrtest(args)
571 self.assertEqual(regrtest.num_workers, 0)
572 self.assertTrue(regrtest.single_process)
573
574
575 @dataclasses.dataclass(slots=True)
576 class Rerun:
577 name: str
578 match: str | None
579 success: bool
580
581
582 class BaseTestCase(unittest.TestCase):
583 TEST_UNIQUE_ID = 1
584 TESTNAME_PREFIX = 'test_regrtest_'
585 TESTNAME_REGEX = r'test_[a-zA-Z0-9_]+'
586
587 def setUp(self):
588 self.testdir = os.path.realpath(os.path.dirname(__file__))
589
590 self.tmptestdir = tempfile.mkdtemp()
591 self.addCleanup(os_helper.rmtree, self.tmptestdir)
592
593 def create_test(self, name=None, code=None):
594 if not name:
595 name = 'noop%s' % BaseTestCase.TEST_UNIQUE_ID
596 BaseTestCase.TEST_UNIQUE_ID += 1
597
598 if code is None:
599 code = textwrap.dedent("""
600 import unittest
601
602 class Tests(unittest.TestCase):
603 def test_empty_test(self):
604 pass
605 """)
606
607 # test_regrtest cannot be run twice in parallel because
608 # of setUp() and create_test()
609 name = self.TESTNAME_PREFIX + name
610 path = os.path.join(self.tmptestdir, name + '.py')
611
612 self.addCleanup(os_helper.unlink, path)
613 # Use 'x' mode to ensure that we do not override existing tests
614 try:
615 with open(path, 'x', encoding='utf-8') as fp:
616 fp.write(code)
617 except PermissionError as exc:
618 if not sysconfig.is_python_build():
619 self.skipTest("cannot write %s: %s" % (path, exc))
620 raise
621 return name
622
623 def regex_search(self, regex, output):
624 match = re.search(regex, output, re.MULTILINE)
625 if not match:
626 self.fail("%r not found in %r" % (regex, output))
627 return match
628
629 def check_line(self, output, pattern, full=False, regex=True):
630 if not regex:
631 pattern = re.escape(pattern)
632 if full:
633 pattern += '\n'
634 regex = re.compile(r'^' + pattern, re.MULTILINE)
635 self.assertRegex(output, regex)
636
637 def parse_executed_tests(self, output):
638 regex = (fr'^{LOG_PREFIX}\[ *[0-9]+(?:/ *[0-9]+)*\] '
639 fr'({self.TESTNAME_REGEX}) {RESULT_REGEX}')
640 parser = re.finditer(regex, output, re.MULTILINE)
641 return list(match.group(1) for match in parser)
642
643 def check_executed_tests(self, output, tests, *, stats,
644 skipped=(), failed=(),
645 env_changed=(), omitted=(),
646 rerun=None, run_no_tests=(),
647 resource_denied=(),
648 randomize=False, parallel=False, interrupted=False,
649 fail_env_changed=False,
650 forever=False, filtered=False):
651 if isinstance(tests, str):
652 tests = [tests]
653 if isinstance(skipped, str):
654 skipped = [skipped]
655 if isinstance(resource_denied, str):
656 resource_denied = [resource_denied]
657 if isinstance(failed, str):
658 failed = [failed]
659 if isinstance(env_changed, str):
660 env_changed = [env_changed]
661 if isinstance(omitted, str):
662 omitted = [omitted]
663 if isinstance(run_no_tests, str):
664 run_no_tests = [run_no_tests]
665 if isinstance(stats, int):
666 stats = TestStats(stats)
667 if parallel:
668 randomize = True
669
670 rerun_failed = []
671 if rerun is not None and not env_changed:
672 failed = [rerun.name]
673 if not rerun.success:
674 rerun_failed.append(rerun.name)
675
676 executed = self.parse_executed_tests(output)
677 total_tests = list(tests)
678 if rerun is not None:
679 total_tests.append(rerun.name)
680 if randomize:
681 self.assertEqual(set(executed), set(total_tests), output)
682 else:
683 self.assertEqual(executed, total_tests, output)
684
685 def plural(count):
686 return 's' if count != 1 else ''
687
688 def list_regex(line_format, tests):
689 count = len(tests)
690 names = ' '.join(sorted(tests))
691 regex = line_format % (count, plural(count))
692 regex = r'%s:\n %s$' % (regex, names)
693 return regex
694
695 if skipped:
696 regex = list_regex('%s test%s skipped', skipped)
697 self.check_line(output, regex)
698
699 if resource_denied:
700 regex = list_regex(r'%s test%s skipped \(resource denied\)', resource_denied)
701 self.check_line(output, regex)
702
703 if failed:
704 regex = list_regex('%s test%s failed', failed)
705 self.check_line(output, regex)
706
707 if env_changed:
708 regex = list_regex(r'%s test%s altered the execution environment '
709 r'\(env changed\)',
710 env_changed)
711 self.check_line(output, regex)
712
713 if omitted:
714 regex = list_regex('%s test%s omitted', omitted)
715 self.check_line(output, regex)
716
717 if rerun is not None:
718 regex = list_regex('%s re-run test%s', [rerun.name])
719 self.check_line(output, regex)
720 regex = LOG_PREFIX + r"Re-running 1 failed tests in verbose mode"
721 self.check_line(output, regex)
722 regex = fr"Re-running {rerun.name} in verbose mode"
723 if rerun.match:
724 regex = fr"{regex} \(matching: {rerun.match}\)"
725 self.check_line(output, regex)
726
727 if run_no_tests:
728 regex = list_regex('%s test%s run no tests', run_no_tests)
729 self.check_line(output, regex)
730
731 good = (len(tests) - len(skipped) - len(resource_denied) - len(failed)
732 - len(omitted) - len(env_changed) - len(run_no_tests))
733 if good:
734 regex = r'%s test%s OK\.' % (good, plural(good))
735 if not skipped and not failed and (rerun is None or rerun.success) and good > 1:
736 regex = 'All %s' % regex
737 self.check_line(output, regex, full=True)
738
739 if interrupted:
740 self.check_line(output, 'Test suite interrupted by signal SIGINT.')
741
742 # Total tests
743 text = f'run={stats.tests_run:,}'
744 if filtered:
745 text = fr'{text} \(filtered\)'
746 parts = [text]
747 if stats.failures:
748 parts.append(f'failures={stats.failures:,}')
749 if stats.skipped:
750 parts.append(f'skipped={stats.skipped:,}')
751 line = fr'Total tests: {" ".join(parts)}'
752 self.check_line(output, line, full=True)
753
754 # Total test files
755 run = len(total_tests) - len(resource_denied)
756 if rerun is not None:
757 total_failed = len(rerun_failed)
758 total_rerun = 1
759 else:
760 total_failed = len(failed)
761 total_rerun = 0
762 if interrupted:
763 run = 0
764 text = f'run={run}'
765 if not forever:
766 text = f'{text}/{len(tests)}'
767 if filtered:
768 text = fr'{text} \(filtered\)'
769 report = [text]
770 for name, ntest in (
771 ('failed', total_failed),
772 ('env_changed', len(env_changed)),
773 ('skipped', len(skipped)),
774 ('resource_denied', len(resource_denied)),
775 ('rerun', total_rerun),
776 ('run_no_tests', len(run_no_tests)),
777 ):
778 if ntest:
779 report.append(f'{name}={ntest}')
780 line = fr'Total test files: {" ".join(report)}'
781 self.check_line(output, line, full=True)
782
783 # Result
784 state = []
785 if failed:
786 state.append('FAILURE')
787 elif fail_env_changed and env_changed:
788 state.append('ENV CHANGED')
789 if interrupted:
790 state.append('INTERRUPTED')
791 if not any((good, failed, interrupted, skipped,
792 env_changed, fail_env_changed)):
793 state.append("NO TESTS RAN")
794 elif not state:
795 state.append('SUCCESS')
796 state = ', '.join(state)
797 if rerun is not None:
798 new_state = 'SUCCESS' if rerun.success else 'FAILURE'
799 state = f'{state} then {new_state}'
800 self.check_line(output, f'Result: {state}', full=True)
801
802 def parse_random_seed(self, output: str) -> str:
803 match = self.regex_search(r'Using random seed: (.*)', output)
804 return match.group(1)
805
806 def run_command(self, args, input=None, exitcode=0, **kw):
807 if not input:
808 input = ''
809 if 'stderr' not in kw:
810 kw['stderr'] = subprocess.STDOUT
811
812 env = kw.pop('env', None)
813 if env is None:
814 env = dict(os.environ)
815 env.pop('SOURCE_DATE_EPOCH', None)
816
817 proc = subprocess.run(args,
818 text=True,
819 input=input,
820 stdout=subprocess.PIPE,
821 env=env,
822 **kw)
823 if proc.returncode != exitcode:
824 msg = ("Command %s failed with exit code %s, but exit code %s expected!\n"
825 "\n"
826 "stdout:\n"
827 "---\n"
828 "%s\n"
829 "---\n"
830 % (str(args), proc.returncode, exitcode, proc.stdout))
831 if proc.stderr:
832 msg += ("\n"
833 "stderr:\n"
834 "---\n"
835 "%s"
836 "---\n"
837 % proc.stderr)
838 self.fail(msg)
839 return proc
840
841 def run_python(self, args, isolated=True, **kw):
842 extraargs = []
843 if 'uops' in sys._xoptions:
844 # Pass -X uops along
845 extraargs.extend(['-X', 'uops'])
846 cmd = [sys.executable, *extraargs, '-X', 'faulthandler']
847 if isolated:
848 cmd.append('-I')
849 cmd.extend(args)
850 proc = self.run_command(cmd, **kw)
851 return proc.stdout
852
853
854 class CheckActualTests(BaseTestCase):
855 def test_finds_expected_number_of_tests(self):
856 """
857 Check that regrtest appears to find the expected set of tests.
858 """
859 args = ['-Wd', '-E', '-bb', '-m', 'test.regrtest', '--list-tests']
860 output = self.run_python(args)
861 rough_number_of_tests_found = len(output.splitlines())
862 actual_testsuite_glob = os.path.join(glob.escape(os.path.dirname(__file__)),
863 'test*.py')
864 rough_counted_test_py_files = len(glob.glob(actual_testsuite_glob))
865 # We're not trying to duplicate test finding logic in here,
866 # just give a rough estimate of how many there should be and
867 # be near that. This is a regression test to prevent mishaps
868 # such as https://bugs.python.org/issue37667 in the future.
869 # If you need to change the values in here during some
870 # mythical future test suite reorganization, don't go
871 # overboard with logic and keep that goal in mind.
872 self.assertGreater(rough_number_of_tests_found,
873 rough_counted_test_py_files*9//10,
874 msg='Unexpectedly low number of tests found in:\n'
875 f'{", ".join(output.splitlines())}')
876
877
878 @support.force_not_colorized_test_class
879 class ProgramsTestCase(BaseTestCase):
880 """
881 Test various ways to run the Python test suite. Use options close
882 to options used on the buildbot.
883 """
884
885 NTEST = 4
886
887 def setUp(self):
888 super().setUp()
889
890 # Create NTEST tests doing nothing
891 self.tests = [self.create_test() for index in range(self.NTEST)]
892
893 self.python_args = ['-Wd', '-E', '-bb']
894 self.regrtest_args = ['-uall', '-rwW',
895 '--testdir=%s' % self.tmptestdir]
896 self.regrtest_args.extend(('--timeout', '3600', '-j4'))
897 if sys.platform == 'win32':
898 self.regrtest_args.append('-n')
899
900 def check_output(self, output):
901 randseed = self.parse_random_seed(output)
902 self.assertTrue(randseed.isdigit(), randseed)
903
904 self.check_executed_tests(output, self.tests,
905 randomize=True, stats=len(self.tests))
906
907 def run_tests(self, args, env=None, isolated=True):
908 output = self.run_python(args, env=env, isolated=isolated)
909 self.check_output(output)
910
911 def test_script_regrtest(self):
912 # Lib/test/regrtest.py
913 script = os.path.join(self.testdir, 'regrtest.py')
914
915 args = [*self.python_args, script, *self.regrtest_args, *self.tests]
916 self.run_tests(args)
917
918 def test_module_test(self):
919 # -m test
920 args = [*self.python_args, '-m', 'test',
921 *self.regrtest_args, *self.tests]
922 self.run_tests(args)
923
924 def test_module_regrtest(self):
925 # -m test.regrtest
926 args = [*self.python_args, '-m', 'test.regrtest',
927 *self.regrtest_args, *self.tests]
928 self.run_tests(args)
929
930 def test_module_autotest(self):
931 # -m test.autotest
932 args = [*self.python_args, '-m', 'test.autotest',
933 *self.regrtest_args, *self.tests]
934 self.run_tests(args)
935
936 def test_module_from_test_autotest(self):
937 # from test import autotest
938 code = 'from test import autotest'
939 args = [*self.python_args, '-c', code,
940 *self.regrtest_args, *self.tests]
941 self.run_tests(args)
942
943 def test_script_autotest(self):
944 # Lib/test/autotest.py
945 script = os.path.join(self.testdir, 'autotest.py')
946 args = [*self.python_args, script, *self.regrtest_args, *self.tests]
947 self.run_tests(args)
948
949 def run_batch(self, *args):
950 proc = self.run_command(args,
951 # gh-133711: cmd.exe uses the OEM code page
952 # to display the non-ASCII current directory
953 errors="backslashreplace")
954 self.check_output(proc.stdout)
955
956 @unittest.skipUnless(sysconfig.is_python_build(),
957 'test.bat script is not installed')
958 @unittest.skipUnless(sys.platform == 'win32', 'Windows only')
959 def test_tools_buildbot_test(self):
960 # Tools\buildbot\test.bat
961 script = os.path.join(ROOT_DIR, 'Tools', 'buildbot', 'test.bat')
962 test_args = ['--testdir=%s' % self.tmptestdir]
963 if platform.machine() == 'ARM64':
964 test_args.append('-arm64') # ARM 64-bit build
965 elif platform.machine() == 'ARM':
966 test_args.append('-arm32') # 32-bit ARM build
967 elif platform.architecture()[0] == '64bit':
968 test_args.append('-x64') # 64-bit build
969 if not support.Py_DEBUG:
970 test_args.append('+d') # Release build, use python.exe
971 if sysconfig.get_config_var("Py_GIL_DISABLED"):
972 test_args.append('--disable-gil')
973 self.run_batch(script, *test_args, *self.tests)
974
975 @unittest.skipUnless(sys.platform == 'win32', 'Windows only')
976 def test_pcbuild_rt(self):
977 # PCbuild\rt.bat
978 script = os.path.join(ROOT_DIR, r'PCbuild\rt.bat')
979 if not os.path.isfile(script):
980 self.skipTest(f'File "{script}" does not exist')
981 rt_args = ["-q"] # Quick, don't run tests twice
982 if platform.machine() == 'ARM64':
983 rt_args.append('-arm64') # ARM 64-bit build
984 elif platform.machine() == 'ARM':
985 rt_args.append('-arm32') # 32-bit ARM build
986 elif platform.architecture()[0] == '64bit':
987 rt_args.append('-x64') # 64-bit build
988 if support.Py_DEBUG:
989 rt_args.append('-d') # Debug build, use python_d.exe
990 if sysconfig.get_config_var("Py_GIL_DISABLED"):
991 rt_args.append('--disable-gil')
992 self.run_batch(script, *rt_args, *self.regrtest_args, *self.tests)
993
994
995 @support.force_not_colorized_test_class
996 class ArgsTestCase(BaseTestCase):
997 """
998 Test arguments of the Python test suite.
999 """
1000
1001 def run_tests(self, *testargs, **kw):
1002 cmdargs = ['-m', 'test', '--testdir=%s' % self.tmptestdir, *testargs]
1003 return self.run_python(cmdargs, **kw)
1004
1005 def test_success(self):
1006 code = textwrap.dedent("""
1007 import unittest
1008
1009 class PassingTests(unittest.TestCase):
1010 def test_test1(self):
1011 pass
1012
1013 def test_test2(self):
1014 pass
1015
1016 def test_test3(self):
1017 pass
1018 """)
1019 tests = [self.create_test(f'ok{i}', code=code) for i in range(1, 6)]
1020
1021 output = self.run_tests(*tests)
1022 self.check_executed_tests(output, tests,
1023 stats=3 * len(tests))
1024
1025 def test_skip(self):
1026 code = textwrap.dedent("""
1027 import unittest
1028 raise unittest.SkipTest("nope")
1029 """)
1030 test_ok = self.create_test('ok')
1031 test_skip = self.create_test('skip', code=code)
1032 tests = [test_ok, test_skip]
1033
1034 output = self.run_tests(*tests)
1035 self.check_executed_tests(output, tests,
1036 skipped=[test_skip],
1037 stats=1)
1038
1039 def test_failing_test(self):
1040 # test a failing test
1041 code = textwrap.dedent("""
1042 import unittest
1043
1044 class FailingTest(unittest.TestCase):
1045 def test_failing(self):
1046 self.fail("bug")
1047 """)
1048 test_ok = self.create_test('ok')
1049 test_failing = self.create_test('failing', code=code)
1050 tests = [test_ok, test_failing]
1051
1052 output = self.run_tests(*tests, exitcode=EXITCODE_BAD_TEST)
1053 self.check_executed_tests(output, tests, failed=test_failing,
1054 stats=TestStats(2, 1))
1055
1056 def test_resources(self):
1057 # test -u command line option
1058 tests = {}
1059 for resource in ('audio', 'network'):
1060 code = textwrap.dedent("""
1061 from test import support; support.requires(%r)
1062 import unittest
1063 class PassingTest(unittest.TestCase):
1064 def test_pass(self):
1065 pass
1066 """ % resource)
1067
1068 tests[resource] = self.create_test(resource, code)
1069 test_names = sorted(tests.values())
1070
1071 # -u all: 2 resources enabled
1072 output = self.run_tests('-u', 'all', *test_names)
1073 self.check_executed_tests(output, test_names, stats=2)
1074
1075 # -u audio: 1 resource enabled
1076 output = self.run_tests('-uaudio', *test_names)
1077 self.check_executed_tests(output, test_names,
1078 resource_denied=tests['network'],
1079 stats=1)
1080
1081 # no option: 0 resources enabled
1082 output = self.run_tests(*test_names, exitcode=EXITCODE_NO_TESTS_RAN)
1083 self.check_executed_tests(output, test_names,
1084 resource_denied=test_names,
1085 stats=0)
1086
1087 def test_random(self):
1088 # test -r and --randseed command line option
1089 code = textwrap.dedent("""
1090 import random
1091 print("TESTRANDOM: %s" % random.randint(1, 1000))
1092 """)
1093 test = self.create_test('random', code)
1094
1095 # first run to get the output with the random seed
1096 output = self.run_tests('-r', test, exitcode=EXITCODE_NO_TESTS_RAN)
1097 randseed = self.parse_random_seed(output)
1098 match = self.regex_search(r'TESTRANDOM: ([0-9]+)', output)
1099 test_random = int(match.group(1))
1100
1101 # try to reproduce with the random seed
1102 output = self.run_tests('-r', f'--randseed={randseed}', test,
1103 exitcode=EXITCODE_NO_TESTS_RAN)
1104 randseed2 = self.parse_random_seed(output)
1105 self.assertEqual(randseed2, randseed)
1106
1107 match = self.regex_search(r'TESTRANDOM: ([0-9]+)', output)
1108 test_random2 = int(match.group(1))
1109 self.assertEqual(test_random2, test_random)
1110
1111 # check that random.seed is used by default
1112 output = self.run_tests(test, exitcode=EXITCODE_NO_TESTS_RAN)
1113 randseed = self.parse_random_seed(output)
1114 self.assertTrue(randseed.isdigit(), randseed)
1115
1116 # check SOURCE_DATE_EPOCH (integer)
1117 timestamp = '1697839080'
1118 env = dict(os.environ, SOURCE_DATE_EPOCH=timestamp)
1119 output = self.run_tests('-r', test, exitcode=EXITCODE_NO_TESTS_RAN,
1120 env=env)
1121 randseed = self.parse_random_seed(output)
1122 self.assertEqual(randseed, timestamp)
1123 self.check_line(output, 'TESTRANDOM: 520')
1124
1125 # check SOURCE_DATE_EPOCH (string)
1126 env = dict(os.environ, SOURCE_DATE_EPOCH='XYZ')
1127 output = self.run_tests('-r', test, exitcode=EXITCODE_NO_TESTS_RAN,
1128 env=env)
1129 randseed = self.parse_random_seed(output)
1130 self.assertEqual(randseed, 'XYZ')
1131 self.check_line(output, 'TESTRANDOM: 22')
1132
1133 # check SOURCE_DATE_EPOCH (empty string): ignore the env var
1134 env = dict(os.environ, SOURCE_DATE_EPOCH='')
1135 output = self.run_tests('-r', test, exitcode=EXITCODE_NO_TESTS_RAN,
1136 env=env)
1137 randseed = self.parse_random_seed(output)
1138 self.assertTrue(randseed.isdigit(), randseed)
1139
1140 def test_fromfile(self):
1141 # test --fromfile
1142 tests = [self.create_test() for index in range(5)]
1143
1144 # Write the list of files using a format similar to regrtest output:
1145 # [1/2] test_1
1146 # [2/2] test_2
1147 filename = os_helper.TESTFN
1148 self.addCleanup(os_helper.unlink, filename)
1149
1150 # test format '0:00:00 [2/7] test_opcodes -- test_grammar took 0 sec'
1151 with open(filename, "w") as fp:
1152 previous = None
1153 for index, name in enumerate(tests, 1):
1154 line = ("00:00:%02i [%s/%s] %s"
1155 % (index, index, len(tests), name))
1156 if previous:
1157 line += " -- %s took 0 sec" % previous
1158 print(line, file=fp)
1159 previous = name
1160
1161 output = self.run_tests('--fromfile', filename)
1162 stats = len(tests)
1163 self.check_executed_tests(output, tests, stats=stats)
1164
1165 # test format '[2/7] test_opcodes'
1166 with open(filename, "w") as fp:
1167 for index, name in enumerate(tests, 1):
1168 print("[%s/%s] %s" % (index, len(tests), name), file=fp)
1169
1170 output = self.run_tests('--fromfile', filename)
1171 self.check_executed_tests(output, tests, stats=stats)
1172
1173 # test format 'test_opcodes'
1174 with open(filename, "w") as fp:
1175 for name in tests:
1176 print(name, file=fp)
1177
1178 output = self.run_tests('--fromfile', filename)
1179 self.check_executed_tests(output, tests, stats=stats)
1180
1181 # test format 'Lib/test/test_opcodes.py'
1182 with open(filename, "w") as fp:
1183 for name in tests:
1184 print('Lib/test/%s.py' % name, file=fp)
1185
1186 output = self.run_tests('--fromfile', filename)
1187 self.check_executed_tests(output, tests, stats=stats)
1188
1189 def test_interrupted(self):
1190 code = TEST_INTERRUPTED
1191 test = self.create_test('sigint', code=code)
1192 output = self.run_tests(test, exitcode=EXITCODE_INTERRUPTED)
1193 self.check_executed_tests(output, test, omitted=test,
1194 interrupted=True, stats=0)
1195
1196 def test_slowest(self):
1197 # test --slowest
1198 tests = [self.create_test() for index in range(3)]
1199 output = self.run_tests("--slowest", *tests)
1200 self.check_executed_tests(output, tests, stats=len(tests))
1201 regex = ('10 slowest tests:\n'
1202 '(?:- %s: .*\n){%s}'
1203 % (self.TESTNAME_REGEX, len(tests)))
1204 self.check_line(output, regex)
1205
1206 def test_slowest_interrupted(self):
1207 # Issue #25373: test --slowest with an interrupted test
1208 code = TEST_INTERRUPTED
1209 test = self.create_test("sigint", code=code)
1210
1211 for multiprocessing in (False, True):
1212 with self.subTest(multiprocessing=multiprocessing):
1213 if multiprocessing:
1214 args = ("--slowest", "-j2", test)
1215 else:
1216 args = ("--slowest", test)
1217 output = self.run_tests(*args, exitcode=EXITCODE_INTERRUPTED)
1218 self.check_executed_tests(output, test,
1219 omitted=test, interrupted=True,
1220 stats=0)
1221
1222 regex = ('10 slowest tests:\n')
1223 self.check_line(output, regex)
1224
1225 def test_coverage(self):
1226 # test --coverage
1227 test = self.create_test('coverage')
1228 output = self.run_tests("--coverage", test)
1229 self.check_executed_tests(output, [test], stats=1)
1230 regex = (r'lines +cov% +module +\(path\)\n'
1231 r'(?: *[0-9]+ *[0-9]{1,2}\.[0-9]% *[^ ]+ +\([^)]+\)+)+')
1232 self.check_line(output, regex)
1233
1234 def test_wait(self):
1235 # test --wait
1236 test = self.create_test('wait')
1237 output = self.run_tests("--wait", test, input='key')
1238 self.check_line(output, 'Press any key to continue')
1239
1240 def test_forever(self):
1241 # test --forever
1242 code = textwrap.dedent("""
1243 import builtins
1244 import unittest
1245
1246 class ForeverTester(unittest.TestCase):
1247 def test_run(self):
1248 # Store the state in the builtins module, because the test
1249 # module is reload at each run
1250 if 'RUN' in builtins.__dict__:
1251 builtins.__dict__['RUN'] += 1
1252 if builtins.__dict__['RUN'] >= 3:
1253 self.fail("fail at the 3rd runs")
1254 else:
1255 builtins.__dict__['RUN'] = 1
1256 """)
1257 test = self.create_test('forever', code=code)
1258
1259 # --forever
1260 output = self.run_tests('--forever', test, exitcode=EXITCODE_BAD_TEST)
1261 self.check_executed_tests(output, [test]*3, failed=test,
1262 stats=TestStats(3, 1),
1263 forever=True)
1264
1265 # --forever --rerun
1266 output = self.run_tests('--forever', '--rerun', test, exitcode=0)
1267 self.check_executed_tests(output, [test]*3,
1268 rerun=Rerun(test,
1269 match='test_run',
1270 success=True),
1271 stats=TestStats(4, 1),
1272 forever=True)
1273
1274 @support.requires_jit_disabled
1275 def check_leak(self, code, what, *, run_workers=False):
1276 test = self.create_test('huntrleaks', code=code)
1277
1278 filename = 'reflog.txt'
1279 self.addCleanup(os_helper.unlink, filename)
1280 cmd = ['--huntrleaks', '3:3:']
1281 if run_workers:
1282 cmd.append('-j1')
1283 cmd.append(test)
1284 output = self.run_tests(*cmd,
1285 exitcode=EXITCODE_BAD_TEST,
1286 stderr=subprocess.STDOUT)
1287 self.check_executed_tests(output, [test], failed=test, stats=1)
1288
1289 line = r'beginning 6 repetitions. .*\n123:456\n[.0-9X]{3} 111\n'
1290 self.check_line(output, line)
1291
1292 line2 = '%s leaked [1, 1, 1] %s, sum=3\n' % (test, what)
1293 self.assertIn(line2, output)
1294
1295 with open(filename) as fp:
1296 reflog = fp.read()
1297 self.assertIn(line2, reflog)
1298
1299 @unittest.skipUnless(support.Py_DEBUG, 'need a debug build')
1300 def check_huntrleaks(self, *, run_workers: bool):
1301 # test --huntrleaks
1302 code = textwrap.dedent("""
1303 import unittest
1304
1305 GLOBAL_LIST = []
1306
1307 class RefLeakTest(unittest.TestCase):
1308 def test_leak(self):
1309 GLOBAL_LIST.append(object())
1310 """)
1311 self.check_leak(code, 'references', run_workers=run_workers)
1312
1313 def test_huntrleaks(self):
1314 self.check_huntrleaks(run_workers=False)
1315
1316 def test_huntrleaks_mp(self):
1317 self.check_huntrleaks(run_workers=True)
1318
1319 @unittest.skipUnless(support.Py_DEBUG, 'need a debug build')
1320 def test_huntrleaks_bisect(self):
1321 # test --huntrleaks --bisect
1322 code = textwrap.dedent("""
1323 import unittest
1324
1325 GLOBAL_LIST = []
1326
1327 class RefLeakTest(unittest.TestCase):
1328 def test1(self):
1329 pass
1330
1331 def test2(self):
1332 pass
1333
1334 def test3(self):
1335 GLOBAL_LIST.append(object())
1336
1337 def test4(self):
1338 pass
1339 """)
1340
1341 test = self.create_test('huntrleaks', code=code)
1342
1343 filename = 'reflog.txt'
1344 self.addCleanup(os_helper.unlink, filename)
1345 cmd = ['--huntrleaks', '3:3:', '--bisect', test]
1346 output = self.run_tests(*cmd,
1347 exitcode=EXITCODE_BAD_TEST,
1348 stderr=subprocess.STDOUT)
1349
1350 self.assertIn(f"Bisect {test}", output)
1351 self.assertIn(f"Bisect {test}: exit code 0", output)
1352
1353 # test3 is the one which leaks
1354 self.assertIn("Bisection completed in", output)
1355 self.assertIn(
1356 "Tests (1):\n"
1357 f"* {test}.RefLeakTest.test3\n",
1358 output)
1359
1360 @unittest.skipUnless(support.Py_DEBUG, 'need a debug build')
1361 def test_huntrleaks_fd_leak(self):
1362 # test --huntrleaks for file descriptor leak
1363 code = textwrap.dedent("""
1364 import os
1365 import unittest
1366
1367 class FDLeakTest(unittest.TestCase):
1368 def test_leak(self):
1369 fd = os.open(__file__, os.O_RDONLY)
1370 # bug: never close the file descriptor
1371 """)
1372 self.check_leak(code, 'file descriptors')
1373
1374 def test_list_tests(self):
1375 # test --list-tests
1376 tests = [self.create_test() for i in range(5)]
1377 output = self.run_tests('--list-tests', *tests)
1378 self.assertEqual(output.rstrip().splitlines(),
1379 tests)
1380
1381 def test_list_cases(self):
1382 # test --list-cases
1383 code = textwrap.dedent("""
1384 import unittest
1385
1386 class Tests(unittest.TestCase):
1387 def test_method1(self):
1388 pass
1389 def test_method2(self):
1390 pass
1391 """)
1392 testname = self.create_test(code=code)
1393
1394 # Test --list-cases
1395 all_methods = ['%s.Tests.test_method1' % testname,
1396 '%s.Tests.test_method2' % testname]
1397 output = self.run_tests('--list-cases', testname)
1398 self.assertEqual(output.splitlines(), all_methods)
1399
1400 # Test --list-cases with --match
1401 all_methods = ['%s.Tests.test_method1' % testname]
1402 output = self.run_tests('--list-cases',
1403 '-m', 'test_method1',
1404 testname)
1405 self.assertEqual(output.splitlines(), all_methods)
1406
1407 @support.cpython_only
1408 def test_crashed(self):
1409 # Any code which causes a crash
1410 code = 'import faulthandler; faulthandler._sigsegv()'
1411 crash_test = self.create_test(name="crash", code=code)
1412
1413 tests = [crash_test]
1414 output = self.run_tests("-j2", *tests, exitcode=EXITCODE_BAD_TEST)
1415 self.check_executed_tests(output, tests, failed=crash_test,
1416 parallel=True, stats=0)
1417
1418 def parse_methods(self, output):
1419 regex = re.compile("^(test[^ ]+).*ok$", flags=re.MULTILINE)
1420 return [match.group(1) for match in regex.finditer(output)]
1421
1422 def test_ignorefile(self):
1423 code = textwrap.dedent("""
1424 import unittest
1425
1426 class Tests(unittest.TestCase):
1427 def test_method1(self):
1428 pass
1429 def test_method2(self):
1430 pass
1431 def test_method3(self):
1432 pass
1433 def test_method4(self):
1434 pass
1435 """)
1436 testname = self.create_test(code=code)
1437
1438 # only run a subset
1439 filename = os_helper.TESTFN
1440 self.addCleanup(os_helper.unlink, filename)
1441
1442 subset = [
1443 # only ignore the method name
1444 'test_method1',
1445 # ignore the full identifier
1446 '%s.Tests.test_method3' % testname]
1447 with open(filename, "w") as fp:
1448 for name in subset:
1449 print(name, file=fp)
1450
1451 output = self.run_tests("-v", "--ignorefile", filename, testname)
1452 methods = self.parse_methods(output)
1453 subset = ['test_method2', 'test_method4']
1454 self.assertEqual(methods, subset)
1455
1456 def test_matchfile(self):
1457 code = textwrap.dedent("""
1458 import unittest
1459
1460 class Tests(unittest.TestCase):
1461 def test_method1(self):
1462 pass
1463 def test_method2(self):
1464 pass
1465 def test_method3(self):
1466 pass
1467 def test_method4(self):
1468 pass
1469 """)
1470 all_methods = ['test_method1', 'test_method2',
1471 'test_method3', 'test_method4']
1472 testname = self.create_test(code=code)
1473
1474 # by default, all methods should be run
1475 output = self.run_tests("-v", testname)
1476 methods = self.parse_methods(output)
1477 self.assertEqual(methods, all_methods)
1478
1479 # only run a subset
1480 filename = os_helper.TESTFN
1481 self.addCleanup(os_helper.unlink, filename)
1482
1483 subset = [
1484 # only match the method name
1485 'test_method1',
1486 # match the full identifier
1487 '%s.Tests.test_method3' % testname]
1488 with open(filename, "w") as fp:
1489 for name in subset:
1490 print(name, file=fp)
1491
1492 output = self.run_tests("-v", "--matchfile", filename, testname)
1493 methods = self.parse_methods(output)
1494 subset = ['test_method1', 'test_method3']
1495 self.assertEqual(methods, subset)
1496
1497 def test_env_changed(self):
1498 code = textwrap.dedent("""
1499 import unittest
1500
1501 class Tests(unittest.TestCase):
1502 def test_env_changed(self):
1503 open("env_changed", "w").close()
1504 """)
1505 testname = self.create_test(code=code)
1506
1507 # don't fail by default
1508 output = self.run_tests(testname)
1509 self.check_executed_tests(output, [testname],
1510 env_changed=testname, stats=1)
1511
1512 # fail with --fail-env-changed
1513 output = self.run_tests("--fail-env-changed", testname,
1514 exitcode=EXITCODE_ENV_CHANGED)
1515 self.check_executed_tests(output, [testname], env_changed=testname,
1516 fail_env_changed=True, stats=1)
1517
1518 # rerun
1519 output = self.run_tests("--rerun", testname)
1520 self.check_executed_tests(output, [testname],
1521 env_changed=testname,
1522 rerun=Rerun(testname,
1523 match=None,
1524 success=True),
1525 stats=2)
1526
1527 def test_rerun_fail(self):
1528 # FAILURE then FAILURE
1529 code = textwrap.dedent("""
1530 import unittest
1531
1532 class Tests(unittest.TestCase):
1533 def test_succeed(self):
1534 return
1535
1536 def test_fail_always(self):
1537 # test that always fails
1538 self.fail("bug")
1539 """)
1540 testname = self.create_test(code=code)
1541
1542 output = self.run_tests("--rerun", testname, exitcode=EXITCODE_BAD_TEST)
1543 self.check_executed_tests(output, [testname],
1544 rerun=Rerun(testname,
1545 "test_fail_always",
1546 success=False),
1547 stats=TestStats(3, 2))
1548
1549 def test_rerun_success(self):
1550 # FAILURE then SUCCESS
1551 marker_filename = os.path.abspath("regrtest_marker_filename")
1552 self.addCleanup(os_helper.unlink, marker_filename)
1553 self.assertFalse(os.path.exists(marker_filename))
1554
1555 code = textwrap.dedent(f"""
1556 import os.path
1557 import unittest
1558
1559 marker_filename = {marker_filename!r}
1560
1561 class Tests(unittest.TestCase):
1562 def test_succeed(self):
1563 return
1564
1565 def test_fail_once(self):
1566 if not os.path.exists(marker_filename):
1567 open(marker_filename, "w").close()
1568 self.fail("bug")
1569 """)
1570 testname = self.create_test(code=code)
1571
1572 # FAILURE then SUCCESS => exit code 0
1573 output = self.run_tests("--rerun", testname, exitcode=0)
1574 self.check_executed_tests(output, [testname],
1575 rerun=Rerun(testname,
1576 match="test_fail_once",
1577 success=True),
1578 stats=TestStats(3, 1))
1579 os_helper.unlink(marker_filename)
1580
1581 # with --fail-rerun, exit code EXITCODE_RERUN_FAIL
1582 # on "FAILURE then SUCCESS" state.
1583 output = self.run_tests("--rerun", "--fail-rerun", testname,
1584 exitcode=EXITCODE_RERUN_FAIL)
1585 self.check_executed_tests(output, [testname],
1586 rerun=Rerun(testname,
1587 match="test_fail_once",
1588 success=True),
1589 stats=TestStats(3, 1))
1590 os_helper.unlink(marker_filename)
1591
1592 def test_rerun_setup_class_hook_failure(self):
1593 # FAILURE then FAILURE
1594 code = textwrap.dedent("""
1595 import unittest
1596
1597 class ExampleTests(unittest.TestCase):
1598 @classmethod
1599 def setUpClass(self):
1600 raise RuntimeError('Fail')
1601
1602 def test_success(self):
1603 return
1604 """)
1605 testname = self.create_test(code=code)
1606
1607 output = self.run_tests("--rerun", testname, exitcode=EXITCODE_BAD_TEST)
1608 self.check_executed_tests(output, testname,
1609 failed=[testname],
1610 rerun=Rerun(testname,
1611 match="ExampleTests",
1612 success=False),
1613 stats=0)
1614
1615 def test_rerun_teardown_class_hook_failure(self):
1616 # FAILURE then FAILURE
1617 code = textwrap.dedent("""
1618 import unittest
1619
1620 class ExampleTests(unittest.TestCase):
1621 @classmethod
1622 def tearDownClass(self):
1623 raise RuntimeError('Fail')
1624
1625 def test_success(self):
1626 return
1627 """)
1628 testname = self.create_test(code=code)
1629
1630 output = self.run_tests("--rerun", testname, exitcode=EXITCODE_BAD_TEST)
1631 self.check_executed_tests(output, testname,
1632 failed=[testname],
1633 rerun=Rerun(testname,
1634 match="ExampleTests",
1635 success=False),
1636 stats=2)
1637
1638 def test_rerun_setup_module_hook_failure(self):
1639 # FAILURE then FAILURE
1640 code = textwrap.dedent("""
1641 import unittest
1642
1643 def setUpModule():
1644 raise RuntimeError('Fail')
1645
1646 class ExampleTests(unittest.TestCase):
1647 def test_success(self):
1648 return
1649 """)
1650 testname = self.create_test(code=code)
1651
1652 output = self.run_tests("--rerun", testname, exitcode=EXITCODE_BAD_TEST)
1653 self.check_executed_tests(output, testname,
1654 failed=[testname],
1655 rerun=Rerun(testname,
1656 match=None,
1657 success=False),
1658 stats=0)
1659
1660 def test_rerun_teardown_module_hook_failure(self):
1661 # FAILURE then FAILURE
1662 code = textwrap.dedent("""
1663 import unittest
1664
1665 def tearDownModule():
1666 raise RuntimeError('Fail')
1667
1668 class ExampleTests(unittest.TestCase):
1669 def test_success(self):
1670 return
1671 """)
1672 testname = self.create_test(code=code)
1673
1674 output = self.run_tests("--rerun", testname, exitcode=EXITCODE_BAD_TEST)
1675 self.check_executed_tests(output, [testname],
1676 failed=[testname],
1677 rerun=Rerun(testname,
1678 match=None,
1679 success=False),
1680 stats=2)
1681
1682 def test_rerun_setup_hook_failure(self):
1683 # FAILURE then FAILURE
1684 code = textwrap.dedent("""
1685 import unittest
1686
1687 class ExampleTests(unittest.TestCase):
1688 def setUp(self):
1689 raise RuntimeError('Fail')
1690
1691 def test_success(self):
1692 return
1693 """)
1694 testname = self.create_test(code=code)
1695
1696 output = self.run_tests("--rerun", testname, exitcode=EXITCODE_BAD_TEST)
1697 self.check_executed_tests(output, testname,
1698 failed=[testname],
1699 rerun=Rerun(testname,
1700 match="test_success",
1701 success=False),
1702 stats=2)
1703
1704 def test_rerun_teardown_hook_failure(self):
1705 # FAILURE then FAILURE
1706 code = textwrap.dedent("""
1707 import unittest
1708
1709 class ExampleTests(unittest.TestCase):
1710 def tearDown(self):
1711 raise RuntimeError('Fail')
1712
1713 def test_success(self):
1714 return
1715 """)
1716 testname = self.create_test(code=code)
1717
1718 output = self.run_tests("--rerun", testname, exitcode=EXITCODE_BAD_TEST)
1719 self.check_executed_tests(output, testname,
1720 failed=[testname],
1721 rerun=Rerun(testname,
1722 match="test_success",
1723 success=False),
1724 stats=2)
1725
1726 def test_rerun_async_setup_hook_failure(self):
1727 # FAILURE then FAILURE
1728 code = textwrap.dedent("""
1729 import unittest
1730
1731 class ExampleTests(unittest.IsolatedAsyncioTestCase):
1732 async def asyncSetUp(self):
1733 raise RuntimeError('Fail')
1734
1735 async def test_success(self):
1736 return
1737 """)
1738 testname = self.create_test(code=code)
1739
1740 output = self.run_tests("--rerun", testname, exitcode=EXITCODE_BAD_TEST)
1741 self.check_executed_tests(output, testname,
1742 rerun=Rerun(testname,
1743 match="test_success",
1744 success=False),
1745 stats=2)
1746
1747 def test_rerun_async_teardown_hook_failure(self):
1748 # FAILURE then FAILURE
1749 code = textwrap.dedent("""
1750 import unittest
1751
1752 class ExampleTests(unittest.IsolatedAsyncioTestCase):
1753 async def asyncTearDown(self):
1754 raise RuntimeError('Fail')
1755
1756 async def test_success(self):
1757 return
1758 """)
1759 testname = self.create_test(code=code)
1760
1761 output = self.run_tests("--rerun", testname, exitcode=EXITCODE_BAD_TEST)
1762 self.check_executed_tests(output, testname,
1763 failed=[testname],
1764 rerun=Rerun(testname,
1765 match="test_success",
1766 success=False),
1767 stats=2)
1768
1769 def test_no_tests_ran(self):
1770 code = textwrap.dedent("""
1771 import unittest
1772
1773 class Tests(unittest.TestCase):
1774 def test_bug(self):
1775 pass
1776 """)
1777 testname = self.create_test(code=code)
1778
1779 output = self.run_tests(testname, "-m", "nosuchtest",
1780 exitcode=EXITCODE_NO_TESTS_RAN)
1781 self.check_executed_tests(output, [testname],
1782 run_no_tests=testname,
1783 stats=0, filtered=True)
1784
1785 def test_no_tests_ran_skip(self):
1786 code = textwrap.dedent("""
1787 import unittest
1788
1789 class Tests(unittest.TestCase):
1790 def test_skipped(self):
1791 self.skipTest("because")
1792 """)
1793 testname = self.create_test(code=code)
1794
1795 output = self.run_tests(testname)
1796 self.check_executed_tests(output, [testname],
1797 stats=TestStats(1, skipped=1))
1798
1799 def test_no_tests_ran_multiple_tests_nonexistent(self):
1800 code = textwrap.dedent("""
1801 import unittest
1802
1803 class Tests(unittest.TestCase):
1804 def test_bug(self):
1805 pass
1806 """)
1807 testname = self.create_test(code=code)
1808 testname2 = self.create_test(code=code)
1809
1810 output = self.run_tests(testname, testname2, "-m", "nosuchtest",
1811 exitcode=EXITCODE_NO_TESTS_RAN)
1812 self.check_executed_tests(output, [testname, testname2],
1813 run_no_tests=[testname, testname2],
1814 stats=0, filtered=True)
1815
1816 def test_no_test_ran_some_test_exist_some_not(self):
1817 code = textwrap.dedent("""
1818 import unittest
1819
1820 class Tests(unittest.TestCase):
1821 def test_bug(self):
1822 pass
1823 """)
1824 testname = self.create_test(code=code)
1825 other_code = textwrap.dedent("""
1826 import unittest
1827
1828 class Tests(unittest.TestCase):
1829 def test_other_bug(self):
1830 pass
1831 """)
1832 testname2 = self.create_test(code=other_code)
1833
1834 output = self.run_tests(testname, testname2, "-m", "nosuchtest",
1835 "-m", "test_other_bug", exitcode=0)
1836 self.check_executed_tests(output, [testname, testname2],
1837 run_no_tests=[testname],
1838 stats=1, filtered=True)
1839
1840 @support.cpython_only
1841 def test_uncollectable(self):
1842 # Skip test if _testcapi is missing
1843 import_helper.import_module('_testcapi')
1844
1845 code = textwrap.dedent(r"""
1846 import _testcapi
1847 import gc
1848 import unittest
1849
1850 @_testcapi.with_tp_del
1851 class Garbage:
1852 def __tp_del__(self):
1853 pass
1854
1855 class Tests(unittest.TestCase):
1856 def test_garbage(self):
1857 # create an uncollectable object
1858 obj = Garbage()
1859 obj.ref_cycle = obj
1860 obj = None
1861 """)
1862 testname = self.create_test(code=code)
1863
1864 output = self.run_tests("--fail-env-changed", testname,
1865 exitcode=EXITCODE_ENV_CHANGED)
1866 self.check_executed_tests(output, [testname],
1867 env_changed=[testname],
1868 fail_env_changed=True,
1869 stats=1)
1870
1871 def test_multiprocessing_timeout(self):
1872 code = textwrap.dedent(r"""
1873 import time
1874 import unittest
1875 try:
1876 import faulthandler
1877 except ImportError:
1878 faulthandler = None
1879
1880 class Tests(unittest.TestCase):
1881 # test hangs and so should be stopped by the timeout
1882 def test_sleep(self):
1883 # we want to test regrtest multiprocessing timeout,
1884 # not faulthandler timeout
1885 if faulthandler is not None:
1886 faulthandler.cancel_dump_traceback_later()
1887
1888 time.sleep(60 * 5)
1889 """)
1890 testname = self.create_test(code=code)
1891
1892 output = self.run_tests("-j2", "--timeout=1.0", testname,
1893 exitcode=EXITCODE_BAD_TEST)
1894 self.check_executed_tests(output, [testname],
1895 failed=testname, stats=0)
1896 self.assertRegex(output,
1897 re.compile('%s timed out' % testname, re.MULTILINE))
1898
1899 def test_unraisable_exc(self):
1900 # --fail-env-changed must catch unraisable exception.
1901 # The exception must be displayed even if sys.stderr is redirected.
1902 code = textwrap.dedent(r"""
1903 import unittest
1904 import weakref
1905 from test.support import captured_stderr
1906
1907 class MyObject:
1908 pass
1909
1910 def weakref_callback(obj):
1911 raise Exception("weakref callback bug")
1912
1913 class Tests(unittest.TestCase):
1914 def test_unraisable_exc(self):
1915 obj = MyObject()
1916 ref = weakref.ref(obj, weakref_callback)
1917 with captured_stderr() as stderr:
1918 # call weakref_callback() which logs
1919 # an unraisable exception
1920 obj = None
1921 self.assertEqual(stderr.getvalue(), '')
1922 """)
1923 testname = self.create_test(code=code)
1924
1925 output = self.run_tests("--fail-env-changed", "-v", testname,
1926 exitcode=EXITCODE_ENV_CHANGED)
1927 self.check_executed_tests(output, [testname],
1928 env_changed=[testname],
1929 fail_env_changed=True,
1930 stats=1)
1931 self.assertIn("Warning -- Unraisable exception", output)
1932 self.assertIn("Exception: weakref callback bug", output)
1933
1934 def test_threading_excepthook(self):
1935 # --fail-env-changed must catch uncaught thread exception.
1936 # The exception must be displayed even if sys.stderr is redirected.
1937 code = textwrap.dedent(r"""
1938 import threading
1939 import unittest
1940 from test.support import captured_stderr
1941
1942 class MyObject:
1943 pass
1944
1945 def func_bug():
1946 raise Exception("bug in thread")
1947
1948 class Tests(unittest.TestCase):
1949 def test_threading_excepthook(self):
1950 with captured_stderr() as stderr:
1951 thread = threading.Thread(target=func_bug)
1952 thread.start()
1953 thread.join()
1954 self.assertEqual(stderr.getvalue(), '')
1955 """)
1956 testname = self.create_test(code=code)
1957
1958 output = self.run_tests("--fail-env-changed", "-v", testname,
1959 exitcode=EXITCODE_ENV_CHANGED)
1960 self.check_executed_tests(output, [testname],
1961 env_changed=[testname],
1962 fail_env_changed=True,
1963 stats=1)
1964 self.assertIn("Warning -- Uncaught thread exception", output)
1965 self.assertIn("Exception: bug in thread", output)
1966
1967 def test_print_warning(self):
1968 # bpo-45410: The order of messages must be preserved when -W and
1969 # support.print_warning() are used.
1970 code = textwrap.dedent(r"""
1971 import sys
1972 import unittest
1973 from test import support
1974
1975 class MyObject:
1976 pass
1977
1978 def func_bug():
1979 raise Exception("bug in thread")
1980
1981 class Tests(unittest.TestCase):
1982 def test_print_warning(self):
1983 print("msg1: stdout")
1984 support.print_warning("msg2: print_warning")
1985 # Fail with ENV CHANGED to see print_warning() log
1986 support.environment_altered = True
1987 """)
1988 testname = self.create_test(code=code)
1989
1990 # Expect an output like:
1991 #
1992 # test_threading_excepthook (test.test_x.Tests) ... msg1: stdout
1993 # Warning -- msg2: print_warning
1994 # ok
1995 regex = (r"test_print_warning.*msg1: stdout\n"
1996 r"Warning -- msg2: print_warning\n"
1997 r"ok\n")
1998 for option in ("-v", "-W"):
1999 with self.subTest(option=option):
2000 cmd = ["--fail-env-changed", option, testname]
2001 output = self.run_tests(*cmd, exitcode=EXITCODE_ENV_CHANGED)
2002 self.check_executed_tests(output, [testname],
2003 env_changed=[testname],
2004 fail_env_changed=True,
2005 stats=1)
2006 self.assertRegex(output, regex)
2007
2008 def test_unicode_guard_env(self):
2009 guard = os.environ.get(setup.UNICODE_GUARD_ENV)
2010 self.assertIsNotNone(guard, f"{setup.UNICODE_GUARD_ENV} not set")
2011 if guard.isascii():
2012 # Skip to signify that the env var value was changed by the user;
2013 # possibly to something ASCII to work around Unicode issues.
2014 self.skipTest("Modified guard")
2015
2016 def test_cleanup(self):
2017 dirname = os.path.join(self.tmptestdir, "test_python_123")
2018 os.mkdir(dirname)
2019 filename = os.path.join(self.tmptestdir, "test_python_456")
2020 open(filename, "wb").close()
2021 names = [dirname, filename]
2022
2023 cmdargs = ['-m', 'test',
2024 '--tempdir=%s' % self.tmptestdir,
2025 '--cleanup']
2026 self.run_python(cmdargs)
2027
2028 for name in names:
2029 self.assertFalse(os.path.exists(name), name)
2030
2031 @unittest.skipIf(support.is_wasi,
2032 'checking temp files is not implemented on WASI')
2033 def test_leak_tmp_file(self):
2034 code = textwrap.dedent(r"""
2035 import os.path
2036 import tempfile
2037 import unittest
2038
2039 class FileTests(unittest.TestCase):
2040 def test_leak_tmp_file(self):
2041 filename = os.path.join(tempfile.gettempdir(), 'mytmpfile')
2042 with open(filename, "wb") as fp:
2043 fp.write(b'content')
2044 """)
2045 testnames = [self.create_test(code=code) for _ in range(3)]
2046
2047 output = self.run_tests("--fail-env-changed", "-v", "-j2", *testnames,
2048 exitcode=EXITCODE_ENV_CHANGED)
2049 self.check_executed_tests(output, testnames,
2050 env_changed=testnames,
2051 fail_env_changed=True,
2052 parallel=True,
2053 stats=len(testnames))
2054 for testname in testnames:
2055 self.assertIn(f"Warning -- {testname} leaked temporary "
2056 f"files (1): mytmpfile",
2057 output)
2058
2059 def test_worker_decode_error(self):
2060 # gh-109425: Use "backslashreplace" error handler to decode stdout.
2061 if sys.platform == 'win32':
2062 encoding = locale.getencoding()
2063 else:
2064 encoding = sys.stdout.encoding
2065 if encoding is None:
2066 encoding = sys.__stdout__.encoding
2067 if encoding is None:
2068 self.skipTest("cannot get regrtest worker encoding")
2069
2070 nonascii = bytes(ch for ch in range(128, 256))
2071 corrupted_output = b"nonascii:%s\n" % (nonascii,)
2072 # gh-108989: On Windows, assertion errors are written in UTF-16: when
2073 # decoded each letter is follow by a NUL character.
2074 assertion_failed = 'Assertion failed: tstate_is_alive(tstate)\n'
2075 corrupted_output += assertion_failed.encode('utf-16-le')
2076 try:
2077 corrupted_output.decode(encoding)
2078 except UnicodeDecodeError:
2079 pass
2080 else:
2081 self.skipTest(f"{encoding} can decode non-ASCII bytes")
2082
2083 expected_line = corrupted_output.decode(encoding, 'backslashreplace')
2084
2085 code = textwrap.dedent(fr"""
2086 import sys
2087 import unittest
2088
2089 class Tests(unittest.TestCase):
2090 def test_pass(self):
2091 pass
2092
2093 # bytes which cannot be decoded from UTF-8
2094 corrupted_output = {corrupted_output!a}
2095 sys.stdout.buffer.write(corrupted_output)
2096 sys.stdout.buffer.flush()
2097 """)
2098 testname = self.create_test(code=code)
2099
2100 output = self.run_tests("--fail-env-changed", "-v", "-j1", testname)
2101 self.check_executed_tests(output, [testname],
2102 parallel=True,
2103 stats=1)
2104 self.check_line(output, expected_line, regex=False)
2105
2106 def test_doctest(self):
2107 code = textwrap.dedent(r'''
2108 import doctest
2109 import sys
2110 from test import support
2111
2112 def my_function():
2113 """
2114 Pass:
2115
2116 >>> 1 + 1
2117 2
2118
2119 Failure:
2120
2121 >>> 2 + 3
2122 23
2123 >>> 1 + 1
2124 11
2125
2126 Skipped test (ignored):
2127
2128 >>> id(1.0) # doctest: +SKIP
2129 7948648
2130 """
2131
2132 def load_tests(loader, tests, pattern):
2133 tests.addTest(doctest.DocTestSuite())
2134 return tests
2135 ''')
2136 testname = self.create_test(code=code)
2137
2138 output = self.run_tests("--fail-env-changed", "-v", "-j1", testname,
2139 exitcode=EXITCODE_BAD_TEST)
2140 self.check_executed_tests(output, [testname],
2141 failed=[testname],
2142 parallel=True,
2143 stats=TestStats(1, 1, 0))
2144
2145 def _check_random_seed(self, run_workers: bool):
2146 # gh-109276: When -r/--randomize is used, random.seed() is called
2147 # with the same random seed before running each test file.
2148 code = textwrap.dedent(r'''
2149 import random
2150 import unittest
2151
2152 class RandomSeedTest(unittest.TestCase):
2153 def test_randint(self):
2154 numbers = [random.randint(0, 1000) for _ in range(10)]
2155 print(f"Random numbers: {numbers}")
2156 ''')
2157 tests = [self.create_test(name=f'test_random{i}', code=code)
2158 for i in range(1, 3+1)]
2159
2160 random_seed = 856_656_202
2161 cmd = ["--randomize", f"--randseed={random_seed}"]
2162 if run_workers:
2163 # run as many worker processes than the number of tests
2164 cmd.append(f'-j{len(tests)}')
2165 cmd.extend(tests)
2166 output = self.run_tests(*cmd)
2167
2168 random.seed(random_seed)
2169 # Make the assumption that nothing consume entropy between libregrest
2170 # setup_tests() which calls random.seed() and RandomSeedTest calling
2171 # random.randint().
2172 numbers = [random.randint(0, 1000) for _ in range(10)]
2173 expected = f"Random numbers: {numbers}"
2174
2175 regex = r'^Random numbers: .*$'
2176 matches = re.findall(regex, output, flags=re.MULTILINE)
2177 self.assertEqual(matches, [expected] * len(tests))
2178
2179 def test_random_seed(self):
2180 self._check_random_seed(run_workers=False)
2181
2182 def test_random_seed_workers(self):
2183 self._check_random_seed(run_workers=True)
2184
2185 def test_python_command(self):
2186 code = textwrap.dedent(r"""
2187 import sys
2188 import unittest
2189
2190 class WorkerTests(unittest.TestCase):
2191 def test_dev_mode(self):
2192 self.assertTrue(sys.flags.dev_mode)
2193 """)
2194 tests = [self.create_test(code=code) for _ in range(3)]
2195
2196 # Custom Python command: "python -X dev"
2197 python_cmd = [sys.executable, '-X', 'dev']
2198 # test.libregrtest.cmdline uses shlex.split() to parse the Python
2199 # command line string
2200 python_cmd = shlex.join(python_cmd)
2201
2202 output = self.run_tests("--python", python_cmd, "-j0", *tests)
2203 self.check_executed_tests(output, tests,
2204 stats=len(tests), parallel=True)
2205
2206 def test_unload_tests(self):
2207 # Test that unloading test modules does not break tests
2208 # that import from other tests.
2209 # The test execution order matters for this test.
2210 # Both test_regrtest_a and test_regrtest_c which are executed before
2211 # and after test_regrtest_b import a submodule from the test_regrtest_b
2212 # package and use it in testing. test_regrtest_b itself does not import
2213 # that submodule.
2214 # Previously test_regrtest_c failed because test_regrtest_b.util in
2215 # sys.modules was left after test_regrtest_a (making the import
2216 # statement no-op), but new test_regrtest_b without the util attribute
2217 # was imported for test_regrtest_b.
2218 testdir = os.path.join(os.path.dirname(__file__),
2219 'regrtestdata', 'import_from_tests')
2220 tests = [f'test_regrtest_{name}' for name in ('a', 'b', 'c')]
2221 args = ['-Wd', '-E', '-bb', '-m', 'test', '--testdir=%s' % testdir, *tests]
2222 output = self.run_python(args)
2223 self.check_executed_tests(output, tests, stats=3)
2224
2225 def check_add_python_opts(self, option):
2226 # --fast-ci and --slow-ci add "-u -W error -bb -E" options to Python
2227
2228 code = textwrap.dedent(r"""
2229 import sys
2230 import unittest
2231 from test import support
2232 try:
2233 from _testcapi import config_get
2234 except ImportError:
2235 config_get = None
2236
2237 # WASI/WASM buildbots don't use -E option
2238 use_environment = (support.is_emscripten or support.is_wasi)
2239
2240 class WorkerTests(unittest.TestCase):
2241 @unittest.skipIf(config_get is None, 'need config_get()')
2242 def test_config(self):
2243 # -u option
2244 self.assertEqual(config_get('buffered_stdio'), 0)
2245 # -W error option
2246 self.assertEqual(config_get('warnoptions'),
2247 ['error', 'error::BytesWarning'])
2248 # -bb option
2249 self.assertEqual(config_get('bytes_warning'), 2)
2250 # -E option
2251 self.assertEqual(config_get('use_environment'), use_environment)
2252
2253 def test_python_opts(self):
2254 # -u option
2255 self.assertTrue(sys.__stdout__.write_through)
2256 self.assertTrue(sys.__stderr__.write_through)
2257
2258 # -W error option
2259 self.assertEqual(sys.warnoptions,
2260 ['error', 'error::BytesWarning'])
2261
2262 # -bb option
2263 self.assertEqual(sys.flags.bytes_warning, 2)
2264
2265 # -E option
2266 self.assertEqual(not sys.flags.ignore_environment,
2267 use_environment)
2268 """)
2269 testname = self.create_test(code=code)
2270
2271 # Use directly subprocess to control the exact command line
2272 cmd = [sys.executable,
2273 "-m", "test", option,
2274 f'--testdir={self.tmptestdir}',
2275 testname]
2276 proc = subprocess.run(cmd,
2277 stdout=subprocess.PIPE,
2278 stderr=subprocess.STDOUT,
2279 text=True)
2280 self.assertEqual(proc.returncode, 0, proc)
2281
2282 def test_add_python_opts(self):
2283 for opt in ("--fast-ci", "--slow-ci"):
2284 with self.subTest(opt=opt):
2285 self.check_add_python_opts(opt)
2286
2287 # gh-76319: Raising SIGSEGV on Android may not cause a crash.
2288 @unittest.skipIf(support.is_android,
2289 'raising SIGSEGV on Android is unreliable')
2290 def test_worker_output_on_failure(self):
2291 # Skip test if faulthandler is missing
2292 import_helper.import_module('faulthandler')
2293
2294 code = textwrap.dedent(r"""
2295 import faulthandler
2296 import unittest
2297 from test import support
2298
2299 class CrashTests(unittest.TestCase):
2300 def test_crash(self):
2301 print("just before crash!", flush=True)
2302
2303 with support.SuppressCrashReport():
2304 faulthandler._sigsegv(True)
2305 """)
2306 testname = self.create_test(code=code)
2307
2308 # Sanitizers must not handle SIGSEGV (ex: for test_enable_fd())
2309 env = dict(os.environ)
2310 option = 'handle_segv=0'
2311 support.set_sanitizer_env_var(env, option)
2312
2313 output = self.run_tests("-j1", testname,
2314 exitcode=EXITCODE_BAD_TEST,
2315 env=env)
2316 self.check_executed_tests(output, testname,
2317 failed=[testname],
2318 stats=0, parallel=True)
2319 if not support.MS_WINDOWS:
2320 exitcode = -int(signal.SIGSEGV)
2321 self.assertIn(f"Exit code {exitcode} (SIGSEGV)", output)
2322 self.check_line(output, "just before crash!", full=True, regex=False)
2323
2324 def test_verbose3(self):
2325 code = textwrap.dedent(r"""
2326 import unittest
2327 from test import support
2328
2329 class VerboseTests(unittest.TestCase):
2330 def test_pass(self):
2331 print("SPAM SPAM SPAM")
2332 """)
2333 testname = self.create_test(code=code)
2334
2335 # Run sequentially
2336 output = self.run_tests("--verbose3", testname)
2337 self.check_executed_tests(output, testname, stats=1)
2338 self.assertNotIn('SPAM SPAM SPAM', output)
2339
2340 # -R option needs a debug build
2341 if support.Py_DEBUG:
2342 # Check for reference leaks, run in parallel
2343 output = self.run_tests("-R", "3:3", "-j1", "--verbose3", testname)
2344 self.check_executed_tests(output, testname, stats=1, parallel=True)
2345 self.assertNotIn('SPAM SPAM SPAM', output)
2346
2347 def test_xml(self):
2348 code = textwrap.dedent(r"""
2349 import unittest
2350
2351 class VerboseTests(unittest.TestCase):
2352 def test_failed(self):
2353 print("abc \x1b def")
2354 self.fail()
2355 """)
2356 testname = self.create_test(code=code)
2357
2358 # Run sequentially
2359 filename = os_helper.TESTFN
2360 self.addCleanup(os_helper.unlink, filename)
2361
2362 output = self.run_tests(testname, "--junit-xml", filename,
2363 exitcode=EXITCODE_BAD_TEST)
2364 self.check_executed_tests(output, testname,
2365 failed=testname,
2366 stats=TestStats(1, 1, 0))
2367
2368 # Test generated XML
2369 with open(filename, encoding="utf8") as fp:
2370 content = fp.read()
2371
2372 testsuite = ElementTree.fromstring(content)
2373 self.assertEqual(int(testsuite.get('tests')), 1)
2374 self.assertEqual(int(testsuite.get('errors')), 0)
2375 self.assertEqual(int(testsuite.get('failures')), 1)
2376
2377 testcase = testsuite[0][0]
2378 self.assertEqual(testcase.get('status'), 'run')
2379 self.assertEqual(testcase.get('result'), 'completed')
2380 self.assertGreater(float(testcase.get('time')), 0)
2381 for out in testcase.iter('system-out'):
2382 self.assertEqual(out.text, r"abc \x1b def")
2383
2384 def test_nonascii(self):
2385 code = textwrap.dedent(r"""
2386 import unittest
2387
2388 class NonASCIITests(unittest.TestCase):
2389 def test_docstring(self):
2390 '''docstring:\u20ac'''
2391
2392 def test_subtest(self):
2393 with self.subTest(param='subtest:\u20ac'):
2394 pass
2395
2396 def test_skip(self):
2397 self.skipTest('skipped:\u20ac')
2398 """)
2399 testname = self.create_test(code=code)
2400
2401 env = dict(os.environ)
2402 env['PYTHONIOENCODING'] = 'ascii'
2403
2404 def check(output):
2405 self.check_executed_tests(output, testname, stats=TestStats(3, 0, 1))
2406 self.assertIn(r'docstring:\u20ac', output)
2407 self.assertIn(r'skipped:\u20ac', output)
2408
2409 # Run sequentially
2410 output = self.run_tests('-v', testname, env=env, isolated=False)
2411 check(output)
2412
2413 # Run in parallel
2414 output = self.run_tests('-j1', '-v', testname, env=env, isolated=False)
2415 check(output)
2416
2417 def test_pgo_exclude(self):
2418 # Get PGO tests
2419 output = self.run_tests('--pgo', '--list-tests')
2420 pgo_tests = output.strip().split()
2421
2422 # Exclude test_re
2423 output = self.run_tests('--pgo', '--list-tests', '-x', 'test_re')
2424 tests = output.strip().split()
2425 self.assertNotIn('test_re', tests)
2426 self.assertEqual(len(tests), len(pgo_tests) - 1)
2427
2428
2429 class TestUtils(unittest.TestCase):
2430 def test_format_duration(self):
2431 self.assertEqual(utils.format_duration(0),
2432 '0 ms')
2433 self.assertEqual(utils.format_duration(1e-9),
2434 '1 ms')
2435 self.assertEqual(utils.format_duration(10e-3),
2436 '10 ms')
2437 self.assertEqual(utils.format_duration(1.5),
2438 '1.5 sec')
2439 self.assertEqual(utils.format_duration(1),
2440 '1.0 sec')
2441 self.assertEqual(utils.format_duration(2 * 60),
2442 '2 min')
2443 self.assertEqual(utils.format_duration(2 * 60 + 1),
2444 '2 min 1 sec')
2445 self.assertEqual(utils.format_duration(3 * 3600),
2446 '3 hour')
2447 self.assertEqual(utils.format_duration(3 * 3600 + 2 * 60 + 1),
2448 '3 hour 2 min')
2449 self.assertEqual(utils.format_duration(3 * 3600 + 1),
2450 '3 hour 1 sec')
2451
2452 def test_normalize_test_name(self):
2453 normalize = normalize_test_name
2454 self.assertEqual(normalize('test_access (test.test_os.FileTests.test_access)'),
2455 'test_access')
2456 self.assertEqual(normalize('setUpClass (test.test_os.ChownFileTests)', is_error=True),
2457 'ChownFileTests')
2458 self.assertEqual(normalize('test_success (test.test_bug.ExampleTests.test_success)', is_error=True),
2459 'test_success')
2460 self.assertIsNone(normalize('setUpModule (test.test_x)', is_error=True))
2461 self.assertIsNone(normalize('tearDownModule (test.test_module)', is_error=True))
2462
2463 def test_format_resources(self):
2464 format_resources = utils.format_resources
2465 ALL_RESOURCES = utils.ALL_RESOURCES
2466 self.assertEqual(
2467 format_resources({"network": None}),
2468 'resources (1): network')
2469 self.assertEqual(
2470 format_resources(dict.fromkeys(("audio", "decimal", "network"))),
2471 'resources (3): audio,decimal,network')
2472 self.assertEqual(
2473 format_resources(dict.fromkeys(ALL_RESOURCES)),
2474 'resources: all')
2475 self.assertEqual(
2476 format_resources({name: None for name in ALL_RESOURCES
2477 if name != "cpu"}),
2478 'resources: all,-cpu')
2479 self.assertEqual(
2480 format_resources({**dict.fromkeys(ALL_RESOURCES), "tzdata": None}),
2481 'resources: all,tzdata')
2482
2483 def test_match_test(self):
2484 class Test:
2485 def __init__(self, test_id):
2486 self.test_id = test_id
2487
2488 def id(self):
2489 return self.test_id
2490
2491 # Restore patterns once the test completes
2492 patterns = get_match_tests()
2493 self.addCleanup(set_match_tests, patterns)
2494
2495 test_access = Test('test.test_os.FileTests.test_access')
2496 test_chdir = Test('test.test_os.Win32ErrorTests.test_chdir')
2497 test_copy = Test('test.test_shutil.TestCopy.test_copy')
2498
2499 # Test acceptance
2500 with support.swap_attr(support, '_test_matchers', ()):
2501 # match all
2502 set_match_tests([])
2503 self.assertTrue(match_test(test_access))
2504 self.assertTrue(match_test(test_chdir))
2505
2506 # match all using None
2507 set_match_tests(None)
2508 self.assertTrue(match_test(test_access))
2509 self.assertTrue(match_test(test_chdir))
2510
2511 # match the full test identifier
2512 set_match_tests([(test_access.id(), True)])
2513 self.assertTrue(match_test(test_access))
2514 self.assertFalse(match_test(test_chdir))
2515
2516 # match the module name
2517 set_match_tests([('test_os', True)])
2518 self.assertTrue(match_test(test_access))
2519 self.assertTrue(match_test(test_chdir))
2520 self.assertFalse(match_test(test_copy))
2521
2522 # Test '*' pattern
2523 set_match_tests([('test_*', True)])
2524 self.assertTrue(match_test(test_access))
2525 self.assertTrue(match_test(test_chdir))
2526
2527 # Test case sensitivity
2528 set_match_tests([('filetests', True)])
2529 self.assertFalse(match_test(test_access))
2530 set_match_tests([('FileTests', True)])
2531 self.assertTrue(match_test(test_access))
2532
2533 # Test pattern containing '.' and a '*' metacharacter
2534 set_match_tests([('*test_os.*.test_*', True)])
2535 self.assertTrue(match_test(test_access))
2536 self.assertTrue(match_test(test_chdir))
2537 self.assertFalse(match_test(test_copy))
2538
2539 # Multiple patterns
2540 set_match_tests([(test_access.id(), True), (test_chdir.id(), True)])
2541 self.assertTrue(match_test(test_access))
2542 self.assertTrue(match_test(test_chdir))
2543 self.assertFalse(match_test(test_copy))
2544
2545 set_match_tests([('test_access', True), ('DONTMATCH', True)])
2546 self.assertTrue(match_test(test_access))
2547 self.assertFalse(match_test(test_chdir))
2548
2549 # Test rejection
2550 with support.swap_attr(support, '_test_matchers', ()):
2551 # match the full test identifier
2552 set_match_tests([(test_access.id(), False)])
2553 self.assertFalse(match_test(test_access))
2554 self.assertTrue(match_test(test_chdir))
2555
2556 # match the module name
2557 set_match_tests([('test_os', False)])
2558 self.assertFalse(match_test(test_access))
2559 self.assertFalse(match_test(test_chdir))
2560 self.assertTrue(match_test(test_copy))
2561
2562 # Test '*' pattern
2563 set_match_tests([('test_*', False)])
2564 self.assertFalse(match_test(test_access))
2565 self.assertFalse(match_test(test_chdir))
2566
2567 # Test case sensitivity
2568 set_match_tests([('filetests', False)])
2569 self.assertTrue(match_test(test_access))
2570 set_match_tests([('FileTests', False)])
2571 self.assertFalse(match_test(test_access))
2572
2573 # Test pattern containing '.' and a '*' metacharacter
2574 set_match_tests([('*test_os.*.test_*', False)])
2575 self.assertFalse(match_test(test_access))
2576 self.assertFalse(match_test(test_chdir))
2577 self.assertTrue(match_test(test_copy))
2578
2579 # Multiple patterns
2580 set_match_tests([(test_access.id(), False), (test_chdir.id(), False)])
2581 self.assertFalse(match_test(test_access))
2582 self.assertFalse(match_test(test_chdir))
2583 self.assertTrue(match_test(test_copy))
2584
2585 set_match_tests([('test_access', False), ('DONTMATCH', False)])
2586 self.assertFalse(match_test(test_access))
2587 self.assertTrue(match_test(test_chdir))
2588
2589 # Test mixed filters
2590 with support.swap_attr(support, '_test_matchers', ()):
2591 set_match_tests([('*test_os', False), ('test_access', True)])
2592 self.assertTrue(match_test(test_access))
2593 self.assertFalse(match_test(test_chdir))
2594 self.assertTrue(match_test(test_copy))
2595
2596 set_match_tests([('*test_os', True), ('test_access', False)])
2597 self.assertFalse(match_test(test_access))
2598 self.assertTrue(match_test(test_chdir))
2599 self.assertFalse(match_test(test_copy))
2600
2601 def test_sanitize_xml(self):
2602 sanitize_xml = utils.sanitize_xml
2603
2604 # escape invalid XML characters
2605 self.assertEqual(sanitize_xml('abc \x1b\x1f def'),
2606 r'abc \x1b\x1f def')
2607 self.assertEqual(sanitize_xml('nul:\x00, bell:\x07'),
2608 r'nul:\x00, bell:\x07')
2609 self.assertEqual(sanitize_xml('surrogate:\uDC80'),
2610 r'surrogate:\udc80')
2611 self.assertEqual(sanitize_xml('illegal \uFFFE and \uFFFF'),
2612 r'illegal \ufffe and \uffff')
2613
2614 # no escape for valid XML characters
2615 self.assertEqual(sanitize_xml('a\n\tb'),
2616 'a\n\tb')
2617 self.assertEqual(sanitize_xml('valid t\xe9xt \u20ac'),
2618 'valid t\xe9xt \u20ac')
2619
2620
2621 from test.libregrtest.results import TestResults
2622
2623
2624 class TestColorized(unittest.TestCase):
2625 def test_test_result_get_state(self):
2626 # Arrange
2627 green = _colorize.ANSIColors.GREEN
2628 red = _colorize.ANSIColors.BOLD_RED
2629 reset = _colorize.ANSIColors.RESET
2630 yellow = _colorize.ANSIColors.YELLOW
2631
2632 good_results = TestResults()
2633 good_results.good = ["good1", "good2"]
2634 bad_results = TestResults()
2635 bad_results.bad = ["bad1", "bad2"]
2636 no_results = TestResults()
2637 no_results.bad = []
2638 interrupted_results = TestResults()
2639 interrupted_results.interrupted = True
2640 interrupted_worker_bug = TestResults()
2641 interrupted_worker_bug.interrupted = True
2642 interrupted_worker_bug.worker_bug = True
2643
2644 for results, expected in (
2645 (good_results, f"{green}SUCCESS{reset}"),
2646 (bad_results, f"{red}FAILURE{reset}"),
2647 (no_results, f"{yellow}NO TESTS RAN{reset}"),
2648 (interrupted_results, f"{yellow}INTERRUPTED{reset}"),
2649 (
2650 interrupted_worker_bug,
2651 f"{yellow}INTERRUPTED{reset}, {red}WORKER BUG{reset}",
2652 ),
2653 ):
2654 with self.subTest(results=results, expected=expected):
2655 # Act
2656 with unittest.mock.patch(
2657 "_colorize.can_colorize", return_value=True
2658 ):
2659 result = results.get_state(fail_env_changed=False)
2660
2661 # Assert
2662 self.assertEqual(result, expected)
2663
2664
2665 if __name__ == '__main__':
2666 setup.setup_process()
2667 unittest.main()