Debian Codegraph @

Lib/test/libregrtest/cmdline.py

raw
1 import argparse
2 import os.path
3 import shlex
4 import sys
5 from test.support import os_helper, Py_DEBUG
6 from .utils import ALL_RESOURCES, RESOURCE_NAMES, TestFilter
7
8
9 USAGE = """\
10 python -m test [options] [test_name1 [test_name2 ...]]
11 python path/to/Lib/test/regrtest.py [options] [test_name1 [test_name2 ...]]
12 """
13
14 DESCRIPTION = """\
15 Run Python regression tests.
16
17 If no arguments or options are provided, finds all files matching
18 the pattern "test_*" in the Lib/test subdirectory and runs
19 them in alphabetical order (but see -M and -u, below, for exceptions).
20
21 For more rigorous testing, it is useful to use the following
22 command line:
23
24 python -E -Wd -m test [options] [test_name1 ...]
25 """
26
27 EPILOG = """\
28 Additional option details:
29
30 -r randomizes test execution order. You can use --randseed=int to provide an
31 int seed value for the randomizer. The randseed value will be used
32 to set seeds for all random usages in tests
33 (including randomizing the tests order if -r is set).
34 By default we always set random seed, but do not randomize test order.
35
36 -s On the first invocation of regrtest using -s, the first test file found
37 or the first test file given on the command line is run, and the name of
38 the next test is recorded in a file named pynexttest. If run from the
39 Python build directory, pynexttest is located in the 'build' subdirectory,
40 otherwise it is located in tempfile.gettempdir(). On subsequent runs,
41 the test in pynexttest is run, and the next test is written to pynexttest.
42 When the last test has been run, pynexttest is deleted. In this way it
43 is possible to single step through the test files. This is useful when
44 doing memory analysis on the Python interpreter, which process tends to
45 consume too many resources to run the full regression test non-stop.
46
47 -S is used to resume running tests after an interrupted run. It will
48 maintain the order a standard run (i.e. it assumes -r is not used).
49 This is useful after the tests have prematurely stopped for some external
50 reason and you want to resume the run from where you left off rather
51 than starting from the beginning. Note: this is different from --prioritize.
52
53 --prioritize is used to influence the order of selected tests, such that
54 the tests listed as an argument are executed first. This is especially
55 useful when combined with -j and -r to pin the longest-running tests
56 to start at the beginning of a test run. Pass --prioritize=test_a,test_b
57 to make test_a run first, followed by test_b, and then the other tests.
58 If test_a wasn't selected for execution by regular means, --prioritize will
59 not make it execute.
60
61 -f reads the names of tests from the file given as f's argument, one
62 or more test names per line. Whitespace is ignored. Blank lines and
63 lines beginning with '#' are ignored. This is especially useful for
64 whittling down failures involving interactions among tests.
65
66 -L causes the leaks(1) command to be run just before exit if it exists.
67 leaks(1) is available on Mac OS X and presumably on some other
68 FreeBSD-derived systems.
69
70 -R runs each test several times and examines sys.gettotalrefcount() to
71 see if the test appears to be leaking references. The argument should
72 be of the form stab:run:fname where 'stab' is the number of times the
73 test is run to let gettotalrefcount settle down, 'run' is the number
74 of times further it is run and 'fname' is the name of the file the
75 reports are written to. These parameters all have defaults (5, 4 and
76 "reflog.txt" respectively), and the minimal invocation is '-R :'.
77
78 -M runs tests that require an exorbitant amount of memory. These tests
79 typically try to ascertain containers keep working when containing more than
80 2 billion objects, which only works on 64-bit systems. There are also some
81 tests that try to exhaust the address space of the process, which only makes
82 sense on 32-bit systems with at least 2Gb of memory. The passed-in memlimit,
83 which is a string in the form of '2.5Gb', determines how much memory the
84 tests will limit themselves to (but they may go slightly over.) The number
85 shouldn't be more memory than the machine has (including swap memory). You
86 should also keep in mind that swap memory is generally much, much slower
87 than RAM, and setting memlimit to all available RAM or higher will heavily
88 tax the machine. On the other hand, it is no use running these tests with a
89 limit of less than 2.5Gb, and many require more than 20Gb. Tests that expect
90 to use more than memlimit memory will be skipped. The big-memory tests
91 generally run very, very long.
92
93 -u is used to specify which special resource intensive tests to run,
94 such as those requiring large file support or network connectivity.
95 The argument is a comma-separated list of words indicating the
96 resources to test. Currently only the following are defined:
97
98 all - Enable all special resources.
99
100 none - Disable all special resources (this is the default).
101
102 audio - Tests that use the audio device. (There are known
103 cases of broken audio drivers that can crash Python or
104 even the Linux kernel.)
105
106 curses - Tests that use curses and will modify the terminal's
107 state and output modes.
108
109 largefile - It is okay to run some test that may create huge
110 files. These tests can take a long time and may
111 consume >2 GiB of disk space temporarily.
112
113 extralargefile - Like 'largefile', but even larger (and slower).
114
115 network - It is okay to run tests that use external network
116 resource, e.g. testing SSL support for sockets.
117
118 decimal - Test the decimal module against a large suite that
119 verifies compliance with standards.
120
121 cpu - Used for certain CPU-heavy tests.
122
123 walltime - Long running but not CPU-bound tests.
124
125 subprocess Run all tests for the subprocess module.
126
127 urlfetch - It is okay to download files required on testing.
128
129 gui - Run tests that require a running GUI.
130
131 tzdata - Run tests that require timezone data.
132
133 xpickle - Test pickle and _pickle against Python 3.6, 3.7, 3.8
134 and 3.9 to test backwards compatibility. These tests
135 may take very long to complete.
136
137 wantobjects - Allows to run Tkinter tests with the specified value of
138 tkinter.wantobjects.
139
140 To enable all resources except one, use '-uall,-<resource>'. For
141 example, to run all the tests except for the gui tests, give the
142 option '-uall,-gui'.
143
144 --matchfile filters tests using a text file, one pattern per line.
145 Pattern examples:
146
147 - test method: test_stat_attributes
148 - test class: FileTests
149 - test identifier: test_os.FileTests.test_stat_attributes
150 """
151
152
153 class Namespace(argparse.Namespace):
154 def __init__(self, **kwargs) -> None:
155 self.ci = False
156 self.testdir = None
157 self.verbose = 0
158 self.quiet = False
159 self.exclude = False
160 self.cleanup = False
161 self.wait = False
162 self.list_cases = False
163 self.list_tests = False
164 self.single = False
165 self.randomize = False
166 self.fromfile = None
167 self.fail_env_changed = False
168 self.use_resources: dict[str, str | None] = {}
169 self.trace = False
170 self.coverdir = 'coverage'
171 self.runleaks = False
172 self.huntrleaks: tuple[int, int, str] | None = None
173 self.rerun = False
174 self.verbose3 = False
175 self.print_slow = False
176 self.random_seed = None
177 self.use_mp = None
178 self.parallel_threads = None
179 self.forever = False
180 self.header = False
181 self.failfast = False
182 self.match_tests: TestFilter = []
183 self.pgo = False
184 self.pgo_extended = False
185 self.tsan = False
186 self.tsan_parallel = False
187 self.worker_json = None
188 self.start = None
189 self.timeout = None
190 self.memlimit = None
191 self.threshold = None
192 self.fail_rerun = False
193 self.tempdir = None
194 self._add_python_opts = True
195 self.xmlpath = None
196 self.single_process = False
197
198 super().__init__(**kwargs)
199
200
201 class _ArgParser(argparse.ArgumentParser):
202
203 def error(self, message):
204 super().error(message + "\nPass -h or --help for complete help.")
205
206
207 class FilterAction(argparse.Action):
208 def __call__(self, parser, namespace, value, option_string=None):
209 items = getattr(namespace, self.dest)
210 items.append((value, self.const))
211
212
213 class FromFileFilterAction(argparse.Action):
214 def __call__(self, parser, namespace, value, option_string=None):
215 items = getattr(namespace, self.dest)
216 with open(value, encoding='utf-8') as fp:
217 for line in fp:
218 items.append((line.strip(), self.const))
219
220
221 def _create_parser():
222 # Set prog to prevent the uninformative "__main__.py" from displaying in
223 # error messages when using "python -m test ...".
224 parser = _ArgParser(prog='regrtest.py',
225 usage=USAGE,
226 description=DESCRIPTION,
227 epilog=EPILOG,
228 add_help=False,
229 formatter_class=argparse.RawDescriptionHelpFormatter)
230 parser.set_defaults(match_tests=[])
231
232 # Arguments with this clause added to its help are described further in
233 # the epilog's "Additional option details" section.
234 more_details = ' See the section at bottom for more details.'
235
236 group = parser.add_argument_group('General options')
237 # We add help explicitly to control what argument group it renders under.
238 group.add_argument('-h', '--help', action='help',
239 help='show this help message and exit')
240 group.add_argument('--fast-ci', action='store_true',
241 help='Fast Continuous Integration (CI) mode used by '
242 'GitHub Actions')
243 group.add_argument('--slow-ci', action='store_true',
244 help='Slow Continuous Integration (CI) mode used by '
245 'buildbot workers')
246 group.add_argument('--timeout', metavar='TIMEOUT',
247 help='dump the traceback and exit if a test takes '
248 'more than TIMEOUT seconds; disabled if TIMEOUT '
249 'is negative or equals to zero')
250 group.add_argument('--wait', action='store_true',
251 help='wait for user input, e.g., allow a debugger '
252 'to be attached')
253 group.add_argument('-S', '--start', metavar='START',
254 help='resume an interrupted run at the following test.' +
255 more_details)
256 group.add_argument('-p', '--python', metavar='PYTHON',
257 help='Command to run Python test subprocesses with.')
258 group.add_argument('--randseed', metavar='SEED',
259 dest='random_seed', type=int,
260 help='pass a global random seed')
261
262 group = parser.add_argument_group('Verbosity')
263 group.add_argument('-v', '--verbose', action='count',
264 help='run tests in verbose mode with output to stdout')
265 group.add_argument('-w', '--rerun', action='store_true',
266 help='re-run failed tests in verbose mode')
267 group.add_argument('--verbose2', action='store_true', dest='rerun',
268 help='deprecated alias to --rerun')
269 group.add_argument('-W', '--verbose3', action='store_true',
270 help='display test output on failure')
271 group.add_argument('-q', '--quiet', action='store_true',
272 help='no output unless one or more tests fail')
273 group.add_argument('-o', '--slowest', action='store_true', dest='print_slow',
274 help='print the slowest 10 tests')
275 group.add_argument('--header', action='store_true',
276 help='print header with interpreter info')
277
278 group = parser.add_argument_group('Selecting tests')
279 group.add_argument('-r', '--randomize', action='store_true',
280 help='randomize test execution order.' + more_details)
281 group.add_argument('--no-randomize', dest='no_randomize', action='store_true',
282 help='do not randomize test execution order, even if '
283 'it would be implied by another option')
284 group.add_argument('--prioritize', metavar='TEST1,TEST2,...',
285 action='append', type=priority_list,
286 help='select these tests first, even if the order is'
287 ' randomized.' + more_details)
288 group.add_argument('-f', '--fromfile', metavar='FILE',
289 help='read names of tests to run from a file.' +
290 more_details)
291 group.add_argument('-x', '--exclude', action='store_true',
292 help='arguments are tests to *exclude*')
293 group.add_argument('-s', '--single', action='store_true',
294 help='single step through a set of tests.' +
295 more_details)
296 group.add_argument('-m', '--match', metavar='PAT',
297 dest='match_tests', action=FilterAction, const=True,
298 help='match test cases and methods with glob pattern PAT')
299 group.add_argument('-i', '--ignore', metavar='PAT',
300 dest='match_tests', action=FilterAction, const=False,
301 help='ignore test cases and methods with glob pattern PAT')
302 group.add_argument('--matchfile', metavar='FILENAME',
303 dest='match_tests',
304 action=FromFileFilterAction, const=True,
305 help='similar to --match but get patterns from a '
306 'text file, one pattern per line')
307 group.add_argument('--ignorefile', metavar='FILENAME',
308 dest='match_tests',
309 action=FromFileFilterAction, const=False,
310 help='similar to --matchfile but it receives patterns '
311 'from text file to ignore')
312 group.add_argument('-G', '--failfast', action='store_true',
313 help='fail as soon as a test fails (only with -v or -W)')
314 group.add_argument('-u', '--use', metavar='RES1,RES2,...',
315 action='extend', type=resources_list,
316 help='specify which special resource intensive tests '
317 'to run.' + more_details)
318 group.add_argument('-M', '--memlimit', metavar='LIMIT',
319 help='run very large memory-consuming tests.' +
320 more_details)
321 group.add_argument('--testdir', metavar='DIR',
322 type=relative_filename,
323 help='execute test files in the specified directory '
324 '(instead of the Python stdlib test suite)')
325
326 group = parser.add_argument_group('Special runs')
327 group.add_argument('-L', '--runleaks', action='store_true',
328 help='run the leaks(1) command just before exit.' +
329 more_details)
330 group.add_argument('-R', '--huntrleaks', metavar='RUNCOUNTS',
331 type=huntrleaks,
332 help='search for reference leaks (needs debug build, '
333 'very slow).' + more_details)
334 group.add_argument('-j', '--multiprocess', metavar='PROCESSES',
335 dest='use_mp', type=int,
336 help='run PROCESSES processes at once')
337 group.add_argument('--single-process', action='store_true',
338 dest='single_process',
339 help='always run all tests sequentially in '
340 'a single process, ignore -jN option, '
341 'and failed tests are also rerun sequentially '
342 'in the same process')
343 group.add_argument('--parallel-threads', metavar='PARALLEL_THREADS',
344 type=int,
345 help='run copies of each test in PARALLEL_THREADS at '
346 'once')
347 group.add_argument('-T', '--coverage', action='store_true',
348 dest='trace',
349 help='turn on code coverage tracing using the trace '
350 'module')
351 group.add_argument('-D', '--coverdir', metavar='DIR',
352 type=relative_filename,
353 help='directory where coverage files are put')
354 group.add_argument('-N', '--nocoverdir',
355 action='store_const', const=None, dest='coverdir',
356 help='put coverage files alongside modules')
357 group.add_argument('-t', '--threshold', metavar='THRESHOLD',
358 type=int,
359 help='call gc.set_threshold(THRESHOLD)')
360 group.add_argument('-n', '--nowindows', action='store_true',
361 help='suppress error message boxes on Windows')
362 group.add_argument('-F', '--forever', action='store_true',
363 help='run the specified tests in a loop, until an '
364 'error happens; imply --failfast')
365 group.add_argument('--list-tests', action='store_true',
366 help="only write the name of tests that will be run, "
367 "don't execute them")
368 group.add_argument('--list-cases', action='store_true',
369 help='only write the name of test cases that will be run'
370 ' , don\'t execute them')
371 group.add_argument('-P', '--pgo', dest='pgo', action='store_true',
372 help='enable Profile Guided Optimization (PGO) training')
373 group.add_argument('--pgo-extended', action='store_true',
374 help='enable extended PGO training (slower training)')
375 group.add_argument('--tsan', dest='tsan', action='store_true',
376 help='run a subset of test cases that are proper for the TSAN test')
377 group.add_argument('--tsan-parallel', action='store_true',
378 help='run a subset of test cases that are appropriate '
379 'for TSAN with `--parallel-threads=N`')
380 group.add_argument('--fail-env-changed', action='store_true',
381 help='if a test file alters the environment, mark '
382 'the test as failed')
383 group.add_argument('--fail-rerun', action='store_true',
384 help='if a test failed and then passed when re-run, '
385 'mark the tests as failed')
386
387 group.add_argument('--junit-xml', dest='xmlpath', metavar='FILENAME',
388 help='writes JUnit-style XML results to the specified '
389 'file')
390 group.add_argument('--tempdir', metavar='PATH',
391 help='override the working directory for the test run')
392 group.add_argument('--cleanup', action='store_true',
393 help='remove old test_python_* directories')
394 group.add_argument('--bisect', action='store_true',
395 help='if some tests fail, run test.bisect_cmd on them')
396 group.add_argument('--dont-add-python-opts', dest='_add_python_opts',
397 action='store_false',
398 help="internal option, don't use it")
399 return parser
400
401
402 def relative_filename(string):
403 # CWD is replaced with a temporary dir before calling main(), so we
404 # join it with the saved CWD so it ends up where the user expects.
405 return os.path.join(os_helper.SAVEDCWD, string)
406
407
408 def huntrleaks(string):
409 args = string.split(':')
410 if len(args) not in (2, 3):
411 raise argparse.ArgumentTypeError(
412 'needs 2 or 3 colon-separated arguments')
413 nwarmup = int(args[0]) if args[0] else 5
414 ntracked = int(args[1]) if args[1] else 4
415 fname = args[2] if len(args) > 2 and args[2] else 'reflog.txt'
416 return nwarmup, ntracked, fname
417
418
419 def resources_list(string):
420 u = []
421 for x in string.split(','):
422 r, eq, v = x.partition('=')
423 r = r.lower()
424 u.append((r, v if eq else None))
425 if r == 'all' or r == 'none':
426 if eq:
427 raise argparse.ArgumentTypeError('invalid resource: ' + x)
428 continue
429 if r[0] == '-':
430 if eq:
431 raise argparse.ArgumentTypeError('invalid resource: ' + x)
432 r = r[1:]
433 if r not in RESOURCE_NAMES:
434 raise argparse.ArgumentTypeError('invalid resource: ' + r)
435 return u
436
437
438 def priority_list(string):
439 return string.split(",")
440
441
442 def _parse_args(args, **kwargs):
443 # Defaults
444 ns = Namespace()
445 for k, v in kwargs.items():
446 if not hasattr(ns, k):
447 raise TypeError('%r is an invalid keyword argument '
448 'for this function' % k)
449 setattr(ns, k, v)
450
451 parser = _create_parser()
452 # Issue #14191: argparse doesn't support "intermixed" positional and
453 # optional arguments. Use parse_known_args() as workaround.
454 ns.args = parser.parse_known_args(args=args, namespace=ns)[1]
455 for arg in ns.args:
456 if arg.startswith('-'):
457 parser.error("unrecognized arguments: %s" % arg)
458
459 if ns.timeout is not None:
460 # Support "--timeout=" (no value) so Makefile.pre.pre TESTTIMEOUT
461 # can be used by "make buildbottest" and "make test".
462 if ns.timeout != "":
463 try:
464 ns.timeout = float(ns.timeout)
465 except ValueError:
466 parser.error(f"invalid timeout value: {ns.timeout!r}")
467 else:
468 ns.timeout = None
469
470 # Continuous Integration (CI): common options for fast/slow CI modes
471 if ns.slow_ci or ns.fast_ci:
472 # Similar to options:
473 # -j0 --randomize --fail-env-changed --rerun --slowest --verbose3
474 if ns.use_mp is None:
475 ns.use_mp = 0
476 ns.randomize = True
477 ns.fail_env_changed = True
478 if ns.python is None:
479 ns.rerun = True
480 ns.print_slow = True
481 if not ns.verbose:
482 ns.verbose3 = True
483 else:
484 # --verbose has the priority over --verbose3
485 pass
486 else:
487 ns._add_python_opts = False
488
489 # --singleprocess overrides -jN option
490 if ns.single_process:
491 ns.use_mp = None
492
493 # When both --slow-ci and --fast-ci options are present,
494 # --slow-ci has the priority
495 if ns.slow_ci:
496 # Similar to: -u "all" --timeout=1200
497 if ns.use is None:
498 ns.use = []
499 ns.use[:0] = [('all', None)]
500 if ns.timeout is None:
501 ns.timeout = 1200 # 20 minutes
502 elif ns.fast_ci:
503 # Similar to: -u "all,-cpu" --timeout=600
504 if ns.use is None:
505 ns.use = []
506 ns.use[:0] = [('all', None), ('-cpu', None)]
507 if ns.timeout is None:
508 ns.timeout = 600 # 10 minutes
509
510 if ns.single and ns.fromfile:
511 parser.error("-s and -f don't go together!")
512 if ns.trace:
513 if ns.use_mp is not None:
514 if not Py_DEBUG:
515 parser.error("need --with-pydebug to use -T and -j together")
516 else:
517 print(
518 "Warning: collecting coverage without -j is imprecise. Configure"
519 " --with-pydebug and run -m test -T -j for best results.",
520 file=sys.stderr
521 )
522 if ns.python is not None:
523 if ns.use_mp is None:
524 parser.error("-p requires -j!")
525 # The "executable" may be two or more parts, e.g. "node python.js"
526 ns.python = shlex.split(ns.python)
527 if ns.failfast and not (ns.verbose or ns.verbose3):
528 parser.error("-G/--failfast needs either -v or -W")
529 if ns.pgo and (ns.verbose or ns.rerun or ns.verbose3):
530 parser.error("--pgo/-v don't go together!")
531 if ns.pgo_extended:
532 ns.pgo = True # pgo_extended implies pgo
533
534 if ns.nowindows:
535 print("Warning: the --nowindows (-n) option is deprecated. "
536 "Use -vv to display assertions in stderr.", file=sys.stderr)
537
538 if ns.quiet:
539 ns.verbose = 0
540 if ns.timeout is not None:
541 if ns.timeout <= 0:
542 ns.timeout = None
543 if ns.use:
544 for r, v in ns.use:
545 if r == 'all':
546 for r in ALL_RESOURCES:
547 ns.use_resources[r] = None
548 elif r == 'none':
549 ns.use_resources.clear()
550 elif r[0] == '-':
551 r = r[1:]
552 ns.use_resources.pop(r, None)
553 else:
554 ns.use_resources[r] = v
555 if ns.random_seed is not None:
556 ns.randomize = True
557 if ns.no_randomize:
558 ns.randomize = False
559 if ns.verbose:
560 ns.header = True
561
562 # When -jN option is used, a worker process does not use --verbose3
563 # and so -R 3:3 -jN --verbose3 just works as expected: there is no false
564 # alarm about memory leak.
565 if ns.huntrleaks and ns.verbose3 and ns.use_mp is None:
566 # run_single_test() replaces sys.stdout with io.StringIO if verbose3
567 # is true. In this case, huntrleaks sees an write into StringIO as
568 # a memory leak, whereas it is not (gh-71290).
569 ns.verbose3 = False
570 print("WARNING: Disable --verbose3 because it's incompatible with "
571 "--huntrleaks without -jN option",
572 file=sys.stderr)
573
574 if ns.forever:
575 # --forever implies --failfast
576 ns.failfast = True
577
578 if ns.huntrleaks:
579 warmup, repetitions, _ = ns.huntrleaks
580 if warmup < 1 or repetitions < 1:
581 msg = ("Invalid values for the --huntrleaks/-R parameters. The "
582 "number of warmups and repetitions must be at least 1 "
583 "each (1:1).")
584 print(msg, file=sys.stderr, flush=True)
585 sys.exit(2)
586
587 ns.prioritize = [
588 test
589 for test_list in (ns.prioritize or ())
590 for test in test_list
591 ]
592
593 return ns