Debian Codegraph @

Lib/test/bisect_cmd.py

raw
1 #!/usr/bin/env python3
2 """
3 Command line tool to bisect failing CPython tests.
4
5 Find the test_os test method which alters the environment:
6
7 ./python -m test.bisect_cmd --fail-env-changed test_os
8
9 Find a reference leak in "test_os", write the list of failing tests into the
10 "bisect" file:
11
12 ./python -m test.bisect_cmd -o bisect -R 3:3 test_os
13
14 Load an existing list of tests from a file using -i option:
15
16 ./python -m test --list-cases -m FileTests test_os > tests
17 ./python -m test.bisect_cmd -i tests test_os
18 """
19
20 import argparse
21 import datetime
22 import os.path
23 import math
24 import random
25 import subprocess
26 import sys
27 import tempfile
28 import time
29
30
31 def write_tests(filename, tests):
32 with open(filename, "w") as fp:
33 for name in tests:
34 print(name, file=fp)
35 fp.flush()
36
37
38 def write_output(filename, tests):
39 if not filename:
40 return
41 print("Writing %s tests into %s" % (len(tests), filename))
42 write_tests(filename, tests)
43 return filename
44
45
46 def format_shell_args(args):
47 return ' '.join(args)
48
49
50 def python_cmd():
51 cmd = [sys.executable]
52 cmd.extend(subprocess._args_from_interpreter_flags())
53 cmd.extend(subprocess._optim_args_from_interpreter_flags())
54 cmd.extend(('-X', 'faulthandler'))
55 return cmd
56
57
58 def list_cases(args):
59 cmd = python_cmd()
60 cmd.extend(['-m', 'test', '--list-cases'])
61 cmd.extend(args.test_args)
62 proc = subprocess.run(cmd,
63 stdout=subprocess.PIPE,
64 universal_newlines=True)
65 exitcode = proc.returncode
66 if exitcode:
67 cmd = format_shell_args(cmd)
68 print("Failed to list tests: %s failed with exit code %s"
69 % (cmd, exitcode))
70 sys.exit(exitcode)
71 tests = proc.stdout.splitlines()
72 return tests
73
74
75 def run_tests(args, tests, huntrleaks=None):
76 tmp = tempfile.mktemp()
77 try:
78 write_tests(tmp, tests)
79
80 cmd = python_cmd()
81 cmd.extend(['-u', '-m', 'test', '--matchfile', tmp])
82 cmd.extend(args.test_args)
83 print("+ %s" % format_shell_args(cmd))
84
85 sys.stdout.flush()
86 sys.stderr.flush()
87
88 proc = subprocess.run(cmd)
89 return proc.returncode
90 finally:
91 if os.path.exists(tmp):
92 os.unlink(tmp)
93
94
95 def parse_args():
96 parser = argparse.ArgumentParser()
97 parser.add_argument('-i', '--input',
98 help='Test names produced by --list-tests written '
99 'into a file. If not set, run --list-tests')
100 parser.add_argument('-o', '--output',
101 help='Result of the bisection')
102 parser.add_argument('-n', '--max-tests', type=int, default=1,
103 help='Maximum number of tests to stop the bisection '
104 '(default: 1)')
105 parser.add_argument('-N', '--max-iter', type=int, default=100,
106 help='Maximum number of bisection iterations '
107 '(default: 100)')
108 # FIXME: document that following arguments are test arguments
109
110 args, test_args = parser.parse_known_args()
111 args.test_args = test_args
112 return args
113
114
115 def main():
116 args = parse_args()
117 for opt in ('-w', '--rerun', '--verbose2'):
118 if opt in args.test_args:
119 print(f"WARNING: {opt} option should not be used to bisect!")
120 print()
121
122 if args.input:
123 with open(args.input) as fp:
124 tests = [line.strip() for line in fp]
125 else:
126 tests = list_cases(args)
127
128 print("Start bisection with %s tests" % len(tests))
129 print("Test arguments: %s" % format_shell_args(args.test_args))
130 print("Bisection will stop when getting %s or less tests "
131 "(-n/--max-tests option), or after %s iterations "
132 "(-N/--max-iter option)"
133 % (args.max_tests, args.max_iter))
134 output = write_output(args.output, tests)
135 print()
136
137 start_time = time.monotonic()
138 iteration = 1
139 try:
140 while len(tests) > args.max_tests and iteration <= args.max_iter:
141 ntest = len(tests)
142 ntest = max(ntest // 2, 1)
143 subtests = random.sample(tests, ntest)
144
145 print(f"[+] Iteration {iteration}/{args.max_iter}: "
146 f"run {len(subtests)} tests/{len(tests)}")
147 print()
148
149 exitcode = run_tests(args, subtests)
150
151 print("ran %s tests/%s" % (ntest, len(tests)))
152 print("exit", exitcode)
153 if exitcode:
154 print("Tests failed: continuing with this subtest")
155 tests = subtests
156 output = write_output(args.output, tests)
157 else:
158 print("Tests succeeded: skipping this subtest, trying a new subset")
159 print()
160 iteration += 1
161 except KeyboardInterrupt:
162 print()
163 print("Bisection interrupted!")
164 print()
165
166 print("Tests (%s):" % len(tests))
167 for test in tests:
168 print("* %s" % test)
169 print()
170
171 if output:
172 print("Output written into %s" % output)
173
174 dt = math.ceil(time.monotonic() - start_time)
175 if len(tests) <= args.max_tests:
176 print("Bisection completed in %s iterations and %s"
177 % (iteration, datetime.timedelta(seconds=dt)))
178 else:
179 print("Bisection failed after %s iterations and %s"
180 % (iteration, datetime.timedelta(seconds=dt)))
181 sys.exit(1)
182
183
184 if __name__ == "__main__":
185 main()