Debian Codegraph @

Lib/test/support/script_helper.py

raw
1 # Common utility functions used by various script execution tests
2 # e.g. test_cmd_line, test_cmd_line_script and test_runpy
3
4 import collections
5 import importlib
6 import sys
7 import os
8 import os.path
9 import subprocess
10 import py_compile
11
12 from importlib.util import source_from_cache
13 from test import support
14 from test.support.import_helper import make_legacy_pyc
15
16
17 # Cached result of the expensive test performed in the function below.
18 __cached_interp_requires_environment = None
19
20
21 def interpreter_requires_environment():
22 """
23 Returns True if our sys.executable interpreter requires environment
24 variables in order to be able to run at all.
25
26 This is designed to be used with @unittest.skipIf() to annotate tests
27 that need to use an assert_python*() function to launch an isolated
28 mode (-I) or no environment mode (-E) sub-interpreter process.
29
30 A normal build & test does not run into this situation but it can happen
31 when trying to run the standard library test suite from an interpreter that
32 doesn't have an obvious home with Python's current home finding logic.
33
34 Setting PYTHONHOME is one way to get most of the testsuite to run in that
35 situation. PYTHONPATH or PYTHONUSERSITE are other common environment
36 variables that might impact whether or not the interpreter can start.
37 """
38 global __cached_interp_requires_environment
39 if __cached_interp_requires_environment is None:
40 # If PYTHONHOME is set, assume that we need it
41 if 'PYTHONHOME' in os.environ:
42 __cached_interp_requires_environment = True
43 return True
44 # cannot run subprocess, assume we don't need it
45 if not support.has_subprocess_support:
46 __cached_interp_requires_environment = False
47 return False
48
49 # Try running an interpreter with -E to see if it works or not.
50 try:
51 subprocess.check_call([sys.executable, '-E',
52 '-c', 'import sys; sys.exit(0)'])
53 except subprocess.CalledProcessError:
54 __cached_interp_requires_environment = True
55 else:
56 __cached_interp_requires_environment = False
57
58 return __cached_interp_requires_environment
59
60
61 class _PythonRunResult(collections.namedtuple("_PythonRunResult",
62 ("rc", "out", "err"))):
63 """Helper for reporting Python subprocess run results"""
64 def fail(self, cmd_line):
65 """Provide helpful details about failed subcommand runs"""
66 # Limit to 300 lines of ASCII characters
67 maxlen = 300 * 100
68 out, err = self.out, self.err
69 if len(out) > maxlen:
70 out = b'(... truncated stdout ...)' + out[-maxlen:]
71 if len(err) > maxlen:
72 err = b'(... truncated stderr ...)' + err[-maxlen:]
73 out = out.decode('utf8', 'replace').rstrip()
74 err = err.decode('utf8', 'replace').rstrip()
75
76 exitcode = self.rc
77 signame = support.get_signal_name(exitcode)
78 if signame:
79 exitcode = f"{exitcode} ({signame})"
80 raise AssertionError(f"Process return code is {exitcode}\n"
81 f"command line: {cmd_line!r}\n"
82 f"\n"
83 f"stdout:\n"
84 f"---\n"
85 f"{out}\n"
86 f"---\n"
87 f"\n"
88 f"stderr:\n"
89 f"---\n"
90 f"{err}\n"
91 f"---")
92
93
94 # Executing the interpreter in a subprocess
95 @support.requires_subprocess()
96 def run_python_until_end(*args, **env_vars):
97 """Used to implement assert_python_*.
98
99 *args are the command line flags to pass to the python interpreter.
100 **env_vars keyword arguments are environment variables to set on the process.
101
102 If __run_using_command= is supplied, it must be a list of
103 command line arguments to prepend to the command line used.
104 Useful when you want to run another command that should launch the
105 python interpreter via its own arguments. ["/bin/echo", "--"] for
106 example could print the unquoted python command line instead of
107 run it.
108 """
109 env_required = interpreter_requires_environment()
110 run_using_command = env_vars.pop('__run_using_command', None)
111 cwd = env_vars.pop('__cwd', None)
112 if '__isolated' in env_vars:
113 isolated = env_vars.pop('__isolated')
114 else:
115 isolated = not env_vars and not env_required
116 cmd_line = [sys.executable, '-X', 'faulthandler']
117 if run_using_command:
118 cmd_line = run_using_command + cmd_line
119 if isolated:
120 # isolated mode: ignore Python environment variables, ignore user
121 # site-packages, and don't add the current directory to sys.path
122 cmd_line.append('-I')
123 elif not env_vars and not env_required:
124 # ignore Python environment variables
125 cmd_line.append('-E')
126
127 # But a special flag that can be set to override -- in this case, the
128 # caller is responsible to pass the full environment.
129 if env_vars.pop('__cleanenv', None):
130 env = {}
131 if sys.platform == 'win32':
132 # Windows requires at least the SYSTEMROOT environment variable to
133 # start Python.
134 env['SYSTEMROOT'] = os.environ['SYSTEMROOT']
135
136 # Other interesting environment variables, not copied currently:
137 # COMSPEC, HOME, PATH, TEMP, TMPDIR, TMP.
138 else:
139 # Need to preserve the original environment, for in-place testing of
140 # shared library builds.
141 env = os.environ.copy()
142
143 # set TERM='' unless the TERM environment variable is passed explicitly
144 # see issues #11390 and #18300
145 if 'TERM' not in env_vars:
146 env['TERM'] = ''
147
148 env.update(env_vars)
149 cmd_line.extend(args)
150 proc = subprocess.Popen(cmd_line, stdin=subprocess.PIPE,
151 stdout=subprocess.PIPE, stderr=subprocess.PIPE,
152 env=env, cwd=cwd)
153 with proc:
154 try:
155 out, err = proc.communicate()
156 finally:
157 proc.kill()
158 subprocess._cleanup()
159 rc = proc.returncode
160 return _PythonRunResult(rc, out, err), cmd_line
161
162
163 @support.requires_subprocess()
164 def _assert_python(expected_success, /, *args, **env_vars):
165 res, cmd_line = run_python_until_end(*args, **env_vars)
166 if (res.rc and expected_success) or (not res.rc and not expected_success):
167 res.fail(cmd_line)
168 return res
169
170
171 def assert_python_ok(*args, **env_vars):
172 """
173 Assert that running the interpreter with `args` and optional environment
174 variables `env_vars` succeeds (rc == 0) and return a (return code, stdout,
175 stderr) tuple.
176
177 If the __cleanenv keyword is set, env_vars is used as a fresh environment.
178
179 Python is started in isolated mode (command line option -I),
180 except if the __isolated keyword is set to False.
181 """
182 return _assert_python(True, *args, **env_vars)
183
184
185 def assert_python_failure(*args, **env_vars):
186 """
187 Assert that running the interpreter with `args` and optional environment
188 variables `env_vars` fails (rc != 0) and return a (return code, stdout,
189 stderr) tuple.
190
191 See assert_python_ok() for more options.
192 """
193 return _assert_python(False, *args, **env_vars)
194
195
196 @support.requires_subprocess()
197 def spawn_python(*args, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, **kw):
198 """Run a Python subprocess with the given arguments.
199
200 kw is extra keyword args to pass to subprocess.Popen. Returns a Popen
201 object.
202 """
203 cmd_line = [sys.executable]
204 if not interpreter_requires_environment():
205 cmd_line.append('-E')
206 cmd_line.extend(args)
207 # Under Fedora (?), GNU readline can output junk on stderr when initialized,
208 # depending on the TERM setting. Setting TERM=vt100 is supposed to disable
209 # that. References:
210 # - http://reinout.vanrees.org/weblog/2009/08/14/readline-invisible-character-hack.html
211 # - http://stackoverflow.com/questions/15760712/python-readline-module-prints-escape-character-during-import
212 # - http://lists.gnu.org/archive/html/bug-readline/2007-08/msg00004.html
213 env = kw.setdefault('env', dict(os.environ))
214 env['TERM'] = 'vt100'
215 return subprocess.Popen(cmd_line, stdin=subprocess.PIPE,
216 stdout=stdout, stderr=stderr,
217 **kw)
218
219
220 def kill_python(p):
221 """Run the given Popen process until completion and return stdout."""
222 p.stdin.close()
223 data = p.stdout.read()
224 p.stdout.close()
225 # try to cleanup the child so we don't appear to leak when running
226 # with regrtest -R.
227 p.wait()
228 subprocess._cleanup()
229 return data
230
231
232 def make_script(script_dir, script_basename, source, omit_suffix=False):
233 script_filename = script_basename
234 if not omit_suffix:
235 script_filename += os.extsep + 'py'
236 script_name = os.path.join(script_dir, script_filename)
237 if isinstance(source, str):
238 # The script should be encoded to UTF-8, the default string encoding
239 with open(script_name, 'w', encoding='utf-8') as script_file:
240 script_file.write(source)
241 else:
242 with open(script_name, 'wb') as script_file:
243 script_file.write(source)
244 importlib.invalidate_caches()
245 return script_name
246
247
248 def make_zip_script(zip_dir, zip_basename, script_name, name_in_zip=None):
249 import zipfile
250 zip_filename = zip_basename+os.extsep+'zip'
251 zip_name = os.path.join(zip_dir, zip_filename)
252 with zipfile.ZipFile(zip_name, 'w') as zip_file:
253 if name_in_zip is None:
254 parts = script_name.split(os.sep)
255 if len(parts) >= 2 and parts[-2] == '__pycache__':
256 legacy_pyc = make_legacy_pyc(source_from_cache(script_name))
257 name_in_zip = os.path.basename(legacy_pyc)
258 script_name = legacy_pyc
259 else:
260 name_in_zip = os.path.basename(script_name)
261 zip_file.write(script_name, name_in_zip)
262 #if test.support.verbose:
263 # with zipfile.ZipFile(zip_name, 'r') as zip_file:
264 # print 'Contents of %r:' % zip_name
265 # zip_file.printdir()
266 return zip_name, os.path.join(zip_name, name_in_zip)
267
268
269 def make_pkg(pkg_dir, init_source=''):
270 os.mkdir(pkg_dir)
271 make_script(pkg_dir, '__init__', init_source)
272
273
274 def make_zip_pkg(zip_dir, zip_basename, pkg_name, script_basename,
275 source, depth=1, compiled=False):
276 import zipfile
277 unlink = []
278 init_name = make_script(zip_dir, '__init__', '')
279 unlink.append(init_name)
280 init_basename = os.path.basename(init_name)
281 script_name = make_script(zip_dir, script_basename, source)
282 unlink.append(script_name)
283 if compiled:
284 init_name = py_compile.compile(init_name, doraise=True)
285 script_name = py_compile.compile(script_name, doraise=True)
286 unlink.extend((init_name, script_name))
287 pkg_names = [os.sep.join([pkg_name]*i) for i in range(1, depth+1)]
288 script_name_in_zip = os.path.join(pkg_names[-1], os.path.basename(script_name))
289 zip_filename = zip_basename+os.extsep+'zip'
290 zip_name = os.path.join(zip_dir, zip_filename)
291 with zipfile.ZipFile(zip_name, 'w') as zip_file:
292 for name in pkg_names:
293 init_name_in_zip = os.path.join(name, init_basename)
294 zip_file.write(init_name, init_name_in_zip)
295 zip_file.write(script_name, script_name_in_zip)
296 for name in unlink:
297 os.unlink(name)
298 #if test.support.verbose:
299 # with zipfile.ZipFile(zip_name, 'r') as zip_file:
300 # print 'Contents of %r:' % zip_name
301 # zip_file.printdir()
302 return zip_name, os.path.join(zip_name, script_name_in_zip)
303
304
305 @support.requires_subprocess()
306 def run_test_script(script):
307 # use -u to try to get the full output if the test hangs or crash
308 if support.verbose:
309 def title(text):
310 return f"===== {text} ======"
311
312 name = f"script {os.path.basename(script)}"
313 print()
314 print(title(name), flush=True)
315 # In verbose mode, the child process inherit stdout and stdout,
316 # to see output in realtime and reduce the risk of losing output.
317 args = [sys.executable, "-E", "-X", "faulthandler", "-u", script, "-v"]
318 proc = subprocess.run(args)
319 print(title(f"{name} completed: exit code {proc.returncode}"),
320 flush=True)
321 if proc.returncode:
322 raise AssertionError(f"{name} failed")
323 else:
324 assert_python_ok("-u", script, "-v")