Debian Codegraph @

Lib/test/test_venv.py

raw
1 """
2 Test harness for the venv module.
3
4 Copyright (C) 2011-2012 Vinay Sajip.
5 Licensed to the PSF under a contributor agreement.
6 """
7
8 import contextlib
9 import ensurepip
10 import os
11 import os.path
12 import pathlib
13 import re
14 import shlex
15 import shutil
16 import struct
17 import subprocess
18 import sys
19 import sysconfig
20 import tempfile
21 from test.support import (captured_stdout, captured_stderr,
22 skip_if_broken_multiprocessing_synchronize, verbose,
23 requires_subprocess, is_android, is_apple_mobile,
24 is_wasm32,
25 requires_venv_with_pip, TEST_HOME_DIR,
26 requires_resource, copy_python_src_ignore)
27 from test.support.os_helper import (can_symlink, EnvironmentVarGuard, rmtree,
28 TESTFN, FakePath)
29 import unittest
30 import venv
31 from unittest.mock import patch, Mock
32
33 try:
34 import ctypes
35 except ImportError:
36 ctypes = None
37
38 # Platforms that set sys._base_executable can create venvs from within
39 # another venv, so no need to skip tests that require venv.create().
40 requireVenvCreate = unittest.skipUnless(
41 sys.prefix == sys.base_prefix
42 or sys._base_executable != sys.executable,
43 'cannot run venv.create from within a venv on this platform')
44
45 if is_android or is_apple_mobile or is_wasm32:
46 raise unittest.SkipTest("venv is not available on this platform")
47
48 @requires_subprocess()
49 def check_output(cmd, encoding=None):
50 p = subprocess.Popen(cmd,
51 stdout=subprocess.PIPE,
52 stderr=subprocess.PIPE,
53 env={**os.environ, "PYTHONHOME": ""})
54 out, err = p.communicate()
55 if p.returncode:
56 if verbose and err:
57 print(err.decode(encoding or 'utf-8', 'backslashreplace'))
58 raise subprocess.CalledProcessError(
59 p.returncode, cmd, out, err)
60 if encoding:
61 return (
62 out.decode(encoding, 'backslashreplace'),
63 err.decode(encoding, 'backslashreplace'),
64 )
65 return out, err
66
67 class BaseTest(unittest.TestCase):
68 """Base class for venv tests."""
69 maxDiff = 80 * 50
70
71 def setUp(self):
72 self.env_dir = os.path.realpath(tempfile.mkdtemp())
73 if os.name == 'nt':
74 self.bindir = 'Scripts'
75 self.lib = ('Lib',)
76 self.include = 'Include'
77 else:
78 self.bindir = 'bin'
79 self.lib = ('lib', f'python{sysconfig._get_python_version_abi()}')
80 self.include = 'include'
81 executable = sys._base_executable
82 self.exe = os.path.split(executable)[-1]
83 if (sys.platform == 'win32'
84 and os.path.lexists(executable)
85 and not os.path.exists(executable)):
86 self.cannot_link_exe = True
87 else:
88 self.cannot_link_exe = False
89
90 def tearDown(self):
91 rmtree(self.env_dir)
92
93 def envpy(self, *, real_env_dir=False):
94 if real_env_dir:
95 env_dir = os.path.realpath(self.env_dir)
96 else:
97 env_dir = self.env_dir
98 return os.path.join(env_dir, self.bindir, self.exe)
99
100 def run_with_capture(self, func, *args, **kwargs):
101 with captured_stdout() as output:
102 with captured_stderr() as error:
103 func(*args, **kwargs)
104 return output.getvalue(), error.getvalue()
105
106 def get_env_file(self, *args):
107 return os.path.join(self.env_dir, *args)
108
109 def get_text_file_contents(self, *args, encoding='utf-8'):
110 with open(self.get_env_file(*args), 'r', encoding=encoding) as f:
111 result = f.read()
112 return result
113
114 class BasicTest(BaseTest):
115 """Test venv module functionality."""
116
117 def isdir(self, *args):
118 fn = self.get_env_file(*args)
119 self.assertTrue(os.path.isdir(fn))
120
121 def test_defaults_with_str_path(self):
122 """
123 Test the create function with default arguments and a str path.
124 """
125 rmtree(self.env_dir)
126 self.run_with_capture(venv.create, self.env_dir)
127 self._check_output_of_default_create()
128
129 def test_defaults_with_pathlike(self):
130 """
131 Test the create function with default arguments and a path-like path.
132 """
133 rmtree(self.env_dir)
134 self.run_with_capture(venv.create, FakePath(self.env_dir))
135 self._check_output_of_default_create()
136
137 def _check_output_of_default_create(self):
138 self.isdir(self.bindir)
139 self.isdir(self.include)
140 self.isdir(*self.lib)
141 # Issue 21197
142 p = self.get_env_file('lib64')
143 conditions = ((struct.calcsize('P') == 8) and (os.name == 'posix') and
144 (sys.platform != 'darwin'))
145 if conditions:
146 self.assertTrue(os.path.islink(p))
147 else:
148 self.assertFalse(os.path.exists(p))
149 data = self.get_text_file_contents('pyvenv.cfg')
150 executable = sys._base_executable
151 path = os.path.dirname(executable)
152 self.assertIn('home = %s' % path, data)
153 self.assertIn('executable = %s' %
154 os.path.realpath(sys.executable), data)
155 copies = '' if os.name=='nt' else ' --copies'
156 cmd = (f'command = {sys.executable} -m venv{copies} --without-pip '
157 f'--without-scm-ignore-files {self.env_dir}')
158 self.assertIn(cmd, data)
159 fn = self.get_env_file(self.bindir, self.exe)
160 if not os.path.exists(fn): # diagnostics for Windows buildbot failures
161 bd = self.get_env_file(self.bindir)
162 print('Contents of %r:' % bd)
163 print(' %r' % os.listdir(bd))
164 self.assertTrue(os.path.exists(fn), 'File %r should exist.' % fn)
165
166 def test_config_file_command_key(self):
167 options = [
168 (None, None, None), # Default case.
169 ('--copies', 'symlinks', False),
170 ('--without-pip', 'with_pip', False),
171 ('--system-site-packages', 'system_site_packages', True),
172 ('--clear', 'clear', True),
173 ('--upgrade', 'upgrade', True),
174 ('--upgrade-deps', 'upgrade_deps', True),
175 ('--prompt="foobar"', 'prompt', 'foobar'),
176 ('--without-scm-ignore-files', 'scm_ignore_files', frozenset()),
177 ]
178 for opt, attr, value in options:
179 with self.subTest(opt=opt, attr=attr, value=value):
180 rmtree(self.env_dir)
181 if not attr:
182 kwargs = {}
183 else:
184 kwargs = {attr: value}
185 b = venv.EnvBuilder(**kwargs)
186 b.upgrade_dependencies = Mock() # avoid pip command to upgrade deps
187 b._setup_pip = Mock() # avoid pip setup
188 self.run_with_capture(b.create, self.env_dir)
189 data = self.get_text_file_contents('pyvenv.cfg')
190 if not attr or opt.endswith('git'):
191 for opt in ('--system-site-packages', '--clear', '--upgrade',
192 '--upgrade-deps', '--prompt'):
193 self.assertNotRegex(data, rf'command = .* {opt}')
194 elif os.name=='nt' and attr=='symlinks':
195 pass
196 else:
197 self.assertRegex(data, rf'command = .* {opt}')
198
199 def test_prompt(self):
200 env_name = os.path.split(self.env_dir)[1]
201
202 rmtree(self.env_dir)
203 builder = venv.EnvBuilder()
204 self.run_with_capture(builder.create, self.env_dir)
205 context = builder.ensure_directories(self.env_dir)
206 data = self.get_text_file_contents('pyvenv.cfg')
207 self.assertEqual(context.prompt, env_name)
208 self.assertNotIn("prompt = ", data)
209
210 rmtree(self.env_dir)
211 builder = venv.EnvBuilder(prompt='My prompt')
212 self.run_with_capture(builder.create, self.env_dir)
213 context = builder.ensure_directories(self.env_dir)
214 data = self.get_text_file_contents('pyvenv.cfg')
215 self.assertEqual(context.prompt, 'My prompt')
216 self.assertIn("prompt = 'My prompt'\n", data)
217
218 rmtree(self.env_dir)
219 builder = venv.EnvBuilder(prompt='.')
220 cwd = os.path.basename(os.getcwd())
221 self.run_with_capture(builder.create, self.env_dir)
222 context = builder.ensure_directories(self.env_dir)
223 data = self.get_text_file_contents('pyvenv.cfg')
224 self.assertEqual(context.prompt, cwd)
225 self.assertIn("prompt = '%s'\n" % cwd, data)
226
227 def test_upgrade_dependencies(self):
228 builder = venv.EnvBuilder()
229 bin_path = 'bin'
230 python_exe = os.path.split(sys.executable)[1]
231 expected_exe = os.path.basename(sys._base_executable)
232
233 if sys.platform == 'win32':
234 bin_path = 'Scripts'
235 if os.path.normcase(os.path.splitext(python_exe)[0]).endswith('_d'):
236 expected_exe = 'python_d'
237 else:
238 expected_exe = 'python'
239 python_exe = expected_exe + '.exe'
240
241 with tempfile.TemporaryDirectory() as fake_env_dir:
242 expect_exe = os.path.normcase(
243 os.path.join(fake_env_dir, bin_path, expected_exe)
244 )
245 if sys.platform == 'win32':
246 expect_exe = os.path.normcase(os.path.realpath(expect_exe))
247
248 def pip_cmd_checker(cmd, **kwargs):
249 self.assertEqual(
250 cmd[1:],
251 [
252 '-m',
253 'pip',
254 'install',
255 '--upgrade',
256 'pip',
257 ]
258 )
259 exe_dir = os.path.normcase(os.path.dirname(cmd[0]))
260 expected_dir = os.path.normcase(os.path.dirname(expect_exe))
261 self.assertEqual(exe_dir, expected_dir)
262
263 fake_context = builder.ensure_directories(fake_env_dir)
264 with patch('venv.subprocess.check_output', pip_cmd_checker):
265 builder.upgrade_dependencies(fake_context)
266
267 @requireVenvCreate
268 def test_prefixes(self):
269 """
270 Test that the prefix values are as expected.
271 """
272 # check a venv's prefixes
273 rmtree(self.env_dir)
274 self.run_with_capture(venv.create, self.env_dir)
275 cmd = [self.envpy(), '-c', None]
276 for prefix, expected in (
277 ('prefix', self.env_dir),
278 ('exec_prefix', self.env_dir),
279 ('base_prefix', sys.base_prefix),
280 ('base_exec_prefix', sys.base_exec_prefix)):
281 cmd[2] = 'import sys; print(sys.%s)' % prefix
282 out, err = check_output(cmd)
283 self.assertEqual(pathlib.Path(out.strip().decode()),
284 pathlib.Path(expected), prefix)
285
286 @requireVenvCreate
287 def test_sysconfig(self):
288 """
289 Test that the sysconfig functions work in a virtual environment.
290 """
291 rmtree(self.env_dir)
292 self.run_with_capture(venv.create, self.env_dir, symlinks=False)
293 cmd = [self.envpy(), '-c', None]
294 for call, expected in (
295 # installation scheme
296 ('get_preferred_scheme("prefix")', 'venv'),
297 ('get_default_scheme()', 'venv'),
298 # build environment
299 ('is_python_build()', str(sysconfig.is_python_build())),
300 ('get_makefile_filename()', sysconfig.get_makefile_filename()),
301 ('get_config_h_filename()', sysconfig.get_config_h_filename()),
302 ('get_config_var("Py_GIL_DISABLED")',
303 str(sysconfig.get_config_var("Py_GIL_DISABLED")))):
304 with self.subTest(call):
305 cmd[2] = 'import sysconfig; print(sysconfig.%s)' % call
306 out, err = check_output(cmd, encoding='utf-8')
307 self.assertEqual(out.strip(), expected, err)
308 for attr, expected in (
309 ('executable', self.envpy()),
310 # Usually compare to sys.executable, but if we're running in our own
311 # venv then we really need to compare to our base executable
312 ('_base_executable', sys._base_executable),
313 ):
314 with self.subTest(attr):
315 cmd[2] = f'import sys; print(sys.{attr})'
316 out, err = check_output(cmd, encoding='utf-8')
317 self.assertEqual(out.strip(), expected, err)
318
319 @requireVenvCreate
320 @unittest.skipUnless(can_symlink(), 'Needs symlinks')
321 def test_sysconfig_symlinks(self):
322 """
323 Test that the sysconfig functions work in a virtual environment.
324 """
325 rmtree(self.env_dir)
326 self.run_with_capture(venv.create, self.env_dir, symlinks=True)
327 cmd = [self.envpy(), '-c', None]
328 for call, expected in (
329 # installation scheme
330 ('get_preferred_scheme("prefix")', 'venv'),
331 ('get_default_scheme()', 'venv'),
332 # build environment
333 ('is_python_build()', str(sysconfig.is_python_build())),
334 ('get_makefile_filename()', sysconfig.get_makefile_filename()),
335 ('get_config_h_filename()', sysconfig.get_config_h_filename()),
336 ('get_config_var("Py_GIL_DISABLED")',
337 str(sysconfig.get_config_var("Py_GIL_DISABLED")))):
338 with self.subTest(call):
339 cmd[2] = 'import sysconfig; print(sysconfig.%s)' % call
340 out, err = check_output(cmd, encoding='utf-8')
341 self.assertEqual(out.strip(), expected, err)
342 for attr, expected in (
343 ('executable', self.envpy()),
344 # Usually compare to sys.executable, but if we're running in our own
345 # venv then we really need to compare to our base executable
346 # HACK: Test fails on POSIX with unversioned binary (PR gh-113033)
347 #('_base_executable', sys._base_executable),
348 ):
349 with self.subTest(attr):
350 cmd[2] = f'import sys; print(sys.{attr})'
351 out, err = check_output(cmd, encoding='utf-8')
352 self.assertEqual(out.strip(), expected, err)
353
354 if sys.platform == 'win32':
355 ENV_SUBDIRS = (
356 ('Scripts',),
357 ('Include',),
358 ('Lib',),
359 ('Lib', 'site-packages'),
360 )
361 else:
362 ENV_SUBDIRS = (
363 ('bin',),
364 ('include',),
365 ('lib',),
366 ('lib', 'python%d.%d' % sys.version_info[:2]),
367 ('lib', 'python%d.%d' % sys.version_info[:2], 'site-packages'),
368 )
369
370 def create_contents(self, paths, filename):
371 """
372 Create some files in the environment which are unrelated
373 to the virtual environment.
374 """
375 for subdirs in paths:
376 d = os.path.join(self.env_dir, *subdirs)
377 os.mkdir(d)
378 fn = os.path.join(d, filename)
379 with open(fn, 'wb') as f:
380 f.write(b'Still here?')
381
382 @unittest.skipUnless(hasattr(os, 'listxattr'), 'test requires os.listxattr')
383 def test_install_scripts_selinux(self):
384 """
385 gh-145417: Test that install_scripts does not copy SELinux context
386 when copying scripts.
387 """
388 with patch('os.listxattr') as listxattr_mock:
389 venv.create(self.env_dir)
390 listxattr_mock.assert_not_called()
391
392 def test_overwrite_existing(self):
393 """
394 Test creating environment in an existing directory.
395 """
396 self.create_contents(self.ENV_SUBDIRS, 'foo')
397 venv.create(self.env_dir)
398 for subdirs in self.ENV_SUBDIRS:
399 fn = os.path.join(self.env_dir, *(subdirs + ('foo',)))
400 self.assertTrue(os.path.exists(fn))
401 with open(fn, 'rb') as f:
402 self.assertEqual(f.read(), b'Still here?')
403
404 builder = venv.EnvBuilder(clear=True)
405 builder.create(self.env_dir)
406 for subdirs in self.ENV_SUBDIRS:
407 fn = os.path.join(self.env_dir, *(subdirs + ('foo',)))
408 self.assertFalse(os.path.exists(fn))
409
410 def clear_directory(self, path):
411 for fn in os.listdir(path):
412 fn = os.path.join(path, fn)
413 if os.path.islink(fn) or os.path.isfile(fn):
414 os.remove(fn)
415 elif os.path.isdir(fn):
416 rmtree(fn)
417
418 def test_unoverwritable_fails(self):
419 #create a file clashing with directories in the env dir
420 for paths in self.ENV_SUBDIRS[:3]:
421 fn = os.path.join(self.env_dir, *paths)
422 with open(fn, 'wb') as f:
423 f.write(b'')
424 self.assertRaises((ValueError, OSError), venv.create, self.env_dir)
425 self.clear_directory(self.env_dir)
426
427 def test_upgrade(self):
428 """
429 Test upgrading an existing environment directory.
430 """
431 # See Issue #21643: the loop needs to run twice to ensure
432 # that everything works on the upgrade (the first run just creates
433 # the venv).
434 for upgrade in (False, True):
435 builder = venv.EnvBuilder(upgrade=upgrade)
436 self.run_with_capture(builder.create, self.env_dir)
437 self.isdir(self.bindir)
438 self.isdir(self.include)
439 self.isdir(*self.lib)
440 fn = self.get_env_file(self.bindir, self.exe)
441 if not os.path.exists(fn):
442 # diagnostics for Windows buildbot failures
443 bd = self.get_env_file(self.bindir)
444 print('Contents of %r:' % bd)
445 print(' %r' % os.listdir(bd))
446 self.assertTrue(os.path.exists(fn), 'File %r should exist.' % fn)
447
448 def test_isolation(self):
449 """
450 Test isolation from system site-packages
451 """
452 for ssp, s in ((True, 'true'), (False, 'false')):
453 builder = venv.EnvBuilder(clear=True, system_site_packages=ssp)
454 builder.create(self.env_dir)
455 data = self.get_text_file_contents('pyvenv.cfg')
456 self.assertIn('include-system-site-packages = %s\n' % s, data)
457
458 @unittest.skipUnless(can_symlink(), 'Needs symlinks')
459 def test_symlinking(self):
460 """
461 Test symlinking works as expected
462 """
463 for usl in (False, True):
464 builder = venv.EnvBuilder(clear=True, symlinks=usl)
465 builder.create(self.env_dir)
466 fn = self.get_env_file(self.bindir, self.exe)
467 # Don't test when False, because e.g. 'python' is always
468 # symlinked to 'python3.3' in the env, even when symlinking in
469 # general isn't wanted.
470 if usl:
471 if self.cannot_link_exe:
472 # Symlinking is skipped when our executable is already a
473 # special app symlink
474 self.assertFalse(os.path.islink(fn))
475 else:
476 self.assertTrue(os.path.islink(fn))
477
478 # If a venv is created from a source build and that venv is used to
479 # run the test, the pyvenv.cfg in the venv created in the test will
480 # point to the venv being used to run the test, and we lose the link
481 # to the source build - so Python can't initialise properly.
482 @requireVenvCreate
483 def test_executable(self):
484 """
485 Test that the sys.executable value is as expected.
486 """
487 rmtree(self.env_dir)
488 self.run_with_capture(venv.create, self.env_dir)
489 envpy = self.envpy(real_env_dir=True)
490 out, err = check_output([envpy, '-c',
491 'import sys; print(sys.executable)'])
492 self.assertEqual(out.strip(), envpy.encode())
493
494 @unittest.skipUnless(can_symlink(), 'Needs symlinks')
495 def test_executable_symlinks(self):
496 """
497 Test that the sys.executable value is as expected.
498 """
499 rmtree(self.env_dir)
500 builder = venv.EnvBuilder(clear=True, symlinks=True)
501 builder.create(self.env_dir)
502 envpy = self.envpy(real_env_dir=True)
503 out, err = check_output([envpy, '-c',
504 'import sys; print(sys.executable)'])
505 self.assertEqual(out.strip(), envpy.encode())
506
507 # gh-124651: test quoted strings
508 @unittest.skipIf(os.name == 'nt', 'contains invalid characters on Windows')
509 def test_special_chars_bash(self):
510 """
511 Test that the template strings are quoted properly (bash)
512 """
513 rmtree(self.env_dir)
514 bash = shutil.which('bash')
515 if bash is None:
516 self.skipTest('bash required for this test')
517 env_name = '"\';&&$e|\'"'
518 env_dir = os.path.join(os.path.realpath(self.env_dir), env_name)
519 builder = venv.EnvBuilder(clear=True)
520 builder.create(env_dir)
521 activate = os.path.join(env_dir, self.bindir, 'activate')
522 test_script = os.path.join(self.env_dir, 'test_special_chars.sh')
523 with open(test_script, "w") as f:
524 f.write(f'source {shlex.quote(activate)}\n'
525 'python -c \'import sys; print(sys.executable)\'\n'
526 'python -c \'import os; print(os.environ["VIRTUAL_ENV"])\'\n'
527 'deactivate\n')
528 out, err = check_output([bash, test_script])
529 lines = out.splitlines()
530 self.assertTrue(env_name.encode() in lines[0])
531 self.assertEndsWith(lines[1], env_name.encode())
532
533 # gh-124651: test quoted strings
534 @unittest.skipIf(os.name == 'nt', 'contains invalid characters on Windows')
535 @unittest.skipIf(sys.platform.startswith('netbsd'),
536 "NetBSD csh fails with quoted special chars; see gh-139308")
537 def test_special_chars_csh(self):
538 """
539 Test that the template strings are quoted properly (csh)
540 """
541 rmtree(self.env_dir)
542 csh = shutil.which('tcsh') or shutil.which('csh')
543 if csh is None:
544 self.skipTest('csh required for this test')
545 env_name = '"\';&&$e|\'"'
546 env_dir = os.path.join(os.path.realpath(self.env_dir), env_name)
547 builder = venv.EnvBuilder(clear=True)
548 builder.create(env_dir)
549 activate = os.path.join(env_dir, self.bindir, 'activate.csh')
550 test_script = os.path.join(self.env_dir, 'test_special_chars.csh')
551 with open(test_script, "w") as f:
552 f.write(f'source {shlex.quote(activate)}\n'
553 'python -c \'import sys; print(sys.executable)\'\n'
554 'python -c \'import os; print(os.environ["VIRTUAL_ENV"])\'\n'
555 'deactivate\n')
556 out, err = check_output([csh, test_script])
557 lines = out.splitlines()
558 self.assertTrue(env_name.encode() in lines[0])
559 self.assertEndsWith(lines[1], env_name.encode())
560
561 # gh-124651: test quoted strings on Windows
562 @unittest.skipUnless(os.name == 'nt', 'only relevant on Windows')
563 def test_special_chars_windows(self):
564 """
565 Test that the template strings are quoted properly on Windows
566 """
567 rmtree(self.env_dir)
568 env_name = "'&&^$e"
569 env_dir = os.path.join(os.path.realpath(self.env_dir), env_name)
570 builder = venv.EnvBuilder(clear=True)
571 builder.create(env_dir)
572 activate = os.path.join(env_dir, self.bindir, 'activate.bat')
573 test_batch = os.path.join(self.env_dir, 'test_special_chars.bat')
574 with open(test_batch, "w") as f:
575 f.write('@echo off\n'
576 f'"{activate}" & '
577 f'{self.exe} -c "import sys; print(sys.executable)" & '
578 f'{self.exe} -c "import os; print(os.environ[\'VIRTUAL_ENV\'])" & '
579 'deactivate')
580 out, err = check_output([test_batch])
581 lines = out.splitlines()
582 self.assertTrue(env_name.encode() in lines[0])
583 self.assertEndsWith(lines[1], env_name.encode())
584
585 @unittest.skipUnless(os.name == 'nt', 'only relevant on Windows')
586 def test_unicode_in_batch_file(self):
587 """
588 Test handling of Unicode paths
589 """
590 rmtree(self.env_dir)
591 env_dir = os.path.join(os.path.realpath(self.env_dir), 'ϼўТλФЙ')
592 builder = venv.EnvBuilder(clear=True)
593 builder.create(env_dir)
594 activate = os.path.join(env_dir, self.bindir, 'activate.bat')
595 out, err = check_output(
596 [activate, '&', self.exe, '-c', 'print(0)'],
597 encoding='oem',
598 )
599 self.assertEqual(out.strip(), '0')
600
601 @unittest.skipUnless(os.name == 'nt' and can_symlink(),
602 'symlinks on Windows')
603 def test_failed_symlink(self):
604 """
605 Test handling of failed symlinks on Windows.
606 """
607 rmtree(self.env_dir)
608 env_dir = os.path.join(os.path.realpath(self.env_dir), 'venv')
609 with patch('os.symlink') as mock_symlink:
610 mock_symlink.side_effect = OSError()
611 builder = venv.EnvBuilder(clear=True, symlinks=True)
612 _, err = self.run_with_capture(builder.create, env_dir)
613 filepath_regex = r"'[A-Z]:\\\\(?:[^\\\\]+\\\\)*[^\\\\]+'"
614 self.assertRegex(err, rf"Unable to symlink {filepath_regex} to {filepath_regex}")
615
616 @requireVenvCreate
617 def test_multiprocessing(self):
618 """
619 Test that the multiprocessing is able to spawn.
620 """
621 # bpo-36342: Instantiation of a Pool object imports the
622 # multiprocessing.synchronize module. Skip the test if this module
623 # cannot be imported.
624 skip_if_broken_multiprocessing_synchronize()
625
626 rmtree(self.env_dir)
627 self.run_with_capture(venv.create, self.env_dir)
628 out, err = check_output([self.envpy(real_env_dir=True), '-c',
629 'from multiprocessing import Pool; '
630 'pool = Pool(1); '
631 'print(pool.apply_async("Python".lower).get(3)); '
632 'pool.terminate()'])
633 self.assertEqual(out.strip(), "python".encode())
634
635 @requireVenvCreate
636 def test_multiprocessing_recursion(self):
637 """
638 Test that the multiprocessing is able to spawn itself
639 """
640 skip_if_broken_multiprocessing_synchronize()
641
642 rmtree(self.env_dir)
643 self.run_with_capture(venv.create, self.env_dir)
644 script = os.path.join(TEST_HOME_DIR, '_test_venv_multiprocessing.py')
645 subprocess.check_call([self.envpy(real_env_dir=True), "-I", script])
646
647 @unittest.skipIf(os.name == 'nt', 'not relevant on Windows')
648 def test_deactivate_with_strict_bash_opts(self):
649 bash = shutil.which("bash")
650 if bash is None:
651 self.skipTest("bash required for this test")
652 rmtree(self.env_dir)
653 builder = venv.EnvBuilder(clear=True)
654 builder.create(self.env_dir)
655 activate = os.path.join(self.env_dir, self.bindir, "activate")
656 test_script = os.path.join(self.env_dir, "test_strict.sh")
657 with open(test_script, "w") as f:
658 f.write("set -euo pipefail\n"
659 f"source {activate}\n"
660 "deactivate\n")
661 out, err = check_output([bash, test_script])
662 self.assertEqual(out, "".encode())
663 self.assertEqual(err, "".encode())
664
665
666 @unittest.skipUnless(sys.platform == 'darwin', 'only relevant on macOS')
667 def test_macos_env(self):
668 rmtree(self.env_dir)
669 builder = venv.EnvBuilder()
670 builder.create(self.env_dir)
671
672 out, err = check_output([self.envpy(real_env_dir=True), '-c',
673 'import os; print("__PYVENV_LAUNCHER__" in os.environ)'])
674 self.assertEqual(out.strip(), 'False'.encode())
675
676 def test_pathsep_error(self):
677 """
678 Test that venv creation fails when the target directory contains
679 the path separator.
680 """
681 rmtree(self.env_dir)
682 bad_itempath = self.env_dir + os.pathsep
683 self.assertRaises(ValueError, venv.create, bad_itempath)
684 self.assertRaises(ValueError, venv.create, FakePath(bad_itempath))
685
686 @unittest.skipIf(os.name == 'nt', 'not relevant on Windows')
687 @requireVenvCreate
688 def test_zippath_from_non_installed_posix(self):
689 """
690 Test that when create venv from non-installed python, the zip path
691 value is as expected.
692 """
693 rmtree(self.env_dir)
694 # First try to create a non-installed python. It's not a real full
695 # functional non-installed python, but enough for this test.
696 platlibdir = sys.platlibdir
697 non_installed_dir = os.path.realpath(tempfile.mkdtemp())
698 self.addCleanup(rmtree, non_installed_dir)
699 bindir = os.path.join(non_installed_dir, self.bindir)
700 os.mkdir(bindir)
701 python_exe = os.path.basename(sys.executable)
702 shutil.copy2(sys.executable, os.path.join(bindir, python_exe))
703 libdir = os.path.join(non_installed_dir, platlibdir, self.lib[1])
704 os.makedirs(libdir)
705 landmark = os.path.join(libdir, "os.py")
706 abi_thread = "t" if sysconfig.get_config_var("Py_GIL_DISABLED") else ""
707 stdlib_zip = f"python{sys.version_info.major}{sys.version_info.minor}{abi_thread}"
708 zip_landmark = os.path.join(non_installed_dir,
709 platlibdir,
710 stdlib_zip)
711 additional_pythonpath_for_non_installed = []
712
713 # Copy stdlib files to the non-installed python so venv can
714 # correctly calculate the prefix.
715 for eachpath in sys.path:
716 if eachpath.endswith(".zip"):
717 if os.path.isfile(eachpath):
718 shutil.copyfile(
719 eachpath,
720 os.path.join(non_installed_dir, platlibdir))
721 elif os.path.isfile(os.path.join(eachpath, "os.py")):
722 names = os.listdir(eachpath)
723 ignored_names = copy_python_src_ignore(eachpath, names)
724 for name in names:
725 if name in ignored_names:
726 continue
727 if name == "site-packages":
728 continue
729 fn = os.path.join(eachpath, name)
730 if os.path.isfile(fn):
731 shutil.copy(fn, libdir)
732 elif os.path.isdir(fn):
733 shutil.copytree(fn, os.path.join(libdir, name),
734 ignore=copy_python_src_ignore)
735 else:
736 additional_pythonpath_for_non_installed.append(
737 eachpath)
738 cmd = [os.path.join(non_installed_dir, self.bindir, python_exe),
739 "-m",
740 "venv",
741 "--without-pip",
742 "--without-scm-ignore-files",
743 self.env_dir]
744 # Our fake non-installed python is not fully functional because
745 # it cannot find the extensions. Set PYTHONPATH so it can run the
746 # venv module correctly.
747 pythonpath = os.pathsep.join(
748 additional_pythonpath_for_non_installed)
749 # For python built with shared enabled. We need to set
750 # LD_LIBRARY_PATH so the non-installed python can find and link
751 # libpython.so
752 ld_library_path = sysconfig.get_config_var("LIBDIR")
753 if not ld_library_path or sysconfig.is_python_build():
754 ld_library_path = os.path.abspath(os.path.dirname(sys.executable))
755 if sys.platform == 'darwin':
756 ld_library_path_env = "DYLD_LIBRARY_PATH"
757 else:
758 ld_library_path_env = "LD_LIBRARY_PATH"
759 child_env = {
760 "PYTHONPATH": pythonpath,
761 ld_library_path_env: ld_library_path,
762 }
763 if asan_options := os.environ.get("ASAN_OPTIONS"):
764 # prevent https://github.com/python/cpython/issues/104839
765 child_env["ASAN_OPTIONS"] = asan_options
766 subprocess.check_call(cmd, env=child_env)
767 # Now check the venv created from the non-installed python has
768 # correct zip path in pythonpath.
769 target_python = os.path.join(self.env_dir, self.bindir, python_exe)
770 cmd = [target_python, '-S', '-c', 'import sys; print(sys.path)']
771 out, err = check_output(cmd)
772 self.assertTrue(zip_landmark.encode() in out)
773
774 @requireVenvCreate
775 def test_activate_shell_script_has_no_dos_newlines(self):
776 """
777 Test that the `activate` shell script contains no CR LF.
778 This is relevant for Cygwin, as the Windows build might have
779 converted line endings accidentally.
780 """
781 venv_dir = pathlib.Path(self.env_dir)
782 rmtree(venv_dir)
783 [[scripts_dir], *_] = self.ENV_SUBDIRS
784 script_path = venv_dir / scripts_dir / "activate"
785 venv.create(venv_dir)
786 with open(script_path, 'rb') as script:
787 for i, line in enumerate(script, 1):
788 error_message = f"CR LF found in line {i}"
789 self.assertNotEndsWith(line, b'\r\n', error_message)
790
791 @requireVenvCreate
792 def test_scm_ignore_files_git(self):
793 """
794 Test that a .gitignore file is created when "git" is specified.
795 The file should contain a `*\n` line.
796 """
797 self.run_with_capture(venv.create, self.env_dir,
798 scm_ignore_files={'git'})
799 file_lines = self.get_text_file_contents('.gitignore').splitlines()
800 self.assertIn('*', file_lines)
801
802 @requireVenvCreate
803 def test_create_scm_ignore_files_multiple(self):
804 """
805 Test that ``scm_ignore_files`` can work with multiple SCMs.
806 """
807 bzrignore_name = ".bzrignore"
808 contents = "# For Bazaar.\n*\n"
809
810 class BzrEnvBuilder(venv.EnvBuilder):
811 def create_bzr_ignore_file(self, context):
812 gitignore_path = os.path.join(context.env_dir, bzrignore_name)
813 with open(gitignore_path, 'w', encoding='utf-8') as file:
814 file.write(contents)
815
816 builder = BzrEnvBuilder(scm_ignore_files={'git', 'bzr'})
817 self.run_with_capture(builder.create, self.env_dir)
818
819 gitignore_lines = self.get_text_file_contents('.gitignore').splitlines()
820 self.assertIn('*', gitignore_lines)
821
822 bzrignore = self.get_text_file_contents(bzrignore_name)
823 self.assertEqual(bzrignore, contents)
824
825 @requireVenvCreate
826 def test_create_scm_ignore_files_empty(self):
827 """
828 Test that no default ignore files are created when ``scm_ignore_files``
829 is empty.
830 """
831 # scm_ignore_files is set to frozenset() by default.
832 self.run_with_capture(venv.create, self.env_dir)
833 with self.assertRaises(FileNotFoundError):
834 self.get_text_file_contents('.gitignore')
835
836 self.assertIn("--without-scm-ignore-files",
837 self.get_text_file_contents('pyvenv.cfg'))
838
839 @requireVenvCreate
840 def test_cli_with_scm_ignore_files(self):
841 """
842 Test that default SCM ignore files are created by default via the CLI.
843 """
844 self.run_with_capture(venv.main, ['--without-pip', self.env_dir])
845
846 gitignore_lines = self.get_text_file_contents('.gitignore').splitlines()
847 self.assertIn('*', gitignore_lines)
848
849 @requireVenvCreate
850 def test_cli_without_scm_ignore_files(self):
851 """
852 Test that ``--without-scm-ignore-files`` doesn't create SCM ignore files.
853 """
854 args = ['--without-pip', '--without-scm-ignore-files', self.env_dir]
855 self.run_with_capture(venv.main, args)
856
857 with self.assertRaises(FileNotFoundError):
858 self.get_text_file_contents('.gitignore')
859
860 def test_venv_same_path(self):
861 same_path = venv.EnvBuilder._same_path
862 if sys.platform == 'win32':
863 # Case-insensitive, and handles short/long names
864 tests = [
865 (True, TESTFN, TESTFN),
866 (True, TESTFN.lower(), TESTFN.upper()),
867 ]
868 import _winapi
869 # ProgramFiles is the most reliable path that will have short/long
870 progfiles = os.getenv('ProgramFiles')
871 if progfiles:
872 tests = [
873 *tests,
874 (True, progfiles, progfiles),
875 (True, _winapi.GetShortPathName(progfiles), _winapi.GetLongPathName(progfiles)),
876 ]
877 else:
878 # Just a simple case-sensitive comparison
879 tests = [
880 (True, TESTFN, TESTFN),
881 (False, TESTFN.lower(), TESTFN.upper()),
882 ]
883 for r, path1, path2 in tests:
884 with self.subTest(f"{path1}-{path2}"):
885 if r:
886 self.assertTrue(same_path(path1, path2))
887 else:
888 self.assertFalse(same_path(path1, path2))
889
890 # gh-126084: venvwlauncher should run pythonw, not python
891 @requireVenvCreate
892 @unittest.skipUnless(os.name == 'nt', 'only relevant on Windows')
893 def test_venvwlauncher(self):
894 """
895 Test that the GUI launcher runs the GUI python.
896 """
897 rmtree(self.env_dir)
898 venv.create(self.env_dir)
899 exename = self.exe
900 # Retain the debug suffix if present
901 if "python" in exename and not "pythonw" in exename:
902 exename = exename.replace("python", "pythonw")
903 envpyw = os.path.join(self.env_dir, self.bindir, exename)
904 try:
905 subprocess.check_call([envpyw, "-c", "import sys; "
906 "assert sys._base_executable.endswith('%s')" % exename])
907 except subprocess.CalledProcessError:
908 self.fail("venvwlauncher.exe did not run %s" % exename)
909
910
911 @requireVenvCreate
912 class EnsurePipTest(BaseTest):
913 """Test venv module installation of pip."""
914 def assert_pip_not_installed(self):
915 out, err = check_output([self.envpy(real_env_dir=True), '-c',
916 'try:\n import pip\nexcept ImportError:\n print("OK")'])
917 # We force everything to text, so unittest gives the detailed diff
918 # if we get unexpected results
919 err = err.decode("latin-1") # Force to text, prevent decoding errors
920 self.assertEqual(err, "")
921 out = out.decode("latin-1") # Force to text, prevent decoding errors
922 self.assertEqual(out.strip(), "OK")
923
924
925 def test_no_pip_by_default(self):
926 rmtree(self.env_dir)
927 self.run_with_capture(venv.create, self.env_dir)
928 self.assert_pip_not_installed()
929
930 def test_explicit_no_pip(self):
931 rmtree(self.env_dir)
932 self.run_with_capture(venv.create, self.env_dir, with_pip=False)
933 self.assert_pip_not_installed()
934
935 def test_devnull(self):
936 # Fix for issue #20053 uses os.devnull to force a config file to
937 # appear empty. However http://bugs.python.org/issue20541 means
938 # that doesn't currently work properly on Windows. Once that is
939 # fixed, the "win_location" part of test_with_pip should be restored
940 with open(os.devnull, "rb") as f:
941 self.assertEqual(f.read(), b"")
942
943 self.assertTrue(os.path.exists(os.devnull))
944
945 def do_test_with_pip(self, system_site_packages):
946 rmtree(self.env_dir)
947 with EnvironmentVarGuard() as envvars:
948 # pip's cross-version compatibility may trigger deprecation
949 # warnings in current versions of Python. Ensure related
950 # environment settings don't cause venv to fail.
951 envvars["PYTHONWARNINGS"] = "ignore"
952 # ensurepip is different enough from a normal pip invocation
953 # that we want to ensure it ignores the normal pip environment
954 # variable settings. We set PIP_NO_INSTALL here specifically
955 # to check that ensurepip (and hence venv) ignores it.
956 # See http://bugs.python.org/issue19734
957 envvars["PIP_NO_INSTALL"] = "1"
958 # Also check that we ignore the pip configuration file
959 # See http://bugs.python.org/issue20053
960 with tempfile.TemporaryDirectory() as home_dir:
961 envvars["HOME"] = home_dir
962 bad_config = "[global]\nno-install=1"
963 # Write to both config file names on all platforms to reduce
964 # cross-platform variation in test code behaviour
965 win_location = ("pip", "pip.ini")
966 posix_location = (".pip", "pip.conf")
967 # Skips win_location due to http://bugs.python.org/issue20541
968 for dirname, fname in (posix_location,):
969 dirpath = os.path.join(home_dir, dirname)
970 os.mkdir(dirpath)
971 fpath = os.path.join(dirpath, fname)
972 with open(fpath, 'w') as f:
973 f.write(bad_config)
974
975 # Actually run the create command with all that unhelpful
976 # config in place to ensure we ignore it
977 with self.nicer_error():
978 self.run_with_capture(venv.create, self.env_dir,
979 system_site_packages=system_site_packages,
980 with_pip=True)
981 # Ensure pip is available in the virtual environment
982 # Ignore DeprecationWarning since pip code is not part of Python
983 out, err = check_output([self.envpy(real_env_dir=True),
984 '-W', 'ignore::DeprecationWarning',
985 '-W', 'ignore::ImportWarning', '-I',
986 '-m', 'pip', '--version'])
987 # We force everything to text, so unittest gives the detailed diff
988 # if we get unexpected results
989 err = err.decode("latin-1") # Force to text, prevent decoding errors
990 self.assertEqual(err, "")
991 out = out.decode("latin-1") # Force to text, prevent decoding errors
992 expected_version = "pip {}".format(ensurepip.version())
993 self.assertStartsWith(out, expected_version)
994 env_dir = os.fsencode(self.env_dir).decode("latin-1")
995 self.assertIn(env_dir, out)
996
997 # http://bugs.python.org/issue19728
998 # Check the private uninstall command provided for the Windows
999 # installers works (at least in a virtual environment)
1000 with EnvironmentVarGuard() as envvars:
1001 with self.nicer_error():
1002 # It seems ensurepip._uninstall calls subprocesses which do not
1003 # inherit the interpreter settings.
1004 envvars["PYTHONWARNINGS"] = "ignore"
1005 out, err = check_output([self.envpy(real_env_dir=True),
1006 '-W', 'ignore::DeprecationWarning',
1007 '-W', 'ignore::ImportWarning', '-I',
1008 '-m', 'ensurepip._uninstall'])
1009 # We force everything to text, so unittest gives the detailed diff
1010 # if we get unexpected results
1011 err = err.decode("latin-1") # Force to text, prevent decoding errors
1012 # Ignore the warning:
1013 # "The directory '$HOME/.cache/pip/http' or its parent directory
1014 # is not owned by the current user and the cache has been disabled.
1015 # Please check the permissions and owner of that directory. If
1016 # executing pip with sudo, you may want sudo's -H flag."
1017 # where $HOME is replaced by the HOME environment variable.
1018 err = re.sub("^(WARNING: )?The directory .* or its parent directory "
1019 "is not owned or is not writable by the current user.*$", "",
1020 err, flags=re.MULTILINE)
1021 # Ignore warning about missing optional module:
1022 try:
1023 import ssl
1024 except ImportError:
1025 err = re.sub(
1026 "^WARNING: Disabling truststore since ssl support is missing$",
1027 "",
1028 err, flags=re.MULTILINE)
1029 self.assertEqual(err.rstrip(), "")
1030 # Being fairly specific regarding the expected behaviour for the
1031 # initial bundling phase in Python 3.4. If the output changes in
1032 # future pip versions, this test can likely be relaxed further.
1033 out = out.decode("latin-1") # Force to text, prevent decoding errors
1034 self.assertIn("Successfully uninstalled pip", out)
1035 # Check pip is now gone from the virtual environment. This only
1036 # applies in the system_site_packages=False case, because in the
1037 # other case, pip may still be available in the system site-packages
1038 if not system_site_packages:
1039 self.assert_pip_not_installed()
1040
1041 @contextlib.contextmanager
1042 def nicer_error(self):
1043 """
1044 Capture output from a failed subprocess for easier debugging.
1045
1046 The output this handler produces can be a little hard to read,
1047 but at least it has all the details.
1048 """
1049 try:
1050 yield
1051 except subprocess.CalledProcessError as exc:
1052 out = (exc.output or b'').decode(errors="replace")
1053 err = (exc.stderr or b'').decode(errors="replace")
1054 self.fail(
1055 f"{exc}\n\n"
1056 f"**Subprocess Output**\n{out}\n\n"
1057 f"**Subprocess Error**\n{err}"
1058 )
1059
1060 @requires_venv_with_pip()
1061 @requires_resource('cpu')
1062 def test_with_pip(self):
1063 self.do_test_with_pip(False)
1064 self.do_test_with_pip(True)
1065
1066
1067 if __name__ == "__main__":
1068 unittest.main()