Debian Codegraph @

Lib/test/support/import_helper.py

raw
1 import contextlib
2 import _imp
3 import importlib
4 import importlib.machinery
5 import importlib.util
6 import os
7 import shutil
8 import sys
9 import textwrap
10 import unittest
11 import warnings
12
13 from .os_helper import unlink, temp_dir
14
15
16 @contextlib.contextmanager
17 def _ignore_deprecated_imports(ignore=True):
18 """Context manager to suppress package and module deprecation
19 warnings when importing them.
20
21 If ignore is False, this context manager has no effect.
22 """
23 if ignore:
24 with warnings.catch_warnings():
25 warnings.filterwarnings("ignore", ".+ (module|package)",
26 DeprecationWarning)
27 yield
28 else:
29 yield
30
31
32 def unload(name):
33 try:
34 del sys.modules[name]
35 except KeyError:
36 pass
37
38
39 def forget(modname):
40 """'Forget' a module was ever imported.
41
42 This removes the module from sys.modules and deletes any PEP 3147/488 or
43 legacy .pyc files.
44 """
45 unload(modname)
46 for dirname in sys.path:
47 source = os.path.join(dirname, modname + '.py')
48 # It doesn't matter if they exist or not, unlink all possible
49 # combinations of PEP 3147/488 and legacy pyc files.
50 unlink(source + 'c')
51 for opt in ('', 1, 2):
52 unlink(importlib.util.cache_from_source(source, optimization=opt))
53
54
55 def make_legacy_pyc(source):
56 """Move a PEP 3147/488 pyc file to its legacy pyc location.
57
58 :param source: The file system path to the source file. The source file
59 does not need to exist, however the PEP 3147/488 pyc file must exist.
60 :return: The file system path to the legacy pyc file.
61 """
62 pyc_file = importlib.util.cache_from_source(source)
63 assert source.endswith('.py')
64 legacy_pyc = source + 'c'
65 shutil.move(pyc_file, legacy_pyc)
66 return legacy_pyc
67
68
69 def import_module(name, deprecated=False, *, required_on=()):
70 """Import and return the module to be tested, raising SkipTest if
71 it is not available.
72
73 If deprecated is True, any module or package deprecation messages
74 will be suppressed. If a module is required on a platform but optional for
75 others, set required_on to an iterable of platform prefixes which will be
76 compared against sys.platform.
77 """
78 with _ignore_deprecated_imports(deprecated):
79 try:
80 return importlib.import_module(name)
81 except ImportError as msg:
82 if sys.platform.startswith(tuple(required_on)):
83 raise
84 raise unittest.SkipTest(str(msg))
85
86
87 def _save_and_remove_modules(names):
88 orig_modules = {}
89 prefixes = tuple(name + '.' for name in names)
90 for modname in list(sys.modules):
91 if modname in names or modname.startswith(prefixes):
92 orig_modules[modname] = sys.modules.pop(modname)
93 return orig_modules
94
95
96 @contextlib.contextmanager
97 def frozen_modules(enabled=True):
98 """Force frozen modules to be used (or not).
99
100 This only applies to modules that haven't been imported yet.
101 Also, some essential modules will always be imported frozen.
102 """
103 _imp._override_frozen_modules_for_tests(1 if enabled else -1)
104 try:
105 yield
106 finally:
107 _imp._override_frozen_modules_for_tests(0)
108
109
110 @contextlib.contextmanager
111 def multi_interp_extensions_check(enabled=True):
112 """Force legacy modules to be allowed in subinterpreters (or not).
113
114 ("legacy" == single-phase init)
115
116 This only applies to modules that haven't been imported yet.
117 It overrides the PyInterpreterConfig.check_multi_interp_extensions
118 setting (see support.run_in_subinterp_with_config() and
119 _interpreters.create()).
120
121 Also see importlib.utils.allowing_all_extensions().
122 """
123 old = _imp._override_multi_interp_extensions_check(1 if enabled else -1)
124 try:
125 yield
126 finally:
127 _imp._override_multi_interp_extensions_check(old)
128
129
130 def import_fresh_module(name, fresh=(), blocked=(), *,
131 deprecated=False,
132 usefrozen=False,
133 ):
134 """Import and return a module, deliberately bypassing sys.modules.
135
136 This function imports and returns a fresh copy of the named Python module
137 by removing the named module from sys.modules before doing the import.
138 Note that unlike reload, the original module is not affected by
139 this operation.
140
141 *fresh* is an iterable of additional module names that are also removed
142 from the sys.modules cache before doing the import. If one of these
143 modules can't be imported, None is returned.
144
145 *blocked* is an iterable of module names that are replaced with None
146 in the module cache during the import to ensure that attempts to import
147 them raise ImportError.
148
149 The named module and any modules named in the *fresh* and *blocked*
150 parameters are saved before starting the import and then reinserted into
151 sys.modules when the fresh import is complete.
152
153 Module and package deprecation messages are suppressed during this import
154 if *deprecated* is True.
155
156 This function will raise ImportError if the named module cannot be
157 imported.
158
159 If "usefrozen" is False (the default) then the frozen importer is
160 disabled (except for essential modules like importlib._bootstrap).
161 """
162 # NOTE: test_heapq, test_json and test_warnings include extra sanity checks
163 # to make sure that this utility function is working as expected
164 with _ignore_deprecated_imports(deprecated):
165 # Keep track of modules saved for later restoration as well
166 # as those which just need a blocking entry removed
167 fresh = list(fresh)
168 blocked = list(blocked)
169 names = {name, *fresh, *blocked}
170 orig_modules = _save_and_remove_modules(names)
171 for modname in blocked:
172 sys.modules[modname] = None
173
174 try:
175 with frozen_modules(usefrozen):
176 # Return None when one of the "fresh" modules can not be imported.
177 try:
178 for modname in fresh:
179 __import__(modname)
180 except ImportError:
181 return None
182 return importlib.import_module(name)
183 finally:
184 _save_and_remove_modules(names)
185 sys.modules.update(orig_modules)
186
187
188 class CleanImport(object):
189 """Context manager to force import to return a new module reference.
190
191 This is useful for testing module-level behaviours, such as
192 the emission of a DeprecationWarning on import.
193
194 Use like this:
195
196 with CleanImport("foo"):
197 importlib.import_module("foo") # new reference
198
199 If "usefrozen" is False (the default) then the frozen importer is
200 disabled (except for essential modules like importlib._bootstrap).
201 """
202
203 def __init__(self, *module_names, usefrozen=False):
204 self.original_modules = sys.modules.copy()
205 for module_name in module_names:
206 if module_name in sys.modules:
207 module = sys.modules[module_name]
208 # It is possible that module_name is just an alias for
209 # another module (e.g. stub for modules renamed in 3.x).
210 # In that case, we also need delete the real module to clear
211 # the import cache.
212 if module.__name__ != module_name:
213 del sys.modules[module.__name__]
214 del sys.modules[module_name]
215 self._frozen_modules = frozen_modules(usefrozen)
216
217 def __enter__(self):
218 self._frozen_modules.__enter__()
219 return self
220
221 def __exit__(self, *ignore_exc):
222 sys.modules.update(self.original_modules)
223 self._frozen_modules.__exit__(*ignore_exc)
224
225
226 class DirsOnSysPath(object):
227 """Context manager to temporarily add directories to sys.path.
228
229 This makes a copy of sys.path, appends any directories given
230 as positional arguments, then reverts sys.path to the copied
231 settings when the context ends.
232
233 Note that *all* sys.path modifications in the body of the
234 context manager, including replacement of the object,
235 will be reverted at the end of the block.
236 """
237
238 def __init__(self, *paths):
239 self.original_value = sys.path[:]
240 self.original_object = sys.path
241 sys.path.extend(paths)
242
243 def __enter__(self):
244 return self
245
246 def __exit__(self, *ignore_exc):
247 sys.path = self.original_object
248 sys.path[:] = self.original_value
249
250
251 def modules_setup():
252 return sys.modules.copy(),
253
254
255 def modules_cleanup(oldmodules):
256 # Encoders/decoders are registered permanently within the internal
257 # codec cache. If we destroy the corresponding modules their
258 # globals will be set to None which will trip up the cached functions.
259 encodings = [(k, v) for k, v in sys.modules.items()
260 if k.startswith('encodings.')]
261 sys.modules.clear()
262 sys.modules.update(encodings)
263 # XXX: This kind of problem can affect more than just encodings.
264 # In particular extension modules (such as _ssl) don't cope
265 # with reloading properly. Really, test modules should be cleaning
266 # out the test specific modules they know they added (ala test_runpy)
267 # rather than relying on this function (as test_importhooks and test_pkg
268 # do currently). Implicitly imported *real* modules should be left alone
269 # (see issue 10556).
270 sys.modules.update(oldmodules)
271
272
273 @contextlib.contextmanager
274 def isolated_modules():
275 """
276 Save modules on entry and cleanup on exit.
277 """
278 (saved,) = modules_setup()
279 try:
280 yield
281 finally:
282 modules_cleanup(saved)
283
284
285 def mock_register_at_fork(func):
286 # bpo-30599: Mock os.register_at_fork() when importing the random module,
287 # since this function doesn't allow to unregister callbacks and would leak
288 # memory.
289 from unittest import mock
290 return mock.patch('os.register_at_fork', create=True)(func)
291
292
293 @contextlib.contextmanager
294 def ready_to_import(name=None, source=""):
295 from test.support import script_helper
296
297 # 1. Sets up a temporary directory and removes it afterwards
298 # 2. Creates the module file
299 # 3. Temporarily clears the module from sys.modules (if any)
300 # 4. Reverts or removes the module when cleaning up
301 name = name or "spam"
302 with temp_dir() as tempdir:
303 path = script_helper.make_script(tempdir, name, source)
304 old_module = sys.modules.pop(name, None)
305 try:
306 sys.path.insert(0, tempdir)
307 yield name, path
308 finally:
309 sys.path.remove(tempdir)
310 if old_module is not None:
311 sys.modules[name] = old_module
312 else:
313 sys.modules.pop(name, None)
314
315
316 def ensure_lazy_imports(imported_module, modules_to_block):
317 """Test that when imported_module is imported, none of the modules in
318 modules_to_block are imported as a side effect."""
319 modules_to_block = frozenset(modules_to_block)
320 script = textwrap.dedent(
321 f"""
322 import sys
323 modules_to_block = {modules_to_block}
324 if unexpected := modules_to_block & sys.modules.keys():
325 startup = ", ".join(unexpected)
326 raise AssertionError(f'unexpectedly imported at startup: {{startup}}')
327
328 import {imported_module}
329 if unexpected := modules_to_block & sys.modules.keys():
330 after = ", ".join(unexpected)
331 raise AssertionError(f'unexpectedly imported after importing {imported_module}: {{after}}')
332 """
333 )
334 from .script_helper import assert_python_ok
335 assert_python_ok("-S", "-c", script)
336
337
338 @contextlib.contextmanager
339 def module_restored(name):
340 """A context manager that restores a module to the original state."""
341 missing = object()
342 orig = sys.modules.get(name, missing)
343 if orig is None:
344 mod = importlib.import_module(name)
345 else:
346 mod = type(sys)(name)
347 mod.__dict__.update(orig.__dict__)
348 sys.modules[name] = mod
349 try:
350 yield mod
351 finally:
352 if orig is missing:
353 sys.modules.pop(name, None)
354 else:
355 sys.modules[name] = orig
356
357
358 def create_module(name, loader=None, *, ispkg=False):
359 """Return a new, empty module."""
360 spec = importlib.machinery.ModuleSpec(
361 name,
362 loader,
363 origin='<import_helper>',
364 is_package=ispkg,
365 )
366 return importlib.util.module_from_spec(spec)
367
368
369 def _ensure_module(name, ispkg, addparent, clearnone):
370 try:
371 mod = orig = sys.modules[name]
372 except KeyError:
373 mod = orig = None
374 missing = True
375 else:
376 missing = False
377 if mod is not None:
378 # It was already imported.
379 return mod, orig, missing
380 # Otherwise, None means it was explicitly disabled.
381
382 assert name != '__main__'
383 if not missing:
384 assert orig is None, (name, sys.modules[name])
385 if not clearnone:
386 raise ModuleNotFoundError(name)
387 del sys.modules[name]
388 # Try normal import, then fall back to adding the module.
389 try:
390 mod = importlib.import_module(name)
391 except ModuleNotFoundError:
392 if addparent and not clearnone:
393 addparent = None
394 mod = _add_module(name, ispkg, addparent)
395 return mod, orig, missing
396
397
398 def _add_module(spec, ispkg, addparent):
399 if isinstance(spec, str):
400 name = spec
401 mod = create_module(name, ispkg=ispkg)
402 spec = mod.__spec__
403 else:
404 name = spec.name
405 mod = importlib.util.module_from_spec(spec)
406 sys.modules[name] = mod
407 if addparent is not False and spec.parent:
408 _ensure_module(spec.parent, True, addparent, bool(addparent))
409 return mod
410
411
412 def add_module(spec, *, parents=True):
413 """Return the module after creating it and adding it to sys.modules.
414
415 If parents is True then also create any missing parents.
416 """
417 return _add_module(spec, False, parents)
418
419
420 def add_package(spec, *, parents=True):
421 """Return the module after creating it and adding it to sys.modules.
422
423 If parents is True then also create any missing parents.
424 """
425 return _add_module(spec, True, parents)
426
427
428 def ensure_module_imported(name, *, clearnone=True):
429 """Return the corresponding module.
430
431 If it was already imported then return that. Otherwise, try
432 importing it (optionally clear it first if None). If that fails
433 then create a new empty module.
434
435 It can be helpful to combine this with ready_to_import() and/or
436 isolated_modules().
437 """
438 if sys.modules.get(name) is not None:
439 mod = sys.modules[name]
440 else:
441 mod, _, _ = _ensure_module(name, False, True, clearnone)
442 return mod