| 1 |
"""Utility code for constructing importers, etc.""" |
| 2 |
from ._abc import Loader |
| 3 |
from ._bootstrap import module_from_spec |
| 4 |
from ._bootstrap import _resolve_name |
| 5 |
from ._bootstrap import spec_from_loader |
| 6 |
from ._bootstrap import _find_spec |
| 7 |
from ._bootstrap_external import MAGIC_NUMBER |
| 8 |
from ._bootstrap_external import cache_from_source |
| 9 |
from ._bootstrap_external import decode_source |
| 10 |
from ._bootstrap_external import source_from_cache |
| 11 |
from ._bootstrap_external import spec_from_file_location |
| 12 |
|
| 13 |
import _imp |
| 14 |
import sys |
| 15 |
import types |
| 16 |
|
| 17 |
|
| 18 |
def source_hash(source_bytes): |
| 19 |
"Return the hash of *source_bytes* as used in hash-based pyc files." |
| 20 |
return _imp.source_hash(_imp.pyc_magic_number_token, source_bytes) |
| 21 |
|
| 22 |
|
| 23 |
def resolve_name(name, package): |
| 24 |
"""Resolve a relative module name to an absolute one.""" |
| 25 |
if not name.startswith('.'): |
| 26 |
return name |
| 27 |
elif not package: |
| 28 |
raise ImportError(f'no package specified for {repr(name)} ' |
| 29 |
'(required for relative module names)') |
| 30 |
level = 0 |
| 31 |
for character in name: |
| 32 |
if character != '.': |
| 33 |
break |
| 34 |
level += 1 |
| 35 |
return _resolve_name(name[level:], package, level) |
| 36 |
|
| 37 |
|
| 38 |
def _find_spec_from_path(name, path=None): |
| 39 |
"""Return the spec for the specified module. |
| 40 |
|
| 41 |
First, sys.modules is checked to see if the module was already imported. If |
| 42 |
so, then sys.modules[name].__spec__ is returned. If that happens to be |
| 43 |
set to None, then ValueError is raised. If the module is not in |
| 44 |
sys.modules, then sys.meta_path is searched for a suitable spec with the |
| 45 |
value of 'path' given to the finders. None is returned if no spec could |
| 46 |
be found. |
| 47 |
|
| 48 |
Dotted names do not have their parent packages implicitly imported. You will |
| 49 |
most likely need to explicitly import all parent packages in the proper |
| 50 |
order for a submodule to get the correct spec. |
| 51 |
|
| 52 |
""" |
| 53 |
if name not in sys.modules: |
| 54 |
return _find_spec(name, path) |
| 55 |
else: |
| 56 |
module = sys.modules[name] |
| 57 |
if module is None: |
| 58 |
return None |
| 59 |
try: |
| 60 |
spec = module.__spec__ |
| 61 |
except AttributeError: |
| 62 |
raise ValueError(f'{name}.__spec__ is not set') from None |
| 63 |
else: |
| 64 |
if spec is None: |
| 65 |
raise ValueError(f'{name}.__spec__ is None') |
| 66 |
return spec |
| 67 |
|
| 68 |
|
| 69 |
def find_spec(name, package=None): |
| 70 |
"""Return the spec for the specified module. |
| 71 |
|
| 72 |
First, sys.modules is checked to see if the module was already imported. If |
| 73 |
so, then sys.modules[name].__spec__ is returned. If that happens to be |
| 74 |
set to None, then ValueError is raised. If the module is not in |
| 75 |
sys.modules, then sys.meta_path is searched for a suitable spec with the |
| 76 |
value of 'path' given to the finders. None is returned if no spec could |
| 77 |
be found. |
| 78 |
|
| 79 |
If the name is for submodule (contains a dot), the parent module is |
| 80 |
automatically imported. |
| 81 |
|
| 82 |
The name and package arguments work the same as importlib.import_module(). |
| 83 |
In other words, relative module names (with leading dots) work. |
| 84 |
|
| 85 |
""" |
| 86 |
fullname = resolve_name(name, package) if name.startswith('.') else name |
| 87 |
if fullname not in sys.modules: |
| 88 |
parent_name = fullname.rpartition('.')[0] |
| 89 |
if parent_name: |
| 90 |
parent = __import__(parent_name, fromlist=['__path__']) |
| 91 |
try: |
| 92 |
parent_path = parent.__path__ |
| 93 |
except AttributeError as e: |
| 94 |
raise ModuleNotFoundError( |
| 95 |
f"__path__ attribute not found on {parent_name!r} " |
| 96 |
f"while trying to find {fullname!r}", name=fullname) from e |
| 97 |
else: |
| 98 |
parent_path = None |
| 99 |
return _find_spec(fullname, parent_path) |
| 100 |
else: |
| 101 |
module = sys.modules[fullname] |
| 102 |
if module is None: |
| 103 |
return None |
| 104 |
try: |
| 105 |
spec = module.__spec__ |
| 106 |
except AttributeError: |
| 107 |
raise ValueError(f'{name}.__spec__ is not set') from None |
| 108 |
else: |
| 109 |
if spec is None: |
| 110 |
raise ValueError(f'{name}.__spec__ is None') |
| 111 |
return spec |
| 112 |
|
| 113 |
|
| 114 |
# Normally we would use contextlib.contextmanager. However, this module |
| 115 |
# is imported by runpy, which means we want to avoid any unnecessary |
| 116 |
# dependencies. Thus we use a class. |
| 117 |
|
| 118 |
class _incompatible_extension_module_restrictions: |
| 119 |
"""A context manager that can temporarily skip the compatibility check. |
| 120 |
|
| 121 |
NOTE: This function is meant to accommodate an unusual case; one |
| 122 |
which is likely to eventually go away. There's is a pretty good |
| 123 |
chance this is not what you were looking for. |
| 124 |
|
| 125 |
WARNING: Using this function to disable the check can lead to |
| 126 |
unexpected behavior and even crashes. It should only be used during |
| 127 |
extension module development. |
| 128 |
|
| 129 |
If "disable_check" is True then the compatibility check will not |
| 130 |
happen while the context manager is active. Otherwise the check |
| 131 |
*will* happen. |
| 132 |
|
| 133 |
Normally, extensions that do not support multiple interpreters |
| 134 |
may not be imported in a subinterpreter. That implies modules |
| 135 |
that do not implement multi-phase init or that explicitly of out. |
| 136 |
|
| 137 |
Likewise for modules import in a subinterpreter with its own GIL |
| 138 |
when the extension does not support a per-interpreter GIL. This |
| 139 |
implies the module does not have a Py_mod_multiple_interpreters slot |
| 140 |
set to Py_MOD_PER_INTERPRETER_GIL_SUPPORTED. |
| 141 |
|
| 142 |
In both cases, this context manager may be used to temporarily |
| 143 |
disable the check for compatible extension modules. |
| 144 |
|
| 145 |
You can get the same effect as this function by implementing the |
| 146 |
basic interface of multi-phase init (PEP 489) and lying about |
| 147 |
support for multiple interpreters (or per-interpreter GIL). |
| 148 |
""" |
| 149 |
|
| 150 |
def __init__(self, *, disable_check): |
| 151 |
self.disable_check = bool(disable_check) |
| 152 |
|
| 153 |
def __enter__(self): |
| 154 |
self.old = _imp._override_multi_interp_extensions_check(self.override) |
| 155 |
return self |
| 156 |
|
| 157 |
def __exit__(self, *args): |
| 158 |
old = self.old |
| 159 |
del self.old |
| 160 |
_imp._override_multi_interp_extensions_check(old) |
| 161 |
|
| 162 |
@property |
| 163 |
def override(self): |
| 164 |
return -1 if self.disable_check else 1 |
| 165 |
|
| 166 |
|
| 167 |
class _LazyModule(types.ModuleType): |
| 168 |
|
| 169 |
"""A subclass of the module type which triggers loading upon attribute access.""" |
| 170 |
|
| 171 |
def __getattribute__(self, attr): |
| 172 |
"""Trigger the load of the module and return the attribute.""" |
| 173 |
__spec__ = object.__getattribute__(self, '__spec__') |
| 174 |
loader_state = __spec__.loader_state |
| 175 |
with loader_state['lock']: |
| 176 |
# Only the first thread to get the lock should trigger the load |
| 177 |
# and reset the module's class. The rest can now getattr(). |
| 178 |
if object.__getattribute__(self, '__class__') is _LazyModule: |
| 179 |
__class__ = loader_state['__class__'] |
| 180 |
|
| 181 |
# Reentrant calls from the same thread must be allowed to proceed without |
| 182 |
# triggering the load again. |
| 183 |
# exec_module() and self-referential imports are the primary ways this can |
| 184 |
# happen, but in any case we must return something to avoid deadlock. |
| 185 |
if loader_state['is_loading']: |
| 186 |
return __class__.__getattribute__(self, attr) |
| 187 |
loader_state['is_loading'] = True |
| 188 |
|
| 189 |
__dict__ = __class__.__getattribute__(self, '__dict__') |
| 190 |
|
| 191 |
# All module metadata must be gathered from __spec__ in order to avoid |
| 192 |
# using mutated values. |
| 193 |
# Get the original name to make sure no object substitution occurred |
| 194 |
# in sys.modules. |
| 195 |
original_name = __spec__.name |
| 196 |
# Figure out exactly what attributes were mutated between the creation |
| 197 |
# of the module and now. |
| 198 |
attrs_then = loader_state['__dict__'] |
| 199 |
attrs_now = __dict__ |
| 200 |
attrs_updated = {} |
| 201 |
for key, value in attrs_now.items(): |
| 202 |
# Code that set an attribute may have kept a reference to the |
| 203 |
# assigned object, making identity more important than equality. |
| 204 |
if key not in attrs_then: |
| 205 |
attrs_updated[key] = value |
| 206 |
elif id(attrs_now[key]) != id(attrs_then[key]): |
| 207 |
attrs_updated[key] = value |
| 208 |
__spec__.loader.exec_module(self) |
| 209 |
# If exec_module() was used directly there is no guarantee the module |
| 210 |
# object was put into sys.modules. |
| 211 |
if original_name in sys.modules: |
| 212 |
if id(self) != id(sys.modules[original_name]): |
| 213 |
raise ValueError(f"module object for {original_name!r} " |
| 214 |
"substituted in sys.modules during a lazy " |
| 215 |
"load") |
| 216 |
# Update after loading since that's what would happen in an eager |
| 217 |
# loading situation. |
| 218 |
__dict__.update(attrs_updated) |
| 219 |
# Finally, stop triggering this method, if the module did not |
| 220 |
# already update its own __class__. |
| 221 |
if isinstance(self, _LazyModule): |
| 222 |
object.__setattr__(self, '__class__', __class__) |
| 223 |
|
| 224 |
return getattr(self, attr) |
| 225 |
|
| 226 |
def __delattr__(self, attr): |
| 227 |
"""Trigger the load and then perform the deletion.""" |
| 228 |
# To trigger the load and raise an exception if the attribute |
| 229 |
# doesn't exist. |
| 230 |
self.__getattribute__(attr) |
| 231 |
delattr(self, attr) |
| 232 |
|
| 233 |
|
| 234 |
class LazyLoader(Loader): |
| 235 |
|
| 236 |
"""A loader that creates a module which defers loading until attribute access.""" |
| 237 |
|
| 238 |
@staticmethod |
| 239 |
def __check_eager_loader(loader): |
| 240 |
if not hasattr(loader, 'exec_module'): |
| 241 |
raise TypeError('loader must define exec_module()') |
| 242 |
|
| 243 |
@classmethod |
| 244 |
def factory(cls, loader): |
| 245 |
"""Construct a callable which returns the eager loader made lazy.""" |
| 246 |
cls.__check_eager_loader(loader) |
| 247 |
return lambda *args, **kwargs: cls(loader(*args, **kwargs)) |
| 248 |
|
| 249 |
def __init__(self, loader): |
| 250 |
self.__check_eager_loader(loader) |
| 251 |
self.loader = loader |
| 252 |
|
| 253 |
def create_module(self, spec): |
| 254 |
return self.loader.create_module(spec) |
| 255 |
|
| 256 |
def exec_module(self, module): |
| 257 |
"""Make the module load lazily.""" |
| 258 |
# Threading is only needed for lazy loading, and importlib.util can |
| 259 |
# be pulled in at interpreter startup, so defer until needed. |
| 260 |
import threading |
| 261 |
module.__spec__.loader = self.loader |
| 262 |
module.__loader__ = self.loader |
| 263 |
# Don't need to worry about deep-copying as trying to set an attribute |
| 264 |
# on an object would have triggered the load, |
| 265 |
# e.g. ``module.__spec__.loader = None`` would trigger a load from |
| 266 |
# trying to access module.__spec__. |
| 267 |
loader_state = {} |
| 268 |
loader_state['__dict__'] = module.__dict__.copy() |
| 269 |
loader_state['__class__'] = module.__class__ |
| 270 |
loader_state['lock'] = threading.RLock() |
| 271 |
loader_state['is_loading'] = False |
| 272 |
module.__spec__.loader_state = loader_state |
| 273 |
module.__class__ = _LazyModule |
| 274 |
|
| 275 |
|
| 276 |
__all__ = ['LazyLoader', 'Loader', 'MAGIC_NUMBER', |
| 277 |
'cache_from_source', 'decode_source', 'find_spec', |
| 278 |
'module_from_spec', 'resolve_name', 'source_from_cache', |
| 279 |
'source_hash', 'spec_from_file_location', 'spec_from_loader'] |