| 1 |
"""Core implementation of path-based import. |
| 2 |
|
| 3 |
This module is NOT meant to be directly imported! It has been designed such |
| 4 |
that it can be bootstrapped into Python as the implementation of import. As |
| 5 |
such it requires the injection of specific modules and attributes in order to |
| 6 |
work. One should use importlib as the public-facing version of this module. |
| 7 |
|
| 8 |
""" |
| 9 |
# IMPORTANT: Whenever making changes to this module, be sure to run a top-level |
| 10 |
# `make regen-importlib` followed by `make` in order to get the frozen version |
| 11 |
# of the module updated. Not doing so will result in the Makefile to fail for |
| 12 |
# all others who don't have a ./python around to freeze the module in the early |
| 13 |
# stages of compilation. |
| 14 |
# |
| 15 |
|
| 16 |
# See importlib._setup() for what is injected into the global namespace. |
| 17 |
|
| 18 |
# When editing this code be aware that code executed at import time CANNOT |
| 19 |
# reference any injected objects! This includes not only global code but also |
| 20 |
# anything specified at the class level. |
| 21 |
|
| 22 |
# Module injected manually by _set_bootstrap_module() |
| 23 |
_bootstrap = None |
| 24 |
|
| 25 |
# Import builtin modules |
| 26 |
import _imp |
| 27 |
import _io |
| 28 |
import sys |
| 29 |
import _warnings |
| 30 |
import marshal |
| 31 |
|
| 32 |
|
| 33 |
_MS_WINDOWS = (sys.platform == 'win32') |
| 34 |
if _MS_WINDOWS: |
| 35 |
import nt as _os |
| 36 |
import winreg |
| 37 |
else: |
| 38 |
import posix as _os |
| 39 |
|
| 40 |
|
| 41 |
if _MS_WINDOWS: |
| 42 |
path_separators = ['\\', '/'] |
| 43 |
else: |
| 44 |
path_separators = ['/'] |
| 45 |
# Assumption made in _path_join() |
| 46 |
assert all(len(sep) == 1 for sep in path_separators) |
| 47 |
path_sep = path_separators[0] |
| 48 |
path_sep_tuple = tuple(path_separators) |
| 49 |
path_separators = ''.join(path_separators) |
| 50 |
_pathseps_with_colon = {f':{s}' for s in path_separators} |
| 51 |
|
| 52 |
|
| 53 |
# Bootstrap-related code ###################################################### |
| 54 |
_CASE_INSENSITIVE_PLATFORMS_STR_KEY = 'win', |
| 55 |
_CASE_INSENSITIVE_PLATFORMS_BYTES_KEY = 'cygwin', 'darwin', 'ios', 'tvos', 'watchos' |
| 56 |
_CASE_INSENSITIVE_PLATFORMS = (_CASE_INSENSITIVE_PLATFORMS_BYTES_KEY |
| 57 |
+ _CASE_INSENSITIVE_PLATFORMS_STR_KEY) |
| 58 |
|
| 59 |
|
| 60 |
def _make_relax_case(): |
| 61 |
if sys.platform.startswith(_CASE_INSENSITIVE_PLATFORMS): |
| 62 |
if sys.platform.startswith(_CASE_INSENSITIVE_PLATFORMS_STR_KEY): |
| 63 |
key = 'PYTHONCASEOK' |
| 64 |
else: |
| 65 |
key = b'PYTHONCASEOK' |
| 66 |
|
| 67 |
def _relax_case(): |
| 68 |
"""True if filenames must be checked case-insensitively and ignore environment flags are not set.""" |
| 69 |
return not sys.flags.ignore_environment and key in _os.environ |
| 70 |
else: |
| 71 |
def _relax_case(): |
| 72 |
"""True if filenames must be checked case-insensitively.""" |
| 73 |
return False |
| 74 |
return _relax_case |
| 75 |
|
| 76 |
_relax_case = _make_relax_case() |
| 77 |
|
| 78 |
|
| 79 |
def _pack_uint32(x): |
| 80 |
"""Convert a 32-bit integer to little-endian.""" |
| 81 |
return (int(x) & 0xFFFFFFFF).to_bytes(4, 'little') |
| 82 |
|
| 83 |
|
| 84 |
def _unpack_uint64(data): |
| 85 |
"""Convert 8 bytes in little-endian to an integer.""" |
| 86 |
assert len(data) == 8 |
| 87 |
return int.from_bytes(data, 'little') |
| 88 |
|
| 89 |
def _unpack_uint32(data): |
| 90 |
"""Convert 4 bytes in little-endian to an integer.""" |
| 91 |
assert len(data) == 4 |
| 92 |
return int.from_bytes(data, 'little') |
| 93 |
|
| 94 |
def _unpack_uint16(data): |
| 95 |
"""Convert 2 bytes in little-endian to an integer.""" |
| 96 |
assert len(data) == 2 |
| 97 |
return int.from_bytes(data, 'little') |
| 98 |
|
| 99 |
|
| 100 |
if _MS_WINDOWS: |
| 101 |
def _path_join(*path_parts): |
| 102 |
"""Replacement for os.path.join().""" |
| 103 |
if not path_parts: |
| 104 |
return "" |
| 105 |
if len(path_parts) == 1: |
| 106 |
return path_parts[0] |
| 107 |
root = "" |
| 108 |
path = [] |
| 109 |
for new_root, tail in map(_os._path_splitroot, path_parts): |
| 110 |
if new_root.startswith(path_sep_tuple) or new_root.endswith(path_sep_tuple): |
| 111 |
root = new_root.rstrip(path_separators) or root |
| 112 |
path = [path_sep + tail] |
| 113 |
elif new_root.endswith(':'): |
| 114 |
if root.casefold() != new_root.casefold(): |
| 115 |
# Drive relative paths have to be resolved by the OS, so we reset the |
| 116 |
# tail but do not add a path_sep prefix. |
| 117 |
root = new_root |
| 118 |
path = [tail] |
| 119 |
else: |
| 120 |
path.append(tail) |
| 121 |
else: |
| 122 |
root = new_root or root |
| 123 |
path.append(tail) |
| 124 |
path = [p.rstrip(path_separators) for p in path if p] |
| 125 |
if len(path) == 1 and not path[0]: |
| 126 |
# Avoid losing the root's trailing separator when joining with nothing |
| 127 |
return root + path_sep |
| 128 |
return root + path_sep.join(path) |
| 129 |
|
| 130 |
else: |
| 131 |
def _path_join(*path_parts): |
| 132 |
"""Replacement for os.path.join().""" |
| 133 |
return path_sep.join([part.rstrip(path_separators) |
| 134 |
for part in path_parts if part]) |
| 135 |
|
| 136 |
|
| 137 |
def _path_split(path): |
| 138 |
"""Replacement for os.path.split().""" |
| 139 |
i = max(path.rfind(p) for p in path_separators) |
| 140 |
if i < 0: |
| 141 |
return '', path |
| 142 |
return path[:i], path[i + 1:] |
| 143 |
|
| 144 |
|
| 145 |
def _path_stat(path): |
| 146 |
"""Stat the path. |
| 147 |
|
| 148 |
Made a separate function to make it easier to override in experiments |
| 149 |
(e.g. cache stat results). |
| 150 |
|
| 151 |
""" |
| 152 |
return _os.stat(path) |
| 153 |
|
| 154 |
|
| 155 |
def _path_is_mode_type(path, mode): |
| 156 |
"""Test whether the path is the specified mode type.""" |
| 157 |
try: |
| 158 |
stat_info = _path_stat(path) |
| 159 |
except OSError: |
| 160 |
return False |
| 161 |
return (stat_info.st_mode & 0o170000) == mode |
| 162 |
|
| 163 |
|
| 164 |
def _path_isfile(path): |
| 165 |
"""Replacement for os.path.isfile.""" |
| 166 |
return _path_is_mode_type(path, 0o100000) |
| 167 |
|
| 168 |
|
| 169 |
def _path_isdir(path): |
| 170 |
"""Replacement for os.path.isdir.""" |
| 171 |
if not path: |
| 172 |
path = _os.getcwd() |
| 173 |
return _path_is_mode_type(path, 0o040000) |
| 174 |
|
| 175 |
|
| 176 |
if _MS_WINDOWS: |
| 177 |
def _path_isabs(path): |
| 178 |
"""Replacement for os.path.isabs.""" |
| 179 |
if not path: |
| 180 |
return False |
| 181 |
root = _os._path_splitroot(path)[0].replace('/', '\\') |
| 182 |
return len(root) > 1 and (root.startswith('\\\\') or root.endswith('\\')) |
| 183 |
|
| 184 |
else: |
| 185 |
def _path_isabs(path): |
| 186 |
"""Replacement for os.path.isabs.""" |
| 187 |
return path.startswith(path_separators) |
| 188 |
|
| 189 |
|
| 190 |
def _path_abspath(path): |
| 191 |
"""Replacement for os.path.abspath.""" |
| 192 |
if not _path_isabs(path): |
| 193 |
for sep in path_separators: |
| 194 |
path = path.removeprefix(f".{sep}") |
| 195 |
return _path_join(_os.getcwd(), path) |
| 196 |
else: |
| 197 |
return path |
| 198 |
|
| 199 |
|
| 200 |
def _write_atomic(path, data, mode=0o666): |
| 201 |
"""Best-effort function to write data to a path atomically. |
| 202 |
Be prepared to handle a FileExistsError if concurrent writing of the |
| 203 |
temporary file is attempted.""" |
| 204 |
# id() is used to generate a pseudo-random filename. |
| 205 |
path_tmp = f'{path}.{id(path)}' |
| 206 |
fd = _os.open(path_tmp, |
| 207 |
_os.O_EXCL | _os.O_CREAT | _os.O_WRONLY, mode & 0o666) |
| 208 |
try: |
| 209 |
# We first write data to a temporary file, and then use os.replace() to |
| 210 |
# perform an atomic rename. |
| 211 |
with _io.open(fd, 'wb') as file: |
| 212 |
file.write(data) |
| 213 |
_os.replace(path_tmp, path) |
| 214 |
except OSError: |
| 215 |
try: |
| 216 |
_os.unlink(path_tmp) |
| 217 |
except OSError: |
| 218 |
pass |
| 219 |
raise |
| 220 |
|
| 221 |
|
| 222 |
_code_type = type(_write_atomic.__code__) |
| 223 |
|
| 224 |
|
| 225 |
# Finder/loader utility code ############################################### |
| 226 |
|
| 227 |
# Magic word to reject .pyc files generated by other Python versions. |
| 228 |
# It should change for each incompatible change to the bytecode. |
| 229 |
# |
| 230 |
# The value of CR and LF is incorporated so if you ever read or write |
| 231 |
# a .pyc file in text mode the magic number will be wrong; also, the |
| 232 |
# Apple MPW compiler swaps their values, botching string constants. |
| 233 |
# |
| 234 |
# There were a variety of old schemes for setting the magic number. |
| 235 |
# The current working scheme is to increment the previous value by |
| 236 |
# 10. |
| 237 |
# |
| 238 |
# Starting with the adoption of PEP 3147 in Python 3.2, every bump in magic |
| 239 |
# number also includes a new "magic tag", i.e. a human readable string used |
| 240 |
# to represent the magic number in __pycache__ directories. When you change |
| 241 |
# the magic number, you must also set a new unique magic tag. Generally this |
| 242 |
# can be named after the Python major version of the magic number bump, but |
| 243 |
# it can really be anything, as long as it's different than anything else |
| 244 |
# that's come before. The tags are included in the following table, starting |
| 245 |
# with Python 3.2a0. |
| 246 |
# |
| 247 |
# Known values: |
| 248 |
# Python 1.5: 20121 |
| 249 |
# Python 1.5.1: 20121 |
| 250 |
# Python 1.5.2: 20121 |
| 251 |
# Python 1.6: 50428 |
| 252 |
# Python 2.0: 50823 |
| 253 |
# Python 2.0.1: 50823 |
| 254 |
# Python 2.1: 60202 |
| 255 |
# Python 2.1.1: 60202 |
| 256 |
# Python 2.1.2: 60202 |
| 257 |
# Python 2.2: 60717 |
| 258 |
# Python 2.3a0: 62011 |
| 259 |
# Python 2.3a0: 62021 |
| 260 |
# Python 2.3a0: 62011 (!) |
| 261 |
# Python 2.4a0: 62041 |
| 262 |
# Python 2.4a3: 62051 |
| 263 |
# Python 2.4b1: 62061 |
| 264 |
# Python 2.5a0: 62071 |
| 265 |
# Python 2.5a0: 62081 (ast-branch) |
| 266 |
# Python 2.5a0: 62091 (with) |
| 267 |
# Python 2.5a0: 62092 (changed WITH_CLEANUP opcode) |
| 268 |
# Python 2.5b3: 62101 (fix wrong code: for x, in ...) |
| 269 |
# Python 2.5b3: 62111 (fix wrong code: x += yield) |
| 270 |
# Python 2.5c1: 62121 (fix wrong lnotab with for loops and |
| 271 |
# storing constants that should have been removed) |
| 272 |
# Python 2.5c2: 62131 (fix wrong code: for x, in ... in listcomp/genexp) |
| 273 |
# Python 2.6a0: 62151 (peephole optimizations and STORE_MAP opcode) |
| 274 |
# Python 2.6a1: 62161 (WITH_CLEANUP optimization) |
| 275 |
# Python 2.7a0: 62171 (optimize list comprehensions/change LIST_APPEND) |
| 276 |
# Python 2.7a0: 62181 (optimize conditional branches: |
| 277 |
# introduce POP_JUMP_IF_FALSE and POP_JUMP_IF_TRUE) |
| 278 |
# Python 2.7a0 62191 (introduce SETUP_WITH) |
| 279 |
# Python 2.7a0 62201 (introduce BUILD_SET) |
| 280 |
# Python 2.7a0 62211 (introduce MAP_ADD and SET_ADD) |
| 281 |
# Python 3000: 3000 |
| 282 |
# 3010 (removed UNARY_CONVERT) |
| 283 |
# 3020 (added BUILD_SET) |
| 284 |
# 3030 (added keyword-only parameters) |
| 285 |
# 3040 (added signature annotations) |
| 286 |
# 3050 (print becomes a function) |
| 287 |
# 3060 (PEP 3115 metaclass syntax) |
| 288 |
# 3061 (string literals become unicode) |
| 289 |
# 3071 (PEP 3109 raise changes) |
| 290 |
# 3081 (PEP 3137 make __file__ and __name__ unicode) |
| 291 |
# 3091 (kill str8 interning) |
| 292 |
# 3101 (merge from 2.6a0, see 62151) |
| 293 |
# 3103 (__file__ points to source file) |
| 294 |
# Python 3.0a4: 3111 (WITH_CLEANUP optimization). |
| 295 |
# Python 3.0b1: 3131 (lexical exception stacking, including POP_EXCEPT |
| 296 |
#3021) |
| 297 |
# Python 3.1a1: 3141 (optimize list, set and dict comprehensions: |
| 298 |
# change LIST_APPEND and SET_ADD, add MAP_ADD #2183) |
| 299 |
# Python 3.1a1: 3151 (optimize conditional branches: |
| 300 |
# introduce POP_JUMP_IF_FALSE and POP_JUMP_IF_TRUE |
| 301 |
#4715) |
| 302 |
# Python 3.2a1: 3160 (add SETUP_WITH #6101) |
| 303 |
# tag: cpython-32 |
| 304 |
# Python 3.2a2: 3170 (add DUP_TOP_TWO, remove DUP_TOPX and ROT_FOUR #9225) |
| 305 |
# tag: cpython-32 |
| 306 |
# Python 3.2a3 3180 (add DELETE_DEREF #4617) |
| 307 |
# Python 3.3a1 3190 (__class__ super closure changed) |
| 308 |
# Python 3.3a1 3200 (PEP 3155 __qualname__ added #13448) |
| 309 |
# Python 3.3a1 3210 (added size modulo 2**32 to the pyc header #13645) |
| 310 |
# Python 3.3a2 3220 (changed PEP 380 implementation #14230) |
| 311 |
# Python 3.3a4 3230 (revert changes to implicit __class__ closure #14857) |
| 312 |
# Python 3.4a1 3250 (evaluate positional default arguments before |
| 313 |
# keyword-only defaults #16967) |
| 314 |
# Python 3.4a1 3260 (add LOAD_CLASSDEREF; allow locals of class to override |
| 315 |
# free vars #17853) |
| 316 |
# Python 3.4a1 3270 (various tweaks to the __class__ closure #12370) |
| 317 |
# Python 3.4a1 3280 (remove implicit class argument) |
| 318 |
# Python 3.4a4 3290 (changes to __qualname__ computation #19301) |
| 319 |
# Python 3.4a4 3300 (more changes to __qualname__ computation #19301) |
| 320 |
# Python 3.4rc2 3310 (alter __qualname__ computation #20625) |
| 321 |
# Python 3.5a1 3320 (PEP 465: Matrix multiplication operator #21176) |
| 322 |
# Python 3.5b1 3330 (PEP 448: Additional Unpacking Generalizations #2292) |
| 323 |
# Python 3.5b2 3340 (fix dictionary display evaluation order #11205) |
| 324 |
# Python 3.5b3 3350 (add GET_YIELD_FROM_ITER opcode #24400) |
| 325 |
# Python 3.5.2 3351 (fix BUILD_MAP_UNPACK_WITH_CALL opcode #27286) |
| 326 |
# Python 3.6a0 3360 (add FORMAT_VALUE opcode #25483) |
| 327 |
# Python 3.6a1 3361 (lineno delta of code.co_lnotab becomes signed #26107) |
| 328 |
# Python 3.6a2 3370 (16 bit wordcode #26647) |
| 329 |
# Python 3.6a2 3371 (add BUILD_CONST_KEY_MAP opcode #27140) |
| 330 |
# Python 3.6a2 3372 (MAKE_FUNCTION simplification, remove MAKE_CLOSURE |
| 331 |
# #27095) |
| 332 |
# Python 3.6b1 3373 (add BUILD_STRING opcode #27078) |
| 333 |
# Python 3.6b1 3375 (add SETUP_ANNOTATIONS and STORE_ANNOTATION opcodes |
| 334 |
# #27985) |
| 335 |
# Python 3.6b1 3376 (simplify CALL_FUNCTIONs & BUILD_MAP_UNPACK_WITH_CALL |
| 336 |
#27213) |
| 337 |
# Python 3.6b1 3377 (set __class__ cell from type.__new__ #23722) |
| 338 |
# Python 3.6b2 3378 (add BUILD_TUPLE_UNPACK_WITH_CALL #28257) |
| 339 |
# Python 3.6rc1 3379 (more thorough __class__ validation #23722) |
| 340 |
# Python 3.7a1 3390 (add LOAD_METHOD and CALL_METHOD opcodes #26110) |
| 341 |
# Python 3.7a2 3391 (update GET_AITER #31709) |
| 342 |
# Python 3.7a4 3392 (PEP 552: Deterministic pycs #31650) |
| 343 |
# Python 3.7b1 3393 (remove STORE_ANNOTATION opcode #32550) |
| 344 |
# Python 3.7b5 3394 (restored docstring as the first stmt in the body; |
| 345 |
# this might affected the first line number #32911) |
| 346 |
# Python 3.8a1 3400 (move frame block handling to compiler #17611) |
| 347 |
# Python 3.8a1 3401 (add END_ASYNC_FOR #33041) |
| 348 |
# Python 3.8a1 3410 (PEP570 Python Positional-Only Parameters #36540) |
| 349 |
# Python 3.8b2 3411 (Reverse evaluation order of key: value in dict |
| 350 |
# comprehensions #35224) |
| 351 |
# Python 3.8b2 3412 (Swap the position of positional args and positional |
| 352 |
# only args in ast.arguments #37593) |
| 353 |
# Python 3.8b4 3413 (Fix "break" and "continue" in "finally" #37830) |
| 354 |
# Python 3.9a0 3420 (add LOAD_ASSERTION_ERROR #34880) |
| 355 |
# Python 3.9a0 3421 (simplified bytecode for with blocks #32949) |
| 356 |
# Python 3.9a0 3422 (remove BEGIN_FINALLY, END_FINALLY, CALL_FINALLY, POP_FINALLY bytecodes #33387) |
| 357 |
# Python 3.9a2 3423 (add IS_OP, CONTAINS_OP and JUMP_IF_NOT_EXC_MATCH bytecodes #39156) |
| 358 |
# Python 3.9a2 3424 (simplify bytecodes for *value unpacking) |
| 359 |
# Python 3.9a2 3425 (simplify bytecodes for **value unpacking) |
| 360 |
# Python 3.10a1 3430 (Make 'annotations' future by default) |
| 361 |
# Python 3.10a1 3431 (New line number table format -- PEP 626) |
| 362 |
# Python 3.10a2 3432 (Function annotation for MAKE_FUNCTION is changed from dict to tuple bpo-42202) |
| 363 |
# Python 3.10a2 3433 (RERAISE restores f_lasti if oparg != 0) |
| 364 |
# Python 3.10a6 3434 (PEP 634: Structural Pattern Matching) |
| 365 |
# Python 3.10a7 3435 Use instruction offsets (as opposed to byte offsets). |
| 366 |
# Python 3.10b1 3436 (Add GEN_START bytecode #43683) |
| 367 |
# Python 3.10b1 3437 (Undo making 'annotations' future by default - We like to dance among core devs!) |
| 368 |
# Python 3.10b1 3438 Safer line number table handling. |
| 369 |
# Python 3.10b1 3439 (Add ROT_N) |
| 370 |
# Python 3.11a1 3450 Use exception table for unwinding ("zero cost" exception handling) |
| 371 |
# Python 3.11a1 3451 (Add CALL_METHOD_KW) |
| 372 |
# Python 3.11a1 3452 (drop nlocals from marshaled code objects) |
| 373 |
# Python 3.11a1 3453 (add co_fastlocalnames and co_fastlocalkinds) |
| 374 |
# Python 3.11a1 3454 (compute cell offsets relative to locals bpo-43693) |
| 375 |
# Python 3.11a1 3455 (add MAKE_CELL bpo-43693) |
| 376 |
# Python 3.11a1 3456 (interleave cell args bpo-43693) |
| 377 |
# Python 3.11a1 3457 (Change localsplus to a bytes object bpo-43693) |
| 378 |
# Python 3.11a1 3458 (imported objects now don't use LOAD_METHOD/CALL_METHOD) |
| 379 |
# Python 3.11a1 3459 (PEP 657: add end line numbers and column offsets for instructions) |
| 380 |
# Python 3.11a1 3460 (Add co_qualname field to PyCodeObject bpo-44530) |
| 381 |
# Python 3.11a1 3461 (JUMP_ABSOLUTE must jump backwards) |
| 382 |
# Python 3.11a2 3462 (bpo-44511: remove COPY_DICT_WITHOUT_KEYS, change |
| 383 |
# MATCH_CLASS and MATCH_KEYS, and add COPY) |
| 384 |
# Python 3.11a3 3463 (bpo-45711: JUMP_IF_NOT_EXC_MATCH no longer pops the |
| 385 |
# active exception) |
| 386 |
# Python 3.11a3 3464 (bpo-45636: Merge numeric BINARY_*/INPLACE_* into |
| 387 |
# BINARY_OP) |
| 388 |
# Python 3.11a3 3465 (Add COPY_FREE_VARS opcode) |
| 389 |
# Python 3.11a4 3466 (bpo-45292: PEP-654 except*) |
| 390 |
# Python 3.11a4 3467 (Change CALL_xxx opcodes) |
| 391 |
# Python 3.11a4 3468 (Add SEND opcode) |
| 392 |
# Python 3.11a4 3469 (bpo-45711: remove type, traceback from exc_info) |
| 393 |
# Python 3.11a4 3470 (bpo-46221: PREP_RERAISE_STAR no longer pushes lasti) |
| 394 |
# Python 3.11a4 3471 (bpo-46202: remove pop POP_EXCEPT_AND_RERAISE) |
| 395 |
# Python 3.11a4 3472 (bpo-46009: replace GEN_START with POP_TOP) |
| 396 |
# Python 3.11a4 3473 (Add POP_JUMP_IF_NOT_NONE/POP_JUMP_IF_NONE opcodes) |
| 397 |
# Python 3.11a4 3474 (Add RESUME opcode) |
| 398 |
# Python 3.11a5 3475 (Add RETURN_GENERATOR opcode) |
| 399 |
# Python 3.11a5 3476 (Add ASYNC_GEN_WRAP opcode) |
| 400 |
# Python 3.11a5 3477 (Replace DUP_TOP/DUP_TOP_TWO with COPY and |
| 401 |
# ROT_TWO/ROT_THREE/ROT_FOUR/ROT_N with SWAP) |
| 402 |
# Python 3.11a5 3478 (New CALL opcodes) |
| 403 |
# Python 3.11a5 3479 (Add PUSH_NULL opcode) |
| 404 |
# Python 3.11a5 3480 (New CALL opcodes, second iteration) |
| 405 |
# Python 3.11a5 3481 (Use inline cache for BINARY_OP) |
| 406 |
# Python 3.11a5 3482 (Use inline caching for UNPACK_SEQUENCE and LOAD_GLOBAL) |
| 407 |
# Python 3.11a5 3483 (Use inline caching for COMPARE_OP and BINARY_SUBSCR) |
| 408 |
# Python 3.11a5 3484 (Use inline caching for LOAD_ATTR, LOAD_METHOD, and |
| 409 |
# STORE_ATTR) |
| 410 |
# Python 3.11a5 3485 (Add an oparg to GET_AWAITABLE) |
| 411 |
# Python 3.11a6 3486 (Use inline caching for PRECALL and CALL) |
| 412 |
# Python 3.11a6 3487 (Remove the adaptive "oparg counter" mechanism) |
| 413 |
# Python 3.11a6 3488 (LOAD_GLOBAL can push additional NULL) |
| 414 |
# Python 3.11a6 3489 (Add JUMP_BACKWARD, remove JUMP_ABSOLUTE) |
| 415 |
# Python 3.11a6 3490 (remove JUMP_IF_NOT_EXC_MATCH, add CHECK_EXC_MATCH) |
| 416 |
# Python 3.11a6 3491 (remove JUMP_IF_NOT_EG_MATCH, add CHECK_EG_MATCH, |
| 417 |
# add JUMP_BACKWARD_NO_INTERRUPT, make JUMP_NO_INTERRUPT virtual) |
| 418 |
# Python 3.11a7 3492 (make POP_JUMP_IF_NONE/NOT_NONE/TRUE/FALSE relative) |
| 419 |
# Python 3.11a7 3493 (Make JUMP_IF_TRUE_OR_POP/JUMP_IF_FALSE_OR_POP relative) |
| 420 |
# Python 3.11a7 3494 (New location info table) |
| 421 |
# Python 3.11b4 3495 (Set line number of module's RESUME instr to 0 per PEP 626) |
| 422 |
# Python 3.12a1 3500 (Remove PRECALL opcode) |
| 423 |
# Python 3.12a1 3501 (YIELD_VALUE oparg == stack_depth) |
| 424 |
# Python 3.12a1 3502 (LOAD_FAST_CHECK, no NULL-check in LOAD_FAST) |
| 425 |
# Python 3.12a1 3503 (Shrink LOAD_METHOD cache) |
| 426 |
# Python 3.12a1 3504 (Merge LOAD_METHOD back into LOAD_ATTR) |
| 427 |
# Python 3.12a1 3505 (Specialization/Cache for FOR_ITER) |
| 428 |
# Python 3.12a1 3506 (Add BINARY_SLICE and STORE_SLICE instructions) |
| 429 |
# Python 3.12a1 3507 (Set lineno of module's RESUME to 0) |
| 430 |
# Python 3.12a1 3508 (Add CLEANUP_THROW) |
| 431 |
# Python 3.12a1 3509 (Conditional jumps only jump forward) |
| 432 |
# Python 3.12a2 3510 (FOR_ITER leaves iterator on the stack) |
| 433 |
# Python 3.12a2 3511 (Add STOPITERATION_ERROR instruction) |
| 434 |
# Python 3.12a2 3512 (Remove all unused consts from code objects) |
| 435 |
# Python 3.12a4 3513 (Add CALL_INTRINSIC_1 instruction, removed STOPITERATION_ERROR, PRINT_EXPR, IMPORT_STAR) |
| 436 |
# Python 3.12a4 3514 (Remove ASYNC_GEN_WRAP, LIST_TO_TUPLE, and UNARY_POSITIVE) |
| 437 |
# Python 3.12a5 3515 (Embed jump mask in COMPARE_OP oparg) |
| 438 |
# Python 3.12a5 3516 (Add COMPARE_AND_BRANCH instruction) |
| 439 |
# Python 3.12a5 3517 (Change YIELD_VALUE oparg to exception block depth) |
| 440 |
# Python 3.12a6 3518 (Add RETURN_CONST instruction) |
| 441 |
# Python 3.12a6 3519 (Modify SEND instruction) |
| 442 |
# Python 3.12a6 3520 (Remove PREP_RERAISE_STAR, add CALL_INTRINSIC_2) |
| 443 |
# Python 3.12a7 3521 (Shrink the LOAD_GLOBAL caches) |
| 444 |
# Python 3.12a7 3522 (Removed JUMP_IF_FALSE_OR_POP/JUMP_IF_TRUE_OR_POP) |
| 445 |
# Python 3.12a7 3523 (Convert COMPARE_AND_BRANCH back to COMPARE_OP) |
| 446 |
# Python 3.12a7 3524 (Shrink the BINARY_SUBSCR caches) |
| 447 |
# Python 3.12b1 3525 (Shrink the CALL caches) |
| 448 |
# Python 3.12b1 3526 (Add instrumentation support) |
| 449 |
# Python 3.12b1 3527 (Add LOAD_SUPER_ATTR) |
| 450 |
# Python 3.12b1 3528 (Add LOAD_SUPER_ATTR_METHOD specialization) |
| 451 |
# Python 3.12b1 3529 (Inline list/dict/set comprehensions) |
| 452 |
# Python 3.12b1 3530 (Shrink the LOAD_SUPER_ATTR caches) |
| 453 |
# Python 3.12b1 3531 (Add PEP 695 changes) |
| 454 |
# Python 3.13a1 3550 (Plugin optimizer support) |
| 455 |
# Python 3.13a1 3551 (Compact superinstructions) |
| 456 |
# Python 3.13a1 3552 (Remove LOAD_FAST__LOAD_CONST and LOAD_CONST__LOAD_FAST) |
| 457 |
# Python 3.13a1 3553 (Add SET_FUNCTION_ATTRIBUTE) |
| 458 |
# Python 3.13a1 3554 (more efficient bytecodes for f-strings) |
| 459 |
# Python 3.13a1 3555 (generate specialized opcodes metadata from bytecodes.c) |
| 460 |
# Python 3.13a1 3556 (Convert LOAD_CLOSURE to a pseudo-op) |
| 461 |
# Python 3.13a1 3557 (Make the conversion to boolean in jumps explicit) |
| 462 |
# Python 3.13a1 3558 (Reorder the stack items for CALL) |
| 463 |
# Python 3.13a1 3559 (Generate opcode IDs from bytecodes.c) |
| 464 |
# Python 3.13a1 3560 (Add RESUME_CHECK instruction) |
| 465 |
# Python 3.13a1 3561 (Add cache entry to branch instructions) |
| 466 |
# Python 3.13a1 3562 (Assign opcode IDs for internal ops in separate range) |
| 467 |
# Python 3.13a1 3563 (Add CALL_KW and remove KW_NAMES) |
| 468 |
# Python 3.13a1 3564 (Removed oparg from YIELD_VALUE, changed oparg values of RESUME) |
| 469 |
# Python 3.13a1 3565 (Oparg of YIELD_VALUE indicates whether it is in a yield-from) |
| 470 |
# Python 3.13a1 3566 (Emit JUMP_NO_INTERRUPT instead of JUMP for non-loop no-lineno cases) |
| 471 |
# Python 3.13a1 3567 (Reimplement line number propagation by the compiler) |
| 472 |
# Python 3.13a1 3568 (Change semantics of END_FOR) |
| 473 |
# Python 3.13a5 3569 (Specialize CONTAINS_OP) |
| 474 |
# Python 3.13a6 3570 (Add __firstlineno__ class attribute) |
| 475 |
# Python 3.13b1 3571 (Fix miscompilation of private names in generic classes) |
| 476 |
|
| 477 |
# Python 3.14 will start with 3600 |
| 478 |
|
| 479 |
# Please don't copy-paste the same pre-release tag for new entries above!!! |
| 480 |
# You should always use the *upcoming* tag. For example, if 3.12a6 came out |
| 481 |
# a week ago, I should put "Python 3.12a7" next to my new magic number. |
| 482 |
|
| 483 |
# MAGIC must change whenever the bytecode emitted by the compiler may no |
| 484 |
# longer be understood by older implementations of the eval loop (usually |
| 485 |
# due to the addition of new opcodes). |
| 486 |
# |
| 487 |
# Starting with Python 3.11, Python 3.n starts with magic number 2900+50n. |
| 488 |
# |
| 489 |
# Whenever MAGIC_NUMBER is changed, the ranges in the magic_values array |
| 490 |
# in PC/launcher.c must also be updated. |
| 491 |
|
| 492 |
MAGIC_NUMBER = (3571).to_bytes(2, 'little') + b'\r\n' |
| 493 |
|
| 494 |
_RAW_MAGIC_NUMBER = int.from_bytes(MAGIC_NUMBER, 'little') # For import.c |
| 495 |
|
| 496 |
_PYCACHE = '__pycache__' |
| 497 |
_OPT = 'opt-' |
| 498 |
|
| 499 |
SOURCE_SUFFIXES = ['.py'] |
| 500 |
if _MS_WINDOWS: |
| 501 |
SOURCE_SUFFIXES.append('.pyw') |
| 502 |
|
| 503 |
EXTENSION_SUFFIXES = _imp.extension_suffixes() |
| 504 |
|
| 505 |
BYTECODE_SUFFIXES = ['.pyc'] |
| 506 |
# Deprecated. |
| 507 |
DEBUG_BYTECODE_SUFFIXES = OPTIMIZED_BYTECODE_SUFFIXES = BYTECODE_SUFFIXES |
| 508 |
|
| 509 |
def cache_from_source(path, debug_override=None, *, optimization=None): |
| 510 |
"""Given the path to a .py file, return the path to its .pyc file. |
| 511 |
|
| 512 |
The .py file does not need to exist; this simply returns the path to the |
| 513 |
.pyc file calculated as if the .py file were imported. |
| 514 |
|
| 515 |
The 'optimization' parameter controls the presumed optimization level of |
| 516 |
the bytecode file. If 'optimization' is not None, the string representation |
| 517 |
of the argument is taken and verified to be alphanumeric (else ValueError |
| 518 |
is raised). |
| 519 |
|
| 520 |
The debug_override parameter is deprecated. If debug_override is not None, |
| 521 |
a True value is the same as setting 'optimization' to the empty string |
| 522 |
while a False value is equivalent to setting 'optimization' to '1'. |
| 523 |
|
| 524 |
If sys.implementation.cache_tag is None then NotImplementedError is raised. |
| 525 |
|
| 526 |
""" |
| 527 |
if debug_override is not None: |
| 528 |
_warnings.warn('the debug_override parameter is deprecated; use ' |
| 529 |
"'optimization' instead", DeprecationWarning) |
| 530 |
if optimization is not None: |
| 531 |
message = 'debug_override or optimization must be set to None' |
| 532 |
raise TypeError(message) |
| 533 |
optimization = '' if debug_override else 1 |
| 534 |
path = _os.fspath(path) |
| 535 |
head, tail = _path_split(path) |
| 536 |
base, sep, rest = tail.rpartition('.') |
| 537 |
tag = sys.implementation.cache_tag |
| 538 |
if tag is None: |
| 539 |
raise NotImplementedError('sys.implementation.cache_tag is None') |
| 540 |
almost_filename = ''.join([(base if base else rest), sep, tag]) |
| 541 |
if optimization is None: |
| 542 |
if sys.flags.optimize == 0: |
| 543 |
optimization = '' |
| 544 |
else: |
| 545 |
optimization = sys.flags.optimize |
| 546 |
optimization = str(optimization) |
| 547 |
if optimization != '': |
| 548 |
if not optimization.isalnum(): |
| 549 |
raise ValueError(f'{optimization!r} is not alphanumeric') |
| 550 |
almost_filename = f'{almost_filename}.{_OPT}{optimization}' |
| 551 |
filename = almost_filename + BYTECODE_SUFFIXES[0] |
| 552 |
if sys.pycache_prefix is not None: |
| 553 |
# We need an absolute path to the py file to avoid the possibility of |
| 554 |
# collisions within sys.pycache_prefix, if someone has two different |
| 555 |
# `foo/bar.py` on their system and they import both of them using the |
| 556 |
# same sys.pycache_prefix. Let's say sys.pycache_prefix is |
| 557 |
# `C:\Bytecode`; the idea here is that if we get `Foo\Bar`, we first |
| 558 |
# make it absolute (`C:\Somewhere\Foo\Bar`), then make it root-relative |
| 559 |
# (`Somewhere\Foo\Bar`), so we end up placing the bytecode file in an |
| 560 |
# unambiguous `C:\Bytecode\Somewhere\Foo\Bar\`. |
| 561 |
head = _path_abspath(head) |
| 562 |
|
| 563 |
# Strip initial drive from a Windows path. We know we have an absolute |
| 564 |
# path here, so the second part of the check rules out a POSIX path that |
| 565 |
# happens to contain a colon at the second character. |
| 566 |
# Slicing avoids issues with an empty (or short) `head`. |
| 567 |
if head[1:2] == ':' and head[0:1] not in path_separators: |
| 568 |
head = head[2:] |
| 569 |
|
| 570 |
# Strip initial path separator from `head` to complete the conversion |
| 571 |
# back to a root-relative path before joining. |
| 572 |
return _path_join( |
| 573 |
sys.pycache_prefix, |
| 574 |
head.lstrip(path_separators), |
| 575 |
filename, |
| 576 |
) |
| 577 |
return _path_join(head, _PYCACHE, filename) |
| 578 |
|
| 579 |
|
| 580 |
def source_from_cache(path): |
| 581 |
"""Given the path to a .pyc. file, return the path to its .py file. |
| 582 |
|
| 583 |
The .pyc file does not need to exist; this simply returns the path to |
| 584 |
the .py file calculated to correspond to the .pyc file. If path does |
| 585 |
not conform to PEP 3147/488 format, ValueError will be raised. If |
| 586 |
sys.implementation.cache_tag is None then NotImplementedError is raised. |
| 587 |
|
| 588 |
""" |
| 589 |
if sys.implementation.cache_tag is None: |
| 590 |
raise NotImplementedError('sys.implementation.cache_tag is None') |
| 591 |
path = _os.fspath(path) |
| 592 |
head, pycache_filename = _path_split(path) |
| 593 |
found_in_pycache_prefix = False |
| 594 |
if sys.pycache_prefix is not None: |
| 595 |
stripped_path = sys.pycache_prefix.rstrip(path_separators) |
| 596 |
if head.startswith(stripped_path + path_sep): |
| 597 |
head = head[len(stripped_path):] |
| 598 |
found_in_pycache_prefix = True |
| 599 |
if not found_in_pycache_prefix: |
| 600 |
head, pycache = _path_split(head) |
| 601 |
if pycache != _PYCACHE: |
| 602 |
raise ValueError(f'{_PYCACHE} not bottom-level directory in ' |
| 603 |
f'{path!r}') |
| 604 |
dot_count = pycache_filename.count('.') |
| 605 |
if dot_count not in {2, 3}: |
| 606 |
raise ValueError(f'expected only 2 or 3 dots in {pycache_filename!r}') |
| 607 |
elif dot_count == 3: |
| 608 |
optimization = pycache_filename.rsplit('.', 2)[-2] |
| 609 |
if not optimization.startswith(_OPT): |
| 610 |
raise ValueError("optimization portion of filename does not start " |
| 611 |
f"with {_OPT!r}") |
| 612 |
opt_level = optimization[len(_OPT):] |
| 613 |
if not opt_level.isalnum(): |
| 614 |
raise ValueError(f"optimization level {optimization!r} is not an " |
| 615 |
"alphanumeric value") |
| 616 |
base_filename = pycache_filename.partition('.')[0] |
| 617 |
return _path_join(head, base_filename + SOURCE_SUFFIXES[0]) |
| 618 |
|
| 619 |
|
| 620 |
def _get_sourcefile(bytecode_path): |
| 621 |
"""Convert a bytecode file path to a source path (if possible). |
| 622 |
|
| 623 |
This function exists purely for backwards-compatibility for |
| 624 |
PyImport_ExecCodeModuleWithFilenames() in the C API. |
| 625 |
|
| 626 |
""" |
| 627 |
if len(bytecode_path) == 0: |
| 628 |
return None |
| 629 |
rest, _, extension = bytecode_path.rpartition('.') |
| 630 |
if not rest or extension.lower()[-3:-1] != 'py': |
| 631 |
return bytecode_path |
| 632 |
try: |
| 633 |
source_path = source_from_cache(bytecode_path) |
| 634 |
except (NotImplementedError, ValueError): |
| 635 |
source_path = bytecode_path[:-1] |
| 636 |
return source_path if _path_isfile(source_path) else bytecode_path |
| 637 |
|
| 638 |
|
| 639 |
def _get_cached(filename): |
| 640 |
if filename.endswith(tuple(SOURCE_SUFFIXES)): |
| 641 |
try: |
| 642 |
return cache_from_source(filename) |
| 643 |
except NotImplementedError: |
| 644 |
pass |
| 645 |
elif filename.endswith(tuple(BYTECODE_SUFFIXES)): |
| 646 |
return filename |
| 647 |
else: |
| 648 |
return None |
| 649 |
|
| 650 |
|
| 651 |
def _calc_mode(path): |
| 652 |
"""Calculate the mode permissions for a bytecode file.""" |
| 653 |
try: |
| 654 |
mode = _path_stat(path).st_mode |
| 655 |
except OSError: |
| 656 |
mode = 0o666 |
| 657 |
# We always ensure write access so we can update cached files |
| 658 |
# later even when the source files are read-only on Windows (#6074) |
| 659 |
mode |= 0o200 |
| 660 |
return mode |
| 661 |
|
| 662 |
|
| 663 |
def _check_name(method): |
| 664 |
"""Decorator to verify that the module being requested matches the one the |
| 665 |
loader can handle. |
| 666 |
|
| 667 |
The first argument (self) must define _name which the second argument is |
| 668 |
compared against. If the comparison fails then ImportError is raised. |
| 669 |
|
| 670 |
""" |
| 671 |
def _check_name_wrapper(self, name=None, *args, **kwargs): |
| 672 |
if name is None: |
| 673 |
name = self.name |
| 674 |
elif self.name != name: |
| 675 |
raise ImportError('loader for %s cannot handle %s' % |
| 676 |
(self.name, name), name=name) |
| 677 |
return method(self, name, *args, **kwargs) |
| 678 |
|
| 679 |
# FIXME: @_check_name is used to define class methods before the |
| 680 |
# _bootstrap module is set by _set_bootstrap_module(). |
| 681 |
if _bootstrap is not None: |
| 682 |
_wrap = _bootstrap._wrap |
| 683 |
else: |
| 684 |
def _wrap(new, old): |
| 685 |
for replace in ['__module__', '__name__', '__qualname__', '__doc__']: |
| 686 |
if hasattr(old, replace): |
| 687 |
setattr(new, replace, getattr(old, replace)) |
| 688 |
new.__dict__.update(old.__dict__) |
| 689 |
|
| 690 |
_wrap(_check_name_wrapper, method) |
| 691 |
return _check_name_wrapper |
| 692 |
|
| 693 |
|
| 694 |
def _classify_pyc(data, name, exc_details): |
| 695 |
"""Perform basic validity checking of a pyc header and return the flags field, |
| 696 |
which determines how the pyc should be further validated against the source. |
| 697 |
|
| 698 |
*data* is the contents of the pyc file. (Only the first 16 bytes are |
| 699 |
required, though.) |
| 700 |
|
| 701 |
*name* is the name of the module being imported. It is used for logging. |
| 702 |
|
| 703 |
*exc_details* is a dictionary passed to ImportError if it raised for |
| 704 |
improved debugging. |
| 705 |
|
| 706 |
ImportError is raised when the magic number is incorrect or when the flags |
| 707 |
field is invalid. EOFError is raised when the data is found to be truncated. |
| 708 |
|
| 709 |
""" |
| 710 |
magic = data[:4] |
| 711 |
if magic != MAGIC_NUMBER: |
| 712 |
message = f'bad magic number in {name!r}: {magic!r}' |
| 713 |
_bootstrap._verbose_message('{}', message) |
| 714 |
raise ImportError(message, **exc_details) |
| 715 |
if len(data) < 16: |
| 716 |
message = f'reached EOF while reading pyc header of {name!r}' |
| 717 |
_bootstrap._verbose_message('{}', message) |
| 718 |
raise EOFError(message) |
| 719 |
flags = _unpack_uint32(data[4:8]) |
| 720 |
# Only the first two flags are defined. |
| 721 |
if flags & ~0b11: |
| 722 |
message = f'invalid flags {flags!r} in {name!r}' |
| 723 |
raise ImportError(message, **exc_details) |
| 724 |
return flags |
| 725 |
|
| 726 |
|
| 727 |
def _validate_timestamp_pyc(data, source_mtime, source_size, name, |
| 728 |
exc_details): |
| 729 |
"""Validate a pyc against the source last-modified time. |
| 730 |
|
| 731 |
*data* is the contents of the pyc file. (Only the first 16 bytes are |
| 732 |
required.) |
| 733 |
|
| 734 |
*source_mtime* is the last modified timestamp of the source file. |
| 735 |
|
| 736 |
*source_size* is None or the size of the source file in bytes. |
| 737 |
|
| 738 |
*name* is the name of the module being imported. It is used for logging. |
| 739 |
|
| 740 |
*exc_details* is a dictionary passed to ImportError if it raised for |
| 741 |
improved debugging. |
| 742 |
|
| 743 |
An ImportError is raised if the bytecode is stale. |
| 744 |
|
| 745 |
""" |
| 746 |
if _unpack_uint32(data[8:12]) != (source_mtime & 0xFFFFFFFF): |
| 747 |
message = f'bytecode is stale for {name!r}' |
| 748 |
_bootstrap._verbose_message('{}', message) |
| 749 |
raise ImportError(message, **exc_details) |
| 750 |
if (source_size is not None and |
| 751 |
_unpack_uint32(data[12:16]) != (source_size & 0xFFFFFFFF)): |
| 752 |
raise ImportError(f'bytecode is stale for {name!r}', **exc_details) |
| 753 |
|
| 754 |
|
| 755 |
def _validate_hash_pyc(data, source_hash, name, exc_details): |
| 756 |
"""Validate a hash-based pyc by checking the real source hash against the one in |
| 757 |
the pyc header. |
| 758 |
|
| 759 |
*data* is the contents of the pyc file. (Only the first 16 bytes are |
| 760 |
required.) |
| 761 |
|
| 762 |
*source_hash* is the importlib.util.source_hash() of the source file. |
| 763 |
|
| 764 |
*name* is the name of the module being imported. It is used for logging. |
| 765 |
|
| 766 |
*exc_details* is a dictionary passed to ImportError if it raised for |
| 767 |
improved debugging. |
| 768 |
|
| 769 |
An ImportError is raised if the bytecode is stale. |
| 770 |
|
| 771 |
""" |
| 772 |
if data[8:16] != source_hash: |
| 773 |
raise ImportError( |
| 774 |
f'hash in bytecode doesn\'t match hash of source {name!r}', |
| 775 |
**exc_details, |
| 776 |
) |
| 777 |
|
| 778 |
|
| 779 |
def _compile_bytecode(data, name=None, bytecode_path=None, source_path=None): |
| 780 |
"""Compile bytecode as found in a pyc.""" |
| 781 |
code = marshal.loads(data) |
| 782 |
if isinstance(code, _code_type): |
| 783 |
_bootstrap._verbose_message('code object from {!r}', bytecode_path) |
| 784 |
if source_path is not None: |
| 785 |
_imp._fix_co_filename(code, source_path) |
| 786 |
return code |
| 787 |
else: |
| 788 |
raise ImportError(f'Non-code object in {bytecode_path!r}', |
| 789 |
name=name, path=bytecode_path) |
| 790 |
|
| 791 |
|
| 792 |
def _code_to_timestamp_pyc(code, mtime=0, source_size=0): |
| 793 |
"Produce the data for a timestamp-based pyc." |
| 794 |
data = bytearray(MAGIC_NUMBER) |
| 795 |
data.extend(_pack_uint32(0)) |
| 796 |
data.extend(_pack_uint32(mtime)) |
| 797 |
data.extend(_pack_uint32(source_size)) |
| 798 |
data.extend(marshal.dumps(code)) |
| 799 |
return data |
| 800 |
|
| 801 |
|
| 802 |
def _code_to_hash_pyc(code, source_hash, checked=True): |
| 803 |
"Produce the data for a hash-based pyc." |
| 804 |
data = bytearray(MAGIC_NUMBER) |
| 805 |
flags = 0b1 | checked << 1 |
| 806 |
data.extend(_pack_uint32(flags)) |
| 807 |
assert len(source_hash) == 8 |
| 808 |
data.extend(source_hash) |
| 809 |
data.extend(marshal.dumps(code)) |
| 810 |
return data |
| 811 |
|
| 812 |
|
| 813 |
def decode_source(source_bytes): |
| 814 |
"""Decode bytes representing source code and return the string. |
| 815 |
|
| 816 |
Universal newline support is used in the decoding. |
| 817 |
""" |
| 818 |
import tokenize # To avoid bootstrap issues. |
| 819 |
source_bytes_readline = _io.BytesIO(source_bytes).readline |
| 820 |
encoding = tokenize.detect_encoding(source_bytes_readline) |
| 821 |
newline_decoder = _io.IncrementalNewlineDecoder(None, True) |
| 822 |
return newline_decoder.decode(source_bytes.decode(encoding[0])) |
| 823 |
|
| 824 |
|
| 825 |
# Module specifications ####################################################### |
| 826 |
|
| 827 |
_POPULATE = object() |
| 828 |
|
| 829 |
|
| 830 |
def spec_from_file_location(name, location=None, *, loader=None, |
| 831 |
submodule_search_locations=_POPULATE): |
| 832 |
"""Return a module spec based on a file location. |
| 833 |
|
| 834 |
To indicate that the module is a package, set |
| 835 |
submodule_search_locations to a list of directory paths. An |
| 836 |
empty list is sufficient, though its not otherwise useful to the |
| 837 |
import system. |
| 838 |
|
| 839 |
The loader must take a spec as its only __init__() arg. |
| 840 |
|
| 841 |
""" |
| 842 |
if location is None: |
| 843 |
# The caller may simply want a partially populated location- |
| 844 |
# oriented spec. So we set the location to a bogus value and |
| 845 |
# fill in as much as we can. |
| 846 |
location = '<unknown>' |
| 847 |
if hasattr(loader, 'get_filename'): |
| 848 |
# ExecutionLoader |
| 849 |
try: |
| 850 |
location = loader.get_filename(name) |
| 851 |
except ImportError: |
| 852 |
pass |
| 853 |
else: |
| 854 |
location = _os.fspath(location) |
| 855 |
try: |
| 856 |
location = _path_abspath(location) |
| 857 |
except OSError: |
| 858 |
pass |
| 859 |
|
| 860 |
# If the location is on the filesystem, but doesn't actually exist, |
| 861 |
# we could return None here, indicating that the location is not |
| 862 |
# valid. However, we don't have a good way of testing since an |
| 863 |
# indirect location (e.g. a zip file or URL) will look like a |
| 864 |
# non-existent file relative to the filesystem. |
| 865 |
|
| 866 |
spec = _bootstrap.ModuleSpec(name, loader, origin=location) |
| 867 |
spec._set_fileattr = True |
| 868 |
|
| 869 |
# Pick a loader if one wasn't provided. |
| 870 |
if loader is None: |
| 871 |
for loader_class, suffixes in _get_supported_file_loaders(): |
| 872 |
if location.endswith(tuple(suffixes)): |
| 873 |
loader = loader_class(name, location) |
| 874 |
spec.loader = loader |
| 875 |
break |
| 876 |
else: |
| 877 |
return None |
| 878 |
|
| 879 |
# Set submodule_search_paths appropriately. |
| 880 |
if submodule_search_locations is _POPULATE: |
| 881 |
# Check the loader. |
| 882 |
if hasattr(loader, 'is_package'): |
| 883 |
try: |
| 884 |
is_package = loader.is_package(name) |
| 885 |
except ImportError: |
| 886 |
pass |
| 887 |
else: |
| 888 |
if is_package: |
| 889 |
spec.submodule_search_locations = [] |
| 890 |
else: |
| 891 |
spec.submodule_search_locations = submodule_search_locations |
| 892 |
if spec.submodule_search_locations == []: |
| 893 |
if location: |
| 894 |
dirname = _path_split(location)[0] |
| 895 |
spec.submodule_search_locations.append(dirname) |
| 896 |
|
| 897 |
return spec |
| 898 |
|
| 899 |
|
| 900 |
def _bless_my_loader(module_globals): |
| 901 |
"""Helper function for _warnings.c |
| 902 |
|
| 903 |
See GH#97850 for details. |
| 904 |
""" |
| 905 |
# 2022-10-06(warsaw): For now, this helper is only used in _warnings.c and |
| 906 |
# that use case only has the module globals. This function could be |
| 907 |
# extended to accept either that or a module object. However, in the |
| 908 |
# latter case, it would be better to raise certain exceptions when looking |
| 909 |
# at a module, which should have either a __loader__ or __spec__.loader. |
| 910 |
# For backward compatibility, it is possible that we'll get an empty |
| 911 |
# dictionary for the module globals, and that cannot raise an exception. |
| 912 |
if not isinstance(module_globals, dict): |
| 913 |
return None |
| 914 |
|
| 915 |
missing = object() |
| 916 |
loader = module_globals.get('__loader__', None) |
| 917 |
spec = module_globals.get('__spec__', missing) |
| 918 |
|
| 919 |
if loader is None: |
| 920 |
if spec is missing: |
| 921 |
# If working with a module: |
| 922 |
# raise AttributeError('Module globals is missing a __spec__') |
| 923 |
return None |
| 924 |
elif spec is None: |
| 925 |
raise ValueError('Module globals is missing a __spec__.loader') |
| 926 |
|
| 927 |
spec_loader = getattr(spec, 'loader', missing) |
| 928 |
|
| 929 |
if spec_loader in (missing, None): |
| 930 |
if loader is None: |
| 931 |
exc = AttributeError if spec_loader is missing else ValueError |
| 932 |
raise exc('Module globals is missing a __spec__.loader') |
| 933 |
_warnings.warn( |
| 934 |
'Module globals is missing a __spec__.loader', |
| 935 |
DeprecationWarning) |
| 936 |
spec_loader = loader |
| 937 |
|
| 938 |
assert spec_loader is not None |
| 939 |
if loader is not None and loader != spec_loader: |
| 940 |
_warnings.warn( |
| 941 |
'Module globals; __loader__ != __spec__.loader', |
| 942 |
DeprecationWarning) |
| 943 |
return loader |
| 944 |
|
| 945 |
return spec_loader |
| 946 |
|
| 947 |
|
| 948 |
# Loaders ##################################################################### |
| 949 |
|
| 950 |
class WindowsRegistryFinder: |
| 951 |
|
| 952 |
"""Meta path finder for modules declared in the Windows registry.""" |
| 953 |
|
| 954 |
REGISTRY_KEY = ( |
| 955 |
'Software\\Python\\PythonCore\\{sys_version}' |
| 956 |
'\\Modules\\{fullname}') |
| 957 |
REGISTRY_KEY_DEBUG = ( |
| 958 |
'Software\\Python\\PythonCore\\{sys_version}' |
| 959 |
'\\Modules\\{fullname}\\Debug') |
| 960 |
DEBUG_BUILD = (_MS_WINDOWS and '_d.pyd' in EXTENSION_SUFFIXES) |
| 961 |
|
| 962 |
@staticmethod |
| 963 |
def _open_registry(key): |
| 964 |
try: |
| 965 |
return winreg.OpenKey(winreg.HKEY_CURRENT_USER, key) |
| 966 |
except OSError: |
| 967 |
return winreg.OpenKey(winreg.HKEY_LOCAL_MACHINE, key) |
| 968 |
|
| 969 |
@classmethod |
| 970 |
def _search_registry(cls, fullname): |
| 971 |
if cls.DEBUG_BUILD: |
| 972 |
registry_key = cls.REGISTRY_KEY_DEBUG |
| 973 |
else: |
| 974 |
registry_key = cls.REGISTRY_KEY |
| 975 |
key = registry_key.format(fullname=fullname, |
| 976 |
sys_version='%d.%d' % sys.version_info[:2]) |
| 977 |
try: |
| 978 |
with cls._open_registry(key) as hkey: |
| 979 |
filepath = winreg.QueryValue(hkey, '') |
| 980 |
except OSError: |
| 981 |
return None |
| 982 |
return filepath |
| 983 |
|
| 984 |
@classmethod |
| 985 |
def find_spec(cls, fullname, path=None, target=None): |
| 986 |
filepath = cls._search_registry(fullname) |
| 987 |
if filepath is None: |
| 988 |
return None |
| 989 |
try: |
| 990 |
_path_stat(filepath) |
| 991 |
except OSError: |
| 992 |
return None |
| 993 |
for loader, suffixes in _get_supported_file_loaders(): |
| 994 |
if filepath.endswith(tuple(suffixes)): |
| 995 |
spec = _bootstrap.spec_from_loader(fullname, |
| 996 |
loader(fullname, filepath), |
| 997 |
origin=filepath) |
| 998 |
return spec |
| 999 |
|
| 1000 |
|
| 1001 |
class _LoaderBasics: |
| 1002 |
|
| 1003 |
"""Base class of common code needed by both SourceLoader and |
| 1004 |
SourcelessFileLoader.""" |
| 1005 |
|
| 1006 |
def is_package(self, fullname): |
| 1007 |
"""Concrete implementation of InspectLoader.is_package by checking if |
| 1008 |
the path returned by get_filename has a filename of '__init__.py'.""" |
| 1009 |
filename = _path_split(self.get_filename(fullname))[1] |
| 1010 |
filename_base = filename.rsplit('.', 1)[0] |
| 1011 |
tail_name = fullname.rpartition('.')[2] |
| 1012 |
return filename_base == '__init__' and tail_name != '__init__' |
| 1013 |
|
| 1014 |
def create_module(self, spec): |
| 1015 |
"""Use default semantics for module creation.""" |
| 1016 |
|
| 1017 |
def exec_module(self, module): |
| 1018 |
"""Execute the module.""" |
| 1019 |
code = self.get_code(module.__name__) |
| 1020 |
if code is None: |
| 1021 |
raise ImportError(f'cannot load module {module.__name__!r} when ' |
| 1022 |
'get_code() returns None') |
| 1023 |
_bootstrap._call_with_frames_removed(exec, code, module.__dict__) |
| 1024 |
|
| 1025 |
def load_module(self, fullname): |
| 1026 |
"""This method is deprecated.""" |
| 1027 |
# Warning implemented in _load_module_shim(). |
| 1028 |
return _bootstrap._load_module_shim(self, fullname) |
| 1029 |
|
| 1030 |
|
| 1031 |
class SourceLoader(_LoaderBasics): |
| 1032 |
|
| 1033 |
def path_mtime(self, path): |
| 1034 |
"""Optional method that returns the modification time (an int) for the |
| 1035 |
specified path (a str). |
| 1036 |
|
| 1037 |
Raises OSError when the path cannot be handled. |
| 1038 |
""" |
| 1039 |
raise OSError |
| 1040 |
|
| 1041 |
def path_stats(self, path): |
| 1042 |
"""Optional method returning a metadata dict for the specified |
| 1043 |
path (a str). |
| 1044 |
|
| 1045 |
Possible keys: |
| 1046 |
- 'mtime' (mandatory) is the numeric timestamp of last source |
| 1047 |
code modification; |
| 1048 |
- 'size' (optional) is the size in bytes of the source code. |
| 1049 |
|
| 1050 |
Implementing this method allows the loader to read bytecode files. |
| 1051 |
Raises OSError when the path cannot be handled. |
| 1052 |
""" |
| 1053 |
return {'mtime': self.path_mtime(path)} |
| 1054 |
|
| 1055 |
def _cache_bytecode(self, source_path, cache_path, data): |
| 1056 |
"""Optional method which writes data (bytes) to a file path (a str). |
| 1057 |
|
| 1058 |
Implementing this method allows for the writing of bytecode files. |
| 1059 |
|
| 1060 |
The source path is needed in order to correctly transfer permissions |
| 1061 |
""" |
| 1062 |
# For backwards compatibility, we delegate to set_data() |
| 1063 |
return self.set_data(cache_path, data) |
| 1064 |
|
| 1065 |
def set_data(self, path, data): |
| 1066 |
"""Optional method which writes data (bytes) to a file path (a str). |
| 1067 |
|
| 1068 |
Implementing this method allows for the writing of bytecode files. |
| 1069 |
""" |
| 1070 |
|
| 1071 |
|
| 1072 |
def get_source(self, fullname): |
| 1073 |
"""Concrete implementation of InspectLoader.get_source.""" |
| 1074 |
path = self.get_filename(fullname) |
| 1075 |
try: |
| 1076 |
source_bytes = self.get_data(path) |
| 1077 |
except OSError as exc: |
| 1078 |
raise ImportError('source not available through get_data()', |
| 1079 |
name=fullname) from exc |
| 1080 |
return decode_source(source_bytes) |
| 1081 |
|
| 1082 |
def source_to_code(self, data, path, *, _optimize=-1): |
| 1083 |
"""Return the code object compiled from source. |
| 1084 |
|
| 1085 |
The 'data' argument can be any object type that compile() supports. |
| 1086 |
""" |
| 1087 |
return _bootstrap._call_with_frames_removed(compile, data, path, 'exec', |
| 1088 |
dont_inherit=True, optimize=_optimize) |
| 1089 |
|
| 1090 |
def get_code(self, fullname): |
| 1091 |
"""Concrete implementation of InspectLoader.get_code. |
| 1092 |
|
| 1093 |
Reading of bytecode requires path_stats to be implemented. To write |
| 1094 |
bytecode, set_data must also be implemented. |
| 1095 |
|
| 1096 |
""" |
| 1097 |
source_path = self.get_filename(fullname) |
| 1098 |
source_mtime = None |
| 1099 |
source_bytes = None |
| 1100 |
source_hash = None |
| 1101 |
hash_based = False |
| 1102 |
check_source = True |
| 1103 |
try: |
| 1104 |
bytecode_path = cache_from_source(source_path) |
| 1105 |
except NotImplementedError: |
| 1106 |
bytecode_path = None |
| 1107 |
else: |
| 1108 |
try: |
| 1109 |
st = self.path_stats(source_path) |
| 1110 |
except OSError: |
| 1111 |
pass |
| 1112 |
else: |
| 1113 |
source_mtime = int(st['mtime']) |
| 1114 |
try: |
| 1115 |
data = self.get_data(bytecode_path) |
| 1116 |
except OSError: |
| 1117 |
pass |
| 1118 |
else: |
| 1119 |
exc_details = { |
| 1120 |
'name': fullname, |
| 1121 |
'path': bytecode_path, |
| 1122 |
} |
| 1123 |
try: |
| 1124 |
flags = _classify_pyc(data, fullname, exc_details) |
| 1125 |
bytes_data = memoryview(data)[16:] |
| 1126 |
hash_based = flags & 0b1 != 0 |
| 1127 |
if hash_based: |
| 1128 |
check_source = flags & 0b10 != 0 |
| 1129 |
if (_imp.check_hash_based_pycs != 'never' and |
| 1130 |
(check_source or |
| 1131 |
_imp.check_hash_based_pycs == 'always')): |
| 1132 |
source_bytes = self.get_data(source_path) |
| 1133 |
source_hash = _imp.source_hash( |
| 1134 |
_RAW_MAGIC_NUMBER, |
| 1135 |
source_bytes, |
| 1136 |
) |
| 1137 |
_validate_hash_pyc(data, source_hash, fullname, |
| 1138 |
exc_details) |
| 1139 |
else: |
| 1140 |
_validate_timestamp_pyc( |
| 1141 |
data, |
| 1142 |
source_mtime, |
| 1143 |
st['size'], |
| 1144 |
fullname, |
| 1145 |
exc_details, |
| 1146 |
) |
| 1147 |
except (ImportError, EOFError): |
| 1148 |
pass |
| 1149 |
else: |
| 1150 |
_bootstrap._verbose_message('{} matches {}', bytecode_path, |
| 1151 |
source_path) |
| 1152 |
return _compile_bytecode(bytes_data, name=fullname, |
| 1153 |
bytecode_path=bytecode_path, |
| 1154 |
source_path=source_path) |
| 1155 |
if source_bytes is None: |
| 1156 |
source_bytes = self.get_data(source_path) |
| 1157 |
code_object = self.source_to_code(source_bytes, source_path) |
| 1158 |
_bootstrap._verbose_message('code object from {}', source_path) |
| 1159 |
if (not sys.dont_write_bytecode and bytecode_path is not None and |
| 1160 |
source_mtime is not None): |
| 1161 |
if hash_based: |
| 1162 |
if source_hash is None: |
| 1163 |
source_hash = _imp.source_hash(_RAW_MAGIC_NUMBER, |
| 1164 |
source_bytes) |
| 1165 |
data = _code_to_hash_pyc(code_object, source_hash, check_source) |
| 1166 |
else: |
| 1167 |
data = _code_to_timestamp_pyc(code_object, source_mtime, |
| 1168 |
len(source_bytes)) |
| 1169 |
try: |
| 1170 |
self._cache_bytecode(source_path, bytecode_path, data) |
| 1171 |
except NotImplementedError: |
| 1172 |
pass |
| 1173 |
return code_object |
| 1174 |
|
| 1175 |
|
| 1176 |
class FileLoader: |
| 1177 |
|
| 1178 |
"""Base file loader class which implements the loader protocol methods that |
| 1179 |
require file system usage.""" |
| 1180 |
|
| 1181 |
def __init__(self, fullname, path): |
| 1182 |
"""Cache the module name and the path to the file found by the |
| 1183 |
finder.""" |
| 1184 |
self.name = fullname |
| 1185 |
self.path = path |
| 1186 |
|
| 1187 |
def __eq__(self, other): |
| 1188 |
return (self.__class__ == other.__class__ and |
| 1189 |
self.__dict__ == other.__dict__) |
| 1190 |
|
| 1191 |
def __hash__(self): |
| 1192 |
return hash(self.name) ^ hash(self.path) |
| 1193 |
|
| 1194 |
@_check_name |
| 1195 |
def load_module(self, fullname): |
| 1196 |
"""Load a module from a file. |
| 1197 |
|
| 1198 |
This method is deprecated. Use exec_module() instead. |
| 1199 |
|
| 1200 |
""" |
| 1201 |
# The only reason for this method is for the name check. |
| 1202 |
# Issue #14857: Avoid the zero-argument form of super so the implementation |
| 1203 |
# of that form can be updated without breaking the frozen module. |
| 1204 |
return super(FileLoader, self).load_module(fullname) |
| 1205 |
|
| 1206 |
@_check_name |
| 1207 |
def get_filename(self, fullname): |
| 1208 |
"""Return the path to the source file as found by the finder.""" |
| 1209 |
return self.path |
| 1210 |
|
| 1211 |
def get_data(self, path): |
| 1212 |
"""Return the data from path as raw bytes.""" |
| 1213 |
if isinstance(self, (SourceLoader, SourcelessFileLoader, ExtensionFileLoader)): |
| 1214 |
with _io.open_code(str(path)) as file: |
| 1215 |
return file.read() |
| 1216 |
else: |
| 1217 |
with _io.FileIO(path, 'r') as file: |
| 1218 |
return file.read() |
| 1219 |
|
| 1220 |
@_check_name |
| 1221 |
def get_resource_reader(self, module): |
| 1222 |
from importlib.readers import FileReader |
| 1223 |
return FileReader(self) |
| 1224 |
|
| 1225 |
|
| 1226 |
class SourceFileLoader(FileLoader, SourceLoader): |
| 1227 |
|
| 1228 |
"""Concrete implementation of SourceLoader using the file system.""" |
| 1229 |
|
| 1230 |
def path_stats(self, path): |
| 1231 |
"""Return the metadata for the path.""" |
| 1232 |
st = _path_stat(path) |
| 1233 |
return {'mtime': st.st_mtime, 'size': st.st_size} |
| 1234 |
|
| 1235 |
def _cache_bytecode(self, source_path, bytecode_path, data): |
| 1236 |
# Adapt between the two APIs |
| 1237 |
mode = _calc_mode(source_path) |
| 1238 |
return self.set_data(bytecode_path, data, _mode=mode) |
| 1239 |
|
| 1240 |
def set_data(self, path, data, *, _mode=0o666): |
| 1241 |
"""Write bytes data to a file.""" |
| 1242 |
parent, filename = _path_split(path) |
| 1243 |
path_parts = [] |
| 1244 |
# Figure out what directories are missing. |
| 1245 |
while parent and not _path_isdir(parent): |
| 1246 |
parent, part = _path_split(parent) |
| 1247 |
path_parts.append(part) |
| 1248 |
# Create needed directories. |
| 1249 |
for part in reversed(path_parts): |
| 1250 |
parent = _path_join(parent, part) |
| 1251 |
try: |
| 1252 |
_os.mkdir(parent) |
| 1253 |
except FileExistsError: |
| 1254 |
# Probably another Python process already created the dir. |
| 1255 |
continue |
| 1256 |
except OSError as exc: |
| 1257 |
# Could be a permission error, read-only filesystem: just forget |
| 1258 |
# about writing the data. |
| 1259 |
_bootstrap._verbose_message('could not create {!r}: {!r}', |
| 1260 |
parent, exc) |
| 1261 |
return |
| 1262 |
try: |
| 1263 |
_write_atomic(path, data, _mode) |
| 1264 |
_bootstrap._verbose_message('created {!r}', path) |
| 1265 |
except OSError as exc: |
| 1266 |
# Same as above: just don't write the bytecode. |
| 1267 |
_bootstrap._verbose_message('could not create {!r}: {!r}', path, |
| 1268 |
exc) |
| 1269 |
|
| 1270 |
|
| 1271 |
class SourcelessFileLoader(FileLoader, _LoaderBasics): |
| 1272 |
|
| 1273 |
"""Loader which handles sourceless file imports.""" |
| 1274 |
|
| 1275 |
def get_code(self, fullname): |
| 1276 |
path = self.get_filename(fullname) |
| 1277 |
data = self.get_data(path) |
| 1278 |
# Call _classify_pyc to do basic validation of the pyc but ignore the |
| 1279 |
# result. There's no source to check against. |
| 1280 |
exc_details = { |
| 1281 |
'name': fullname, |
| 1282 |
'path': path, |
| 1283 |
} |
| 1284 |
_classify_pyc(data, fullname, exc_details) |
| 1285 |
return _compile_bytecode( |
| 1286 |
memoryview(data)[16:], |
| 1287 |
name=fullname, |
| 1288 |
bytecode_path=path, |
| 1289 |
) |
| 1290 |
|
| 1291 |
def get_source(self, fullname): |
| 1292 |
"""Return None as there is no source code.""" |
| 1293 |
return None |
| 1294 |
|
| 1295 |
|
| 1296 |
class ExtensionFileLoader(FileLoader, _LoaderBasics): |
| 1297 |
|
| 1298 |
"""Loader for extension modules. |
| 1299 |
|
| 1300 |
The constructor is designed to work with FileFinder. |
| 1301 |
|
| 1302 |
""" |
| 1303 |
|
| 1304 |
def __init__(self, name, path): |
| 1305 |
self.name = name |
| 1306 |
self.path = path |
| 1307 |
|
| 1308 |
def __eq__(self, other): |
| 1309 |
return (self.__class__ == other.__class__ and |
| 1310 |
self.__dict__ == other.__dict__) |
| 1311 |
|
| 1312 |
def __hash__(self): |
| 1313 |
return hash(self.name) ^ hash(self.path) |
| 1314 |
|
| 1315 |
def create_module(self, spec): |
| 1316 |
"""Create an uninitialized extension module""" |
| 1317 |
module = _bootstrap._call_with_frames_removed( |
| 1318 |
_imp.create_dynamic, spec) |
| 1319 |
_bootstrap._verbose_message('extension module {!r} loaded from {!r}', |
| 1320 |
spec.name, self.path) |
| 1321 |
return module |
| 1322 |
|
| 1323 |
def exec_module(self, module): |
| 1324 |
"""Initialize an extension module""" |
| 1325 |
_bootstrap._call_with_frames_removed(_imp.exec_dynamic, module) |
| 1326 |
_bootstrap._verbose_message('extension module {!r} executed from {!r}', |
| 1327 |
self.name, self.path) |
| 1328 |
|
| 1329 |
def is_package(self, fullname): |
| 1330 |
"""Return True if the extension module is a package.""" |
| 1331 |
file_name = _path_split(self.path)[1] |
| 1332 |
return any(file_name == '__init__' + suffix |
| 1333 |
for suffix in EXTENSION_SUFFIXES) |
| 1334 |
|
| 1335 |
def get_code(self, fullname): |
| 1336 |
"""Return None as an extension module cannot create a code object.""" |
| 1337 |
return None |
| 1338 |
|
| 1339 |
def get_source(self, fullname): |
| 1340 |
"""Return None as extension modules have no source code.""" |
| 1341 |
return None |
| 1342 |
|
| 1343 |
@_check_name |
| 1344 |
def get_filename(self, fullname): |
| 1345 |
"""Return the path to the source file as found by the finder.""" |
| 1346 |
return self.path |
| 1347 |
|
| 1348 |
|
| 1349 |
class _NamespacePath: |
| 1350 |
"""Represents a namespace package's path. It uses the module name |
| 1351 |
to find its parent module, and from there it looks up the parent's |
| 1352 |
__path__. When this changes, the module's own path is recomputed, |
| 1353 |
using path_finder. For top-level modules, the parent module's path |
| 1354 |
is sys.path.""" |
| 1355 |
|
| 1356 |
# When invalidate_caches() is called, this epoch is incremented |
| 1357 |
# https://bugs.python.org/issue45703 |
| 1358 |
_epoch = 0 |
| 1359 |
|
| 1360 |
def __init__(self, name, path, path_finder): |
| 1361 |
self._name = name |
| 1362 |
self._path = path |
| 1363 |
self._last_parent_path = tuple(self._get_parent_path()) |
| 1364 |
self._last_epoch = self._epoch |
| 1365 |
self._path_finder = path_finder |
| 1366 |
|
| 1367 |
def _find_parent_path_names(self): |
| 1368 |
"""Returns a tuple of (parent-module-name, parent-path-attr-name)""" |
| 1369 |
parent, dot, me = self._name.rpartition('.') |
| 1370 |
if dot == '': |
| 1371 |
# This is a top-level module. sys.path contains the parent path. |
| 1372 |
return 'sys', 'path' |
| 1373 |
# Not a top-level module. parent-module.__path__ contains the |
| 1374 |
# parent path. |
| 1375 |
return parent, '__path__' |
| 1376 |
|
| 1377 |
def _get_parent_path(self): |
| 1378 |
parent_module_name, path_attr_name = self._find_parent_path_names() |
| 1379 |
return getattr(sys.modules[parent_module_name], path_attr_name) |
| 1380 |
|
| 1381 |
def _recalculate(self): |
| 1382 |
# If the parent's path has changed, recalculate _path |
| 1383 |
parent_path = tuple(self._get_parent_path()) # Make a copy |
| 1384 |
if parent_path != self._last_parent_path or self._epoch != self._last_epoch: |
| 1385 |
spec = self._path_finder(self._name, parent_path) |
| 1386 |
# Note that no changes are made if a loader is returned, but we |
| 1387 |
# do remember the new parent path |
| 1388 |
if spec is not None and spec.loader is None: |
| 1389 |
if spec.submodule_search_locations: |
| 1390 |
self._path = spec.submodule_search_locations |
| 1391 |
self._last_parent_path = parent_path # Save the copy |
| 1392 |
self._last_epoch = self._epoch |
| 1393 |
return self._path |
| 1394 |
|
| 1395 |
def __iter__(self): |
| 1396 |
return iter(self._recalculate()) |
| 1397 |
|
| 1398 |
def __getitem__(self, index): |
| 1399 |
return self._recalculate()[index] |
| 1400 |
|
| 1401 |
def __setitem__(self, index, path): |
| 1402 |
self._path[index] = path |
| 1403 |
|
| 1404 |
def __len__(self): |
| 1405 |
return len(self._recalculate()) |
| 1406 |
|
| 1407 |
def __repr__(self): |
| 1408 |
return f'_NamespacePath({self._path!r})' |
| 1409 |
|
| 1410 |
def __contains__(self, item): |
| 1411 |
return item in self._recalculate() |
| 1412 |
|
| 1413 |
def append(self, item): |
| 1414 |
self._path.append(item) |
| 1415 |
|
| 1416 |
|
| 1417 |
# This class is actually exposed publicly in a namespace package's __loader__ |
| 1418 |
# attribute, so it should be available through a non-private name. |
| 1419 |
# https://github.com/python/cpython/issues/92054 |
| 1420 |
class NamespaceLoader: |
| 1421 |
def __init__(self, name, path, path_finder): |
| 1422 |
self._path = _NamespacePath(name, path, path_finder) |
| 1423 |
|
| 1424 |
def is_package(self, fullname): |
| 1425 |
return True |
| 1426 |
|
| 1427 |
def get_source(self, fullname): |
| 1428 |
return '' |
| 1429 |
|
| 1430 |
def get_code(self, fullname): |
| 1431 |
return compile('', '<string>', 'exec', dont_inherit=True) |
| 1432 |
|
| 1433 |
def create_module(self, spec): |
| 1434 |
"""Use default semantics for module creation.""" |
| 1435 |
|
| 1436 |
def exec_module(self, module): |
| 1437 |
pass |
| 1438 |
|
| 1439 |
def load_module(self, fullname): |
| 1440 |
"""Load a namespace module. |
| 1441 |
|
| 1442 |
This method is deprecated. Use exec_module() instead. |
| 1443 |
|
| 1444 |
""" |
| 1445 |
# The import system never calls this method. |
| 1446 |
_bootstrap._verbose_message('namespace module loaded with path {!r}', |
| 1447 |
self._path) |
| 1448 |
# Warning implemented in _load_module_shim(). |
| 1449 |
return _bootstrap._load_module_shim(self, fullname) |
| 1450 |
|
| 1451 |
def get_resource_reader(self, module): |
| 1452 |
from importlib.readers import NamespaceReader |
| 1453 |
return NamespaceReader(self._path) |
| 1454 |
|
| 1455 |
|
| 1456 |
# We use this exclusively in module_from_spec() for backward-compatibility. |
| 1457 |
_NamespaceLoader = NamespaceLoader |
| 1458 |
|
| 1459 |
|
| 1460 |
# Finders ##################################################################### |
| 1461 |
|
| 1462 |
class PathFinder: |
| 1463 |
|
| 1464 |
"""Meta path finder for sys.path and package __path__ attributes.""" |
| 1465 |
|
| 1466 |
@staticmethod |
| 1467 |
def invalidate_caches(): |
| 1468 |
"""Call the invalidate_caches() method on all path entry finders |
| 1469 |
stored in sys.path_importer_cache (where implemented).""" |
| 1470 |
for name, finder in list(sys.path_importer_cache.items()): |
| 1471 |
# Drop entry if finder name is a relative path. The current |
| 1472 |
# working directory may have changed. |
| 1473 |
if finder is None or not _path_isabs(name): |
| 1474 |
del sys.path_importer_cache[name] |
| 1475 |
elif hasattr(finder, 'invalidate_caches'): |
| 1476 |
finder.invalidate_caches() |
| 1477 |
# Also invalidate the caches of _NamespacePaths |
| 1478 |
# https://bugs.python.org/issue45703 |
| 1479 |
_NamespacePath._epoch += 1 |
| 1480 |
|
| 1481 |
from importlib.metadata import MetadataPathFinder |
| 1482 |
MetadataPathFinder.invalidate_caches() |
| 1483 |
|
| 1484 |
@staticmethod |
| 1485 |
def _path_hooks(path): |
| 1486 |
"""Search sys.path_hooks for a finder for 'path'.""" |
| 1487 |
if sys.path_hooks is not None and not sys.path_hooks: |
| 1488 |
_warnings.warn('sys.path_hooks is empty', ImportWarning) |
| 1489 |
for hook in sys.path_hooks: |
| 1490 |
try: |
| 1491 |
return hook(path) |
| 1492 |
except ImportError: |
| 1493 |
continue |
| 1494 |
else: |
| 1495 |
return None |
| 1496 |
|
| 1497 |
@classmethod |
| 1498 |
def _path_importer_cache(cls, path): |
| 1499 |
"""Get the finder for the path entry from sys.path_importer_cache. |
| 1500 |
|
| 1501 |
If the path entry is not in the cache, find the appropriate finder |
| 1502 |
and cache it. If no finder is available, store None. |
| 1503 |
|
| 1504 |
""" |
| 1505 |
if path == '': |
| 1506 |
try: |
| 1507 |
path = _os.getcwd() |
| 1508 |
except FileNotFoundError: |
| 1509 |
# Don't cache the failure as the cwd can easily change to |
| 1510 |
# a valid directory later on. |
| 1511 |
return None |
| 1512 |
try: |
| 1513 |
finder = sys.path_importer_cache[path] |
| 1514 |
except KeyError: |
| 1515 |
finder = cls._path_hooks(path) |
| 1516 |
sys.path_importer_cache[path] = finder |
| 1517 |
return finder |
| 1518 |
|
| 1519 |
@classmethod |
| 1520 |
def _get_spec(cls, fullname, path, target=None): |
| 1521 |
"""Find the loader or namespace_path for this module/package name.""" |
| 1522 |
# If this ends up being a namespace package, namespace_path is |
| 1523 |
# the list of paths that will become its __path__ |
| 1524 |
namespace_path = [] |
| 1525 |
for entry in path: |
| 1526 |
if not isinstance(entry, str): |
| 1527 |
continue |
| 1528 |
finder = cls._path_importer_cache(entry) |
| 1529 |
if finder is not None: |
| 1530 |
spec = finder.find_spec(fullname, target) |
| 1531 |
if spec is None: |
| 1532 |
continue |
| 1533 |
if spec.loader is not None: |
| 1534 |
return spec |
| 1535 |
portions = spec.submodule_search_locations |
| 1536 |
if portions is None: |
| 1537 |
raise ImportError('spec missing loader') |
| 1538 |
# This is possibly part of a namespace package. |
| 1539 |
# Remember these path entries (if any) for when we |
| 1540 |
# create a namespace package, and continue iterating |
| 1541 |
# on path. |
| 1542 |
namespace_path.extend(portions) |
| 1543 |
else: |
| 1544 |
spec = _bootstrap.ModuleSpec(fullname, None) |
| 1545 |
spec.submodule_search_locations = namespace_path |
| 1546 |
return spec |
| 1547 |
|
| 1548 |
@classmethod |
| 1549 |
def find_spec(cls, fullname, path=None, target=None): |
| 1550 |
"""Try to find a spec for 'fullname' on sys.path or 'path'. |
| 1551 |
|
| 1552 |
The search is based on sys.path_hooks and sys.path_importer_cache. |
| 1553 |
""" |
| 1554 |
if path is None: |
| 1555 |
path = sys.path |
| 1556 |
spec = cls._get_spec(fullname, path, target) |
| 1557 |
if spec is None: |
| 1558 |
return None |
| 1559 |
elif spec.loader is None: |
| 1560 |
namespace_path = spec.submodule_search_locations |
| 1561 |
if namespace_path: |
| 1562 |
# We found at least one namespace path. Return a spec which |
| 1563 |
# can create the namespace package. |
| 1564 |
spec.origin = None |
| 1565 |
spec.submodule_search_locations = _NamespacePath(fullname, namespace_path, cls._get_spec) |
| 1566 |
return spec |
| 1567 |
else: |
| 1568 |
return None |
| 1569 |
else: |
| 1570 |
return spec |
| 1571 |
|
| 1572 |
@staticmethod |
| 1573 |
def find_distributions(*args, **kwargs): |
| 1574 |
""" |
| 1575 |
Find distributions. |
| 1576 |
|
| 1577 |
Return an iterable of all Distribution instances capable of |
| 1578 |
loading the metadata for packages matching ``context.name`` |
| 1579 |
(or all names if ``None`` indicated) along the paths in the list |
| 1580 |
of directories ``context.path``. |
| 1581 |
""" |
| 1582 |
from importlib.metadata import MetadataPathFinder |
| 1583 |
return MetadataPathFinder.find_distributions(*args, **kwargs) |
| 1584 |
|
| 1585 |
|
| 1586 |
class FileFinder: |
| 1587 |
|
| 1588 |
"""File-based finder. |
| 1589 |
|
| 1590 |
Interactions with the file system are cached for performance, being |
| 1591 |
refreshed when the directory the finder is handling has been modified. |
| 1592 |
|
| 1593 |
""" |
| 1594 |
|
| 1595 |
def __init__(self, path, *loader_details): |
| 1596 |
"""Initialize with the path to search on and a variable number of |
| 1597 |
2-tuples containing the loader and the file suffixes the loader |
| 1598 |
recognizes.""" |
| 1599 |
loaders = [] |
| 1600 |
for loader, suffixes in loader_details: |
| 1601 |
loaders.extend((suffix, loader) for suffix in suffixes) |
| 1602 |
self._loaders = loaders |
| 1603 |
# Base (directory) path |
| 1604 |
if not path or path == '.': |
| 1605 |
self.path = _os.getcwd() |
| 1606 |
else: |
| 1607 |
self.path = _path_abspath(path) |
| 1608 |
self._path_mtime = -1 |
| 1609 |
self._path_cache = set() |
| 1610 |
self._relaxed_path_cache = set() |
| 1611 |
|
| 1612 |
def invalidate_caches(self): |
| 1613 |
"""Invalidate the directory mtime.""" |
| 1614 |
self._path_mtime = -1 |
| 1615 |
|
| 1616 |
def _get_spec(self, loader_class, fullname, path, smsl, target): |
| 1617 |
loader = loader_class(fullname, path) |
| 1618 |
return spec_from_file_location(fullname, path, loader=loader, |
| 1619 |
submodule_search_locations=smsl) |
| 1620 |
|
| 1621 |
def find_spec(self, fullname, target=None): |
| 1622 |
"""Try to find a spec for the specified module. |
| 1623 |
|
| 1624 |
Returns the matching spec, or None if not found. |
| 1625 |
""" |
| 1626 |
is_namespace = False |
| 1627 |
tail_module = fullname.rpartition('.')[2] |
| 1628 |
try: |
| 1629 |
mtime = _path_stat(self.path or _os.getcwd()).st_mtime |
| 1630 |
except OSError: |
| 1631 |
mtime = -1 |
| 1632 |
if mtime != self._path_mtime: |
| 1633 |
self._fill_cache() |
| 1634 |
self._path_mtime = mtime |
| 1635 |
# tail_module keeps the original casing, for __file__ and friends |
| 1636 |
if _relax_case(): |
| 1637 |
cache = self._relaxed_path_cache |
| 1638 |
cache_module = tail_module.lower() |
| 1639 |
else: |
| 1640 |
cache = self._path_cache |
| 1641 |
cache_module = tail_module |
| 1642 |
# Check if the module is the name of a directory (and thus a package). |
| 1643 |
if cache_module in cache: |
| 1644 |
base_path = _path_join(self.path, tail_module) |
| 1645 |
for suffix, loader_class in self._loaders: |
| 1646 |
init_filename = '__init__' + suffix |
| 1647 |
full_path = _path_join(base_path, init_filename) |
| 1648 |
if _path_isfile(full_path): |
| 1649 |
return self._get_spec(loader_class, fullname, full_path, [base_path], target) |
| 1650 |
else: |
| 1651 |
# If a namespace package, return the path if we don't |
| 1652 |
# find a module in the next section. |
| 1653 |
is_namespace = _path_isdir(base_path) |
| 1654 |
# Check for a file w/ a proper suffix exists. |
| 1655 |
for suffix, loader_class in self._loaders: |
| 1656 |
try: |
| 1657 |
full_path = _path_join(self.path, tail_module + suffix) |
| 1658 |
except ValueError: |
| 1659 |
return None |
| 1660 |
_bootstrap._verbose_message('trying {}', full_path, verbosity=2) |
| 1661 |
if cache_module + suffix in cache: |
| 1662 |
if _path_isfile(full_path): |
| 1663 |
return self._get_spec(loader_class, fullname, full_path, |
| 1664 |
None, target) |
| 1665 |
if is_namespace: |
| 1666 |
_bootstrap._verbose_message('possible namespace for {}', base_path) |
| 1667 |
spec = _bootstrap.ModuleSpec(fullname, None) |
| 1668 |
spec.submodule_search_locations = [base_path] |
| 1669 |
return spec |
| 1670 |
return None |
| 1671 |
|
| 1672 |
def _fill_cache(self): |
| 1673 |
"""Fill the cache of potential modules and packages for this directory.""" |
| 1674 |
path = self.path |
| 1675 |
try: |
| 1676 |
contents = _os.listdir(path or _os.getcwd()) |
| 1677 |
except (FileNotFoundError, PermissionError, NotADirectoryError): |
| 1678 |
# Directory has either been removed, turned into a file, or made |
| 1679 |
# unreadable. |
| 1680 |
contents = [] |
| 1681 |
# We store two cached versions, to handle runtime changes of the |
| 1682 |
# PYTHONCASEOK environment variable. |
| 1683 |
if not sys.platform.startswith('win'): |
| 1684 |
self._path_cache = set(contents) |
| 1685 |
else: |
| 1686 |
# Windows users can import modules with case-insensitive file |
| 1687 |
# suffixes (for legacy reasons). Make the suffix lowercase here |
| 1688 |
# so it's done once instead of for every import. This is safe as |
| 1689 |
# the specified suffixes to check against are always specified in a |
| 1690 |
# case-sensitive manner. |
| 1691 |
lower_suffix_contents = set() |
| 1692 |
for item in contents: |
| 1693 |
name, dot, suffix = item.partition('.') |
| 1694 |
if dot: |
| 1695 |
new_name = f'{name}.{suffix.lower()}' |
| 1696 |
else: |
| 1697 |
new_name = name |
| 1698 |
lower_suffix_contents.add(new_name) |
| 1699 |
self._path_cache = lower_suffix_contents |
| 1700 |
if sys.platform.startswith(_CASE_INSENSITIVE_PLATFORMS): |
| 1701 |
self._relaxed_path_cache = {fn.lower() for fn in contents} |
| 1702 |
|
| 1703 |
@classmethod |
| 1704 |
def path_hook(cls, *loader_details): |
| 1705 |
"""A class method which returns a closure to use on sys.path_hook |
| 1706 |
which will return an instance using the specified loaders and the path |
| 1707 |
called on the closure. |
| 1708 |
|
| 1709 |
If the path called on the closure is not a directory, ImportError is |
| 1710 |
raised. |
| 1711 |
|
| 1712 |
""" |
| 1713 |
def path_hook_for_FileFinder(path): |
| 1714 |
"""Path hook for importlib.machinery.FileFinder.""" |
| 1715 |
if not _path_isdir(path): |
| 1716 |
raise ImportError('only directories are supported', path=path) |
| 1717 |
return cls(path, *loader_details) |
| 1718 |
|
| 1719 |
return path_hook_for_FileFinder |
| 1720 |
|
| 1721 |
def __repr__(self): |
| 1722 |
return f'FileFinder({self.path!r})' |
| 1723 |
|
| 1724 |
|
| 1725 |
class AppleFrameworkLoader(ExtensionFileLoader): |
| 1726 |
"""A loader for modules that have been packaged as frameworks for |
| 1727 |
compatibility with Apple's iOS App Store policies. |
| 1728 |
""" |
| 1729 |
def create_module(self, spec): |
| 1730 |
# If the ModuleSpec has been created by the FileFinder, it will have |
| 1731 |
# been created with an origin pointing to the .fwork file. We need to |
| 1732 |
# redirect this to the location in the Frameworks folder, using the |
| 1733 |
# content of the .fwork file. |
| 1734 |
if spec.origin.endswith(".fwork"): |
| 1735 |
with _io.FileIO(spec.origin, 'r') as file: |
| 1736 |
framework_binary = file.read().decode().strip() |
| 1737 |
bundle_path = _path_split(sys.executable)[0] |
| 1738 |
spec.origin = _path_join(bundle_path, framework_binary) |
| 1739 |
|
| 1740 |
# If the loader is created based on the spec for a loaded module, the |
| 1741 |
# path will be pointing at the Framework location. If this occurs, |
| 1742 |
# get the original .fwork location to use as the module's __file__. |
| 1743 |
if self.path.endswith(".fwork"): |
| 1744 |
path = self.path |
| 1745 |
else: |
| 1746 |
with _io.FileIO(self.path + ".origin", 'r') as file: |
| 1747 |
origin = file.read().decode().strip() |
| 1748 |
bundle_path = _path_split(sys.executable)[0] |
| 1749 |
path = _path_join(bundle_path, origin) |
| 1750 |
|
| 1751 |
module = _bootstrap._call_with_frames_removed(_imp.create_dynamic, spec) |
| 1752 |
|
| 1753 |
_bootstrap._verbose_message( |
| 1754 |
"Apple framework extension module {!r} loaded from {!r} (path {!r})", |
| 1755 |
spec.name, |
| 1756 |
spec.origin, |
| 1757 |
path, |
| 1758 |
) |
| 1759 |
|
| 1760 |
# Ensure that the __file__ points at the .fwork location |
| 1761 |
try: |
| 1762 |
module.__file__ = path |
| 1763 |
except AttributeError: |
| 1764 |
# Not important enough to report. |
| 1765 |
# (The error is also ignored in _bootstrap._init_module_attrs or |
| 1766 |
# import_run_extension in import.c) |
| 1767 |
pass |
| 1768 |
|
| 1769 |
return module |
| 1770 |
|
| 1771 |
# Import setup ############################################################### |
| 1772 |
|
| 1773 |
def _fix_up_module(ns, name, pathname, cpathname=None): |
| 1774 |
# This function is used by PyImport_ExecCodeModuleObject(). |
| 1775 |
loader = ns.get('__loader__') |
| 1776 |
spec = ns.get('__spec__') |
| 1777 |
if not loader: |
| 1778 |
if spec: |
| 1779 |
loader = spec.loader |
| 1780 |
elif pathname == cpathname: |
| 1781 |
loader = SourcelessFileLoader(name, pathname) |
| 1782 |
else: |
| 1783 |
loader = SourceFileLoader(name, pathname) |
| 1784 |
if not spec: |
| 1785 |
spec = spec_from_file_location(name, pathname, loader=loader) |
| 1786 |
if cpathname: |
| 1787 |
spec.cached = _path_abspath(cpathname) |
| 1788 |
try: |
| 1789 |
ns['__spec__'] = spec |
| 1790 |
ns['__loader__'] = loader |
| 1791 |
ns['__file__'] = pathname |
| 1792 |
ns['__cached__'] = cpathname |
| 1793 |
except Exception: |
| 1794 |
# Not important enough to report. |
| 1795 |
pass |
| 1796 |
|
| 1797 |
|
| 1798 |
def _get_supported_file_loaders(): |
| 1799 |
"""Returns a list of file-based module loaders. |
| 1800 |
|
| 1801 |
Each item is a tuple (loader, suffixes). |
| 1802 |
""" |
| 1803 |
extension_loaders = [] |
| 1804 |
if hasattr(_imp, 'create_dynamic'): |
| 1805 |
if sys.platform in {"ios", "tvos", "watchos"}: |
| 1806 |
extension_loaders = [(AppleFrameworkLoader, [ |
| 1807 |
suffix.replace(".so", ".fwork") |
| 1808 |
for suffix in _imp.extension_suffixes() |
| 1809 |
])] |
| 1810 |
extension_loaders.append((ExtensionFileLoader, _imp.extension_suffixes())) |
| 1811 |
source = SourceFileLoader, SOURCE_SUFFIXES |
| 1812 |
bytecode = SourcelessFileLoader, BYTECODE_SUFFIXES |
| 1813 |
return extension_loaders + [source, bytecode] |
| 1814 |
|
| 1815 |
|
| 1816 |
def _set_bootstrap_module(_bootstrap_module): |
| 1817 |
global _bootstrap |
| 1818 |
_bootstrap = _bootstrap_module |
| 1819 |
|
| 1820 |
|
| 1821 |
def _install(_bootstrap_module): |
| 1822 |
"""Install the path-based import components.""" |
| 1823 |
_set_bootstrap_module(_bootstrap_module) |
| 1824 |
supported_loaders = _get_supported_file_loaders() |
| 1825 |
sys.path_hooks.extend([FileFinder.path_hook(*supported_loaders)]) |
| 1826 |
sys.meta_path.append(PathFinder) |