Debian Codegraph @

Lib/importlib/_bootstrap_external.py

raw
1 """Core implementation of path-based import.
2
3 This module is NOT meant to be directly imported! It has been designed such
4 that it can be bootstrapped into Python as the implementation of import. As
5 such it requires the injection of specific modules and attributes in order to
6 work. One should use importlib as the public-facing version of this module.
7
8 """
9 # IMPORTANT: Whenever making changes to this module, be sure to run a top-level
10 # `make regen-importlib` followed by `make` in order to get the frozen version
11 # of the module updated. Not doing so will result in the Makefile to fail for
12 # all others who don't have a ./python around to freeze the module in the early
13 # stages of compilation.
14 #
15
16 # See importlib._setup() for what is injected into the global namespace.
17
18 # When editing this code be aware that code executed at import time CANNOT
19 # reference any injected objects! This includes not only global code but also
20 # anything specified at the class level.
21
22 # Module injected manually by _set_bootstrap_module()
23 _bootstrap = None
24
25 # Import builtin modules
26 import _imp
27 import _io
28 import sys
29 import _warnings
30 import marshal
31
32
33 _MS_WINDOWS = (sys.platform == 'win32')
34 if _MS_WINDOWS:
35 import nt as _os
36 import winreg
37 else:
38 import posix as _os
39
40
41 if _MS_WINDOWS:
42 path_separators = ['\\', '/']
43 else:
44 path_separators = ['/']
45 # Assumption made in _path_join()
46 assert all(len(sep) == 1 for sep in path_separators)
47 path_sep = path_separators[0]
48 path_sep_tuple = tuple(path_separators)
49 path_separators = ''.join(path_separators)
50 _pathseps_with_colon = {f':{s}' for s in path_separators}
51
52
53 # Bootstrap-related code ######################################################
54 _CASE_INSENSITIVE_PLATFORMS_STR_KEY = 'win',
55 _CASE_INSENSITIVE_PLATFORMS_BYTES_KEY = 'cygwin', 'darwin', 'ios', 'tvos', 'watchos'
56 _CASE_INSENSITIVE_PLATFORMS = (_CASE_INSENSITIVE_PLATFORMS_BYTES_KEY
57 + _CASE_INSENSITIVE_PLATFORMS_STR_KEY)
58
59
60 def _make_relax_case():
61 if sys.platform.startswith(_CASE_INSENSITIVE_PLATFORMS):
62 if sys.platform.startswith(_CASE_INSENSITIVE_PLATFORMS_STR_KEY):
63 key = 'PYTHONCASEOK'
64 else:
65 key = b'PYTHONCASEOK'
66
67 def _relax_case():
68 """True if filenames must be checked case-insensitively and ignore environment flags are not set."""
69 return not sys.flags.ignore_environment and key in _os.environ
70 else:
71 def _relax_case():
72 """True if filenames must be checked case-insensitively."""
73 return False
74 return _relax_case
75
76 _relax_case = _make_relax_case()
77
78
79 def _pack_uint32(x):
80 """Convert a 32-bit integer to little-endian."""
81 return (int(x) & 0xFFFFFFFF).to_bytes(4, 'little')
82
83
84 def _unpack_uint64(data):
85 """Convert 8 bytes in little-endian to an integer."""
86 assert len(data) == 8
87 return int.from_bytes(data, 'little')
88
89 def _unpack_uint32(data):
90 """Convert 4 bytes in little-endian to an integer."""
91 assert len(data) == 4
92 return int.from_bytes(data, 'little')
93
94 def _unpack_uint16(data):
95 """Convert 2 bytes in little-endian to an integer."""
96 assert len(data) == 2
97 return int.from_bytes(data, 'little')
98
99
100 if _MS_WINDOWS:
101 def _path_join(*path_parts):
102 """Replacement for os.path.join()."""
103 if not path_parts:
104 return ""
105 if len(path_parts) == 1:
106 return path_parts[0]
107 root = ""
108 path = []
109 for new_root, tail in map(_os._path_splitroot, path_parts):
110 if new_root.startswith(path_sep_tuple) or new_root.endswith(path_sep_tuple):
111 root = new_root.rstrip(path_separators) or root
112 path = [path_sep + tail]
113 elif new_root.endswith(':'):
114 if root.casefold() != new_root.casefold():
115 # Drive relative paths have to be resolved by the OS, so we reset the
116 # tail but do not add a path_sep prefix.
117 root = new_root
118 path = [tail]
119 else:
120 path.append(tail)
121 else:
122 root = new_root or root
123 path.append(tail)
124 path = [p.rstrip(path_separators) for p in path if p]
125 if len(path) == 1 and not path[0]:
126 # Avoid losing the root's trailing separator when joining with nothing
127 return root + path_sep
128 return root + path_sep.join(path)
129
130 else:
131 def _path_join(*path_parts):
132 """Replacement for os.path.join()."""
133 return path_sep.join([part.rstrip(path_separators)
134 for part in path_parts if part])
135
136
137 def _path_split(path):
138 """Replacement for os.path.split()."""
139 i = max(path.rfind(p) for p in path_separators)
140 if i < 0:
141 return '', path
142 return path[:i], path[i + 1:]
143
144
145 def _path_stat(path):
146 """Stat the path.
147
148 Made a separate function to make it easier to override in experiments
149 (e.g. cache stat results).
150
151 """
152 return _os.stat(path)
153
154
155 def _path_is_mode_type(path, mode):
156 """Test whether the path is the specified mode type."""
157 try:
158 stat_info = _path_stat(path)
159 except OSError:
160 return False
161 return (stat_info.st_mode & 0o170000) == mode
162
163
164 def _path_isfile(path):
165 """Replacement for os.path.isfile."""
166 return _path_is_mode_type(path, 0o100000)
167
168
169 def _path_isdir(path):
170 """Replacement for os.path.isdir."""
171 if not path:
172 path = _os.getcwd()
173 return _path_is_mode_type(path, 0o040000)
174
175
176 if _MS_WINDOWS:
177 def _path_isabs(path):
178 """Replacement for os.path.isabs."""
179 if not path:
180 return False
181 root = _os._path_splitroot(path)[0].replace('/', '\\')
182 return len(root) > 1 and (root.startswith('\\\\') or root.endswith('\\'))
183
184 else:
185 def _path_isabs(path):
186 """Replacement for os.path.isabs."""
187 return path.startswith(path_separators)
188
189
190 def _path_abspath(path):
191 """Replacement for os.path.abspath."""
192 if not _path_isabs(path):
193 for sep in path_separators:
194 path = path.removeprefix(f".{sep}")
195 return _path_join(_os.getcwd(), path)
196 else:
197 return path
198
199
200 def _write_atomic(path, data, mode=0o666):
201 """Best-effort function to write data to a path atomically.
202 Be prepared to handle a FileExistsError if concurrent writing of the
203 temporary file is attempted."""
204 # id() is used to generate a pseudo-random filename.
205 path_tmp = f'{path}.{id(path)}'
206 fd = _os.open(path_tmp,
207 _os.O_EXCL | _os.O_CREAT | _os.O_WRONLY, mode & 0o666)
208 try:
209 # We first write data to a temporary file, and then use os.replace() to
210 # perform an atomic rename.
211 with _io.open(fd, 'wb') as file:
212 file.write(data)
213 _os.replace(path_tmp, path)
214 except OSError:
215 try:
216 _os.unlink(path_tmp)
217 except OSError:
218 pass
219 raise
220
221
222 _code_type = type(_write_atomic.__code__)
223
224
225 # Finder/loader utility code ###############################################
226
227 # Magic word to reject .pyc files generated by other Python versions.
228 # It should change for each incompatible change to the bytecode.
229 #
230 # The value of CR and LF is incorporated so if you ever read or write
231 # a .pyc file in text mode the magic number will be wrong; also, the
232 # Apple MPW compiler swaps their values, botching string constants.
233 #
234 # There were a variety of old schemes for setting the magic number.
235 # The current working scheme is to increment the previous value by
236 # 10.
237 #
238 # Starting with the adoption of PEP 3147 in Python 3.2, every bump in magic
239 # number also includes a new "magic tag", i.e. a human readable string used
240 # to represent the magic number in __pycache__ directories. When you change
241 # the magic number, you must also set a new unique magic tag. Generally this
242 # can be named after the Python major version of the magic number bump, but
243 # it can really be anything, as long as it's different than anything else
244 # that's come before. The tags are included in the following table, starting
245 # with Python 3.2a0.
246 #
247 # Known values:
248 # Python 1.5: 20121
249 # Python 1.5.1: 20121
250 # Python 1.5.2: 20121
251 # Python 1.6: 50428
252 # Python 2.0: 50823
253 # Python 2.0.1: 50823
254 # Python 2.1: 60202
255 # Python 2.1.1: 60202
256 # Python 2.1.2: 60202
257 # Python 2.2: 60717
258 # Python 2.3a0: 62011
259 # Python 2.3a0: 62021
260 # Python 2.3a0: 62011 (!)
261 # Python 2.4a0: 62041
262 # Python 2.4a3: 62051
263 # Python 2.4b1: 62061
264 # Python 2.5a0: 62071
265 # Python 2.5a0: 62081 (ast-branch)
266 # Python 2.5a0: 62091 (with)
267 # Python 2.5a0: 62092 (changed WITH_CLEANUP opcode)
268 # Python 2.5b3: 62101 (fix wrong code: for x, in ...)
269 # Python 2.5b3: 62111 (fix wrong code: x += yield)
270 # Python 2.5c1: 62121 (fix wrong lnotab with for loops and
271 # storing constants that should have been removed)
272 # Python 2.5c2: 62131 (fix wrong code: for x, in ... in listcomp/genexp)
273 # Python 2.6a0: 62151 (peephole optimizations and STORE_MAP opcode)
274 # Python 2.6a1: 62161 (WITH_CLEANUP optimization)
275 # Python 2.7a0: 62171 (optimize list comprehensions/change LIST_APPEND)
276 # Python 2.7a0: 62181 (optimize conditional branches:
277 # introduce POP_JUMP_IF_FALSE and POP_JUMP_IF_TRUE)
278 # Python 2.7a0 62191 (introduce SETUP_WITH)
279 # Python 2.7a0 62201 (introduce BUILD_SET)
280 # Python 2.7a0 62211 (introduce MAP_ADD and SET_ADD)
281 # Python 3000: 3000
282 # 3010 (removed UNARY_CONVERT)
283 # 3020 (added BUILD_SET)
284 # 3030 (added keyword-only parameters)
285 # 3040 (added signature annotations)
286 # 3050 (print becomes a function)
287 # 3060 (PEP 3115 metaclass syntax)
288 # 3061 (string literals become unicode)
289 # 3071 (PEP 3109 raise changes)
290 # 3081 (PEP 3137 make __file__ and __name__ unicode)
291 # 3091 (kill str8 interning)
292 # 3101 (merge from 2.6a0, see 62151)
293 # 3103 (__file__ points to source file)
294 # Python 3.0a4: 3111 (WITH_CLEANUP optimization).
295 # Python 3.0b1: 3131 (lexical exception stacking, including POP_EXCEPT
296 #3021)
297 # Python 3.1a1: 3141 (optimize list, set and dict comprehensions:
298 # change LIST_APPEND and SET_ADD, add MAP_ADD #2183)
299 # Python 3.1a1: 3151 (optimize conditional branches:
300 # introduce POP_JUMP_IF_FALSE and POP_JUMP_IF_TRUE
301 #4715)
302 # Python 3.2a1: 3160 (add SETUP_WITH #6101)
303 # tag: cpython-32
304 # Python 3.2a2: 3170 (add DUP_TOP_TWO, remove DUP_TOPX and ROT_FOUR #9225)
305 # tag: cpython-32
306 # Python 3.2a3 3180 (add DELETE_DEREF #4617)
307 # Python 3.3a1 3190 (__class__ super closure changed)
308 # Python 3.3a1 3200 (PEP 3155 __qualname__ added #13448)
309 # Python 3.3a1 3210 (added size modulo 2**32 to the pyc header #13645)
310 # Python 3.3a2 3220 (changed PEP 380 implementation #14230)
311 # Python 3.3a4 3230 (revert changes to implicit __class__ closure #14857)
312 # Python 3.4a1 3250 (evaluate positional default arguments before
313 # keyword-only defaults #16967)
314 # Python 3.4a1 3260 (add LOAD_CLASSDEREF; allow locals of class to override
315 # free vars #17853)
316 # Python 3.4a1 3270 (various tweaks to the __class__ closure #12370)
317 # Python 3.4a1 3280 (remove implicit class argument)
318 # Python 3.4a4 3290 (changes to __qualname__ computation #19301)
319 # Python 3.4a4 3300 (more changes to __qualname__ computation #19301)
320 # Python 3.4rc2 3310 (alter __qualname__ computation #20625)
321 # Python 3.5a1 3320 (PEP 465: Matrix multiplication operator #21176)
322 # Python 3.5b1 3330 (PEP 448: Additional Unpacking Generalizations #2292)
323 # Python 3.5b2 3340 (fix dictionary display evaluation order #11205)
324 # Python 3.5b3 3350 (add GET_YIELD_FROM_ITER opcode #24400)
325 # Python 3.5.2 3351 (fix BUILD_MAP_UNPACK_WITH_CALL opcode #27286)
326 # Python 3.6a0 3360 (add FORMAT_VALUE opcode #25483)
327 # Python 3.6a1 3361 (lineno delta of code.co_lnotab becomes signed #26107)
328 # Python 3.6a2 3370 (16 bit wordcode #26647)
329 # Python 3.6a2 3371 (add BUILD_CONST_KEY_MAP opcode #27140)
330 # Python 3.6a2 3372 (MAKE_FUNCTION simplification, remove MAKE_CLOSURE
331 # #27095)
332 # Python 3.6b1 3373 (add BUILD_STRING opcode #27078)
333 # Python 3.6b1 3375 (add SETUP_ANNOTATIONS and STORE_ANNOTATION opcodes
334 # #27985)
335 # Python 3.6b1 3376 (simplify CALL_FUNCTIONs & BUILD_MAP_UNPACK_WITH_CALL
336 #27213)
337 # Python 3.6b1 3377 (set __class__ cell from type.__new__ #23722)
338 # Python 3.6b2 3378 (add BUILD_TUPLE_UNPACK_WITH_CALL #28257)
339 # Python 3.6rc1 3379 (more thorough __class__ validation #23722)
340 # Python 3.7a1 3390 (add LOAD_METHOD and CALL_METHOD opcodes #26110)
341 # Python 3.7a2 3391 (update GET_AITER #31709)
342 # Python 3.7a4 3392 (PEP 552: Deterministic pycs #31650)
343 # Python 3.7b1 3393 (remove STORE_ANNOTATION opcode #32550)
344 # Python 3.7b5 3394 (restored docstring as the first stmt in the body;
345 # this might affected the first line number #32911)
346 # Python 3.8a1 3400 (move frame block handling to compiler #17611)
347 # Python 3.8a1 3401 (add END_ASYNC_FOR #33041)
348 # Python 3.8a1 3410 (PEP570 Python Positional-Only Parameters #36540)
349 # Python 3.8b2 3411 (Reverse evaluation order of key: value in dict
350 # comprehensions #35224)
351 # Python 3.8b2 3412 (Swap the position of positional args and positional
352 # only args in ast.arguments #37593)
353 # Python 3.8b4 3413 (Fix "break" and "continue" in "finally" #37830)
354 # Python 3.9a0 3420 (add LOAD_ASSERTION_ERROR #34880)
355 # Python 3.9a0 3421 (simplified bytecode for with blocks #32949)
356 # Python 3.9a0 3422 (remove BEGIN_FINALLY, END_FINALLY, CALL_FINALLY, POP_FINALLY bytecodes #33387)
357 # Python 3.9a2 3423 (add IS_OP, CONTAINS_OP and JUMP_IF_NOT_EXC_MATCH bytecodes #39156)
358 # Python 3.9a2 3424 (simplify bytecodes for *value unpacking)
359 # Python 3.9a2 3425 (simplify bytecodes for **value unpacking)
360 # Python 3.10a1 3430 (Make 'annotations' future by default)
361 # Python 3.10a1 3431 (New line number table format -- PEP 626)
362 # Python 3.10a2 3432 (Function annotation for MAKE_FUNCTION is changed from dict to tuple bpo-42202)
363 # Python 3.10a2 3433 (RERAISE restores f_lasti if oparg != 0)
364 # Python 3.10a6 3434 (PEP 634: Structural Pattern Matching)
365 # Python 3.10a7 3435 Use instruction offsets (as opposed to byte offsets).
366 # Python 3.10b1 3436 (Add GEN_START bytecode #43683)
367 # Python 3.10b1 3437 (Undo making 'annotations' future by default - We like to dance among core devs!)
368 # Python 3.10b1 3438 Safer line number table handling.
369 # Python 3.10b1 3439 (Add ROT_N)
370 # Python 3.11a1 3450 Use exception table for unwinding ("zero cost" exception handling)
371 # Python 3.11a1 3451 (Add CALL_METHOD_KW)
372 # Python 3.11a1 3452 (drop nlocals from marshaled code objects)
373 # Python 3.11a1 3453 (add co_fastlocalnames and co_fastlocalkinds)
374 # Python 3.11a1 3454 (compute cell offsets relative to locals bpo-43693)
375 # Python 3.11a1 3455 (add MAKE_CELL bpo-43693)
376 # Python 3.11a1 3456 (interleave cell args bpo-43693)
377 # Python 3.11a1 3457 (Change localsplus to a bytes object bpo-43693)
378 # Python 3.11a1 3458 (imported objects now don't use LOAD_METHOD/CALL_METHOD)
379 # Python 3.11a1 3459 (PEP 657: add end line numbers and column offsets for instructions)
380 # Python 3.11a1 3460 (Add co_qualname field to PyCodeObject bpo-44530)
381 # Python 3.11a1 3461 (JUMP_ABSOLUTE must jump backwards)
382 # Python 3.11a2 3462 (bpo-44511: remove COPY_DICT_WITHOUT_KEYS, change
383 # MATCH_CLASS and MATCH_KEYS, and add COPY)
384 # Python 3.11a3 3463 (bpo-45711: JUMP_IF_NOT_EXC_MATCH no longer pops the
385 # active exception)
386 # Python 3.11a3 3464 (bpo-45636: Merge numeric BINARY_*/INPLACE_* into
387 # BINARY_OP)
388 # Python 3.11a3 3465 (Add COPY_FREE_VARS opcode)
389 # Python 3.11a4 3466 (bpo-45292: PEP-654 except*)
390 # Python 3.11a4 3467 (Change CALL_xxx opcodes)
391 # Python 3.11a4 3468 (Add SEND opcode)
392 # Python 3.11a4 3469 (bpo-45711: remove type, traceback from exc_info)
393 # Python 3.11a4 3470 (bpo-46221: PREP_RERAISE_STAR no longer pushes lasti)
394 # Python 3.11a4 3471 (bpo-46202: remove pop POP_EXCEPT_AND_RERAISE)
395 # Python 3.11a4 3472 (bpo-46009: replace GEN_START with POP_TOP)
396 # Python 3.11a4 3473 (Add POP_JUMP_IF_NOT_NONE/POP_JUMP_IF_NONE opcodes)
397 # Python 3.11a4 3474 (Add RESUME opcode)
398 # Python 3.11a5 3475 (Add RETURN_GENERATOR opcode)
399 # Python 3.11a5 3476 (Add ASYNC_GEN_WRAP opcode)
400 # Python 3.11a5 3477 (Replace DUP_TOP/DUP_TOP_TWO with COPY and
401 # ROT_TWO/ROT_THREE/ROT_FOUR/ROT_N with SWAP)
402 # Python 3.11a5 3478 (New CALL opcodes)
403 # Python 3.11a5 3479 (Add PUSH_NULL opcode)
404 # Python 3.11a5 3480 (New CALL opcodes, second iteration)
405 # Python 3.11a5 3481 (Use inline cache for BINARY_OP)
406 # Python 3.11a5 3482 (Use inline caching for UNPACK_SEQUENCE and LOAD_GLOBAL)
407 # Python 3.11a5 3483 (Use inline caching for COMPARE_OP and BINARY_SUBSCR)
408 # Python 3.11a5 3484 (Use inline caching for LOAD_ATTR, LOAD_METHOD, and
409 # STORE_ATTR)
410 # Python 3.11a5 3485 (Add an oparg to GET_AWAITABLE)
411 # Python 3.11a6 3486 (Use inline caching for PRECALL and CALL)
412 # Python 3.11a6 3487 (Remove the adaptive "oparg counter" mechanism)
413 # Python 3.11a6 3488 (LOAD_GLOBAL can push additional NULL)
414 # Python 3.11a6 3489 (Add JUMP_BACKWARD, remove JUMP_ABSOLUTE)
415 # Python 3.11a6 3490 (remove JUMP_IF_NOT_EXC_MATCH, add CHECK_EXC_MATCH)
416 # Python 3.11a6 3491 (remove JUMP_IF_NOT_EG_MATCH, add CHECK_EG_MATCH,
417 # add JUMP_BACKWARD_NO_INTERRUPT, make JUMP_NO_INTERRUPT virtual)
418 # Python 3.11a7 3492 (make POP_JUMP_IF_NONE/NOT_NONE/TRUE/FALSE relative)
419 # Python 3.11a7 3493 (Make JUMP_IF_TRUE_OR_POP/JUMP_IF_FALSE_OR_POP relative)
420 # Python 3.11a7 3494 (New location info table)
421 # Python 3.11b4 3495 (Set line number of module's RESUME instr to 0 per PEP 626)
422 # Python 3.12a1 3500 (Remove PRECALL opcode)
423 # Python 3.12a1 3501 (YIELD_VALUE oparg == stack_depth)
424 # Python 3.12a1 3502 (LOAD_FAST_CHECK, no NULL-check in LOAD_FAST)
425 # Python 3.12a1 3503 (Shrink LOAD_METHOD cache)
426 # Python 3.12a1 3504 (Merge LOAD_METHOD back into LOAD_ATTR)
427 # Python 3.12a1 3505 (Specialization/Cache for FOR_ITER)
428 # Python 3.12a1 3506 (Add BINARY_SLICE and STORE_SLICE instructions)
429 # Python 3.12a1 3507 (Set lineno of module's RESUME to 0)
430 # Python 3.12a1 3508 (Add CLEANUP_THROW)
431 # Python 3.12a1 3509 (Conditional jumps only jump forward)
432 # Python 3.12a2 3510 (FOR_ITER leaves iterator on the stack)
433 # Python 3.12a2 3511 (Add STOPITERATION_ERROR instruction)
434 # Python 3.12a2 3512 (Remove all unused consts from code objects)
435 # Python 3.12a4 3513 (Add CALL_INTRINSIC_1 instruction, removed STOPITERATION_ERROR, PRINT_EXPR, IMPORT_STAR)
436 # Python 3.12a4 3514 (Remove ASYNC_GEN_WRAP, LIST_TO_TUPLE, and UNARY_POSITIVE)
437 # Python 3.12a5 3515 (Embed jump mask in COMPARE_OP oparg)
438 # Python 3.12a5 3516 (Add COMPARE_AND_BRANCH instruction)
439 # Python 3.12a5 3517 (Change YIELD_VALUE oparg to exception block depth)
440 # Python 3.12a6 3518 (Add RETURN_CONST instruction)
441 # Python 3.12a6 3519 (Modify SEND instruction)
442 # Python 3.12a6 3520 (Remove PREP_RERAISE_STAR, add CALL_INTRINSIC_2)
443 # Python 3.12a7 3521 (Shrink the LOAD_GLOBAL caches)
444 # Python 3.12a7 3522 (Removed JUMP_IF_FALSE_OR_POP/JUMP_IF_TRUE_OR_POP)
445 # Python 3.12a7 3523 (Convert COMPARE_AND_BRANCH back to COMPARE_OP)
446 # Python 3.12a7 3524 (Shrink the BINARY_SUBSCR caches)
447 # Python 3.12b1 3525 (Shrink the CALL caches)
448 # Python 3.12b1 3526 (Add instrumentation support)
449 # Python 3.12b1 3527 (Add LOAD_SUPER_ATTR)
450 # Python 3.12b1 3528 (Add LOAD_SUPER_ATTR_METHOD specialization)
451 # Python 3.12b1 3529 (Inline list/dict/set comprehensions)
452 # Python 3.12b1 3530 (Shrink the LOAD_SUPER_ATTR caches)
453 # Python 3.12b1 3531 (Add PEP 695 changes)
454 # Python 3.13a1 3550 (Plugin optimizer support)
455 # Python 3.13a1 3551 (Compact superinstructions)
456 # Python 3.13a1 3552 (Remove LOAD_FAST__LOAD_CONST and LOAD_CONST__LOAD_FAST)
457 # Python 3.13a1 3553 (Add SET_FUNCTION_ATTRIBUTE)
458 # Python 3.13a1 3554 (more efficient bytecodes for f-strings)
459 # Python 3.13a1 3555 (generate specialized opcodes metadata from bytecodes.c)
460 # Python 3.13a1 3556 (Convert LOAD_CLOSURE to a pseudo-op)
461 # Python 3.13a1 3557 (Make the conversion to boolean in jumps explicit)
462 # Python 3.13a1 3558 (Reorder the stack items for CALL)
463 # Python 3.13a1 3559 (Generate opcode IDs from bytecodes.c)
464 # Python 3.13a1 3560 (Add RESUME_CHECK instruction)
465 # Python 3.13a1 3561 (Add cache entry to branch instructions)
466 # Python 3.13a1 3562 (Assign opcode IDs for internal ops in separate range)
467 # Python 3.13a1 3563 (Add CALL_KW and remove KW_NAMES)
468 # Python 3.13a1 3564 (Removed oparg from YIELD_VALUE, changed oparg values of RESUME)
469 # Python 3.13a1 3565 (Oparg of YIELD_VALUE indicates whether it is in a yield-from)
470 # Python 3.13a1 3566 (Emit JUMP_NO_INTERRUPT instead of JUMP for non-loop no-lineno cases)
471 # Python 3.13a1 3567 (Reimplement line number propagation by the compiler)
472 # Python 3.13a1 3568 (Change semantics of END_FOR)
473 # Python 3.13a5 3569 (Specialize CONTAINS_OP)
474 # Python 3.13a6 3570 (Add __firstlineno__ class attribute)
475 # Python 3.13b1 3571 (Fix miscompilation of private names in generic classes)
476
477 # Python 3.14 will start with 3600
478
479 # Please don't copy-paste the same pre-release tag for new entries above!!!
480 # You should always use the *upcoming* tag. For example, if 3.12a6 came out
481 # a week ago, I should put "Python 3.12a7" next to my new magic number.
482
483 # MAGIC must change whenever the bytecode emitted by the compiler may no
484 # longer be understood by older implementations of the eval loop (usually
485 # due to the addition of new opcodes).
486 #
487 # Starting with Python 3.11, Python 3.n starts with magic number 2900+50n.
488 #
489 # Whenever MAGIC_NUMBER is changed, the ranges in the magic_values array
490 # in PC/launcher.c must also be updated.
491
492 MAGIC_NUMBER = (3571).to_bytes(2, 'little') + b'\r\n'
493
494 _RAW_MAGIC_NUMBER = int.from_bytes(MAGIC_NUMBER, 'little') # For import.c
495
496 _PYCACHE = '__pycache__'
497 _OPT = 'opt-'
498
499 SOURCE_SUFFIXES = ['.py']
500 if _MS_WINDOWS:
501 SOURCE_SUFFIXES.append('.pyw')
502
503 EXTENSION_SUFFIXES = _imp.extension_suffixes()
504
505 BYTECODE_SUFFIXES = ['.pyc']
506 # Deprecated.
507 DEBUG_BYTECODE_SUFFIXES = OPTIMIZED_BYTECODE_SUFFIXES = BYTECODE_SUFFIXES
508
509 def cache_from_source(path, debug_override=None, *, optimization=None):
510 """Given the path to a .py file, return the path to its .pyc file.
511
512 The .py file does not need to exist; this simply returns the path to the
513 .pyc file calculated as if the .py file were imported.
514
515 The 'optimization' parameter controls the presumed optimization level of
516 the bytecode file. If 'optimization' is not None, the string representation
517 of the argument is taken and verified to be alphanumeric (else ValueError
518 is raised).
519
520 The debug_override parameter is deprecated. If debug_override is not None,
521 a True value is the same as setting 'optimization' to the empty string
522 while a False value is equivalent to setting 'optimization' to '1'.
523
524 If sys.implementation.cache_tag is None then NotImplementedError is raised.
525
526 """
527 if debug_override is not None:
528 _warnings.warn('the debug_override parameter is deprecated; use '
529 "'optimization' instead", DeprecationWarning)
530 if optimization is not None:
531 message = 'debug_override or optimization must be set to None'
532 raise TypeError(message)
533 optimization = '' if debug_override else 1
534 path = _os.fspath(path)
535 head, tail = _path_split(path)
536 base, sep, rest = tail.rpartition('.')
537 tag = sys.implementation.cache_tag
538 if tag is None:
539 raise NotImplementedError('sys.implementation.cache_tag is None')
540 almost_filename = ''.join([(base if base else rest), sep, tag])
541 if optimization is None:
542 if sys.flags.optimize == 0:
543 optimization = ''
544 else:
545 optimization = sys.flags.optimize
546 optimization = str(optimization)
547 if optimization != '':
548 if not optimization.isalnum():
549 raise ValueError(f'{optimization!r} is not alphanumeric')
550 almost_filename = f'{almost_filename}.{_OPT}{optimization}'
551 filename = almost_filename + BYTECODE_SUFFIXES[0]
552 if sys.pycache_prefix is not None:
553 # We need an absolute path to the py file to avoid the possibility of
554 # collisions within sys.pycache_prefix, if someone has two different
555 # `foo/bar.py` on their system and they import both of them using the
556 # same sys.pycache_prefix. Let's say sys.pycache_prefix is
557 # `C:\Bytecode`; the idea here is that if we get `Foo\Bar`, we first
558 # make it absolute (`C:\Somewhere\Foo\Bar`), then make it root-relative
559 # (`Somewhere\Foo\Bar`), so we end up placing the bytecode file in an
560 # unambiguous `C:\Bytecode\Somewhere\Foo\Bar\`.
561 head = _path_abspath(head)
562
563 # Strip initial drive from a Windows path. We know we have an absolute
564 # path here, so the second part of the check rules out a POSIX path that
565 # happens to contain a colon at the second character.
566 # Slicing avoids issues with an empty (or short) `head`.
567 if head[1:2] == ':' and head[0:1] not in path_separators:
568 head = head[2:]
569
570 # Strip initial path separator from `head` to complete the conversion
571 # back to a root-relative path before joining.
572 return _path_join(
573 sys.pycache_prefix,
574 head.lstrip(path_separators),
575 filename,
576 )
577 return _path_join(head, _PYCACHE, filename)
578
579
580 def source_from_cache(path):
581 """Given the path to a .pyc. file, return the path to its .py file.
582
583 The .pyc file does not need to exist; this simply returns the path to
584 the .py file calculated to correspond to the .pyc file. If path does
585 not conform to PEP 3147/488 format, ValueError will be raised. If
586 sys.implementation.cache_tag is None then NotImplementedError is raised.
587
588 """
589 if sys.implementation.cache_tag is None:
590 raise NotImplementedError('sys.implementation.cache_tag is None')
591 path = _os.fspath(path)
592 head, pycache_filename = _path_split(path)
593 found_in_pycache_prefix = False
594 if sys.pycache_prefix is not None:
595 stripped_path = sys.pycache_prefix.rstrip(path_separators)
596 if head.startswith(stripped_path + path_sep):
597 head = head[len(stripped_path):]
598 found_in_pycache_prefix = True
599 if not found_in_pycache_prefix:
600 head, pycache = _path_split(head)
601 if pycache != _PYCACHE:
602 raise ValueError(f'{_PYCACHE} not bottom-level directory in '
603 f'{path!r}')
604 dot_count = pycache_filename.count('.')
605 if dot_count not in {2, 3}:
606 raise ValueError(f'expected only 2 or 3 dots in {pycache_filename!r}')
607 elif dot_count == 3:
608 optimization = pycache_filename.rsplit('.', 2)[-2]
609 if not optimization.startswith(_OPT):
610 raise ValueError("optimization portion of filename does not start "
611 f"with {_OPT!r}")
612 opt_level = optimization[len(_OPT):]
613 if not opt_level.isalnum():
614 raise ValueError(f"optimization level {optimization!r} is not an "
615 "alphanumeric value")
616 base_filename = pycache_filename.partition('.')[0]
617 return _path_join(head, base_filename + SOURCE_SUFFIXES[0])
618
619
620 def _get_sourcefile(bytecode_path):
621 """Convert a bytecode file path to a source path (if possible).
622
623 This function exists purely for backwards-compatibility for
624 PyImport_ExecCodeModuleWithFilenames() in the C API.
625
626 """
627 if len(bytecode_path) == 0:
628 return None
629 rest, _, extension = bytecode_path.rpartition('.')
630 if not rest or extension.lower()[-3:-1] != 'py':
631 return bytecode_path
632 try:
633 source_path = source_from_cache(bytecode_path)
634 except (NotImplementedError, ValueError):
635 source_path = bytecode_path[:-1]
636 return source_path if _path_isfile(source_path) else bytecode_path
637
638
639 def _get_cached(filename):
640 if filename.endswith(tuple(SOURCE_SUFFIXES)):
641 try:
642 return cache_from_source(filename)
643 except NotImplementedError:
644 pass
645 elif filename.endswith(tuple(BYTECODE_SUFFIXES)):
646 return filename
647 else:
648 return None
649
650
651 def _calc_mode(path):
652 """Calculate the mode permissions for a bytecode file."""
653 try:
654 mode = _path_stat(path).st_mode
655 except OSError:
656 mode = 0o666
657 # We always ensure write access so we can update cached files
658 # later even when the source files are read-only on Windows (#6074)
659 mode |= 0o200
660 return mode
661
662
663 def _check_name(method):
664 """Decorator to verify that the module being requested matches the one the
665 loader can handle.
666
667 The first argument (self) must define _name which the second argument is
668 compared against. If the comparison fails then ImportError is raised.
669
670 """
671 def _check_name_wrapper(self, name=None, *args, **kwargs):
672 if name is None:
673 name = self.name
674 elif self.name != name:
675 raise ImportError('loader for %s cannot handle %s' %
676 (self.name, name), name=name)
677 return method(self, name, *args, **kwargs)
678
679 # FIXME: @_check_name is used to define class methods before the
680 # _bootstrap module is set by _set_bootstrap_module().
681 if _bootstrap is not None:
682 _wrap = _bootstrap._wrap
683 else:
684 def _wrap(new, old):
685 for replace in ['__module__', '__name__', '__qualname__', '__doc__']:
686 if hasattr(old, replace):
687 setattr(new, replace, getattr(old, replace))
688 new.__dict__.update(old.__dict__)
689
690 _wrap(_check_name_wrapper, method)
691 return _check_name_wrapper
692
693
694 def _classify_pyc(data, name, exc_details):
695 """Perform basic validity checking of a pyc header and return the flags field,
696 which determines how the pyc should be further validated against the source.
697
698 *data* is the contents of the pyc file. (Only the first 16 bytes are
699 required, though.)
700
701 *name* is the name of the module being imported. It is used for logging.
702
703 *exc_details* is a dictionary passed to ImportError if it raised for
704 improved debugging.
705
706 ImportError is raised when the magic number is incorrect or when the flags
707 field is invalid. EOFError is raised when the data is found to be truncated.
708
709 """
710 magic = data[:4]
711 if magic != MAGIC_NUMBER:
712 message = f'bad magic number in {name!r}: {magic!r}'
713 _bootstrap._verbose_message('{}', message)
714 raise ImportError(message, **exc_details)
715 if len(data) < 16:
716 message = f'reached EOF while reading pyc header of {name!r}'
717 _bootstrap._verbose_message('{}', message)
718 raise EOFError(message)
719 flags = _unpack_uint32(data[4:8])
720 # Only the first two flags are defined.
721 if flags & ~0b11:
722 message = f'invalid flags {flags!r} in {name!r}'
723 raise ImportError(message, **exc_details)
724 return flags
725
726
727 def _validate_timestamp_pyc(data, source_mtime, source_size, name,
728 exc_details):
729 """Validate a pyc against the source last-modified time.
730
731 *data* is the contents of the pyc file. (Only the first 16 bytes are
732 required.)
733
734 *source_mtime* is the last modified timestamp of the source file.
735
736 *source_size* is None or the size of the source file in bytes.
737
738 *name* is the name of the module being imported. It is used for logging.
739
740 *exc_details* is a dictionary passed to ImportError if it raised for
741 improved debugging.
742
743 An ImportError is raised if the bytecode is stale.
744
745 """
746 if _unpack_uint32(data[8:12]) != (source_mtime & 0xFFFFFFFF):
747 message = f'bytecode is stale for {name!r}'
748 _bootstrap._verbose_message('{}', message)
749 raise ImportError(message, **exc_details)
750 if (source_size is not None and
751 _unpack_uint32(data[12:16]) != (source_size & 0xFFFFFFFF)):
752 raise ImportError(f'bytecode is stale for {name!r}', **exc_details)
753
754
755 def _validate_hash_pyc(data, source_hash, name, exc_details):
756 """Validate a hash-based pyc by checking the real source hash against the one in
757 the pyc header.
758
759 *data* is the contents of the pyc file. (Only the first 16 bytes are
760 required.)
761
762 *source_hash* is the importlib.util.source_hash() of the source file.
763
764 *name* is the name of the module being imported. It is used for logging.
765
766 *exc_details* is a dictionary passed to ImportError if it raised for
767 improved debugging.
768
769 An ImportError is raised if the bytecode is stale.
770
771 """
772 if data[8:16] != source_hash:
773 raise ImportError(
774 f'hash in bytecode doesn\'t match hash of source {name!r}',
775 **exc_details,
776 )
777
778
779 def _compile_bytecode(data, name=None, bytecode_path=None, source_path=None):
780 """Compile bytecode as found in a pyc."""
781 code = marshal.loads(data)
782 if isinstance(code, _code_type):
783 _bootstrap._verbose_message('code object from {!r}', bytecode_path)
784 if source_path is not None:
785 _imp._fix_co_filename(code, source_path)
786 return code
787 else:
788 raise ImportError(f'Non-code object in {bytecode_path!r}',
789 name=name, path=bytecode_path)
790
791
792 def _code_to_timestamp_pyc(code, mtime=0, source_size=0):
793 "Produce the data for a timestamp-based pyc."
794 data = bytearray(MAGIC_NUMBER)
795 data.extend(_pack_uint32(0))
796 data.extend(_pack_uint32(mtime))
797 data.extend(_pack_uint32(source_size))
798 data.extend(marshal.dumps(code))
799 return data
800
801
802 def _code_to_hash_pyc(code, source_hash, checked=True):
803 "Produce the data for a hash-based pyc."
804 data = bytearray(MAGIC_NUMBER)
805 flags = 0b1 | checked << 1
806 data.extend(_pack_uint32(flags))
807 assert len(source_hash) == 8
808 data.extend(source_hash)
809 data.extend(marshal.dumps(code))
810 return data
811
812
813 def decode_source(source_bytes):
814 """Decode bytes representing source code and return the string.
815
816 Universal newline support is used in the decoding.
817 """
818 import tokenize # To avoid bootstrap issues.
819 source_bytes_readline = _io.BytesIO(source_bytes).readline
820 encoding = tokenize.detect_encoding(source_bytes_readline)
821 newline_decoder = _io.IncrementalNewlineDecoder(None, True)
822 return newline_decoder.decode(source_bytes.decode(encoding[0]))
823
824
825 # Module specifications #######################################################
826
827 _POPULATE = object()
828
829
830 def spec_from_file_location(name, location=None, *, loader=None,
831 submodule_search_locations=_POPULATE):
832 """Return a module spec based on a file location.
833
834 To indicate that the module is a package, set
835 submodule_search_locations to a list of directory paths. An
836 empty list is sufficient, though its not otherwise useful to the
837 import system.
838
839 The loader must take a spec as its only __init__() arg.
840
841 """
842 if location is None:
843 # The caller may simply want a partially populated location-
844 # oriented spec. So we set the location to a bogus value and
845 # fill in as much as we can.
846 location = '<unknown>'
847 if hasattr(loader, 'get_filename'):
848 # ExecutionLoader
849 try:
850 location = loader.get_filename(name)
851 except ImportError:
852 pass
853 else:
854 location = _os.fspath(location)
855 try:
856 location = _path_abspath(location)
857 except OSError:
858 pass
859
860 # If the location is on the filesystem, but doesn't actually exist,
861 # we could return None here, indicating that the location is not
862 # valid. However, we don't have a good way of testing since an
863 # indirect location (e.g. a zip file or URL) will look like a
864 # non-existent file relative to the filesystem.
865
866 spec = _bootstrap.ModuleSpec(name, loader, origin=location)
867 spec._set_fileattr = True
868
869 # Pick a loader if one wasn't provided.
870 if loader is None:
871 for loader_class, suffixes in _get_supported_file_loaders():
872 if location.endswith(tuple(suffixes)):
873 loader = loader_class(name, location)
874 spec.loader = loader
875 break
876 else:
877 return None
878
879 # Set submodule_search_paths appropriately.
880 if submodule_search_locations is _POPULATE:
881 # Check the loader.
882 if hasattr(loader, 'is_package'):
883 try:
884 is_package = loader.is_package(name)
885 except ImportError:
886 pass
887 else:
888 if is_package:
889 spec.submodule_search_locations = []
890 else:
891 spec.submodule_search_locations = submodule_search_locations
892 if spec.submodule_search_locations == []:
893 if location:
894 dirname = _path_split(location)[0]
895 spec.submodule_search_locations.append(dirname)
896
897 return spec
898
899
900 def _bless_my_loader(module_globals):
901 """Helper function for _warnings.c
902
903 See GH#97850 for details.
904 """
905 # 2022-10-06(warsaw): For now, this helper is only used in _warnings.c and
906 # that use case only has the module globals. This function could be
907 # extended to accept either that or a module object. However, in the
908 # latter case, it would be better to raise certain exceptions when looking
909 # at a module, which should have either a __loader__ or __spec__.loader.
910 # For backward compatibility, it is possible that we'll get an empty
911 # dictionary for the module globals, and that cannot raise an exception.
912 if not isinstance(module_globals, dict):
913 return None
914
915 missing = object()
916 loader = module_globals.get('__loader__', None)
917 spec = module_globals.get('__spec__', missing)
918
919 if loader is None:
920 if spec is missing:
921 # If working with a module:
922 # raise AttributeError('Module globals is missing a __spec__')
923 return None
924 elif spec is None:
925 raise ValueError('Module globals is missing a __spec__.loader')
926
927 spec_loader = getattr(spec, 'loader', missing)
928
929 if spec_loader in (missing, None):
930 if loader is None:
931 exc = AttributeError if spec_loader is missing else ValueError
932 raise exc('Module globals is missing a __spec__.loader')
933 _warnings.warn(
934 'Module globals is missing a __spec__.loader',
935 DeprecationWarning)
936 spec_loader = loader
937
938 assert spec_loader is not None
939 if loader is not None and loader != spec_loader:
940 _warnings.warn(
941 'Module globals; __loader__ != __spec__.loader',
942 DeprecationWarning)
943 return loader
944
945 return spec_loader
946
947
948 # Loaders #####################################################################
949
950 class WindowsRegistryFinder:
951
952 """Meta path finder for modules declared in the Windows registry."""
953
954 REGISTRY_KEY = (
955 'Software\\Python\\PythonCore\\{sys_version}'
956 '\\Modules\\{fullname}')
957 REGISTRY_KEY_DEBUG = (
958 'Software\\Python\\PythonCore\\{sys_version}'
959 '\\Modules\\{fullname}\\Debug')
960 DEBUG_BUILD = (_MS_WINDOWS and '_d.pyd' in EXTENSION_SUFFIXES)
961
962 @staticmethod
963 def _open_registry(key):
964 try:
965 return winreg.OpenKey(winreg.HKEY_CURRENT_USER, key)
966 except OSError:
967 return winreg.OpenKey(winreg.HKEY_LOCAL_MACHINE, key)
968
969 @classmethod
970 def _search_registry(cls, fullname):
971 if cls.DEBUG_BUILD:
972 registry_key = cls.REGISTRY_KEY_DEBUG
973 else:
974 registry_key = cls.REGISTRY_KEY
975 key = registry_key.format(fullname=fullname,
976 sys_version='%d.%d' % sys.version_info[:2])
977 try:
978 with cls._open_registry(key) as hkey:
979 filepath = winreg.QueryValue(hkey, '')
980 except OSError:
981 return None
982 return filepath
983
984 @classmethod
985 def find_spec(cls, fullname, path=None, target=None):
986 filepath = cls._search_registry(fullname)
987 if filepath is None:
988 return None
989 try:
990 _path_stat(filepath)
991 except OSError:
992 return None
993 for loader, suffixes in _get_supported_file_loaders():
994 if filepath.endswith(tuple(suffixes)):
995 spec = _bootstrap.spec_from_loader(fullname,
996 loader(fullname, filepath),
997 origin=filepath)
998 return spec
999
1000
1001 class _LoaderBasics:
1002
1003 """Base class of common code needed by both SourceLoader and
1004 SourcelessFileLoader."""
1005
1006 def is_package(self, fullname):
1007 """Concrete implementation of InspectLoader.is_package by checking if
1008 the path returned by get_filename has a filename of '__init__.py'."""
1009 filename = _path_split(self.get_filename(fullname))[1]
1010 filename_base = filename.rsplit('.', 1)[0]
1011 tail_name = fullname.rpartition('.')[2]
1012 return filename_base == '__init__' and tail_name != '__init__'
1013
1014 def create_module(self, spec):
1015 """Use default semantics for module creation."""
1016
1017 def exec_module(self, module):
1018 """Execute the module."""
1019 code = self.get_code(module.__name__)
1020 if code is None:
1021 raise ImportError(f'cannot load module {module.__name__!r} when '
1022 'get_code() returns None')
1023 _bootstrap._call_with_frames_removed(exec, code, module.__dict__)
1024
1025 def load_module(self, fullname):
1026 """This method is deprecated."""
1027 # Warning implemented in _load_module_shim().
1028 return _bootstrap._load_module_shim(self, fullname)
1029
1030
1031 class SourceLoader(_LoaderBasics):
1032
1033 def path_mtime(self, path):
1034 """Optional method that returns the modification time (an int) for the
1035 specified path (a str).
1036
1037 Raises OSError when the path cannot be handled.
1038 """
1039 raise OSError
1040
1041 def path_stats(self, path):
1042 """Optional method returning a metadata dict for the specified
1043 path (a str).
1044
1045 Possible keys:
1046 - 'mtime' (mandatory) is the numeric timestamp of last source
1047 code modification;
1048 - 'size' (optional) is the size in bytes of the source code.
1049
1050 Implementing this method allows the loader to read bytecode files.
1051 Raises OSError when the path cannot be handled.
1052 """
1053 return {'mtime': self.path_mtime(path)}
1054
1055 def _cache_bytecode(self, source_path, cache_path, data):
1056 """Optional method which writes data (bytes) to a file path (a str).
1057
1058 Implementing this method allows for the writing of bytecode files.
1059
1060 The source path is needed in order to correctly transfer permissions
1061 """
1062 # For backwards compatibility, we delegate to set_data()
1063 return self.set_data(cache_path, data)
1064
1065 def set_data(self, path, data):
1066 """Optional method which writes data (bytes) to a file path (a str).
1067
1068 Implementing this method allows for the writing of bytecode files.
1069 """
1070
1071
1072 def get_source(self, fullname):
1073 """Concrete implementation of InspectLoader.get_source."""
1074 path = self.get_filename(fullname)
1075 try:
1076 source_bytes = self.get_data(path)
1077 except OSError as exc:
1078 raise ImportError('source not available through get_data()',
1079 name=fullname) from exc
1080 return decode_source(source_bytes)
1081
1082 def source_to_code(self, data, path, *, _optimize=-1):
1083 """Return the code object compiled from source.
1084
1085 The 'data' argument can be any object type that compile() supports.
1086 """
1087 return _bootstrap._call_with_frames_removed(compile, data, path, 'exec',
1088 dont_inherit=True, optimize=_optimize)
1089
1090 def get_code(self, fullname):
1091 """Concrete implementation of InspectLoader.get_code.
1092
1093 Reading of bytecode requires path_stats to be implemented. To write
1094 bytecode, set_data must also be implemented.
1095
1096 """
1097 source_path = self.get_filename(fullname)
1098 source_mtime = None
1099 source_bytes = None
1100 source_hash = None
1101 hash_based = False
1102 check_source = True
1103 try:
1104 bytecode_path = cache_from_source(source_path)
1105 except NotImplementedError:
1106 bytecode_path = None
1107 else:
1108 try:
1109 st = self.path_stats(source_path)
1110 except OSError:
1111 pass
1112 else:
1113 source_mtime = int(st['mtime'])
1114 try:
1115 data = self.get_data(bytecode_path)
1116 except OSError:
1117 pass
1118 else:
1119 exc_details = {
1120 'name': fullname,
1121 'path': bytecode_path,
1122 }
1123 try:
1124 flags = _classify_pyc(data, fullname, exc_details)
1125 bytes_data = memoryview(data)[16:]
1126 hash_based = flags & 0b1 != 0
1127 if hash_based:
1128 check_source = flags & 0b10 != 0
1129 if (_imp.check_hash_based_pycs != 'never' and
1130 (check_source or
1131 _imp.check_hash_based_pycs == 'always')):
1132 source_bytes = self.get_data(source_path)
1133 source_hash = _imp.source_hash(
1134 _RAW_MAGIC_NUMBER,
1135 source_bytes,
1136 )
1137 _validate_hash_pyc(data, source_hash, fullname,
1138 exc_details)
1139 else:
1140 _validate_timestamp_pyc(
1141 data,
1142 source_mtime,
1143 st['size'],
1144 fullname,
1145 exc_details,
1146 )
1147 except (ImportError, EOFError):
1148 pass
1149 else:
1150 _bootstrap._verbose_message('{} matches {}', bytecode_path,
1151 source_path)
1152 return _compile_bytecode(bytes_data, name=fullname,
1153 bytecode_path=bytecode_path,
1154 source_path=source_path)
1155 if source_bytes is None:
1156 source_bytes = self.get_data(source_path)
1157 code_object = self.source_to_code(source_bytes, source_path)
1158 _bootstrap._verbose_message('code object from {}', source_path)
1159 if (not sys.dont_write_bytecode and bytecode_path is not None and
1160 source_mtime is not None):
1161 if hash_based:
1162 if source_hash is None:
1163 source_hash = _imp.source_hash(_RAW_MAGIC_NUMBER,
1164 source_bytes)
1165 data = _code_to_hash_pyc(code_object, source_hash, check_source)
1166 else:
1167 data = _code_to_timestamp_pyc(code_object, source_mtime,
1168 len(source_bytes))
1169 try:
1170 self._cache_bytecode(source_path, bytecode_path, data)
1171 except NotImplementedError:
1172 pass
1173 return code_object
1174
1175
1176 class FileLoader:
1177
1178 """Base file loader class which implements the loader protocol methods that
1179 require file system usage."""
1180
1181 def __init__(self, fullname, path):
1182 """Cache the module name and the path to the file found by the
1183 finder."""
1184 self.name = fullname
1185 self.path = path
1186
1187 def __eq__(self, other):
1188 return (self.__class__ == other.__class__ and
1189 self.__dict__ == other.__dict__)
1190
1191 def __hash__(self):
1192 return hash(self.name) ^ hash(self.path)
1193
1194 @_check_name
1195 def load_module(self, fullname):
1196 """Load a module from a file.
1197
1198 This method is deprecated. Use exec_module() instead.
1199
1200 """
1201 # The only reason for this method is for the name check.
1202 # Issue #14857: Avoid the zero-argument form of super so the implementation
1203 # of that form can be updated without breaking the frozen module.
1204 return super(FileLoader, self).load_module(fullname)
1205
1206 @_check_name
1207 def get_filename(self, fullname):
1208 """Return the path to the source file as found by the finder."""
1209 return self.path
1210
1211 def get_data(self, path):
1212 """Return the data from path as raw bytes."""
1213 if isinstance(self, (SourceLoader, SourcelessFileLoader, ExtensionFileLoader)):
1214 with _io.open_code(str(path)) as file:
1215 return file.read()
1216 else:
1217 with _io.FileIO(path, 'r') as file:
1218 return file.read()
1219
1220 @_check_name
1221 def get_resource_reader(self, module):
1222 from importlib.readers import FileReader
1223 return FileReader(self)
1224
1225
1226 class SourceFileLoader(FileLoader, SourceLoader):
1227
1228 """Concrete implementation of SourceLoader using the file system."""
1229
1230 def path_stats(self, path):
1231 """Return the metadata for the path."""
1232 st = _path_stat(path)
1233 return {'mtime': st.st_mtime, 'size': st.st_size}
1234
1235 def _cache_bytecode(self, source_path, bytecode_path, data):
1236 # Adapt between the two APIs
1237 mode = _calc_mode(source_path)
1238 return self.set_data(bytecode_path, data, _mode=mode)
1239
1240 def set_data(self, path, data, *, _mode=0o666):
1241 """Write bytes data to a file."""
1242 parent, filename = _path_split(path)
1243 path_parts = []
1244 # Figure out what directories are missing.
1245 while parent and not _path_isdir(parent):
1246 parent, part = _path_split(parent)
1247 path_parts.append(part)
1248 # Create needed directories.
1249 for part in reversed(path_parts):
1250 parent = _path_join(parent, part)
1251 try:
1252 _os.mkdir(parent)
1253 except FileExistsError:
1254 # Probably another Python process already created the dir.
1255 continue
1256 except OSError as exc:
1257 # Could be a permission error, read-only filesystem: just forget
1258 # about writing the data.
1259 _bootstrap._verbose_message('could not create {!r}: {!r}',
1260 parent, exc)
1261 return
1262 try:
1263 _write_atomic(path, data, _mode)
1264 _bootstrap._verbose_message('created {!r}', path)
1265 except OSError as exc:
1266 # Same as above: just don't write the bytecode.
1267 _bootstrap._verbose_message('could not create {!r}: {!r}', path,
1268 exc)
1269
1270
1271 class SourcelessFileLoader(FileLoader, _LoaderBasics):
1272
1273 """Loader which handles sourceless file imports."""
1274
1275 def get_code(self, fullname):
1276 path = self.get_filename(fullname)
1277 data = self.get_data(path)
1278 # Call _classify_pyc to do basic validation of the pyc but ignore the
1279 # result. There's no source to check against.
1280 exc_details = {
1281 'name': fullname,
1282 'path': path,
1283 }
1284 _classify_pyc(data, fullname, exc_details)
1285 return _compile_bytecode(
1286 memoryview(data)[16:],
1287 name=fullname,
1288 bytecode_path=path,
1289 )
1290
1291 def get_source(self, fullname):
1292 """Return None as there is no source code."""
1293 return None
1294
1295
1296 class ExtensionFileLoader(FileLoader, _LoaderBasics):
1297
1298 """Loader for extension modules.
1299
1300 The constructor is designed to work with FileFinder.
1301
1302 """
1303
1304 def __init__(self, name, path):
1305 self.name = name
1306 self.path = path
1307
1308 def __eq__(self, other):
1309 return (self.__class__ == other.__class__ and
1310 self.__dict__ == other.__dict__)
1311
1312 def __hash__(self):
1313 return hash(self.name) ^ hash(self.path)
1314
1315 def create_module(self, spec):
1316 """Create an uninitialized extension module"""
1317 module = _bootstrap._call_with_frames_removed(
1318 _imp.create_dynamic, spec)
1319 _bootstrap._verbose_message('extension module {!r} loaded from {!r}',
1320 spec.name, self.path)
1321 return module
1322
1323 def exec_module(self, module):
1324 """Initialize an extension module"""
1325 _bootstrap._call_with_frames_removed(_imp.exec_dynamic, module)
1326 _bootstrap._verbose_message('extension module {!r} executed from {!r}',
1327 self.name, self.path)
1328
1329 def is_package(self, fullname):
1330 """Return True if the extension module is a package."""
1331 file_name = _path_split(self.path)[1]
1332 return any(file_name == '__init__' + suffix
1333 for suffix in EXTENSION_SUFFIXES)
1334
1335 def get_code(self, fullname):
1336 """Return None as an extension module cannot create a code object."""
1337 return None
1338
1339 def get_source(self, fullname):
1340 """Return None as extension modules have no source code."""
1341 return None
1342
1343 @_check_name
1344 def get_filename(self, fullname):
1345 """Return the path to the source file as found by the finder."""
1346 return self.path
1347
1348
1349 class _NamespacePath:
1350 """Represents a namespace package's path. It uses the module name
1351 to find its parent module, and from there it looks up the parent's
1352 __path__. When this changes, the module's own path is recomputed,
1353 using path_finder. For top-level modules, the parent module's path
1354 is sys.path."""
1355
1356 # When invalidate_caches() is called, this epoch is incremented
1357 # https://bugs.python.org/issue45703
1358 _epoch = 0
1359
1360 def __init__(self, name, path, path_finder):
1361 self._name = name
1362 self._path = path
1363 self._last_parent_path = tuple(self._get_parent_path())
1364 self._last_epoch = self._epoch
1365 self._path_finder = path_finder
1366
1367 def _find_parent_path_names(self):
1368 """Returns a tuple of (parent-module-name, parent-path-attr-name)"""
1369 parent, dot, me = self._name.rpartition('.')
1370 if dot == '':
1371 # This is a top-level module. sys.path contains the parent path.
1372 return 'sys', 'path'
1373 # Not a top-level module. parent-module.__path__ contains the
1374 # parent path.
1375 return parent, '__path__'
1376
1377 def _get_parent_path(self):
1378 parent_module_name, path_attr_name = self._find_parent_path_names()
1379 return getattr(sys.modules[parent_module_name], path_attr_name)
1380
1381 def _recalculate(self):
1382 # If the parent's path has changed, recalculate _path
1383 parent_path = tuple(self._get_parent_path()) # Make a copy
1384 if parent_path != self._last_parent_path or self._epoch != self._last_epoch:
1385 spec = self._path_finder(self._name, parent_path)
1386 # Note that no changes are made if a loader is returned, but we
1387 # do remember the new parent path
1388 if spec is not None and spec.loader is None:
1389 if spec.submodule_search_locations:
1390 self._path = spec.submodule_search_locations
1391 self._last_parent_path = parent_path # Save the copy
1392 self._last_epoch = self._epoch
1393 return self._path
1394
1395 def __iter__(self):
1396 return iter(self._recalculate())
1397
1398 def __getitem__(self, index):
1399 return self._recalculate()[index]
1400
1401 def __setitem__(self, index, path):
1402 self._path[index] = path
1403
1404 def __len__(self):
1405 return len(self._recalculate())
1406
1407 def __repr__(self):
1408 return f'_NamespacePath({self._path!r})'
1409
1410 def __contains__(self, item):
1411 return item in self._recalculate()
1412
1413 def append(self, item):
1414 self._path.append(item)
1415
1416
1417 # This class is actually exposed publicly in a namespace package's __loader__
1418 # attribute, so it should be available through a non-private name.
1419 # https://github.com/python/cpython/issues/92054
1420 class NamespaceLoader:
1421 def __init__(self, name, path, path_finder):
1422 self._path = _NamespacePath(name, path, path_finder)
1423
1424 def is_package(self, fullname):
1425 return True
1426
1427 def get_source(self, fullname):
1428 return ''
1429
1430 def get_code(self, fullname):
1431 return compile('', '<string>', 'exec', dont_inherit=True)
1432
1433 def create_module(self, spec):
1434 """Use default semantics for module creation."""
1435
1436 def exec_module(self, module):
1437 pass
1438
1439 def load_module(self, fullname):
1440 """Load a namespace module.
1441
1442 This method is deprecated. Use exec_module() instead.
1443
1444 """
1445 # The import system never calls this method.
1446 _bootstrap._verbose_message('namespace module loaded with path {!r}',
1447 self._path)
1448 # Warning implemented in _load_module_shim().
1449 return _bootstrap._load_module_shim(self, fullname)
1450
1451 def get_resource_reader(self, module):
1452 from importlib.readers import NamespaceReader
1453 return NamespaceReader(self._path)
1454
1455
1456 # We use this exclusively in module_from_spec() for backward-compatibility.
1457 _NamespaceLoader = NamespaceLoader
1458
1459
1460 # Finders #####################################################################
1461
1462 class PathFinder:
1463
1464 """Meta path finder for sys.path and package __path__ attributes."""
1465
1466 @staticmethod
1467 def invalidate_caches():
1468 """Call the invalidate_caches() method on all path entry finders
1469 stored in sys.path_importer_cache (where implemented)."""
1470 for name, finder in list(sys.path_importer_cache.items()):
1471 # Drop entry if finder name is a relative path. The current
1472 # working directory may have changed.
1473 if finder is None or not _path_isabs(name):
1474 del sys.path_importer_cache[name]
1475 elif hasattr(finder, 'invalidate_caches'):
1476 finder.invalidate_caches()
1477 # Also invalidate the caches of _NamespacePaths
1478 # https://bugs.python.org/issue45703
1479 _NamespacePath._epoch += 1
1480
1481 from importlib.metadata import MetadataPathFinder
1482 MetadataPathFinder.invalidate_caches()
1483
1484 @staticmethod
1485 def _path_hooks(path):
1486 """Search sys.path_hooks for a finder for 'path'."""
1487 if sys.path_hooks is not None and not sys.path_hooks:
1488 _warnings.warn('sys.path_hooks is empty', ImportWarning)
1489 for hook in sys.path_hooks:
1490 try:
1491 return hook(path)
1492 except ImportError:
1493 continue
1494 else:
1495 return None
1496
1497 @classmethod
1498 def _path_importer_cache(cls, path):
1499 """Get the finder for the path entry from sys.path_importer_cache.
1500
1501 If the path entry is not in the cache, find the appropriate finder
1502 and cache it. If no finder is available, store None.
1503
1504 """
1505 if path == '':
1506 try:
1507 path = _os.getcwd()
1508 except FileNotFoundError:
1509 # Don't cache the failure as the cwd can easily change to
1510 # a valid directory later on.
1511 return None
1512 try:
1513 finder = sys.path_importer_cache[path]
1514 except KeyError:
1515 finder = cls._path_hooks(path)
1516 sys.path_importer_cache[path] = finder
1517 return finder
1518
1519 @classmethod
1520 def _get_spec(cls, fullname, path, target=None):
1521 """Find the loader or namespace_path for this module/package name."""
1522 # If this ends up being a namespace package, namespace_path is
1523 # the list of paths that will become its __path__
1524 namespace_path = []
1525 for entry in path:
1526 if not isinstance(entry, str):
1527 continue
1528 finder = cls._path_importer_cache(entry)
1529 if finder is not None:
1530 spec = finder.find_spec(fullname, target)
1531 if spec is None:
1532 continue
1533 if spec.loader is not None:
1534 return spec
1535 portions = spec.submodule_search_locations
1536 if portions is None:
1537 raise ImportError('spec missing loader')
1538 # This is possibly part of a namespace package.
1539 # Remember these path entries (if any) for when we
1540 # create a namespace package, and continue iterating
1541 # on path.
1542 namespace_path.extend(portions)
1543 else:
1544 spec = _bootstrap.ModuleSpec(fullname, None)
1545 spec.submodule_search_locations = namespace_path
1546 return spec
1547
1548 @classmethod
1549 def find_spec(cls, fullname, path=None, target=None):
1550 """Try to find a spec for 'fullname' on sys.path or 'path'.
1551
1552 The search is based on sys.path_hooks and sys.path_importer_cache.
1553 """
1554 if path is None:
1555 path = sys.path
1556 spec = cls._get_spec(fullname, path, target)
1557 if spec is None:
1558 return None
1559 elif spec.loader is None:
1560 namespace_path = spec.submodule_search_locations
1561 if namespace_path:
1562 # We found at least one namespace path. Return a spec which
1563 # can create the namespace package.
1564 spec.origin = None
1565 spec.submodule_search_locations = _NamespacePath(fullname, namespace_path, cls._get_spec)
1566 return spec
1567 else:
1568 return None
1569 else:
1570 return spec
1571
1572 @staticmethod
1573 def find_distributions(*args, **kwargs):
1574 """
1575 Find distributions.
1576
1577 Return an iterable of all Distribution instances capable of
1578 loading the metadata for packages matching ``context.name``
1579 (or all names if ``None`` indicated) along the paths in the list
1580 of directories ``context.path``.
1581 """
1582 from importlib.metadata import MetadataPathFinder
1583 return MetadataPathFinder.find_distributions(*args, **kwargs)
1584
1585
1586 class FileFinder:
1587
1588 """File-based finder.
1589
1590 Interactions with the file system are cached for performance, being
1591 refreshed when the directory the finder is handling has been modified.
1592
1593 """
1594
1595 def __init__(self, path, *loader_details):
1596 """Initialize with the path to search on and a variable number of
1597 2-tuples containing the loader and the file suffixes the loader
1598 recognizes."""
1599 loaders = []
1600 for loader, suffixes in loader_details:
1601 loaders.extend((suffix, loader) for suffix in suffixes)
1602 self._loaders = loaders
1603 # Base (directory) path
1604 if not path or path == '.':
1605 self.path = _os.getcwd()
1606 else:
1607 self.path = _path_abspath(path)
1608 self._path_mtime = -1
1609 self._path_cache = set()
1610 self._relaxed_path_cache = set()
1611
1612 def invalidate_caches(self):
1613 """Invalidate the directory mtime."""
1614 self._path_mtime = -1
1615
1616 def _get_spec(self, loader_class, fullname, path, smsl, target):
1617 loader = loader_class(fullname, path)
1618 return spec_from_file_location(fullname, path, loader=loader,
1619 submodule_search_locations=smsl)
1620
1621 def find_spec(self, fullname, target=None):
1622 """Try to find a spec for the specified module.
1623
1624 Returns the matching spec, or None if not found.
1625 """
1626 is_namespace = False
1627 tail_module = fullname.rpartition('.')[2]
1628 try:
1629 mtime = _path_stat(self.path or _os.getcwd()).st_mtime
1630 except OSError:
1631 mtime = -1
1632 if mtime != self._path_mtime:
1633 self._fill_cache()
1634 self._path_mtime = mtime
1635 # tail_module keeps the original casing, for __file__ and friends
1636 if _relax_case():
1637 cache = self._relaxed_path_cache
1638 cache_module = tail_module.lower()
1639 else:
1640 cache = self._path_cache
1641 cache_module = tail_module
1642 # Check if the module is the name of a directory (and thus a package).
1643 if cache_module in cache:
1644 base_path = _path_join(self.path, tail_module)
1645 for suffix, loader_class in self._loaders:
1646 init_filename = '__init__' + suffix
1647 full_path = _path_join(base_path, init_filename)
1648 if _path_isfile(full_path):
1649 return self._get_spec(loader_class, fullname, full_path, [base_path], target)
1650 else:
1651 # If a namespace package, return the path if we don't
1652 # find a module in the next section.
1653 is_namespace = _path_isdir(base_path)
1654 # Check for a file w/ a proper suffix exists.
1655 for suffix, loader_class in self._loaders:
1656 try:
1657 full_path = _path_join(self.path, tail_module + suffix)
1658 except ValueError:
1659 return None
1660 _bootstrap._verbose_message('trying {}', full_path, verbosity=2)
1661 if cache_module + suffix in cache:
1662 if _path_isfile(full_path):
1663 return self._get_spec(loader_class, fullname, full_path,
1664 None, target)
1665 if is_namespace:
1666 _bootstrap._verbose_message('possible namespace for {}', base_path)
1667 spec = _bootstrap.ModuleSpec(fullname, None)
1668 spec.submodule_search_locations = [base_path]
1669 return spec
1670 return None
1671
1672 def _fill_cache(self):
1673 """Fill the cache of potential modules and packages for this directory."""
1674 path = self.path
1675 try:
1676 contents = _os.listdir(path or _os.getcwd())
1677 except (FileNotFoundError, PermissionError, NotADirectoryError):
1678 # Directory has either been removed, turned into a file, or made
1679 # unreadable.
1680 contents = []
1681 # We store two cached versions, to handle runtime changes of the
1682 # PYTHONCASEOK environment variable.
1683 if not sys.platform.startswith('win'):
1684 self._path_cache = set(contents)
1685 else:
1686 # Windows users can import modules with case-insensitive file
1687 # suffixes (for legacy reasons). Make the suffix lowercase here
1688 # so it's done once instead of for every import. This is safe as
1689 # the specified suffixes to check against are always specified in a
1690 # case-sensitive manner.
1691 lower_suffix_contents = set()
1692 for item in contents:
1693 name, dot, suffix = item.partition('.')
1694 if dot:
1695 new_name = f'{name}.{suffix.lower()}'
1696 else:
1697 new_name = name
1698 lower_suffix_contents.add(new_name)
1699 self._path_cache = lower_suffix_contents
1700 if sys.platform.startswith(_CASE_INSENSITIVE_PLATFORMS):
1701 self._relaxed_path_cache = {fn.lower() for fn in contents}
1702
1703 @classmethod
1704 def path_hook(cls, *loader_details):
1705 """A class method which returns a closure to use on sys.path_hook
1706 which will return an instance using the specified loaders and the path
1707 called on the closure.
1708
1709 If the path called on the closure is not a directory, ImportError is
1710 raised.
1711
1712 """
1713 def path_hook_for_FileFinder(path):
1714 """Path hook for importlib.machinery.FileFinder."""
1715 if not _path_isdir(path):
1716 raise ImportError('only directories are supported', path=path)
1717 return cls(path, *loader_details)
1718
1719 return path_hook_for_FileFinder
1720
1721 def __repr__(self):
1722 return f'FileFinder({self.path!r})'
1723
1724
1725 class AppleFrameworkLoader(ExtensionFileLoader):
1726 """A loader for modules that have been packaged as frameworks for
1727 compatibility with Apple's iOS App Store policies.
1728 """
1729 def create_module(self, spec):
1730 # If the ModuleSpec has been created by the FileFinder, it will have
1731 # been created with an origin pointing to the .fwork file. We need to
1732 # redirect this to the location in the Frameworks folder, using the
1733 # content of the .fwork file.
1734 if spec.origin.endswith(".fwork"):
1735 with _io.FileIO(spec.origin, 'r') as file:
1736 framework_binary = file.read().decode().strip()
1737 bundle_path = _path_split(sys.executable)[0]
1738 spec.origin = _path_join(bundle_path, framework_binary)
1739
1740 # If the loader is created based on the spec for a loaded module, the
1741 # path will be pointing at the Framework location. If this occurs,
1742 # get the original .fwork location to use as the module's __file__.
1743 if self.path.endswith(".fwork"):
1744 path = self.path
1745 else:
1746 with _io.FileIO(self.path + ".origin", 'r') as file:
1747 origin = file.read().decode().strip()
1748 bundle_path = _path_split(sys.executable)[0]
1749 path = _path_join(bundle_path, origin)
1750
1751 module = _bootstrap._call_with_frames_removed(_imp.create_dynamic, spec)
1752
1753 _bootstrap._verbose_message(
1754 "Apple framework extension module {!r} loaded from {!r} (path {!r})",
1755 spec.name,
1756 spec.origin,
1757 path,
1758 )
1759
1760 # Ensure that the __file__ points at the .fwork location
1761 try:
1762 module.__file__ = path
1763 except AttributeError:
1764 # Not important enough to report.
1765 # (The error is also ignored in _bootstrap._init_module_attrs or
1766 # import_run_extension in import.c)
1767 pass
1768
1769 return module
1770
1771 # Import setup ###############################################################
1772
1773 def _fix_up_module(ns, name, pathname, cpathname=None):
1774 # This function is used by PyImport_ExecCodeModuleObject().
1775 loader = ns.get('__loader__')
1776 spec = ns.get('__spec__')
1777 if not loader:
1778 if spec:
1779 loader = spec.loader
1780 elif pathname == cpathname:
1781 loader = SourcelessFileLoader(name, pathname)
1782 else:
1783 loader = SourceFileLoader(name, pathname)
1784 if not spec:
1785 spec = spec_from_file_location(name, pathname, loader=loader)
1786 if cpathname:
1787 spec.cached = _path_abspath(cpathname)
1788 try:
1789 ns['__spec__'] = spec
1790 ns['__loader__'] = loader
1791 ns['__file__'] = pathname
1792 ns['__cached__'] = cpathname
1793 except Exception:
1794 # Not important enough to report.
1795 pass
1796
1797
1798 def _get_supported_file_loaders():
1799 """Returns a list of file-based module loaders.
1800
1801 Each item is a tuple (loader, suffixes).
1802 """
1803 extension_loaders = []
1804 if hasattr(_imp, 'create_dynamic'):
1805 if sys.platform in {"ios", "tvos", "watchos"}:
1806 extension_loaders = [(AppleFrameworkLoader, [
1807 suffix.replace(".so", ".fwork")
1808 for suffix in _imp.extension_suffixes()
1809 ])]
1810 extension_loaders.append((ExtensionFileLoader, _imp.extension_suffixes()))
1811 source = SourceFileLoader, SOURCE_SUFFIXES
1812 bytecode = SourcelessFileLoader, BYTECODE_SUFFIXES
1813 return extension_loaders + [source, bytecode]
1814
1815
1816 def _set_bootstrap_module(_bootstrap_module):
1817 global _bootstrap
1818 _bootstrap = _bootstrap_module
1819
1820
1821 def _install(_bootstrap_module):
1822 """Install the path-based import components."""
1823 _set_bootstrap_module(_bootstrap_module)
1824 supported_loaders = _get_supported_file_loaders()
1825 sys.path_hooks.extend([FileFinder.path_hook(*supported_loaders)])
1826 sys.meta_path.append(PathFinder)