Debian Codegraph @

Tools/build/freeze_modules.py

raw
1 """Freeze modules and regen related files (e.g. Python/frozen.c).
2
3 See the notes at the top of Python/frozen.c for more info.
4 """
5
6 import hashlib
7 import ntpath
8 import os
9 import posixpath
10 from collections import namedtuple
11
12 from update_file import updating_file_with_tmpfile
13
14 ROOT_DIR = os.path.dirname(os.path.dirname(os.path.dirname(__file__)))
15 ROOT_DIR = os.path.abspath(ROOT_DIR)
16 FROZEN_ONLY = os.path.join(ROOT_DIR, 'Tools', 'freeze', 'flag.py')
17
18 STDLIB_DIR = os.path.join(ROOT_DIR, 'Lib')
19 # If FROZEN_MODULES_DIR or DEEPFROZEN_MODULES_DIR is changed then the
20 # .gitattributes and .gitignore files needs to be updated.
21 FROZEN_MODULES_DIR = os.path.join(ROOT_DIR, 'Python', 'frozen_modules')
22
23 FROZEN_FILE = os.path.join(ROOT_DIR, 'Python', 'frozen.c')
24 MAKEFILE = os.path.join(ROOT_DIR, 'Makefile.pre.in')
25 PCBUILD_PROJECT = os.path.join(ROOT_DIR, 'PCbuild', '_freeze_module.vcxproj')
26 PCBUILD_FILTERS = os.path.join(ROOT_DIR, 'PCbuild', '_freeze_module.vcxproj.filters')
27 PCBUILD_PYTHONCORE = os.path.join(ROOT_DIR, 'PCbuild', 'pythoncore.vcxproj')
28
29
30 OS_PATH = 'ntpath' if os.name == 'nt' else 'posixpath'
31
32 # These are modules that get frozen.
33 # If you're debugging new bytecode instructions,
34 # you can delete all sections except 'import system'.
35 # This also speeds up building somewhat.
36 TESTS_SECTION = 'Test module'
37 FROZEN = [
38 # See parse_frozen_spec() for the format.
39 # In cases where the frozenid is duplicated, the first one is re-used.
40 ('import system', [
41 # These frozen modules are necessary for bootstrapping
42 # the import system.
43 'importlib._bootstrap : _frozen_importlib',
44 'importlib._bootstrap_external : _frozen_importlib_external',
45 # This module is important because some Python builds rely
46 # on a builtin zip file instead of a filesystem.
47 'zipimport',
48 ]),
49 # (You can delete entries from here down to the end of the list.)
50 ('stdlib - startup, without site (python -S)', [
51 'abc',
52 'codecs',
53 # For now we do not freeze the encodings, due # to the noise all
54 # those extra modules add to the text printed during the build.
55 # (See https://github.com/python/cpython/pull/28398#pullrequestreview-756856469.)
56 #'<encodings.*>',
57 'io',
58 ]),
59 ('stdlib - startup, with site', [
60 '_collections_abc',
61 '_sitebuiltins',
62 'genericpath',
63 'ntpath',
64 'posixpath',
65 # We must explicitly mark os.path as a frozen module
66 # even though it will never be imported.
67 f'{OS_PATH} : os.path',
68 'os',
69 'site',
70 'stat',
71 ]),
72 ('runpy - run module with -m', [
73 "importlib.util",
74 "importlib.machinery",
75 "runpy",
76 ]),
77 (TESTS_SECTION, [
78 '__hello__',
79 '__hello__ : __hello_alias__',
80 '__hello__ : <__phello_alias__>',
81 '__hello__ : __phello_alias__.spam',
82 '<__phello__.**.*>',
83 f'frozen_only : __hello_only__ = {FROZEN_ONLY}',
84 ]),
85 # (End of stuff you could delete.)
86 ]
87 BOOTSTRAP = {
88 'importlib._bootstrap',
89 'importlib._bootstrap_external',
90 'zipimport',
91 }
92
93
94 #######################################
95 # platform-specific helpers
96
97 if os.path is posixpath:
98 relpath_for_posix_display = os.path.relpath
99
100 def relpath_for_windows_display(path, base):
101 return ntpath.relpath(
102 ntpath.join(*path.split(os.path.sep)),
103 ntpath.join(*base.split(os.path.sep)),
104 )
105
106 else:
107 relpath_for_windows_display = ntpath.relpath
108
109 def relpath_for_posix_display(path, base):
110 return posixpath.relpath(
111 posixpath.join(*path.split(os.path.sep)),
112 posixpath.join(*base.split(os.path.sep)),
113 )
114
115
116 #######################################
117 # specs
118
119 def parse_frozen_specs():
120 seen = {}
121 for section, specs in FROZEN:
122 parsed = _parse_specs(specs, section, seen)
123 for item in parsed:
124 frozenid, pyfile, modname, ispkg, section = item
125 try:
126 source = seen[frozenid]
127 except KeyError:
128 source = FrozenSource.from_id(frozenid, pyfile)
129 seen[frozenid] = source
130 else:
131 assert not pyfile or pyfile == source.pyfile, item
132 yield FrozenModule(modname, ispkg, section, source)
133
134
135 def _parse_specs(specs, section, seen):
136 for spec in specs:
137 info, subs = _parse_spec(spec, seen, section)
138 yield info
139 for info in subs or ():
140 yield info
141
142
143 def _parse_spec(spec, knownids=None, section=None):
144 """Yield an info tuple for each module corresponding to the given spec.
145
146 The info consists of: (frozenid, pyfile, modname, ispkg, section).
147
148 Supported formats:
149
150 frozenid
151 frozenid : modname
152 frozenid : modname = pyfile
153
154 "frozenid" and "modname" must be valid module names (dot-separated
155 identifiers). If "modname" is not provided then "frozenid" is used.
156 If "pyfile" is not provided then the filename of the module
157 corresponding to "frozenid" is used.
158
159 Angle brackets around a frozenid (e.g. '<encodings>") indicate
160 it is a package. This also means it must be an actual module
161 (i.e. "pyfile" cannot have been provided). Such values can have
162 patterns to expand submodules:
163
164 <encodings.*> - also freeze all direct submodules
165 <encodings.**.*> - also freeze the full submodule tree
166
167 As with "frozenid", angle brackets around "modname" indicate
168 it is a package. However, in this case "pyfile" should not
169 have been provided and patterns in "modname" are not supported.
170 Also, if "modname" has brackets then "frozenid" should not,
171 and "pyfile" should have been provided..
172 """
173 frozenid, _, remainder = spec.partition(':')
174 modname, _, pyfile = remainder.partition('=')
175 frozenid = frozenid.strip()
176 modname = modname.strip()
177 pyfile = pyfile.strip()
178
179 submodules = None
180 if modname.startswith('<') and modname.endswith('>'):
181 assert check_modname(frozenid), spec
182 modname = modname[1:-1]
183 assert check_modname(modname), spec
184 if frozenid in knownids:
185 pass
186 elif pyfile:
187 assert not os.path.isdir(pyfile), spec
188 else:
189 pyfile = _resolve_module(frozenid, ispkg=False)
190 ispkg = True
191 elif pyfile:
192 assert check_modname(frozenid), spec
193 assert not knownids or frozenid not in knownids, spec
194 assert check_modname(modname), spec
195 assert not os.path.isdir(pyfile), spec
196 ispkg = False
197 elif knownids and frozenid in knownids:
198 assert check_modname(frozenid), spec
199 assert check_modname(modname), spec
200 ispkg = False
201 else:
202 assert not modname or check_modname(modname), spec
203 resolved = iter(resolve_modules(frozenid))
204 frozenid, pyfile, ispkg = next(resolved)
205 if not modname:
206 modname = frozenid
207 if ispkg:
208 pkgid = frozenid
209 pkgname = modname
210 pkgfiles = {pyfile: pkgid}
211 def iter_subs():
212 for frozenid, pyfile, ispkg in resolved:
213 if pkgname:
214 modname = frozenid.replace(pkgid, pkgname, 1)
215 else:
216 modname = frozenid
217 if pyfile:
218 if pyfile in pkgfiles:
219 frozenid = pkgfiles[pyfile]
220 pyfile = None
221 elif ispkg:
222 pkgfiles[pyfile] = frozenid
223 yield frozenid, pyfile, modname, ispkg, section
224 submodules = iter_subs()
225
226 info = (frozenid, pyfile or None, modname, ispkg, section)
227 return info, submodules
228
229
230 #######################################
231 # frozen source files
232
233 class FrozenSource(namedtuple('FrozenSource', 'id pyfile frozenfile')):
234
235 @classmethod
236 def from_id(cls, frozenid, pyfile=None):
237 if not pyfile:
238 pyfile = os.path.join(STDLIB_DIR, *frozenid.split('.')) + '.py'
239 #assert os.path.exists(pyfile), (frozenid, pyfile)
240 frozenfile = resolve_frozen_file(frozenid, FROZEN_MODULES_DIR)
241 return cls(frozenid, pyfile, frozenfile)
242
243 @property
244 def frozenid(self):
245 return self.id
246
247 @property
248 def modname(self):
249 if self.pyfile.startswith(STDLIB_DIR):
250 return self.id
251 return None
252
253 @property
254 def symbol(self):
255 # This matches what we do in Programs/_freeze_module.c:
256 name = self.frozenid.replace('.', '_')
257 return '_Py_M__' + name
258
259 @property
260 def ispkg(self):
261 if not self.pyfile:
262 return False
263 elif self.frozenid.endswith('.__init__'):
264 return False
265 else:
266 return os.path.basename(self.pyfile) == '__init__.py'
267
268 @property
269 def isbootstrap(self):
270 return self.id in BOOTSTRAP
271
272
273 def resolve_frozen_file(frozenid, destdir):
274 """Return the filename corresponding to the given frozen ID.
275
276 For stdlib modules the ID will always be the full name
277 of the source module.
278 """
279 if not isinstance(frozenid, str):
280 try:
281 frozenid = frozenid.frozenid
282 except AttributeError:
283 raise ValueError(f'unsupported frozenid {frozenid!r}')
284 # We use a consistent naming convention for all frozen modules.
285 frozenfile = f'{frozenid}.h'
286 if not destdir:
287 return frozenfile
288 return os.path.join(destdir, frozenfile)
289
290
291 #######################################
292 # frozen modules
293
294 class FrozenModule(namedtuple('FrozenModule', 'name ispkg section source')):
295
296 def __getattr__(self, name):
297 return getattr(self.source, name)
298
299 @property
300 def modname(self):
301 return self.name
302
303 @property
304 def orig(self):
305 return self.source.modname
306
307 @property
308 def isalias(self):
309 orig = self.source.modname
310 if not orig:
311 return True
312 return self.name != orig
313
314 def summarize(self):
315 source = self.source.modname
316 if source:
317 source = f'<{source}>'
318 else:
319 source = relpath_for_posix_display(self.pyfile, ROOT_DIR)
320 return {
321 'module': self.name,
322 'ispkg': self.ispkg,
323 'source': source,
324 'frozen': os.path.basename(self.frozenfile),
325 'checksum': _get_checksum(self.frozenfile),
326 }
327
328
329 def _iter_sources(modules):
330 seen = set()
331 for mod in modules:
332 if mod.source not in seen:
333 yield mod.source
334 seen.add(mod.source)
335
336
337 #######################################
338 # generic helpers
339
340 def _get_checksum(filename):
341 with open(filename, "rb") as infile:
342 contents = infile.read()
343 m = hashlib.sha256()
344 m.update(contents)
345 return m.hexdigest()
346
347
348 def resolve_modules(modname, pyfile=None):
349 if modname.startswith('<') and modname.endswith('>'):
350 if pyfile:
351 assert os.path.isdir(pyfile) or os.path.basename(pyfile) == '__init__.py', pyfile
352 ispkg = True
353 modname = modname[1:-1]
354 rawname = modname
355 # For now, we only expect match patterns at the end of the name.
356 _modname, sep, match = modname.rpartition('.')
357 if sep:
358 if _modname.endswith('.**'):
359 modname = _modname[:-3]
360 match = f'**.{match}'
361 elif match and not match.isidentifier():
362 modname = _modname
363 # Otherwise it's a plain name so we leave it alone.
364 else:
365 match = None
366 else:
367 ispkg = False
368 rawname = modname
369 match = None
370
371 if not check_modname(modname):
372 raise ValueError(f'not a valid module name ({rawname})')
373
374 if not pyfile:
375 pyfile = _resolve_module(modname, ispkg=ispkg)
376 elif os.path.isdir(pyfile):
377 pyfile = _resolve_module(modname, pyfile, ispkg)
378 yield modname, pyfile, ispkg
379
380 if match:
381 pkgdir = os.path.dirname(pyfile)
382 yield from iter_submodules(modname, pkgdir, match)
383
384
385 def check_modname(modname):
386 return all(n.isidentifier() for n in modname.split('.'))
387
388
389 def iter_submodules(pkgname, pkgdir=None, match='*'):
390 if not pkgdir:
391 pkgdir = os.path.join(STDLIB_DIR, *pkgname.split('.'))
392 if not match:
393 match = '**.*'
394 match_modname = _resolve_modname_matcher(match, pkgdir)
395
396 def _iter_submodules(pkgname, pkgdir):
397 for entry in sorted(os.scandir(pkgdir), key=lambda e: e.name):
398 matched, recursive = match_modname(entry.name)
399 if not matched:
400 continue
401 modname = f'{pkgname}.{entry.name}'
402 if modname.endswith('.py'):
403 yield modname[:-3], entry.path, False
404 elif entry.is_dir():
405 pyfile = os.path.join(entry.path, '__init__.py')
406 # We ignore namespace packages.
407 if os.path.exists(pyfile):
408 yield modname, pyfile, True
409 if recursive:
410 yield from _iter_submodules(modname, entry.path)
411
412 return _iter_submodules(pkgname, pkgdir)
413
414
415 def _resolve_modname_matcher(match, rootdir=None):
416 if isinstance(match, str):
417 if match.startswith('**.'):
418 recursive = True
419 pat = match[3:]
420 assert match
421 else:
422 recursive = False
423 pat = match
424
425 if pat == '*':
426 def match_modname(modname):
427 return True, recursive
428 else:
429 raise NotImplementedError(match)
430 elif callable(match):
431 match_modname = match(rootdir)
432 else:
433 raise ValueError(f'unsupported matcher {match!r}')
434 return match_modname
435
436
437 def _resolve_module(modname, pathentry=STDLIB_DIR, ispkg=False):
438 assert pathentry, pathentry
439 pathentry = os.path.normpath(pathentry)
440 assert os.path.isabs(pathentry)
441 if ispkg:
442 return os.path.join(pathentry, *modname.split('.'), '__init__.py')
443 return os.path.join(pathentry, *modname.split('.')) + '.py'
444
445
446 #######################################
447 # regenerating dependent files
448
449 def find_marker(lines, marker, file):
450 for pos, line in enumerate(lines):
451 if marker in line:
452 return pos
453 raise Exception(f"Can't find {marker!r} in file {file}")
454
455
456 def replace_block(lines, start_marker, end_marker, replacements, file):
457 start_pos = find_marker(lines, start_marker, file)
458 end_pos = find_marker(lines, end_marker, file)
459 if end_pos <= start_pos:
460 raise Exception(f"End marker {end_marker!r} "
461 f"occurs before start marker {start_marker!r} "
462 f"in file {file}")
463 replacements = [line.rstrip() + '\n' for line in replacements]
464 return lines[:start_pos + 1] + replacements + lines[end_pos:]
465
466
467 class UniqueList(list):
468 def __init__(self):
469 self._seen = set()
470
471 def append(self, item):
472 if item in self._seen:
473 return
474 super().append(item)
475 self._seen.add(item)
476
477
478 def regen_frozen(modules):
479 headerlines = []
480 parentdir = os.path.dirname(FROZEN_FILE)
481 for src in _iter_sources(modules):
482 # Adding a comment to separate sections here doesn't add much,
483 # so we don't.
484 header = relpath_for_posix_display(src.frozenfile, parentdir)
485 headerlines.append(f'#include "{header}"')
486
487 bootstraplines = []
488 stdliblines = []
489 testlines = []
490 aliaslines = []
491 indent = ' '
492 lastsection = None
493 for mod in modules:
494 if mod.isbootstrap:
495 lines = bootstraplines
496 elif mod.section == TESTS_SECTION:
497 lines = testlines
498 else:
499 lines = stdliblines
500 if mod.section != lastsection:
501 if lastsection is not None:
502 lines.append('')
503 lines.append(f'/* {mod.section} */')
504 lastsection = mod.section
505
506 pkg = 'true' if mod.ispkg else 'false'
507 size = f"(int)sizeof({mod.symbol})"
508 line = f'{{"{mod.name}", {mod.symbol}, {size}, {pkg}}},'
509 lines.append(line)
510
511 if mod.isalias:
512 if not mod.orig:
513 entry = '{"%s", NULL},' % (mod.name,)
514 elif mod.source.ispkg:
515 entry = '{"%s", "<%s"},' % (mod.name, mod.orig)
516 else:
517 entry = '{"%s", "%s"},' % (mod.name, mod.orig)
518 aliaslines.append(indent + entry)
519
520 for lines in (bootstraplines, stdliblines, testlines):
521 # TODO: Is this necessary any more?
522 if lines and not lines[0]:
523 del lines[0]
524 for i, line in enumerate(lines):
525 if line:
526 lines[i] = indent + line
527
528 print(f'# Updating {os.path.relpath(FROZEN_FILE)}')
529 with updating_file_with_tmpfile(FROZEN_FILE) as (infile, outfile):
530 lines = infile.readlines()
531 # TODO: Use more obvious markers, e.g.
532 # $START GENERATED FOOBAR$ / $END GENERATED FOOBAR$
533 lines = replace_block(
534 lines,
535 "/* Includes for frozen modules: */",
536 "/* End includes */",
537 headerlines,
538 FROZEN_FILE,
539 )
540 lines = replace_block(
541 lines,
542 "static const struct _frozen bootstrap_modules[] =",
543 "/* bootstrap sentinel */",
544 bootstraplines,
545 FROZEN_FILE,
546 )
547 lines = replace_block(
548 lines,
549 "static const struct _frozen stdlib_modules[] =",
550 "/* stdlib sentinel */",
551 stdliblines,
552 FROZEN_FILE,
553 )
554 lines = replace_block(
555 lines,
556 "static const struct _frozen test_modules[] =",
557 "/* test sentinel */",
558 testlines,
559 FROZEN_FILE,
560 )
561 lines = replace_block(
562 lines,
563 "const struct _module_alias aliases[] =",
564 "/* aliases sentinel */",
565 aliaslines,
566 FROZEN_FILE,
567 )
568 outfile.writelines(lines)
569
570
571 def regen_makefile(modules):
572 pyfiles = []
573 frozenfiles = []
574 rules = ['']
575 for src in _iter_sources(modules):
576 frozen_header = relpath_for_posix_display(src.frozenfile, ROOT_DIR)
577 frozenfiles.append(f'\t\t{frozen_header} \\')
578
579 pyfile = relpath_for_posix_display(src.pyfile, ROOT_DIR)
580 pyfiles.append(f'\t\t{pyfile} \\')
581
582 if src.isbootstrap:
583 freezecmd = '$(FREEZE_MODULE_BOOTSTRAP)'
584 freezedep = '$(FREEZE_MODULE_BOOTSTRAP_DEPS)'
585 else:
586 freezecmd = '$(FREEZE_MODULE)'
587 freezedep = '$(FREEZE_MODULE_DEPS)'
588
589 freeze = (f'{freezecmd} {src.frozenid} '
590 f'$(srcdir)/{pyfile} {frozen_header}')
591 rules.extend([
592 f'{frozen_header}: {pyfile} {freezedep}',
593 f'\t{freeze}',
594 '',
595 ])
596 pyfiles[-1] = pyfiles[-1].rstrip(" \\")
597 frozenfiles[-1] = frozenfiles[-1].rstrip(" \\")
598
599 print(f'# Updating {os.path.relpath(MAKEFILE)}')
600 with updating_file_with_tmpfile(MAKEFILE) as (infile, outfile):
601 lines = infile.readlines()
602 lines = replace_block(
603 lines,
604 "FROZEN_FILES_IN =",
605 "# End FROZEN_FILES_IN",
606 pyfiles,
607 MAKEFILE,
608 )
609 lines = replace_block(
610 lines,
611 "FROZEN_FILES_OUT =",
612 "# End FROZEN_FILES_OUT",
613 frozenfiles,
614 MAKEFILE,
615 )
616 lines = replace_block(
617 lines,
618 "# BEGIN: freezing modules",
619 "# END: freezing modules",
620 rules,
621 MAKEFILE,
622 )
623 outfile.writelines(lines)
624
625
626 def regen_pcbuild(modules):
627 projlines = []
628 filterlines = []
629 for src in _iter_sources(modules):
630 pyfile = relpath_for_windows_display(src.pyfile, ROOT_DIR)
631 header = relpath_for_windows_display(src.frozenfile, ROOT_DIR)
632 intfile = ntpath.splitext(ntpath.basename(header))[0] + '.g.h'
633 projlines.append(f' <None Include="..\\{pyfile}">')
634 projlines.append(f' <ModName>{src.frozenid}</ModName>')
635 projlines.append(f' <IntFile>$(IntDir){intfile}</IntFile>')
636 projlines.append(f' <OutFile>$(GeneratedFrozenModulesDir){header}</OutFile>')
637 projlines.append(f' </None>')
638
639 filterlines.append(f' <None Include="..\\{pyfile}">')
640 filterlines.append(' <Filter>Python Files</Filter>')
641 filterlines.append(' </None>')
642
643 print(f'# Updating {os.path.relpath(PCBUILD_PROJECT)}')
644 with updating_file_with_tmpfile(PCBUILD_PROJECT) as (infile, outfile):
645 lines = infile.readlines()
646 lines = replace_block(
647 lines,
648 '<!-- BEGIN frozen modules -->',
649 '<!-- END frozen modules -->',
650 projlines,
651 PCBUILD_PROJECT,
652 )
653 outfile.writelines(lines)
654 print(f'# Updating {os.path.relpath(PCBUILD_FILTERS)}')
655 with updating_file_with_tmpfile(PCBUILD_FILTERS) as (infile, outfile):
656 lines = infile.readlines()
657 lines = replace_block(
658 lines,
659 '<!-- BEGIN frozen modules -->',
660 '<!-- END frozen modules -->',
661 filterlines,
662 PCBUILD_FILTERS,
663 )
664 outfile.writelines(lines)
665
666
667 #######################################
668 # the script
669
670 def main():
671 # Expand the raw specs, preserving order.
672 modules = list(parse_frozen_specs())
673
674 # Regen build-related files.
675 regen_makefile(modules)
676 regen_pcbuild(modules)
677 regen_frozen(modules)
678
679
680 if __name__ == '__main__':
681 main()