Debian Codegraph @

Tools/build/freeze_modules.py

raw
1 """Freeze modules and regen related files (e.g. Python/frozen.c).
2
3 See the notes at the top of Python/frozen.c for more info.
4 """
5
6 import hashlib
7 import ntpath
8 import os
9 import posixpath
10 from collections import namedtuple
11
12 from update_file import updating_file_with_tmpfile
13
14 ROOT_DIR = os.path.dirname(os.path.dirname(os.path.dirname(__file__)))
15 ROOT_DIR = os.path.abspath(ROOT_DIR)
16 FROZEN_ONLY = os.path.join(ROOT_DIR, 'Tools', 'freeze', 'flag.py')
17
18 STDLIB_DIR = os.path.join(ROOT_DIR, 'Lib')
19 # If FROZEN_MODULES_DIR or DEEPFROZEN_MODULES_DIR is changed then the
20 # .gitattributes and .gitignore files needs to be updated.
21 FROZEN_MODULES_DIR = os.path.join(ROOT_DIR, 'Python', 'frozen_modules')
22
23 FROZEN_FILE = os.path.join(ROOT_DIR, 'Python', 'frozen.c')
24 MAKEFILE = os.path.join(ROOT_DIR, 'Makefile.pre.in')
25 PCBUILD_PROJECT = os.path.join(ROOT_DIR, 'PCbuild', '_freeze_module.vcxproj')
26 PCBUILD_FILTERS = os.path.join(ROOT_DIR, 'PCbuild', '_freeze_module.vcxproj.filters')
27 PCBUILD_PYTHONCORE = os.path.join(ROOT_DIR, 'PCbuild', 'pythoncore.vcxproj')
28
29
30 OS_PATH = 'ntpath' if os.name == 'nt' else 'posixpath'
31
32 # These are modules that get frozen.
33 # If you're debugging new bytecode instructions,
34 # you can delete all sections except 'import system'.
35 # This also speeds up building somewhat.
36 TESTS_SECTION = 'Test module'
37 FROZEN = [
38 # See parse_frozen_spec() for the format.
39 # In cases where the frozenid is duplicated, the first one is re-used.
40 ('import system', [
41 # These frozen modules are necessary for bootstrapping
42 # the import system.
43 'importlib._bootstrap : _frozen_importlib',
44 'importlib._bootstrap_external : _frozen_importlib_external',
45 # This module is important because some Python builds rely
46 # on a builtin zip file instead of a filesystem.
47 'zipimport',
48 ]),
49 # (You can delete entries from here down to the end of the list.)
50 ('stdlib - startup, without site (python -S)', [
51 'abc',
52 'codecs',
53 # For now we do not freeze the encodings, due # to the noise all
54 # those extra modules add to the text printed during the build.
55 # (See https://github.com/python/cpython/pull/28398#pullrequestreview-756856469.)
56 #'<encodings.*>',
57 'io',
58 ]),
59 ('stdlib - startup, with site', [
60 '_collections_abc',
61 '_sitebuiltins',
62 'genericpath',
63 'ntpath',
64 'posixpath',
65 'os',
66 'site',
67 'stat',
68 ]),
69 ('runpy - run module with -m', [
70 "importlib.util",
71 "importlib.machinery",
72 "runpy",
73 ]),
74 (TESTS_SECTION, [
75 '__hello__',
76 '__hello__ : __hello_alias__',
77 '__hello__ : <__phello_alias__>',
78 '__hello__ : __phello_alias__.spam',
79 '<__phello__.**.*>',
80 f'frozen_only : __hello_only__ = {FROZEN_ONLY}',
81 ]),
82 # (End of stuff you could delete.)
83 ]
84 BOOTSTRAP = {
85 'importlib._bootstrap',
86 'importlib._bootstrap_external',
87 'zipimport',
88 }
89
90
91 #######################################
92 # platform-specific helpers
93
94 if os.path is posixpath:
95 relpath_for_posix_display = os.path.relpath
96
97 def relpath_for_windows_display(path, base):
98 return ntpath.relpath(
99 ntpath.join(*path.split(os.path.sep)),
100 ntpath.join(*base.split(os.path.sep)),
101 )
102
103 else:
104 relpath_for_windows_display = ntpath.relpath
105
106 def relpath_for_posix_display(path, base):
107 return posixpath.relpath(
108 posixpath.join(*path.split(os.path.sep)),
109 posixpath.join(*base.split(os.path.sep)),
110 )
111
112
113 #######################################
114 # specs
115
116 def parse_frozen_specs():
117 seen = {}
118 for section, specs in FROZEN:
119 parsed = _parse_specs(specs, section, seen)
120 for item in parsed:
121 frozenid, pyfile, modname, ispkg, section = item
122 try:
123 source = seen[frozenid]
124 except KeyError:
125 source = FrozenSource.from_id(frozenid, pyfile)
126 seen[frozenid] = source
127 else:
128 assert not pyfile or pyfile == source.pyfile, item
129 yield FrozenModule(modname, ispkg, section, source)
130
131
132 def _parse_specs(specs, section, seen):
133 for spec in specs:
134 info, subs = _parse_spec(spec, seen, section)
135 yield info
136 for info in subs or ():
137 yield info
138
139
140 def _parse_spec(spec, knownids=None, section=None):
141 """Yield an info tuple for each module corresponding to the given spec.
142
143 The info consists of: (frozenid, pyfile, modname, ispkg, section).
144
145 Supported formats:
146
147 frozenid
148 frozenid : modname
149 frozenid : modname = pyfile
150
151 "frozenid" and "modname" must be valid module names (dot-separated
152 identifiers). If "modname" is not provided then "frozenid" is used.
153 If "pyfile" is not provided then the filename of the module
154 corresponding to "frozenid" is used.
155
156 Angle brackets around a frozenid (e.g. '<encodings>") indicate
157 it is a package. This also means it must be an actual module
158 (i.e. "pyfile" cannot have been provided). Such values can have
159 patterns to expand submodules:
160
161 <encodings.*> - also freeze all direct submodules
162 <encodings.**.*> - also freeze the full submodule tree
163
164 As with "frozenid", angle brackets around "modname" indicate
165 it is a package. However, in this case "pyfile" should not
166 have been provided and patterns in "modname" are not supported.
167 Also, if "modname" has brackets then "frozenid" should not,
168 and "pyfile" should have been provided..
169 """
170 frozenid, _, remainder = spec.partition(':')
171 modname, _, pyfile = remainder.partition('=')
172 frozenid = frozenid.strip()
173 modname = modname.strip()
174 pyfile = pyfile.strip()
175
176 submodules = None
177 if modname.startswith('<') and modname.endswith('>'):
178 assert check_modname(frozenid), spec
179 modname = modname[1:-1]
180 assert check_modname(modname), spec
181 if frozenid in knownids:
182 pass
183 elif pyfile:
184 assert not os.path.isdir(pyfile), spec
185 else:
186 pyfile = _resolve_module(frozenid, ispkg=False)
187 ispkg = True
188 elif pyfile:
189 assert check_modname(frozenid), spec
190 assert not knownids or frozenid not in knownids, spec
191 assert check_modname(modname), spec
192 assert not os.path.isdir(pyfile), spec
193 ispkg = False
194 elif knownids and frozenid in knownids:
195 assert check_modname(frozenid), spec
196 assert check_modname(modname), spec
197 ispkg = False
198 else:
199 assert not modname or check_modname(modname), spec
200 resolved = iter(resolve_modules(frozenid))
201 frozenid, pyfile, ispkg = next(resolved)
202 if not modname:
203 modname = frozenid
204 if ispkg:
205 pkgid = frozenid
206 pkgname = modname
207 pkgfiles = {pyfile: pkgid}
208 def iter_subs():
209 for frozenid, pyfile, ispkg in resolved:
210 if pkgname:
211 modname = frozenid.replace(pkgid, pkgname, 1)
212 else:
213 modname = frozenid
214 if pyfile:
215 if pyfile in pkgfiles:
216 frozenid = pkgfiles[pyfile]
217 pyfile = None
218 elif ispkg:
219 pkgfiles[pyfile] = frozenid
220 yield frozenid, pyfile, modname, ispkg, section
221 submodules = iter_subs()
222
223 info = (frozenid, pyfile or None, modname, ispkg, section)
224 return info, submodules
225
226
227 #######################################
228 # frozen source files
229
230 class FrozenSource(namedtuple('FrozenSource', 'id pyfile frozenfile')):
231
232 @classmethod
233 def from_id(cls, frozenid, pyfile=None):
234 if not pyfile:
235 pyfile = os.path.join(STDLIB_DIR, *frozenid.split('.')) + '.py'
236 #assert os.path.exists(pyfile), (frozenid, pyfile)
237 frozenfile = resolve_frozen_file(frozenid, FROZEN_MODULES_DIR)
238 return cls(frozenid, pyfile, frozenfile)
239
240 @property
241 def frozenid(self):
242 return self.id
243
244 @property
245 def modname(self):
246 if self.pyfile.startswith(STDLIB_DIR):
247 return self.id
248 return None
249
250 @property
251 def symbol(self):
252 # This matches what we do in Programs/_freeze_module.c:
253 name = self.frozenid.replace('.', '_')
254 return '_Py_M__' + name
255
256 @property
257 def ispkg(self):
258 if not self.pyfile:
259 return False
260 elif self.frozenid.endswith('.__init__'):
261 return False
262 else:
263 return os.path.basename(self.pyfile) == '__init__.py'
264
265 @property
266 def isbootstrap(self):
267 return self.id in BOOTSTRAP
268
269
270 def resolve_frozen_file(frozenid, destdir):
271 """Return the filename corresponding to the given frozen ID.
272
273 For stdlib modules the ID will always be the full name
274 of the source module.
275 """
276 if not isinstance(frozenid, str):
277 try:
278 frozenid = frozenid.frozenid
279 except AttributeError:
280 raise ValueError(f'unsupported frozenid {frozenid!r}')
281 # We use a consistent naming convention for all frozen modules.
282 frozenfile = f'{frozenid}.h'
283 if not destdir:
284 return frozenfile
285 return os.path.join(destdir, frozenfile)
286
287
288 #######################################
289 # frozen modules
290
291 class FrozenModule(namedtuple('FrozenModule', 'name ispkg section source')):
292
293 def __getattr__(self, name):
294 return getattr(self.source, name)
295
296 @property
297 def modname(self):
298 return self.name
299
300 @property
301 def orig(self):
302 return self.source.modname
303
304 @property
305 def isalias(self):
306 orig = self.source.modname
307 if not orig:
308 return True
309 return self.name != orig
310
311 def summarize(self):
312 source = self.source.modname
313 if source:
314 source = f'<{source}>'
315 else:
316 source = relpath_for_posix_display(self.pyfile, ROOT_DIR)
317 return {
318 'module': self.name,
319 'ispkg': self.ispkg,
320 'source': source,
321 'frozen': os.path.basename(self.frozenfile),
322 'checksum': _get_checksum(self.frozenfile),
323 }
324
325
326 def _iter_sources(modules):
327 seen = set()
328 for mod in modules:
329 if mod.source not in seen:
330 yield mod.source
331 seen.add(mod.source)
332
333
334 #######################################
335 # generic helpers
336
337 def _get_checksum(filename):
338 with open(filename, "rb") as infile:
339 contents = infile.read()
340 m = hashlib.sha256()
341 m.update(contents)
342 return m.hexdigest()
343
344
345 def resolve_modules(modname, pyfile=None):
346 if modname.startswith('<') and modname.endswith('>'):
347 if pyfile:
348 assert os.path.isdir(pyfile) or os.path.basename(pyfile) == '__init__.py', pyfile
349 ispkg = True
350 modname = modname[1:-1]
351 rawname = modname
352 # For now, we only expect match patterns at the end of the name.
353 _modname, sep, match = modname.rpartition('.')
354 if sep:
355 if _modname.endswith('.**'):
356 modname = _modname[:-3]
357 match = f'**.{match}'
358 elif match and not match.isidentifier():
359 modname = _modname
360 # Otherwise it's a plain name so we leave it alone.
361 else:
362 match = None
363 else:
364 ispkg = False
365 rawname = modname
366 match = None
367
368 if not check_modname(modname):
369 raise ValueError(f'not a valid module name ({rawname})')
370
371 if not pyfile:
372 pyfile = _resolve_module(modname, ispkg=ispkg)
373 elif os.path.isdir(pyfile):
374 pyfile = _resolve_module(modname, pyfile, ispkg)
375 yield modname, pyfile, ispkg
376
377 if match:
378 pkgdir = os.path.dirname(pyfile)
379 yield from iter_submodules(modname, pkgdir, match)
380
381
382 def check_modname(modname):
383 return all(n.isidentifier() for n in modname.split('.'))
384
385
386 def iter_submodules(pkgname, pkgdir=None, match='*'):
387 if not pkgdir:
388 pkgdir = os.path.join(STDLIB_DIR, *pkgname.split('.'))
389 if not match:
390 match = '**.*'
391 match_modname = _resolve_modname_matcher(match, pkgdir)
392
393 def _iter_submodules(pkgname, pkgdir):
394 for entry in sorted(os.scandir(pkgdir), key=lambda e: e.name):
395 matched, recursive = match_modname(entry.name)
396 if not matched:
397 continue
398 modname = f'{pkgname}.{entry.name}'
399 if modname.endswith('.py'):
400 yield modname[:-3], entry.path, False
401 elif entry.is_dir():
402 pyfile = os.path.join(entry.path, '__init__.py')
403 # We ignore namespace packages.
404 if os.path.exists(pyfile):
405 yield modname, pyfile, True
406 if recursive:
407 yield from _iter_submodules(modname, entry.path)
408
409 return _iter_submodules(pkgname, pkgdir)
410
411
412 def _resolve_modname_matcher(match, rootdir=None):
413 if isinstance(match, str):
414 if match.startswith('**.'):
415 recursive = True
416 pat = match[3:]
417 assert match
418 else:
419 recursive = False
420 pat = match
421
422 if pat == '*':
423 def match_modname(modname):
424 return True, recursive
425 else:
426 raise NotImplementedError(match)
427 elif callable(match):
428 match_modname = match(rootdir)
429 else:
430 raise ValueError(f'unsupported matcher {match!r}')
431 return match_modname
432
433
434 def _resolve_module(modname, pathentry=STDLIB_DIR, ispkg=False):
435 assert pathentry, pathentry
436 pathentry = os.path.normpath(pathentry)
437 assert os.path.isabs(pathentry)
438 if ispkg:
439 return os.path.join(pathentry, *modname.split('.'), '__init__.py')
440 return os.path.join(pathentry, *modname.split('.')) + '.py'
441
442
443 #######################################
444 # regenerating dependent files
445
446 def find_marker(lines, marker, file):
447 for pos, line in enumerate(lines):
448 if marker in line:
449 return pos
450 raise Exception(f"Can't find {marker!r} in file {file}")
451
452
453 def replace_block(lines, start_marker, end_marker, replacements, file):
454 start_pos = find_marker(lines, start_marker, file)
455 end_pos = find_marker(lines, end_marker, file)
456 if end_pos <= start_pos:
457 raise Exception(f"End marker {end_marker!r} "
458 f"occurs before start marker {start_marker!r} "
459 f"in file {file}")
460 replacements = [line.rstrip() + '\n' for line in replacements]
461 return lines[:start_pos + 1] + replacements + lines[end_pos:]
462
463
464 class UniqueList(list):
465 def __init__(self):
466 self._seen = set()
467
468 def append(self, item):
469 if item in self._seen:
470 return
471 super().append(item)
472 self._seen.add(item)
473
474
475 def regen_frozen(modules):
476 headerlines = []
477 parentdir = os.path.dirname(FROZEN_FILE)
478 for src in _iter_sources(modules):
479 # Adding a comment to separate sections here doesn't add much,
480 # so we don't.
481 header = relpath_for_posix_display(src.frozenfile, parentdir)
482 headerlines.append(f'#include "{header}"')
483
484 bootstraplines = []
485 stdliblines = []
486 testlines = []
487 aliaslines = []
488 indent = ' '
489 lastsection = None
490 for mod in modules:
491 if mod.isbootstrap:
492 lines = bootstraplines
493 elif mod.section == TESTS_SECTION:
494 lines = testlines
495 else:
496 lines = stdliblines
497 if mod.section != lastsection:
498 if lastsection is not None:
499 lines.append('')
500 lines.append(f'/* {mod.section} */')
501 lastsection = mod.section
502
503 pkg = 'true' if mod.ispkg else 'false'
504 size = f"(int)sizeof({mod.symbol})"
505 line = f'{{"{mod.name}", {mod.symbol}, {size}, {pkg}}},'
506 lines.append(line)
507
508 if mod.isalias:
509 if not mod.orig:
510 entry = '{"%s", NULL},' % (mod.name,)
511 elif mod.source.ispkg:
512 entry = '{"%s", "<%s"},' % (mod.name, mod.orig)
513 else:
514 entry = '{"%s", "%s"},' % (mod.name, mod.orig)
515 aliaslines.append(indent + entry)
516
517 for lines in (bootstraplines, stdliblines, testlines):
518 # TODO: Is this necessary any more?
519 if lines and not lines[0]:
520 del lines[0]
521 for i, line in enumerate(lines):
522 if line:
523 lines[i] = indent + line
524
525 print(f'# Updating {os.path.relpath(FROZEN_FILE)}')
526 with updating_file_with_tmpfile(FROZEN_FILE) as (infile, outfile):
527 lines = infile.readlines()
528 # TODO: Use more obvious markers, e.g.
529 # $START GENERATED FOOBAR$ / $END GENERATED FOOBAR$
530 lines = replace_block(
531 lines,
532 "/* Includes for frozen modules: */",
533 "/* End includes */",
534 headerlines,
535 FROZEN_FILE,
536 )
537 lines = replace_block(
538 lines,
539 "static const struct _frozen bootstrap_modules[] =",
540 "/* bootstrap sentinel */",
541 bootstraplines,
542 FROZEN_FILE,
543 )
544 lines = replace_block(
545 lines,
546 "static const struct _frozen stdlib_modules[] =",
547 "/* stdlib sentinel */",
548 stdliblines,
549 FROZEN_FILE,
550 )
551 lines = replace_block(
552 lines,
553 "static const struct _frozen test_modules[] =",
554 "/* test sentinel */",
555 testlines,
556 FROZEN_FILE,
557 )
558 lines = replace_block(
559 lines,
560 "const struct _module_alias aliases[] =",
561 "/* aliases sentinel */",
562 aliaslines,
563 FROZEN_FILE,
564 )
565 outfile.writelines(lines)
566
567
568 def regen_makefile(modules):
569 pyfiles = []
570 frozenfiles = []
571 rules = ['']
572 for src in _iter_sources(modules):
573 frozen_header = relpath_for_posix_display(src.frozenfile, ROOT_DIR)
574 frozenfiles.append(f'\t\t{frozen_header} \\')
575
576 pyfile = relpath_for_posix_display(src.pyfile, ROOT_DIR)
577 pyfiles.append(f'\t\t{pyfile} \\')
578
579 if src.isbootstrap:
580 freezecmd = '$(FREEZE_MODULE_BOOTSTRAP)'
581 freezedep = '$(FREEZE_MODULE_BOOTSTRAP_DEPS)'
582 else:
583 freezecmd = '$(FREEZE_MODULE)'
584 freezedep = '$(FREEZE_MODULE_DEPS)'
585
586 freeze = (f'{freezecmd} {src.frozenid} '
587 f'$(srcdir)/{pyfile} {frozen_header}')
588 rules.extend([
589 f'{frozen_header}: {pyfile} {freezedep}',
590 f'\t{freeze}',
591 '',
592 ])
593 pyfiles[-1] = pyfiles[-1].rstrip(" \\")
594 frozenfiles[-1] = frozenfiles[-1].rstrip(" \\")
595
596 print(f'# Updating {os.path.relpath(MAKEFILE)}')
597 with updating_file_with_tmpfile(MAKEFILE) as (infile, outfile):
598 lines = infile.readlines()
599 lines = replace_block(
600 lines,
601 "FROZEN_FILES_IN =",
602 "# End FROZEN_FILES_IN",
603 pyfiles,
604 MAKEFILE,
605 )
606 lines = replace_block(
607 lines,
608 "FROZEN_FILES_OUT =",
609 "# End FROZEN_FILES_OUT",
610 frozenfiles,
611 MAKEFILE,
612 )
613 lines = replace_block(
614 lines,
615 "# BEGIN: freezing modules",
616 "# END: freezing modules",
617 rules,
618 MAKEFILE,
619 )
620 outfile.writelines(lines)
621
622
623 def regen_pcbuild(modules):
624 projlines = []
625 filterlines = []
626 for src in _iter_sources(modules):
627 pyfile = relpath_for_windows_display(src.pyfile, ROOT_DIR)
628 header = relpath_for_windows_display(src.frozenfile, ROOT_DIR)
629 intfile = ntpath.splitext(ntpath.basename(header))[0] + '.g.h'
630 projlines.append(f' <None Include="..\\{pyfile}">')
631 projlines.append(f' <ModName>{src.frozenid}</ModName>')
632 projlines.append(f' <IntFile>$(IntDir){intfile}</IntFile>')
633 projlines.append(f' <OutFile>$(GeneratedFrozenModulesDir){header}</OutFile>')
634 projlines.append(f' </None>')
635
636 filterlines.append(f' <None Include="..\\{pyfile}">')
637 filterlines.append(' <Filter>Python Files</Filter>')
638 filterlines.append(' </None>')
639
640 print(f'# Updating {os.path.relpath(PCBUILD_PROJECT)}')
641 with updating_file_with_tmpfile(PCBUILD_PROJECT) as (infile, outfile):
642 lines = infile.readlines()
643 lines = replace_block(
644 lines,
645 '<!-- BEGIN frozen modules -->',
646 '<!-- END frozen modules -->',
647 projlines,
648 PCBUILD_PROJECT,
649 )
650 outfile.writelines(lines)
651 print(f'# Updating {os.path.relpath(PCBUILD_FILTERS)}')
652 with updating_file_with_tmpfile(PCBUILD_FILTERS) as (infile, outfile):
653 lines = infile.readlines()
654 lines = replace_block(
655 lines,
656 '<!-- BEGIN frozen modules -->',
657 '<!-- END frozen modules -->',
658 filterlines,
659 PCBUILD_FILTERS,
660 )
661 outfile.writelines(lines)
662
663
664 #######################################
665 # the script
666
667 def main():
668 # Expand the raw specs, preserving order.
669 modules = list(parse_frozen_specs())
670
671 # Regen build-related files.
672 regen_makefile(modules)
673 regen_pcbuild(modules)
674 regen_frozen(modules)
675
676
677 if __name__ == '__main__':
678 main()