| 1 |
"""Freeze modules and regen related files (e.g. Python/frozen.c). |
| 2 |
|
| 3 |
See the notes at the top of Python/frozen.c for more info. |
| 4 |
""" |
| 5 |
|
| 6 |
import hashlib |
| 7 |
import ntpath |
| 8 |
import os |
| 9 |
import posixpath |
| 10 |
from collections import namedtuple |
| 11 |
|
| 12 |
from update_file import updating_file_with_tmpfile |
| 13 |
|
| 14 |
ROOT_DIR = os.path.dirname(os.path.dirname(os.path.dirname(__file__))) |
| 15 |
ROOT_DIR = os.path.abspath(ROOT_DIR) |
| 16 |
FROZEN_ONLY = os.path.join(ROOT_DIR, 'Tools', 'freeze', 'flag.py') |
| 17 |
|
| 18 |
STDLIB_DIR = os.path.join(ROOT_DIR, 'Lib') |
| 19 |
# If FROZEN_MODULES_DIR or DEEPFROZEN_MODULES_DIR is changed then the |
| 20 |
# .gitattributes and .gitignore files needs to be updated. |
| 21 |
FROZEN_MODULES_DIR = os.path.join(ROOT_DIR, 'Python', 'frozen_modules') |
| 22 |
|
| 23 |
FROZEN_FILE = os.path.join(ROOT_DIR, 'Python', 'frozen.c') |
| 24 |
MAKEFILE = os.path.join(ROOT_DIR, 'Makefile.pre.in') |
| 25 |
PCBUILD_PROJECT = os.path.join(ROOT_DIR, 'PCbuild', '_freeze_module.vcxproj') |
| 26 |
PCBUILD_FILTERS = os.path.join(ROOT_DIR, 'PCbuild', '_freeze_module.vcxproj.filters') |
| 27 |
PCBUILD_PYTHONCORE = os.path.join(ROOT_DIR, 'PCbuild', 'pythoncore.vcxproj') |
| 28 |
|
| 29 |
|
| 30 |
OS_PATH = 'ntpath' if os.name == 'nt' else 'posixpath' |
| 31 |
|
| 32 |
# These are modules that get frozen. |
| 33 |
# If you're debugging new bytecode instructions, |
| 34 |
# you can delete all sections except 'import system'. |
| 35 |
# This also speeds up building somewhat. |
| 36 |
TESTS_SECTION = 'Test module' |
| 37 |
FROZEN = [ |
| 38 |
# See parse_frozen_spec() for the format. |
| 39 |
# In cases where the frozenid is duplicated, the first one is re-used. |
| 40 |
('import system', [ |
| 41 |
# These frozen modules are necessary for bootstrapping |
| 42 |
# the import system. |
| 43 |
'importlib._bootstrap : _frozen_importlib', |
| 44 |
'importlib._bootstrap_external : _frozen_importlib_external', |
| 45 |
# This module is important because some Python builds rely |
| 46 |
# on a builtin zip file instead of a filesystem. |
| 47 |
'zipimport', |
| 48 |
]), |
| 49 |
# (You can delete entries from here down to the end of the list.) |
| 50 |
('stdlib - startup, without site (python -S)', [ |
| 51 |
'abc', |
| 52 |
'codecs', |
| 53 |
# For now we do not freeze the encodings, due # to the noise all |
| 54 |
# those extra modules add to the text printed during the build. |
| 55 |
# (See https://github.com/python/cpython/pull/28398#pullrequestreview-756856469.) |
| 56 |
#'<encodings.*>', |
| 57 |
'io', |
| 58 |
]), |
| 59 |
('stdlib - startup, with site', [ |
| 60 |
'_collections_abc', |
| 61 |
'_sitebuiltins', |
| 62 |
'genericpath', |
| 63 |
'ntpath', |
| 64 |
'posixpath', |
| 65 |
'os', |
| 66 |
'site', |
| 67 |
'stat', |
| 68 |
]), |
| 69 |
('runpy - run module with -m', [ |
| 70 |
"importlib.util", |
| 71 |
"importlib.machinery", |
| 72 |
"runpy", |
| 73 |
]), |
| 74 |
(TESTS_SECTION, [ |
| 75 |
'__hello__', |
| 76 |
'__hello__ : __hello_alias__', |
| 77 |
'__hello__ : <__phello_alias__>', |
| 78 |
'__hello__ : __phello_alias__.spam', |
| 79 |
'<__phello__.**.*>', |
| 80 |
f'frozen_only : __hello_only__ = {FROZEN_ONLY}', |
| 81 |
]), |
| 82 |
# (End of stuff you could delete.) |
| 83 |
] |
| 84 |
BOOTSTRAP = { |
| 85 |
'importlib._bootstrap', |
| 86 |
'importlib._bootstrap_external', |
| 87 |
'zipimport', |
| 88 |
} |
| 89 |
|
| 90 |
|
| 91 |
####################################### |
| 92 |
# platform-specific helpers |
| 93 |
|
| 94 |
if os.path is posixpath: |
| 95 |
relpath_for_posix_display = os.path.relpath |
| 96 |
|
| 97 |
def relpath_for_windows_display(path, base): |
| 98 |
return ntpath.relpath( |
| 99 |
ntpath.join(*path.split(os.path.sep)), |
| 100 |
ntpath.join(*base.split(os.path.sep)), |
| 101 |
) |
| 102 |
|
| 103 |
else: |
| 104 |
relpath_for_windows_display = ntpath.relpath |
| 105 |
|
| 106 |
def relpath_for_posix_display(path, base): |
| 107 |
return posixpath.relpath( |
| 108 |
posixpath.join(*path.split(os.path.sep)), |
| 109 |
posixpath.join(*base.split(os.path.sep)), |
| 110 |
) |
| 111 |
|
| 112 |
|
| 113 |
####################################### |
| 114 |
# specs |
| 115 |
|
| 116 |
def parse_frozen_specs(): |
| 117 |
seen = {} |
| 118 |
for section, specs in FROZEN: |
| 119 |
parsed = _parse_specs(specs, section, seen) |
| 120 |
for item in parsed: |
| 121 |
frozenid, pyfile, modname, ispkg, section = item |
| 122 |
try: |
| 123 |
source = seen[frozenid] |
| 124 |
except KeyError: |
| 125 |
source = FrozenSource.from_id(frozenid, pyfile) |
| 126 |
seen[frozenid] = source |
| 127 |
else: |
| 128 |
assert not pyfile or pyfile == source.pyfile, item |
| 129 |
yield FrozenModule(modname, ispkg, section, source) |
| 130 |
|
| 131 |
|
| 132 |
def _parse_specs(specs, section, seen): |
| 133 |
for spec in specs: |
| 134 |
info, subs = _parse_spec(spec, seen, section) |
| 135 |
yield info |
| 136 |
for info in subs or (): |
| 137 |
yield info |
| 138 |
|
| 139 |
|
| 140 |
def _parse_spec(spec, knownids=None, section=None): |
| 141 |
"""Yield an info tuple for each module corresponding to the given spec. |
| 142 |
|
| 143 |
The info consists of: (frozenid, pyfile, modname, ispkg, section). |
| 144 |
|
| 145 |
Supported formats: |
| 146 |
|
| 147 |
frozenid |
| 148 |
frozenid : modname |
| 149 |
frozenid : modname = pyfile |
| 150 |
|
| 151 |
"frozenid" and "modname" must be valid module names (dot-separated |
| 152 |
identifiers). If "modname" is not provided then "frozenid" is used. |
| 153 |
If "pyfile" is not provided then the filename of the module |
| 154 |
corresponding to "frozenid" is used. |
| 155 |
|
| 156 |
Angle brackets around a frozenid (e.g. '<encodings>") indicate |
| 157 |
it is a package. This also means it must be an actual module |
| 158 |
(i.e. "pyfile" cannot have been provided). Such values can have |
| 159 |
patterns to expand submodules: |
| 160 |
|
| 161 |
<encodings.*> - also freeze all direct submodules |
| 162 |
<encodings.**.*> - also freeze the full submodule tree |
| 163 |
|
| 164 |
As with "frozenid", angle brackets around "modname" indicate |
| 165 |
it is a package. However, in this case "pyfile" should not |
| 166 |
have been provided and patterns in "modname" are not supported. |
| 167 |
Also, if "modname" has brackets then "frozenid" should not, |
| 168 |
and "pyfile" should have been provided.. |
| 169 |
""" |
| 170 |
frozenid, _, remainder = spec.partition(':') |
| 171 |
modname, _, pyfile = remainder.partition('=') |
| 172 |
frozenid = frozenid.strip() |
| 173 |
modname = modname.strip() |
| 174 |
pyfile = pyfile.strip() |
| 175 |
|
| 176 |
submodules = None |
| 177 |
if modname.startswith('<') and modname.endswith('>'): |
| 178 |
assert check_modname(frozenid), spec |
| 179 |
modname = modname[1:-1] |
| 180 |
assert check_modname(modname), spec |
| 181 |
if frozenid in knownids: |
| 182 |
pass |
| 183 |
elif pyfile: |
| 184 |
assert not os.path.isdir(pyfile), spec |
| 185 |
else: |
| 186 |
pyfile = _resolve_module(frozenid, ispkg=False) |
| 187 |
ispkg = True |
| 188 |
elif pyfile: |
| 189 |
assert check_modname(frozenid), spec |
| 190 |
assert not knownids or frozenid not in knownids, spec |
| 191 |
assert check_modname(modname), spec |
| 192 |
assert not os.path.isdir(pyfile), spec |
| 193 |
ispkg = False |
| 194 |
elif knownids and frozenid in knownids: |
| 195 |
assert check_modname(frozenid), spec |
| 196 |
assert check_modname(modname), spec |
| 197 |
ispkg = False |
| 198 |
else: |
| 199 |
assert not modname or check_modname(modname), spec |
| 200 |
resolved = iter(resolve_modules(frozenid)) |
| 201 |
frozenid, pyfile, ispkg = next(resolved) |
| 202 |
if not modname: |
| 203 |
modname = frozenid |
| 204 |
if ispkg: |
| 205 |
pkgid = frozenid |
| 206 |
pkgname = modname |
| 207 |
pkgfiles = {pyfile: pkgid} |
| 208 |
def iter_subs(): |
| 209 |
for frozenid, pyfile, ispkg in resolved: |
| 210 |
if pkgname: |
| 211 |
modname = frozenid.replace(pkgid, pkgname, 1) |
| 212 |
else: |
| 213 |
modname = frozenid |
| 214 |
if pyfile: |
| 215 |
if pyfile in pkgfiles: |
| 216 |
frozenid = pkgfiles[pyfile] |
| 217 |
pyfile = None |
| 218 |
elif ispkg: |
| 219 |
pkgfiles[pyfile] = frozenid |
| 220 |
yield frozenid, pyfile, modname, ispkg, section |
| 221 |
submodules = iter_subs() |
| 222 |
|
| 223 |
info = (frozenid, pyfile or None, modname, ispkg, section) |
| 224 |
return info, submodules |
| 225 |
|
| 226 |
|
| 227 |
####################################### |
| 228 |
# frozen source files |
| 229 |
|
| 230 |
class FrozenSource(namedtuple('FrozenSource', 'id pyfile frozenfile')): |
| 231 |
|
| 232 |
@classmethod |
| 233 |
def from_id(cls, frozenid, pyfile=None): |
| 234 |
if not pyfile: |
| 235 |
pyfile = os.path.join(STDLIB_DIR, *frozenid.split('.')) + '.py' |
| 236 |
#assert os.path.exists(pyfile), (frozenid, pyfile) |
| 237 |
frozenfile = resolve_frozen_file(frozenid, FROZEN_MODULES_DIR) |
| 238 |
return cls(frozenid, pyfile, frozenfile) |
| 239 |
|
| 240 |
@property |
| 241 |
def frozenid(self): |
| 242 |
return self.id |
| 243 |
|
| 244 |
@property |
| 245 |
def modname(self): |
| 246 |
if self.pyfile.startswith(STDLIB_DIR): |
| 247 |
return self.id |
| 248 |
return None |
| 249 |
|
| 250 |
@property |
| 251 |
def symbol(self): |
| 252 |
# This matches what we do in Programs/_freeze_module.c: |
| 253 |
name = self.frozenid.replace('.', '_') |
| 254 |
return '_Py_M__' + name |
| 255 |
|
| 256 |
@property |
| 257 |
def ispkg(self): |
| 258 |
if not self.pyfile: |
| 259 |
return False |
| 260 |
elif self.frozenid.endswith('.__init__'): |
| 261 |
return False |
| 262 |
else: |
| 263 |
return os.path.basename(self.pyfile) == '__init__.py' |
| 264 |
|
| 265 |
@property |
| 266 |
def isbootstrap(self): |
| 267 |
return self.id in BOOTSTRAP |
| 268 |
|
| 269 |
|
| 270 |
def resolve_frozen_file(frozenid, destdir): |
| 271 |
"""Return the filename corresponding to the given frozen ID. |
| 272 |
|
| 273 |
For stdlib modules the ID will always be the full name |
| 274 |
of the source module. |
| 275 |
""" |
| 276 |
if not isinstance(frozenid, str): |
| 277 |
try: |
| 278 |
frozenid = frozenid.frozenid |
| 279 |
except AttributeError: |
| 280 |
raise ValueError(f'unsupported frozenid {frozenid!r}') |
| 281 |
# We use a consistent naming convention for all frozen modules. |
| 282 |
frozenfile = f'{frozenid}.h' |
| 283 |
if not destdir: |
| 284 |
return frozenfile |
| 285 |
return os.path.join(destdir, frozenfile) |
| 286 |
|
| 287 |
|
| 288 |
####################################### |
| 289 |
# frozen modules |
| 290 |
|
| 291 |
class FrozenModule(namedtuple('FrozenModule', 'name ispkg section source')): |
| 292 |
|
| 293 |
def __getattr__(self, name): |
| 294 |
return getattr(self.source, name) |
| 295 |
|
| 296 |
@property |
| 297 |
def modname(self): |
| 298 |
return self.name |
| 299 |
|
| 300 |
@property |
| 301 |
def orig(self): |
| 302 |
return self.source.modname |
| 303 |
|
| 304 |
@property |
| 305 |
def isalias(self): |
| 306 |
orig = self.source.modname |
| 307 |
if not orig: |
| 308 |
return True |
| 309 |
return self.name != orig |
| 310 |
|
| 311 |
def summarize(self): |
| 312 |
source = self.source.modname |
| 313 |
if source: |
| 314 |
source = f'<{source}>' |
| 315 |
else: |
| 316 |
source = relpath_for_posix_display(self.pyfile, ROOT_DIR) |
| 317 |
return { |
| 318 |
'module': self.name, |
| 319 |
'ispkg': self.ispkg, |
| 320 |
'source': source, |
| 321 |
'frozen': os.path.basename(self.frozenfile), |
| 322 |
'checksum': _get_checksum(self.frozenfile), |
| 323 |
} |
| 324 |
|
| 325 |
|
| 326 |
def _iter_sources(modules): |
| 327 |
seen = set() |
| 328 |
for mod in modules: |
| 329 |
if mod.source not in seen: |
| 330 |
yield mod.source |
| 331 |
seen.add(mod.source) |
| 332 |
|
| 333 |
|
| 334 |
####################################### |
| 335 |
# generic helpers |
| 336 |
|
| 337 |
def _get_checksum(filename): |
| 338 |
with open(filename, "rb") as infile: |
| 339 |
contents = infile.read() |
| 340 |
m = hashlib.sha256() |
| 341 |
m.update(contents) |
| 342 |
return m.hexdigest() |
| 343 |
|
| 344 |
|
| 345 |
def resolve_modules(modname, pyfile=None): |
| 346 |
if modname.startswith('<') and modname.endswith('>'): |
| 347 |
if pyfile: |
| 348 |
assert os.path.isdir(pyfile) or os.path.basename(pyfile) == '__init__.py', pyfile |
| 349 |
ispkg = True |
| 350 |
modname = modname[1:-1] |
| 351 |
rawname = modname |
| 352 |
# For now, we only expect match patterns at the end of the name. |
| 353 |
_modname, sep, match = modname.rpartition('.') |
| 354 |
if sep: |
| 355 |
if _modname.endswith('.**'): |
| 356 |
modname = _modname[:-3] |
| 357 |
match = f'**.{match}' |
| 358 |
elif match and not match.isidentifier(): |
| 359 |
modname = _modname |
| 360 |
# Otherwise it's a plain name so we leave it alone. |
| 361 |
else: |
| 362 |
match = None |
| 363 |
else: |
| 364 |
ispkg = False |
| 365 |
rawname = modname |
| 366 |
match = None |
| 367 |
|
| 368 |
if not check_modname(modname): |
| 369 |
raise ValueError(f'not a valid module name ({rawname})') |
| 370 |
|
| 371 |
if not pyfile: |
| 372 |
pyfile = _resolve_module(modname, ispkg=ispkg) |
| 373 |
elif os.path.isdir(pyfile): |
| 374 |
pyfile = _resolve_module(modname, pyfile, ispkg) |
| 375 |
yield modname, pyfile, ispkg |
| 376 |
|
| 377 |
if match: |
| 378 |
pkgdir = os.path.dirname(pyfile) |
| 379 |
yield from iter_submodules(modname, pkgdir, match) |
| 380 |
|
| 381 |
|
| 382 |
def check_modname(modname): |
| 383 |
return all(n.isidentifier() for n in modname.split('.')) |
| 384 |
|
| 385 |
|
| 386 |
def iter_submodules(pkgname, pkgdir=None, match='*'): |
| 387 |
if not pkgdir: |
| 388 |
pkgdir = os.path.join(STDLIB_DIR, *pkgname.split('.')) |
| 389 |
if not match: |
| 390 |
match = '**.*' |
| 391 |
match_modname = _resolve_modname_matcher(match, pkgdir) |
| 392 |
|
| 393 |
def _iter_submodules(pkgname, pkgdir): |
| 394 |
for entry in sorted(os.scandir(pkgdir), key=lambda e: e.name): |
| 395 |
matched, recursive = match_modname(entry.name) |
| 396 |
if not matched: |
| 397 |
continue |
| 398 |
modname = f'{pkgname}.{entry.name}' |
| 399 |
if modname.endswith('.py'): |
| 400 |
yield modname[:-3], entry.path, False |
| 401 |
elif entry.is_dir(): |
| 402 |
pyfile = os.path.join(entry.path, '__init__.py') |
| 403 |
# We ignore namespace packages. |
| 404 |
if os.path.exists(pyfile): |
| 405 |
yield modname, pyfile, True |
| 406 |
if recursive: |
| 407 |
yield from _iter_submodules(modname, entry.path) |
| 408 |
|
| 409 |
return _iter_submodules(pkgname, pkgdir) |
| 410 |
|
| 411 |
|
| 412 |
def _resolve_modname_matcher(match, rootdir=None): |
| 413 |
if isinstance(match, str): |
| 414 |
if match.startswith('**.'): |
| 415 |
recursive = True |
| 416 |
pat = match[3:] |
| 417 |
assert match |
| 418 |
else: |
| 419 |
recursive = False |
| 420 |
pat = match |
| 421 |
|
| 422 |
if pat == '*': |
| 423 |
def match_modname(modname): |
| 424 |
return True, recursive |
| 425 |
else: |
| 426 |
raise NotImplementedError(match) |
| 427 |
elif callable(match): |
| 428 |
match_modname = match(rootdir) |
| 429 |
else: |
| 430 |
raise ValueError(f'unsupported matcher {match!r}') |
| 431 |
return match_modname |
| 432 |
|
| 433 |
|
| 434 |
def _resolve_module(modname, pathentry=STDLIB_DIR, ispkg=False): |
| 435 |
assert pathentry, pathentry |
| 436 |
pathentry = os.path.normpath(pathentry) |
| 437 |
assert os.path.isabs(pathentry) |
| 438 |
if ispkg: |
| 439 |
return os.path.join(pathentry, *modname.split('.'), '__init__.py') |
| 440 |
return os.path.join(pathentry, *modname.split('.')) + '.py' |
| 441 |
|
| 442 |
|
| 443 |
####################################### |
| 444 |
# regenerating dependent files |
| 445 |
|
| 446 |
def find_marker(lines, marker, file): |
| 447 |
for pos, line in enumerate(lines): |
| 448 |
if marker in line: |
| 449 |
return pos |
| 450 |
raise Exception(f"Can't find {marker!r} in file {file}") |
| 451 |
|
| 452 |
|
| 453 |
def replace_block(lines, start_marker, end_marker, replacements, file): |
| 454 |
start_pos = find_marker(lines, start_marker, file) |
| 455 |
end_pos = find_marker(lines, end_marker, file) |
| 456 |
if end_pos <= start_pos: |
| 457 |
raise Exception(f"End marker {end_marker!r} " |
| 458 |
f"occurs before start marker {start_marker!r} " |
| 459 |
f"in file {file}") |
| 460 |
replacements = [line.rstrip() + '\n' for line in replacements] |
| 461 |
return lines[:start_pos + 1] + replacements + lines[end_pos:] |
| 462 |
|
| 463 |
|
| 464 |
class UniqueList(list): |
| 465 |
def __init__(self): |
| 466 |
self._seen = set() |
| 467 |
|
| 468 |
def append(self, item): |
| 469 |
if item in self._seen: |
| 470 |
return |
| 471 |
super().append(item) |
| 472 |
self._seen.add(item) |
| 473 |
|
| 474 |
|
| 475 |
def regen_frozen(modules): |
| 476 |
headerlines = [] |
| 477 |
parentdir = os.path.dirname(FROZEN_FILE) |
| 478 |
for src in _iter_sources(modules): |
| 479 |
# Adding a comment to separate sections here doesn't add much, |
| 480 |
# so we don't. |
| 481 |
header = relpath_for_posix_display(src.frozenfile, parentdir) |
| 482 |
headerlines.append(f'#include "{header}"') |
| 483 |
|
| 484 |
bootstraplines = [] |
| 485 |
stdliblines = [] |
| 486 |
testlines = [] |
| 487 |
aliaslines = [] |
| 488 |
indent = ' ' |
| 489 |
lastsection = None |
| 490 |
for mod in modules: |
| 491 |
if mod.isbootstrap: |
| 492 |
lines = bootstraplines |
| 493 |
elif mod.section == TESTS_SECTION: |
| 494 |
lines = testlines |
| 495 |
else: |
| 496 |
lines = stdliblines |
| 497 |
if mod.section != lastsection: |
| 498 |
if lastsection is not None: |
| 499 |
lines.append('') |
| 500 |
lines.append(f'/* {mod.section} */') |
| 501 |
lastsection = mod.section |
| 502 |
|
| 503 |
pkg = 'true' if mod.ispkg else 'false' |
| 504 |
size = f"(int)sizeof({mod.symbol})" |
| 505 |
line = f'{{"{mod.name}", {mod.symbol}, {size}, {pkg}}},' |
| 506 |
lines.append(line) |
| 507 |
|
| 508 |
if mod.isalias: |
| 509 |
if not mod.orig: |
| 510 |
entry = '{"%s", NULL},' % (mod.name,) |
| 511 |
elif mod.source.ispkg: |
| 512 |
entry = '{"%s", "<%s"},' % (mod.name, mod.orig) |
| 513 |
else: |
| 514 |
entry = '{"%s", "%s"},' % (mod.name, mod.orig) |
| 515 |
aliaslines.append(indent + entry) |
| 516 |
|
| 517 |
for lines in (bootstraplines, stdliblines, testlines): |
| 518 |
# TODO: Is this necessary any more? |
| 519 |
if lines and not lines[0]: |
| 520 |
del lines[0] |
| 521 |
for i, line in enumerate(lines): |
| 522 |
if line: |
| 523 |
lines[i] = indent + line |
| 524 |
|
| 525 |
print(f'# Updating {os.path.relpath(FROZEN_FILE)}') |
| 526 |
with updating_file_with_tmpfile(FROZEN_FILE) as (infile, outfile): |
| 527 |
lines = infile.readlines() |
| 528 |
# TODO: Use more obvious markers, e.g. |
| 529 |
# $START GENERATED FOOBAR$ / $END GENERATED FOOBAR$ |
| 530 |
lines = replace_block( |
| 531 |
lines, |
| 532 |
"/* Includes for frozen modules: */", |
| 533 |
"/* End includes */", |
| 534 |
headerlines, |
| 535 |
FROZEN_FILE, |
| 536 |
) |
| 537 |
lines = replace_block( |
| 538 |
lines, |
| 539 |
"static const struct _frozen bootstrap_modules[] =", |
| 540 |
"/* bootstrap sentinel */", |
| 541 |
bootstraplines, |
| 542 |
FROZEN_FILE, |
| 543 |
) |
| 544 |
lines = replace_block( |
| 545 |
lines, |
| 546 |
"static const struct _frozen stdlib_modules[] =", |
| 547 |
"/* stdlib sentinel */", |
| 548 |
stdliblines, |
| 549 |
FROZEN_FILE, |
| 550 |
) |
| 551 |
lines = replace_block( |
| 552 |
lines, |
| 553 |
"static const struct _frozen test_modules[] =", |
| 554 |
"/* test sentinel */", |
| 555 |
testlines, |
| 556 |
FROZEN_FILE, |
| 557 |
) |
| 558 |
lines = replace_block( |
| 559 |
lines, |
| 560 |
"const struct _module_alias aliases[] =", |
| 561 |
"/* aliases sentinel */", |
| 562 |
aliaslines, |
| 563 |
FROZEN_FILE, |
| 564 |
) |
| 565 |
outfile.writelines(lines) |
| 566 |
|
| 567 |
|
| 568 |
def regen_makefile(modules): |
| 569 |
pyfiles = [] |
| 570 |
frozenfiles = [] |
| 571 |
rules = [''] |
| 572 |
for src in _iter_sources(modules): |
| 573 |
frozen_header = relpath_for_posix_display(src.frozenfile, ROOT_DIR) |
| 574 |
frozenfiles.append(f'\t\t{frozen_header} \\') |
| 575 |
|
| 576 |
pyfile = relpath_for_posix_display(src.pyfile, ROOT_DIR) |
| 577 |
pyfiles.append(f'\t\t{pyfile} \\') |
| 578 |
|
| 579 |
if src.isbootstrap: |
| 580 |
freezecmd = '$(FREEZE_MODULE_BOOTSTRAP)' |
| 581 |
freezedep = '$(FREEZE_MODULE_BOOTSTRAP_DEPS)' |
| 582 |
else: |
| 583 |
freezecmd = '$(FREEZE_MODULE)' |
| 584 |
freezedep = '$(FREEZE_MODULE_DEPS)' |
| 585 |
|
| 586 |
freeze = (f'{freezecmd} {src.frozenid} ' |
| 587 |
f'$(srcdir)/{pyfile} {frozen_header}') |
| 588 |
rules.extend([ |
| 589 |
f'{frozen_header}: {pyfile} {freezedep}', |
| 590 |
f'\t{freeze}', |
| 591 |
'', |
| 592 |
]) |
| 593 |
pyfiles[-1] = pyfiles[-1].rstrip(" \\") |
| 594 |
frozenfiles[-1] = frozenfiles[-1].rstrip(" \\") |
| 595 |
|
| 596 |
print(f'# Updating {os.path.relpath(MAKEFILE)}') |
| 597 |
with updating_file_with_tmpfile(MAKEFILE) as (infile, outfile): |
| 598 |
lines = infile.readlines() |
| 599 |
lines = replace_block( |
| 600 |
lines, |
| 601 |
"FROZEN_FILES_IN =", |
| 602 |
"# End FROZEN_FILES_IN", |
| 603 |
pyfiles, |
| 604 |
MAKEFILE, |
| 605 |
) |
| 606 |
lines = replace_block( |
| 607 |
lines, |
| 608 |
"FROZEN_FILES_OUT =", |
| 609 |
"# End FROZEN_FILES_OUT", |
| 610 |
frozenfiles, |
| 611 |
MAKEFILE, |
| 612 |
) |
| 613 |
lines = replace_block( |
| 614 |
lines, |
| 615 |
"# BEGIN: freezing modules", |
| 616 |
"# END: freezing modules", |
| 617 |
rules, |
| 618 |
MAKEFILE, |
| 619 |
) |
| 620 |
outfile.writelines(lines) |
| 621 |
|
| 622 |
|
| 623 |
def regen_pcbuild(modules): |
| 624 |
projlines = [] |
| 625 |
filterlines = [] |
| 626 |
for src in _iter_sources(modules): |
| 627 |
pyfile = relpath_for_windows_display(src.pyfile, ROOT_DIR) |
| 628 |
header = relpath_for_windows_display(src.frozenfile, ROOT_DIR) |
| 629 |
intfile = ntpath.splitext(ntpath.basename(header))[0] + '.g.h' |
| 630 |
projlines.append(f' <None Include="..\\{pyfile}">') |
| 631 |
projlines.append(f' <ModName>{src.frozenid}</ModName>') |
| 632 |
projlines.append(f' <IntFile>$(IntDir){intfile}</IntFile>') |
| 633 |
projlines.append(f' <OutFile>$(GeneratedFrozenModulesDir){header}</OutFile>') |
| 634 |
projlines.append(f' </None>') |
| 635 |
|
| 636 |
filterlines.append(f' <None Include="..\\{pyfile}">') |
| 637 |
filterlines.append(' <Filter>Python Files</Filter>') |
| 638 |
filterlines.append(' </None>') |
| 639 |
|
| 640 |
print(f'# Updating {os.path.relpath(PCBUILD_PROJECT)}') |
| 641 |
with updating_file_with_tmpfile(PCBUILD_PROJECT) as (infile, outfile): |
| 642 |
lines = infile.readlines() |
| 643 |
lines = replace_block( |
| 644 |
lines, |
| 645 |
'<!-- BEGIN frozen modules -->', |
| 646 |
'<!-- END frozen modules -->', |
| 647 |
projlines, |
| 648 |
PCBUILD_PROJECT, |
| 649 |
) |
| 650 |
outfile.writelines(lines) |
| 651 |
print(f'# Updating {os.path.relpath(PCBUILD_FILTERS)}') |
| 652 |
with updating_file_with_tmpfile(PCBUILD_FILTERS) as (infile, outfile): |
| 653 |
lines = infile.readlines() |
| 654 |
lines = replace_block( |
| 655 |
lines, |
| 656 |
'<!-- BEGIN frozen modules -->', |
| 657 |
'<!-- END frozen modules -->', |
| 658 |
filterlines, |
| 659 |
PCBUILD_FILTERS, |
| 660 |
) |
| 661 |
outfile.writelines(lines) |
| 662 |
|
| 663 |
|
| 664 |
####################################### |
| 665 |
# the script |
| 666 |
|
| 667 |
def main(): |
| 668 |
# Expand the raw specs, preserving order. |
| 669 |
modules = list(parse_frozen_specs()) |
| 670 |
|
| 671 |
# Regen build-related files. |
| 672 |
regen_makefile(modules) |
| 673 |
regen_pcbuild(modules) |
| 674 |
regen_frozen(modules) |
| 675 |
|
| 676 |
|
| 677 |
if __name__ == '__main__': |
| 678 |
main() |