Debian Codegraph @

Tools/build/deepfreeze.py

raw
1 """Deep freeze
2
3 The script may be executed by _bootstrap_python interpreter.
4 Shared library extension modules are not available in that case.
5 Requires 3.11+ to be executed,
6 because relies on `code.co_qualname` and `code.co_exceptiontable`.
7 """
8
9 from __future__ import annotations
10
11 import argparse
12 import builtins
13 import collections
14 import contextlib
15 import os
16 import re
17 import time
18 import types
19
20 import umarshal
21
22 TYPE_CHECKING = False
23 if TYPE_CHECKING:
24 from collections.abc import Iterator
25 from typing import Any, TextIO
26
27 ROOT = os.path.dirname(os.path.dirname(os.path.dirname(__file__)))
28
29 verbose = False
30
31 # This must be kept in sync with Tools/cases_generator/analyzer.py
32 RESUME = 128
33
34 def isprintable(b: bytes) -> bool:
35 return all(0x20 <= c < 0x7f for c in b)
36
37
38 def make_string_literal(b: bytes) -> str:
39 res = ['"']
40 if isprintable(b):
41 res.append(b.decode("ascii").replace("\\", "\\\\").replace("\"", "\\\""))
42 else:
43 for i in b:
44 res.append(f"\\x{i:02x}")
45 res.append('"')
46 return "".join(res)
47
48
49 CO_FAST_LOCAL = 0x20
50 CO_FAST_CELL = 0x40
51 CO_FAST_FREE = 0x80
52
53 next_code_version = 1
54
55 def get_localsplus(code: types.CodeType) -> tuple[tuple[str, ...], bytes]:
56 a: collections.defaultdict[str, int] = collections.defaultdict(int)
57 for name in code.co_varnames:
58 a[name] |= CO_FAST_LOCAL
59 for name in code.co_cellvars:
60 a[name] |= CO_FAST_CELL
61 for name in code.co_freevars:
62 a[name] |= CO_FAST_FREE
63 return tuple(a.keys()), bytes(a.values())
64
65
66 def get_localsplus_counts(code: types.CodeType,
67 names: tuple[str, ...],
68 kinds: bytes) -> tuple[int, int, int]:
69 nlocals = 0
70 ncellvars = 0
71 nfreevars = 0
72 assert len(names) == len(kinds)
73 for name, kind in zip(names, kinds):
74 if kind & CO_FAST_LOCAL:
75 nlocals += 1
76 if kind & CO_FAST_CELL:
77 ncellvars += 1
78 elif kind & CO_FAST_CELL:
79 ncellvars += 1
80 elif kind & CO_FAST_FREE:
81 nfreevars += 1
82 assert nlocals == len(code.co_varnames) == code.co_nlocals, \
83 (nlocals, len(code.co_varnames), code.co_nlocals)
84 assert ncellvars == len(code.co_cellvars)
85 assert nfreevars == len(code.co_freevars)
86 return nlocals, ncellvars, nfreevars
87
88
89 PyUnicode_1BYTE_KIND = 1
90 PyUnicode_2BYTE_KIND = 2
91 PyUnicode_4BYTE_KIND = 4
92
93
94 def analyze_character_width(s: str) -> tuple[int, bool]:
95 maxchar = ' '
96 for c in s:
97 maxchar = max(maxchar, c)
98 ascii = False
99 if maxchar <= '\xFF':
100 kind = PyUnicode_1BYTE_KIND
101 ascii = maxchar <= '\x7F'
102 elif maxchar <= '\uFFFF':
103 kind = PyUnicode_2BYTE_KIND
104 else:
105 kind = PyUnicode_4BYTE_KIND
106 return kind, ascii
107
108
109 def removesuffix(base: str, suffix: str) -> str:
110 if base.endswith(suffix):
111 return base[:len(base) - len(suffix)]
112 return base
113
114 class Printer:
115
116 def __init__(self, file: TextIO) -> None:
117 self.level = 0
118 self.file = file
119 self.cache: dict[tuple[type, object, str], str] = {}
120 self.hits, self.misses = 0, 0
121 self.finis: list[str] = []
122 self.inits: list[str] = []
123 self.identifiers, self.strings = self.get_identifiers_and_strings()
124 self.write('#include "Python.h"')
125 self.write('#include "internal/pycore_object.h"')
126 self.write('#include "internal/pycore_gc.h"')
127 self.write('#include "internal/pycore_code.h"')
128 self.write('#include "internal/pycore_frame.h"')
129 self.write('#include "internal/pycore_long.h"')
130 self.write("")
131
132 def get_identifiers_and_strings(self) -> tuple[set[str], dict[str, str]]:
133 filename = os.path.join(ROOT, "Include", "internal", "pycore_global_strings.h")
134 with open(filename) as fp:
135 lines = fp.readlines()
136 identifiers: set[str] = set()
137 strings: dict[str, str] = {}
138 for line in lines:
139 if m := re.search(r"STRUCT_FOR_ID\((\w+)\)", line):
140 identifiers.add(m.group(1))
141 if m := re.search(r'STRUCT_FOR_STR\((\w+), "(.*?)"\)', line):
142 strings[m.group(2)] = m.group(1)
143 return identifiers, strings
144
145 @contextlib.contextmanager
146 def indent(self) -> Iterator[None]:
147 save_level = self.level
148 try:
149 self.level += 1
150 yield
151 finally:
152 self.level = save_level
153
154 def write(self, arg: str) -> None:
155 self.file.writelines((" "*self.level, arg, "\n"))
156
157 @contextlib.contextmanager
158 def block(self, prefix: str, suffix: str = "") -> Iterator[None]:
159 self.write(prefix + " {")
160 with self.indent():
161 yield
162 self.write("}" + suffix)
163
164 def object_head(self, typename: str) -> None:
165 self.write(f".ob_base = _PyObject_HEAD_INIT(&{typename}),")
166
167 def object_var_head(self, typename: str, size: int) -> None:
168 self.write(f".ob_base = _PyVarObject_HEAD_INIT(&{typename}, {size}),")
169
170 def field(self, obj: object, name: str) -> None:
171 self.write(f".{name} = {getattr(obj, name)},")
172
173 def generate_bytes(self, name: str, b: bytes) -> str:
174 if b == b"":
175 return "(PyObject *)&_Py_SINGLETON(bytes_empty)"
176 if len(b) == 1:
177 return f"(PyObject *)&_Py_SINGLETON(bytes_characters[{b[0]}])"
178 self.write("static")
179 with self.indent():
180 with self.block("struct"):
181 self.write("PyObject_VAR_HEAD")
182 self.write("Py_hash_t ob_shash;")
183 self.write(f"char ob_sval[{len(b) + 1}];")
184 with self.block(f"{name} =", ";"):
185 self.object_var_head("PyBytes_Type", len(b))
186 self.write(".ob_shash = -1,")
187 self.write(f".ob_sval = {make_string_literal(b)},")
188 return f"& {name}.ob_base.ob_base"
189
190 def generate_unicode(self, name: str, s: str) -> str:
191 if s in self.strings:
192 return f"&_Py_STR({self.strings[s]})"
193 if s in self.identifiers:
194 return f"&_Py_ID({s})"
195 if len(s) == 1:
196 c = ord(s)
197 if c < 128:
198 return f"(PyObject *)&_Py_SINGLETON(strings).ascii[{c}]"
199 elif c < 256:
200 return f"(PyObject *)&_Py_SINGLETON(strings).latin1[{c - 128}]"
201 if re.match(r'\A[A-Za-z0-9_]+\Z', s):
202 name = f"const_str_{s}"
203 kind, ascii = analyze_character_width(s)
204 if kind == PyUnicode_1BYTE_KIND:
205 datatype = "uint8_t"
206 elif kind == PyUnicode_2BYTE_KIND:
207 datatype = "uint16_t"
208 else:
209 datatype = "uint32_t"
210 self.write("static")
211 with self.indent():
212 with self.block("struct"):
213 if ascii:
214 self.write("PyASCIIObject _ascii;")
215 else:
216 self.write("PyCompactUnicodeObject _compact;")
217 self.write(f"{datatype} _data[{len(s)+1}];")
218 with self.block(f"{name} =", ";"):
219 if ascii:
220 with self.block("._ascii =", ","):
221 self.object_head("PyUnicode_Type")
222 self.write(f".length = {len(s)},")
223 self.write(".hash = -1,")
224 with self.block(".state =", ","):
225 self.write(".kind = 1,")
226 self.write(".compact = 1,")
227 self.write(".ascii = 1,")
228 self.write(".statically_allocated = 1,")
229 self.write(f"._data = {make_string_literal(s.encode('ascii'))},")
230 return f"& {name}._ascii.ob_base"
231 else:
232 with self.block("._compact =", ","):
233 with self.block("._base =", ","):
234 self.object_head("PyUnicode_Type")
235 self.write(f".length = {len(s)},")
236 self.write(".hash = -1,")
237 with self.block(".state =", ","):
238 self.write(f".kind = {kind},")
239 self.write(".compact = 1,")
240 self.write(".ascii = 0,")
241 self.write(".statically_allocated = 1,")
242 utf8 = s.encode('utf-8')
243 self.write(f'.utf8 = {make_string_literal(utf8)},')
244 self.write(f'.utf8_length = {len(utf8)},')
245 with self.block(f"._data =", ","):
246 for i in range(0, len(s), 16):
247 data = s[i:i+16]
248 self.write(", ".join(map(str, map(ord, data))) + ",")
249 return f"& {name}._compact._base.ob_base"
250
251
252 def generate_code(self, name: str, code: types.CodeType) -> str:
253 global next_code_version
254 # The ordering here matches PyCode_NewWithPosOnlyArgs()
255 # (but see below).
256 co_consts = self.generate(name + "_consts", code.co_consts)
257 co_names = self.generate(name + "_names", code.co_names)
258 co_filename = self.generate(name + "_filename", code.co_filename)
259 co_name = self.generate(name + "_name", code.co_name)
260 co_linetable = self.generate(name + "_linetable", code.co_linetable)
261 # We use 3.10 for type checking, but this module requires 3.11
262 # TODO: bump python version for this script.
263 co_qualname = self.generate(
264 name + "_qualname",
265 code.co_qualname, # type: ignore[attr-defined]
266 )
267 co_exceptiontable = self.generate(
268 name + "_exceptiontable",
269 code.co_exceptiontable, # type: ignore[attr-defined]
270 )
271 # These fields are not directly accessible
272 localsplusnames, localspluskinds = get_localsplus(code)
273 co_localsplusnames = self.generate(name + "_localsplusnames", localsplusnames)
274 co_localspluskinds = self.generate(name + "_localspluskinds", localspluskinds)
275 # Derived values
276 nlocals, ncellvars, nfreevars = \
277 get_localsplus_counts(code, localsplusnames, localspluskinds)
278 co_code_adaptive = make_string_literal(code.co_code)
279 self.write("static")
280 with self.indent():
281 self.write(f"struct _PyCode_DEF({len(code.co_code)})")
282 with self.block(f"{name} =", ";"):
283 self.object_var_head("PyCode_Type", len(code.co_code) // 2)
284 # But the ordering here must match that in cpython/code.h
285 # (which is a pain because we tend to reorder those for perf)
286 # otherwise MSVC doesn't like it.
287 self.write(f".co_consts = {co_consts},")
288 self.write(f".co_names = {co_names},")
289 self.write(f".co_exceptiontable = {co_exceptiontable},")
290 self.field(code, "co_flags")
291 self.field(code, "co_argcount")
292 self.field(code, "co_posonlyargcount")
293 self.field(code, "co_kwonlyargcount")
294 # The following should remain in sync with _PyFrame_NumSlotsForCodeObject
295 self.write(f".co_framesize = {code.co_stacksize + len(localsplusnames)} + FRAME_SPECIALS_SIZE,")
296 self.field(code, "co_stacksize")
297 self.field(code, "co_firstlineno")
298 self.write(f".co_nlocalsplus = {len(localsplusnames)},")
299 self.field(code, "co_nlocals")
300 self.write(f".co_ncellvars = {ncellvars},")
301 self.write(f".co_nfreevars = {nfreevars},")
302 self.write(f".co_version = {next_code_version},")
303 next_code_version += 1
304 self.write(f".co_localsplusnames = {co_localsplusnames},")
305 self.write(f".co_localspluskinds = {co_localspluskinds},")
306 self.write(f".co_filename = {co_filename},")
307 self.write(f".co_name = {co_name},")
308 self.write(f".co_qualname = {co_qualname},")
309 self.write(f".co_linetable = {co_linetable},")
310 self.write(f"._co_cached = NULL,")
311 self.write(f".co_code_adaptive = {co_code_adaptive},")
312 first_traceable = 0
313 for op in code.co_code[::2]:
314 if op == RESUME:
315 break
316 first_traceable += 1
317 self.write(f"._co_firsttraceable = {first_traceable},")
318 name_as_code = f"(PyCodeObject *)&{name}"
319 self.finis.append(f"_PyStaticCode_Fini({name_as_code});")
320 self.inits.append(f"_PyStaticCode_Init({name_as_code})")
321 return f"& {name}.ob_base.ob_base"
322
323 def generate_tuple(self, name: str, t: tuple[object, ...]) -> str:
324 if len(t) == 0:
325 return f"(PyObject *)& _Py_SINGLETON(tuple_empty)"
326 items = [self.generate(f"{name}_{i}", it) for i, it in enumerate(t)]
327 self.write("static")
328 with self.indent():
329 with self.block("struct"):
330 self.write("PyGC_Head _gc_head;")
331 with self.block("struct", "_object;"):
332 self.write("PyObject_VAR_HEAD")
333 if t:
334 self.write(f"PyObject *ob_item[{len(t)}];")
335 with self.block(f"{name} =", ";"):
336 with self.block("._object =", ","):
337 self.object_var_head("PyTuple_Type", len(t))
338 if items:
339 with self.block(f".ob_item =", ","):
340 for item in items:
341 self.write(item + ",")
342 return f"& {name}._object.ob_base.ob_base"
343
344 def _generate_int_for_bits(self, name: str, i: int, digit: int) -> None:
345 sign = (i > 0) - (i < 0)
346 i = abs(i)
347 digits: list[int] = []
348 while i:
349 i, rem = divmod(i, digit)
350 digits.append(rem)
351 self.write("static")
352 with self.indent():
353 with self.block("struct"):
354 self.write("PyObject ob_base;")
355 self.write("uintptr_t lv_tag;")
356 self.write(f"digit ob_digit[{max(1, len(digits))}];")
357 with self.block(f"{name} =", ";"):
358 self.object_head("PyLong_Type")
359 self.write(f".lv_tag = TAG_FROM_SIGN_AND_SIZE({sign}, {len(digits)}),")
360 if digits:
361 ds = ", ".join(map(str, digits))
362 self.write(f".ob_digit = {{ {ds} }},")
363
364 def generate_int(self, name: str, i: int) -> str:
365 if -5 <= i <= 256:
366 return f"(PyObject *)&_PyLong_SMALL_INTS[_PY_NSMALLNEGINTS + {i}]"
367 if i >= 0:
368 name = f"const_int_{i}"
369 else:
370 name = f"const_int_negative_{abs(i)}"
371 if abs(i) < 2**15:
372 self._generate_int_for_bits(name, i, 2**15)
373 else:
374 connective = "if"
375 for bits_in_digit in 15, 30:
376 self.write(f"#{connective} PYLONG_BITS_IN_DIGIT == {bits_in_digit}")
377 self._generate_int_for_bits(name, i, 2**bits_in_digit)
378 connective = "elif"
379 self.write("#else")
380 self.write('#error "PYLONG_BITS_IN_DIGIT should be 15 or 30"')
381 self.write("#endif")
382 # If neither clause applies, it won't compile
383 return f"& {name}.ob_base"
384
385 def generate_float(self, name: str, x: float) -> str:
386 with self.block(f"static PyFloatObject {name} =", ";"):
387 self.object_head("PyFloat_Type")
388 self.write(f".ob_fval = {x},")
389 return f"&{name}.ob_base"
390
391 def generate_complex(self, name: str, z: complex) -> str:
392 with self.block(f"static PyComplexObject {name} =", ";"):
393 self.object_head("PyComplex_Type")
394 self.write(f".cval = {{ {z.real}, {z.imag} }},")
395 return f"&{name}.ob_base"
396
397 def generate_frozenset(self, name: str, fs: frozenset[Any]) -> str:
398 try:
399 fs_sorted = sorted(fs)
400 except TypeError:
401 # frozen set with incompatible types, fallback to repr()
402 fs_sorted = sorted(fs, key=repr)
403 ret = self.generate_tuple(name, tuple(fs_sorted))
404 self.write("// TODO: The above tuple should be a frozenset")
405 return ret
406
407 def generate_file(self, module: str, code: object)-> None:
408 module = module.replace(".", "_")
409 self.generate(f"{module}_toplevel", code)
410 self.write(EPILOGUE.format(name=module))
411
412 def generate(self, name: str, obj: object) -> str:
413 # Use repr() in the key to distinguish -0.0 from +0.0
414 key = (type(obj), obj, repr(obj))
415 if key in self.cache:
416 self.hits += 1
417 # print(f"Cache hit {key!r:.40}: {self.cache[key]!r:.40}")
418 return self.cache[key]
419 self.misses += 1
420 if isinstance(obj, types.CodeType) :
421 val = self.generate_code(name, obj)
422 elif isinstance(obj, tuple):
423 val = self.generate_tuple(name, obj)
424 elif isinstance(obj, str):
425 val = self.generate_unicode(name, obj)
426 elif isinstance(obj, bytes):
427 val = self.generate_bytes(name, obj)
428 elif obj is True:
429 return "Py_True"
430 elif obj is False:
431 return "Py_False"
432 elif isinstance(obj, int):
433 val = self.generate_int(name, obj)
434 elif isinstance(obj, float):
435 val = self.generate_float(name, obj)
436 elif isinstance(obj, complex):
437 val = self.generate_complex(name, obj)
438 elif isinstance(obj, frozenset):
439 val = self.generate_frozenset(name, obj)
440 elif obj is builtins.Ellipsis:
441 return "Py_Ellipsis"
442 elif obj is None:
443 return "Py_None"
444 else:
445 raise TypeError(
446 f"Cannot generate code for {type(obj).__name__} object")
447 # print(f"Cache store {key!r:.40}: {val!r:.40}")
448 self.cache[key] = val
449 return val
450
451
452 EPILOGUE = """
453 PyObject *
454 _Py_get_{name}_toplevel(void)
455 {{
456 return Py_NewRef((PyObject *) &{name}_toplevel);
457 }}
458 """
459
460 FROZEN_COMMENT_C = "/* Auto-generated by Programs/_freeze_module.c */"
461 FROZEN_COMMENT_PY = "/* Auto-generated by Programs/_freeze_module.py */"
462
463 FROZEN_DATA_LINE = r"\s*(\d+,\s*)+\s*"
464
465
466 def is_frozen_header(source: str) -> bool:
467 return source.startswith((FROZEN_COMMENT_C, FROZEN_COMMENT_PY))
468
469
470 def decode_frozen_data(source: str) -> types.CodeType:
471 values: list[int] = []
472 for line in source.splitlines():
473 if re.match(FROZEN_DATA_LINE, line):
474 values.extend([int(x) for x in line.split(",") if x.strip()])
475 data = bytes(values)
476 return umarshal.loads(data) # type: ignore[no-any-return]
477
478
479 def generate(args: list[str], output: TextIO) -> None:
480 printer = Printer(output)
481 for arg in args:
482 file, modname = arg.rsplit(':', 1)
483 with open(file, encoding="utf8") as fd:
484 source = fd.read()
485 if is_frozen_header(source):
486 code = decode_frozen_data(source)
487 else:
488 code = compile(fd.read(), f"<frozen {modname}>", "exec")
489 printer.generate_file(modname, code)
490 with printer.block(f"void\n_Py_Deepfreeze_Fini(void)"):
491 for p in printer.finis:
492 printer.write(p)
493 with printer.block(f"int\n_Py_Deepfreeze_Init(void)"):
494 for p in printer.inits:
495 with printer.block(f"if ({p} < 0)"):
496 printer.write("return -1;")
497 printer.write("return 0;")
498 printer.write(f"\nuint32_t _Py_next_func_version = {next_code_version};\n")
499 if verbose:
500 print(f"Cache hits: {printer.hits}, misses: {printer.misses}")
501
502
503 parser = argparse.ArgumentParser()
504 parser.add_argument("-o", "--output", help="Defaults to deepfreeze.c", default="deepfreeze.c")
505 parser.add_argument("-v", "--verbose", action="store_true", help="Print diagnostics")
506 group = parser.add_mutually_exclusive_group(required=True)
507 group.add_argument("-f", "--file", help="read rule lines from a file")
508 group.add_argument('args', nargs="*", default=(),
509 help="Input file and module name (required) in file:modname format")
510
511 @contextlib.contextmanager
512 def report_time(label: str) -> Iterator[None]:
513 t0 = time.perf_counter()
514 try:
515 yield
516 finally:
517 t1 = time.perf_counter()
518 if verbose:
519 print(f"{label}: {t1-t0:.3f} sec")
520
521
522 def main() -> None:
523 global verbose
524 args = parser.parse_args()
525 verbose = args.verbose
526 output = args.output
527
528 if args.file:
529 if verbose:
530 print(f"Reading targets from {args.file}")
531 with open(args.file, encoding="utf-8-sig") as fin:
532 rules = [x.strip() for x in fin]
533 else:
534 rules = args.args
535
536 with open(output, "w", encoding="utf-8") as file:
537 with report_time("generate"):
538 generate(rules, file)
539 if verbose:
540 print(f"Wrote {os.path.getsize(output)} bytes to {output}")
541
542
543 if __name__ == "__main__":
544 main()