| 1 |
"""Deep freeze |
| 2 |
|
| 3 |
The script may be executed by _bootstrap_python interpreter. |
| 4 |
Shared library extension modules are not available in that case. |
| 5 |
Requires 3.11+ to be executed, |
| 6 |
because relies on `code.co_qualname` and `code.co_exceptiontable`. |
| 7 |
""" |
| 8 |
|
| 9 |
from __future__ import annotations |
| 10 |
|
| 11 |
import argparse |
| 12 |
import builtins |
| 13 |
import collections |
| 14 |
import contextlib |
| 15 |
import os |
| 16 |
import re |
| 17 |
import time |
| 18 |
import types |
| 19 |
|
| 20 |
import umarshal |
| 21 |
|
| 22 |
TYPE_CHECKING = False |
| 23 |
if TYPE_CHECKING: |
| 24 |
from collections.abc import Iterator |
| 25 |
from typing import Any, TextIO |
| 26 |
|
| 27 |
ROOT = os.path.dirname(os.path.dirname(os.path.dirname(__file__))) |
| 28 |
|
| 29 |
verbose = False |
| 30 |
|
| 31 |
# This must be kept in sync with Tools/cases_generator/analyzer.py |
| 32 |
RESUME = 128 |
| 33 |
|
| 34 |
def isprintable(b: bytes) -> bool: |
| 35 |
return all(0x20 <= c < 0x7f for c in b) |
| 36 |
|
| 37 |
|
| 38 |
def make_string_literal(b: bytes) -> str: |
| 39 |
res = ['"'] |
| 40 |
if isprintable(b): |
| 41 |
res.append(b.decode("ascii").replace("\\", "\\\\").replace("\"", "\\\"")) |
| 42 |
else: |
| 43 |
for i in b: |
| 44 |
res.append(f"\\x{i:02x}") |
| 45 |
res.append('"') |
| 46 |
return "".join(res) |
| 47 |
|
| 48 |
|
| 49 |
CO_FAST_LOCAL = 0x20 |
| 50 |
CO_FAST_CELL = 0x40 |
| 51 |
CO_FAST_FREE = 0x80 |
| 52 |
|
| 53 |
next_code_version = 1 |
| 54 |
|
| 55 |
def get_localsplus(code: types.CodeType) -> tuple[tuple[str, ...], bytes]: |
| 56 |
a: collections.defaultdict[str, int] = collections.defaultdict(int) |
| 57 |
for name in code.co_varnames: |
| 58 |
a[name] |= CO_FAST_LOCAL |
| 59 |
for name in code.co_cellvars: |
| 60 |
a[name] |= CO_FAST_CELL |
| 61 |
for name in code.co_freevars: |
| 62 |
a[name] |= CO_FAST_FREE |
| 63 |
return tuple(a.keys()), bytes(a.values()) |
| 64 |
|
| 65 |
|
| 66 |
def get_localsplus_counts(code: types.CodeType, |
| 67 |
names: tuple[str, ...], |
| 68 |
kinds: bytes) -> tuple[int, int, int]: |
| 69 |
nlocals = 0 |
| 70 |
ncellvars = 0 |
| 71 |
nfreevars = 0 |
| 72 |
assert len(names) == len(kinds) |
| 73 |
for name, kind in zip(names, kinds): |
| 74 |
if kind & CO_FAST_LOCAL: |
| 75 |
nlocals += 1 |
| 76 |
if kind & CO_FAST_CELL: |
| 77 |
ncellvars += 1 |
| 78 |
elif kind & CO_FAST_CELL: |
| 79 |
ncellvars += 1 |
| 80 |
elif kind & CO_FAST_FREE: |
| 81 |
nfreevars += 1 |
| 82 |
assert nlocals == len(code.co_varnames) == code.co_nlocals, \ |
| 83 |
(nlocals, len(code.co_varnames), code.co_nlocals) |
| 84 |
assert ncellvars == len(code.co_cellvars) |
| 85 |
assert nfreevars == len(code.co_freevars) |
| 86 |
return nlocals, ncellvars, nfreevars |
| 87 |
|
| 88 |
|
| 89 |
PyUnicode_1BYTE_KIND = 1 |
| 90 |
PyUnicode_2BYTE_KIND = 2 |
| 91 |
PyUnicode_4BYTE_KIND = 4 |
| 92 |
|
| 93 |
|
| 94 |
def analyze_character_width(s: str) -> tuple[int, bool]: |
| 95 |
maxchar = ' ' |
| 96 |
for c in s: |
| 97 |
maxchar = max(maxchar, c) |
| 98 |
ascii = False |
| 99 |
if maxchar <= '\xFF': |
| 100 |
kind = PyUnicode_1BYTE_KIND |
| 101 |
ascii = maxchar <= '\x7F' |
| 102 |
elif maxchar <= '\uFFFF': |
| 103 |
kind = PyUnicode_2BYTE_KIND |
| 104 |
else: |
| 105 |
kind = PyUnicode_4BYTE_KIND |
| 106 |
return kind, ascii |
| 107 |
|
| 108 |
|
| 109 |
def removesuffix(base: str, suffix: str) -> str: |
| 110 |
if base.endswith(suffix): |
| 111 |
return base[:len(base) - len(suffix)] |
| 112 |
return base |
| 113 |
|
| 114 |
class Printer: |
| 115 |
|
| 116 |
def __init__(self, file: TextIO) -> None: |
| 117 |
self.level = 0 |
| 118 |
self.file = file |
| 119 |
self.cache: dict[tuple[type, object, str], str] = {} |
| 120 |
self.hits, self.misses = 0, 0 |
| 121 |
self.finis: list[str] = [] |
| 122 |
self.inits: list[str] = [] |
| 123 |
self.identifiers, self.strings = self.get_identifiers_and_strings() |
| 124 |
self.write('#include "Python.h"') |
| 125 |
self.write('#include "internal/pycore_object.h"') |
| 126 |
self.write('#include "internal/pycore_gc.h"') |
| 127 |
self.write('#include "internal/pycore_code.h"') |
| 128 |
self.write('#include "internal/pycore_frame.h"') |
| 129 |
self.write('#include "internal/pycore_long.h"') |
| 130 |
self.write("") |
| 131 |
|
| 132 |
def get_identifiers_and_strings(self) -> tuple[set[str], dict[str, str]]: |
| 133 |
filename = os.path.join(ROOT, "Include", "internal", "pycore_global_strings.h") |
| 134 |
with open(filename) as fp: |
| 135 |
lines = fp.readlines() |
| 136 |
identifiers: set[str] = set() |
| 137 |
strings: dict[str, str] = {} |
| 138 |
for line in lines: |
| 139 |
if m := re.search(r"STRUCT_FOR_ID\((\w+)\)", line): |
| 140 |
identifiers.add(m.group(1)) |
| 141 |
if m := re.search(r'STRUCT_FOR_STR\((\w+), "(.*?)"\)', line): |
| 142 |
strings[m.group(2)] = m.group(1) |
| 143 |
return identifiers, strings |
| 144 |
|
| 145 |
@contextlib.contextmanager |
| 146 |
def indent(self) -> Iterator[None]: |
| 147 |
save_level = self.level |
| 148 |
try: |
| 149 |
self.level += 1 |
| 150 |
yield |
| 151 |
finally: |
| 152 |
self.level = save_level |
| 153 |
|
| 154 |
def write(self, arg: str) -> None: |
| 155 |
self.file.writelines((" "*self.level, arg, "\n")) |
| 156 |
|
| 157 |
@contextlib.contextmanager |
| 158 |
def block(self, prefix: str, suffix: str = "") -> Iterator[None]: |
| 159 |
self.write(prefix + " {") |
| 160 |
with self.indent(): |
| 161 |
yield |
| 162 |
self.write("}" + suffix) |
| 163 |
|
| 164 |
def object_head(self, typename: str) -> None: |
| 165 |
self.write(f".ob_base = _PyObject_HEAD_INIT(&{typename}),") |
| 166 |
|
| 167 |
def object_var_head(self, typename: str, size: int) -> None: |
| 168 |
self.write(f".ob_base = _PyVarObject_HEAD_INIT(&{typename}, {size}),") |
| 169 |
|
| 170 |
def field(self, obj: object, name: str) -> None: |
| 171 |
self.write(f".{name} = {getattr(obj, name)},") |
| 172 |
|
| 173 |
def generate_bytes(self, name: str, b: bytes) -> str: |
| 174 |
if b == b"": |
| 175 |
return "(PyObject *)&_Py_SINGLETON(bytes_empty)" |
| 176 |
if len(b) == 1: |
| 177 |
return f"(PyObject *)&_Py_SINGLETON(bytes_characters[{b[0]}])" |
| 178 |
self.write("static") |
| 179 |
with self.indent(): |
| 180 |
with self.block("struct"): |
| 181 |
self.write("PyObject_VAR_HEAD") |
| 182 |
self.write("Py_hash_t ob_shash;") |
| 183 |
self.write(f"char ob_sval[{len(b) + 1}];") |
| 184 |
with self.block(f"{name} =", ";"): |
| 185 |
self.object_var_head("PyBytes_Type", len(b)) |
| 186 |
self.write(".ob_shash = -1,") |
| 187 |
self.write(f".ob_sval = {make_string_literal(b)},") |
| 188 |
return f"& {name}.ob_base.ob_base" |
| 189 |
|
| 190 |
def generate_unicode(self, name: str, s: str) -> str: |
| 191 |
if s in self.strings: |
| 192 |
return f"&_Py_STR({self.strings[s]})" |
| 193 |
if s in self.identifiers: |
| 194 |
return f"&_Py_ID({s})" |
| 195 |
if len(s) == 1: |
| 196 |
c = ord(s) |
| 197 |
if c < 128: |
| 198 |
return f"(PyObject *)&_Py_SINGLETON(strings).ascii[{c}]" |
| 199 |
elif c < 256: |
| 200 |
return f"(PyObject *)&_Py_SINGLETON(strings).latin1[{c - 128}]" |
| 201 |
if re.match(r'\A[A-Za-z0-9_]+\Z', s): |
| 202 |
name = f"const_str_{s}" |
| 203 |
kind, ascii = analyze_character_width(s) |
| 204 |
if kind == PyUnicode_1BYTE_KIND: |
| 205 |
datatype = "uint8_t" |
| 206 |
elif kind == PyUnicode_2BYTE_KIND: |
| 207 |
datatype = "uint16_t" |
| 208 |
else: |
| 209 |
datatype = "uint32_t" |
| 210 |
self.write("static") |
| 211 |
with self.indent(): |
| 212 |
with self.block("struct"): |
| 213 |
if ascii: |
| 214 |
self.write("PyASCIIObject _ascii;") |
| 215 |
else: |
| 216 |
self.write("PyCompactUnicodeObject _compact;") |
| 217 |
self.write(f"{datatype} _data[{len(s)+1}];") |
| 218 |
with self.block(f"{name} =", ";"): |
| 219 |
if ascii: |
| 220 |
with self.block("._ascii =", ","): |
| 221 |
self.object_head("PyUnicode_Type") |
| 222 |
self.write(f".length = {len(s)},") |
| 223 |
self.write(".hash = -1,") |
| 224 |
with self.block(".state =", ","): |
| 225 |
self.write(".kind = 1,") |
| 226 |
self.write(".compact = 1,") |
| 227 |
self.write(".ascii = 1,") |
| 228 |
self.write(".statically_allocated = 1,") |
| 229 |
self.write(f"._data = {make_string_literal(s.encode('ascii'))},") |
| 230 |
return f"& {name}._ascii.ob_base" |
| 231 |
else: |
| 232 |
with self.block("._compact =", ","): |
| 233 |
with self.block("._base =", ","): |
| 234 |
self.object_head("PyUnicode_Type") |
| 235 |
self.write(f".length = {len(s)},") |
| 236 |
self.write(".hash = -1,") |
| 237 |
with self.block(".state =", ","): |
| 238 |
self.write(f".kind = {kind},") |
| 239 |
self.write(".compact = 1,") |
| 240 |
self.write(".ascii = 0,") |
| 241 |
self.write(".statically_allocated = 1,") |
| 242 |
utf8 = s.encode('utf-8') |
| 243 |
self.write(f'.utf8 = {make_string_literal(utf8)},') |
| 244 |
self.write(f'.utf8_length = {len(utf8)},') |
| 245 |
with self.block(f"._data =", ","): |
| 246 |
for i in range(0, len(s), 16): |
| 247 |
data = s[i:i+16] |
| 248 |
self.write(", ".join(map(str, map(ord, data))) + ",") |
| 249 |
return f"& {name}._compact._base.ob_base" |
| 250 |
|
| 251 |
|
| 252 |
def generate_code(self, name: str, code: types.CodeType) -> str: |
| 253 |
global next_code_version |
| 254 |
# The ordering here matches PyCode_NewWithPosOnlyArgs() |
| 255 |
# (but see below). |
| 256 |
co_consts = self.generate(name + "_consts", code.co_consts) |
| 257 |
co_names = self.generate(name + "_names", code.co_names) |
| 258 |
co_filename = self.generate(name + "_filename", code.co_filename) |
| 259 |
co_name = self.generate(name + "_name", code.co_name) |
| 260 |
co_linetable = self.generate(name + "_linetable", code.co_linetable) |
| 261 |
# We use 3.10 for type checking, but this module requires 3.11 |
| 262 |
# TODO: bump python version for this script. |
| 263 |
co_qualname = self.generate( |
| 264 |
name + "_qualname", |
| 265 |
code.co_qualname, # type: ignore[attr-defined] |
| 266 |
) |
| 267 |
co_exceptiontable = self.generate( |
| 268 |
name + "_exceptiontable", |
| 269 |
code.co_exceptiontable, # type: ignore[attr-defined] |
| 270 |
) |
| 271 |
# These fields are not directly accessible |
| 272 |
localsplusnames, localspluskinds = get_localsplus(code) |
| 273 |
co_localsplusnames = self.generate(name + "_localsplusnames", localsplusnames) |
| 274 |
co_localspluskinds = self.generate(name + "_localspluskinds", localspluskinds) |
| 275 |
# Derived values |
| 276 |
nlocals, ncellvars, nfreevars = \ |
| 277 |
get_localsplus_counts(code, localsplusnames, localspluskinds) |
| 278 |
co_code_adaptive = make_string_literal(code.co_code) |
| 279 |
self.write("static") |
| 280 |
with self.indent(): |
| 281 |
self.write(f"struct _PyCode_DEF({len(code.co_code)})") |
| 282 |
with self.block(f"{name} =", ";"): |
| 283 |
self.object_var_head("PyCode_Type", len(code.co_code) // 2) |
| 284 |
# But the ordering here must match that in cpython/code.h |
| 285 |
# (which is a pain because we tend to reorder those for perf) |
| 286 |
# otherwise MSVC doesn't like it. |
| 287 |
self.write(f".co_consts = {co_consts},") |
| 288 |
self.write(f".co_names = {co_names},") |
| 289 |
self.write(f".co_exceptiontable = {co_exceptiontable},") |
| 290 |
self.field(code, "co_flags") |
| 291 |
self.field(code, "co_argcount") |
| 292 |
self.field(code, "co_posonlyargcount") |
| 293 |
self.field(code, "co_kwonlyargcount") |
| 294 |
# The following should remain in sync with _PyFrame_NumSlotsForCodeObject |
| 295 |
self.write(f".co_framesize = {code.co_stacksize + len(localsplusnames)} + FRAME_SPECIALS_SIZE,") |
| 296 |
self.field(code, "co_stacksize") |
| 297 |
self.field(code, "co_firstlineno") |
| 298 |
self.write(f".co_nlocalsplus = {len(localsplusnames)},") |
| 299 |
self.field(code, "co_nlocals") |
| 300 |
self.write(f".co_ncellvars = {ncellvars},") |
| 301 |
self.write(f".co_nfreevars = {nfreevars},") |
| 302 |
self.write(f".co_version = {next_code_version},") |
| 303 |
next_code_version += 1 |
| 304 |
self.write(f".co_localsplusnames = {co_localsplusnames},") |
| 305 |
self.write(f".co_localspluskinds = {co_localspluskinds},") |
| 306 |
self.write(f".co_filename = {co_filename},") |
| 307 |
self.write(f".co_name = {co_name},") |
| 308 |
self.write(f".co_qualname = {co_qualname},") |
| 309 |
self.write(f".co_linetable = {co_linetable},") |
| 310 |
self.write(f"._co_cached = NULL,") |
| 311 |
self.write(f".co_code_adaptive = {co_code_adaptive},") |
| 312 |
first_traceable = 0 |
| 313 |
for op in code.co_code[::2]: |
| 314 |
if op == RESUME: |
| 315 |
break |
| 316 |
first_traceable += 1 |
| 317 |
self.write(f"._co_firsttraceable = {first_traceable},") |
| 318 |
name_as_code = f"(PyCodeObject *)&{name}" |
| 319 |
self.finis.append(f"_PyStaticCode_Fini({name_as_code});") |
| 320 |
self.inits.append(f"_PyStaticCode_Init({name_as_code})") |
| 321 |
return f"& {name}.ob_base.ob_base" |
| 322 |
|
| 323 |
def generate_tuple(self, name: str, t: tuple[object, ...]) -> str: |
| 324 |
if len(t) == 0: |
| 325 |
return f"(PyObject *)& _Py_SINGLETON(tuple_empty)" |
| 326 |
items = [self.generate(f"{name}_{i}", it) for i, it in enumerate(t)] |
| 327 |
self.write("static") |
| 328 |
with self.indent(): |
| 329 |
with self.block("struct"): |
| 330 |
self.write("PyGC_Head _gc_head;") |
| 331 |
with self.block("struct", "_object;"): |
| 332 |
self.write("PyObject_VAR_HEAD") |
| 333 |
if t: |
| 334 |
self.write(f"PyObject *ob_item[{len(t)}];") |
| 335 |
with self.block(f"{name} =", ";"): |
| 336 |
with self.block("._object =", ","): |
| 337 |
self.object_var_head("PyTuple_Type", len(t)) |
| 338 |
if items: |
| 339 |
with self.block(f".ob_item =", ","): |
| 340 |
for item in items: |
| 341 |
self.write(item + ",") |
| 342 |
return f"& {name}._object.ob_base.ob_base" |
| 343 |
|
| 344 |
def _generate_int_for_bits(self, name: str, i: int, digit: int) -> None: |
| 345 |
sign = (i > 0) - (i < 0) |
| 346 |
i = abs(i) |
| 347 |
digits: list[int] = [] |
| 348 |
while i: |
| 349 |
i, rem = divmod(i, digit) |
| 350 |
digits.append(rem) |
| 351 |
self.write("static") |
| 352 |
with self.indent(): |
| 353 |
with self.block("struct"): |
| 354 |
self.write("PyObject ob_base;") |
| 355 |
self.write("uintptr_t lv_tag;") |
| 356 |
self.write(f"digit ob_digit[{max(1, len(digits))}];") |
| 357 |
with self.block(f"{name} =", ";"): |
| 358 |
self.object_head("PyLong_Type") |
| 359 |
self.write(f".lv_tag = TAG_FROM_SIGN_AND_SIZE({sign}, {len(digits)}),") |
| 360 |
if digits: |
| 361 |
ds = ", ".join(map(str, digits)) |
| 362 |
self.write(f".ob_digit = {{ {ds} }},") |
| 363 |
|
| 364 |
def generate_int(self, name: str, i: int) -> str: |
| 365 |
if -5 <= i <= 256: |
| 366 |
return f"(PyObject *)&_PyLong_SMALL_INTS[_PY_NSMALLNEGINTS + {i}]" |
| 367 |
if i >= 0: |
| 368 |
name = f"const_int_{i}" |
| 369 |
else: |
| 370 |
name = f"const_int_negative_{abs(i)}" |
| 371 |
if abs(i) < 2**15: |
| 372 |
self._generate_int_for_bits(name, i, 2**15) |
| 373 |
else: |
| 374 |
connective = "if" |
| 375 |
for bits_in_digit in 15, 30: |
| 376 |
self.write(f"#{connective} PYLONG_BITS_IN_DIGIT == {bits_in_digit}") |
| 377 |
self._generate_int_for_bits(name, i, 2**bits_in_digit) |
| 378 |
connective = "elif" |
| 379 |
self.write("#else") |
| 380 |
self.write('#error "PYLONG_BITS_IN_DIGIT should be 15 or 30"') |
| 381 |
self.write("#endif") |
| 382 |
# If neither clause applies, it won't compile |
| 383 |
return f"& {name}.ob_base" |
| 384 |
|
| 385 |
def generate_float(self, name: str, x: float) -> str: |
| 386 |
with self.block(f"static PyFloatObject {name} =", ";"): |
| 387 |
self.object_head("PyFloat_Type") |
| 388 |
self.write(f".ob_fval = {x},") |
| 389 |
return f"&{name}.ob_base" |
| 390 |
|
| 391 |
def generate_complex(self, name: str, z: complex) -> str: |
| 392 |
with self.block(f"static PyComplexObject {name} =", ";"): |
| 393 |
self.object_head("PyComplex_Type") |
| 394 |
self.write(f".cval = {{ {z.real}, {z.imag} }},") |
| 395 |
return f"&{name}.ob_base" |
| 396 |
|
| 397 |
def generate_frozenset(self, name: str, fs: frozenset[Any]) -> str: |
| 398 |
try: |
| 399 |
fs_sorted = sorted(fs) |
| 400 |
except TypeError: |
| 401 |
# frozen set with incompatible types, fallback to repr() |
| 402 |
fs_sorted = sorted(fs, key=repr) |
| 403 |
ret = self.generate_tuple(name, tuple(fs_sorted)) |
| 404 |
self.write("// TODO: The above tuple should be a frozenset") |
| 405 |
return ret |
| 406 |
|
| 407 |
def generate_file(self, module: str, code: object)-> None: |
| 408 |
module = module.replace(".", "_") |
| 409 |
self.generate(f"{module}_toplevel", code) |
| 410 |
self.write(EPILOGUE.format(name=module)) |
| 411 |
|
| 412 |
def generate(self, name: str, obj: object) -> str: |
| 413 |
# Use repr() in the key to distinguish -0.0 from +0.0 |
| 414 |
key = (type(obj), obj, repr(obj)) |
| 415 |
if key in self.cache: |
| 416 |
self.hits += 1 |
| 417 |
# print(f"Cache hit {key!r:.40}: {self.cache[key]!r:.40}") |
| 418 |
return self.cache[key] |
| 419 |
self.misses += 1 |
| 420 |
if isinstance(obj, types.CodeType) : |
| 421 |
val = self.generate_code(name, obj) |
| 422 |
elif isinstance(obj, tuple): |
| 423 |
val = self.generate_tuple(name, obj) |
| 424 |
elif isinstance(obj, str): |
| 425 |
val = self.generate_unicode(name, obj) |
| 426 |
elif isinstance(obj, bytes): |
| 427 |
val = self.generate_bytes(name, obj) |
| 428 |
elif obj is True: |
| 429 |
return "Py_True" |
| 430 |
elif obj is False: |
| 431 |
return "Py_False" |
| 432 |
elif isinstance(obj, int): |
| 433 |
val = self.generate_int(name, obj) |
| 434 |
elif isinstance(obj, float): |
| 435 |
val = self.generate_float(name, obj) |
| 436 |
elif isinstance(obj, complex): |
| 437 |
val = self.generate_complex(name, obj) |
| 438 |
elif isinstance(obj, frozenset): |
| 439 |
val = self.generate_frozenset(name, obj) |
| 440 |
elif obj is builtins.Ellipsis: |
| 441 |
return "Py_Ellipsis" |
| 442 |
elif obj is None: |
| 443 |
return "Py_None" |
| 444 |
else: |
| 445 |
raise TypeError( |
| 446 |
f"Cannot generate code for {type(obj).__name__} object") |
| 447 |
# print(f"Cache store {key!r:.40}: {val!r:.40}") |
| 448 |
self.cache[key] = val |
| 449 |
return val |
| 450 |
|
| 451 |
|
| 452 |
EPILOGUE = """ |
| 453 |
PyObject * |
| 454 |
_Py_get_{name}_toplevel(void) |
| 455 |
{{ |
| 456 |
return Py_NewRef((PyObject *) &{name}_toplevel); |
| 457 |
}} |
| 458 |
""" |
| 459 |
|
| 460 |
FROZEN_COMMENT_C = "/* Auto-generated by Programs/_freeze_module.c */" |
| 461 |
FROZEN_COMMENT_PY = "/* Auto-generated by Programs/_freeze_module.py */" |
| 462 |
|
| 463 |
FROZEN_DATA_LINE = r"\s*(\d+,\s*)+\s*" |
| 464 |
|
| 465 |
|
| 466 |
def is_frozen_header(source: str) -> bool: |
| 467 |
return source.startswith((FROZEN_COMMENT_C, FROZEN_COMMENT_PY)) |
| 468 |
|
| 469 |
|
| 470 |
def decode_frozen_data(source: str) -> types.CodeType: |
| 471 |
values: list[int] = [] |
| 472 |
for line in source.splitlines(): |
| 473 |
if re.match(FROZEN_DATA_LINE, line): |
| 474 |
values.extend([int(x) for x in line.split(",") if x.strip()]) |
| 475 |
data = bytes(values) |
| 476 |
return umarshal.loads(data) # type: ignore[no-any-return] |
| 477 |
|
| 478 |
|
| 479 |
def generate(args: list[str], output: TextIO) -> None: |
| 480 |
printer = Printer(output) |
| 481 |
for arg in args: |
| 482 |
file, modname = arg.rsplit(':', 1) |
| 483 |
with open(file, encoding="utf8") as fd: |
| 484 |
source = fd.read() |
| 485 |
if is_frozen_header(source): |
| 486 |
code = decode_frozen_data(source) |
| 487 |
else: |
| 488 |
code = compile(fd.read(), f"<frozen {modname}>", "exec") |
| 489 |
printer.generate_file(modname, code) |
| 490 |
with printer.block(f"void\n_Py_Deepfreeze_Fini(void)"): |
| 491 |
for p in printer.finis: |
| 492 |
printer.write(p) |
| 493 |
with printer.block(f"int\n_Py_Deepfreeze_Init(void)"): |
| 494 |
for p in printer.inits: |
| 495 |
with printer.block(f"if ({p} < 0)"): |
| 496 |
printer.write("return -1;") |
| 497 |
printer.write("return 0;") |
| 498 |
printer.write(f"\nuint32_t _Py_next_func_version = {next_code_version};\n") |
| 499 |
if verbose: |
| 500 |
print(f"Cache hits: {printer.hits}, misses: {printer.misses}") |
| 501 |
|
| 502 |
|
| 503 |
parser = argparse.ArgumentParser() |
| 504 |
parser.add_argument("-o", "--output", help="Defaults to deepfreeze.c", default="deepfreeze.c") |
| 505 |
parser.add_argument("-v", "--verbose", action="store_true", help="Print diagnostics") |
| 506 |
group = parser.add_mutually_exclusive_group(required=True) |
| 507 |
group.add_argument("-f", "--file", help="read rule lines from a file") |
| 508 |
group.add_argument('args', nargs="*", default=(), |
| 509 |
help="Input file and module name (required) in file:modname format") |
| 510 |
|
| 511 |
@contextlib.contextmanager |
| 512 |
def report_time(label: str) -> Iterator[None]: |
| 513 |
t0 = time.perf_counter() |
| 514 |
try: |
| 515 |
yield |
| 516 |
finally: |
| 517 |
t1 = time.perf_counter() |
| 518 |
if verbose: |
| 519 |
print(f"{label}: {t1-t0:.3f} sec") |
| 520 |
|
| 521 |
|
| 522 |
def main() -> None: |
| 523 |
global verbose |
| 524 |
args = parser.parse_args() |
| 525 |
verbose = args.verbose |
| 526 |
output = args.output |
| 527 |
|
| 528 |
if args.file: |
| 529 |
if verbose: |
| 530 |
print(f"Reading targets from {args.file}") |
| 531 |
with open(args.file, encoding="utf-8-sig") as fin: |
| 532 |
rules = [x.strip() for x in fin] |
| 533 |
else: |
| 534 |
rules = args.args |
| 535 |
|
| 536 |
with open(output, "w", encoding="utf-8") as file: |
| 537 |
with report_time("generate"): |
| 538 |
generate(rules, file) |
| 539 |
if verbose: |
| 540 |
print(f"Wrote {os.path.getsize(output)} bytes to {output}") |
| 541 |
|
| 542 |
|
| 543 |
if __name__ == "__main__": |
| 544 |
main() |