Debian Codegraph @

Platforms/emscripten/wasm_assets.py

raw
1 #!/usr/bin/env python
2 """Create a WASM asset bundle directory structure.
3
4 The WASM asset bundles are pre-loaded by the final WASM build. The bundle
5 contains:
6
7 - a stripped down, pyc-only stdlib zip file, e.g. {PREFIX}/lib/python311.zip
8 - os.py as marker module {PREFIX}/lib/python3.11/os.py
9 - empty lib-dynload directory, to make sure it is copied into the bundle:
10 {PREFIX}/lib/python3.11/lib-dynload/.empty
11 """
12
13 import argparse
14 import pathlib
15 import sys
16 import sysconfig
17 import zipfile
18
19 # source directory
20 SRCDIR = pathlib.Path(__file__).parents[2].absolute()
21 SRCDIR_LIB = SRCDIR / "Lib"
22
23
24 # Library directory relative to $(prefix).
25 WASM_LIB = pathlib.PurePath("lib")
26 WASM_STDLIB_ZIP = (
27 WASM_LIB / f"python{sys.version_info.major}{sys.version_info.minor}.zip"
28 )
29 WASM_STDLIB = (
30 WASM_LIB / f"python{sys.version_info.major}.{sys.version_info.minor}"
31 )
32 WASM_DYNLOAD = WASM_STDLIB / "lib-dynload"
33
34
35 # Don't ship large files / packages that are not particularly useful at
36 # the moment.
37 OMIT_FILES = (
38 # regression tests
39 "test/",
40 # package management
41 "ensurepip/",
42 "venv/",
43 # other platforms
44 "_aix_support.py",
45 "_osx_support.py",
46 # webbrowser
47 "antigravity.py",
48 "webbrowser.py",
49 # Pure Python implementations of C extensions
50 "_pydecimal.py",
51 "_pyio.py",
52 # concurrent threading
53 "concurrent/futures/thread.py",
54 # Misc unused or large files
55 "pydoc_data/",
56 )
57
58 # Synchronous network I/O and protocols are not supported; for example,
59 # socket.create_connection() raises an exception:
60 # "BlockingIOError: [Errno 26] Operation in progress".
61 OMIT_NETWORKING_FILES = (
62 "ftplib.py",
63 "http/",
64 "imaplib.py",
65 "mailbox.py",
66 "poplib.py",
67 "smtplib.py",
68 "socketserver.py",
69 # keep urllib.parse for pydoc
70 "urllib/error.py",
71 "urllib/request.py",
72 "urllib/response.py",
73 "urllib/robotparser.py",
74 "wsgiref/",
75 )
76
77 OMIT_MODULE_FILES = {
78 "_asyncio": ["asyncio/"],
79 "_curses": ["curses/"],
80 "_ctypes": ["ctypes/"],
81 "_decimal": ["decimal.py"],
82 "_dbm": ["dbm/ndbm.py"],
83 "_gdbm": ["dbm/gnu.py"],
84 "_json": ["json/"],
85 "_multiprocessing": ["concurrent/futures/process.py", "multiprocessing/"],
86 "pyexpat": ["xml/", "xmlrpc/"],
87 "_sqlite3": ["sqlite3/"],
88 "_ssl": ["ssl.py"],
89 "_tkinter": ["idlelib/", "tkinter/", "turtle.py", "turtledemo/"],
90 "_zoneinfo": ["zoneinfo/"],
91 }
92
93
94 def get_builddir(args: argparse.Namespace) -> pathlib.Path:
95 """Get builddir path from pybuilddir.txt"""
96 with open("pybuilddir.txt", encoding="utf-8") as f:
97 builddir = f.read()
98 return pathlib.Path(builddir)
99
100
101 def get_sysconfigdata(args: argparse.Namespace) -> pathlib.Path:
102 """Get path to sysconfigdata relative to build root"""
103 assert isinstance(args.builddir, pathlib.Path)
104 data_name: str = sysconfig._get_sysconfigdata_name() # type: ignore[attr-defined]
105 filename = data_name + ".py"
106 return args.builddir / filename
107
108
109 def create_stdlib_zip(
110 args: argparse.Namespace,
111 *,
112 optimize: int = 0,
113 ) -> None:
114 def filterfunc(filename: str) -> bool:
115 pathname = pathlib.Path(filename).resolve()
116 return pathname not in args.omit_files_absolute
117
118 with zipfile.PyZipFile(
119 args.output,
120 mode="w",
121 compression=args.compression,
122 optimize=optimize,
123 ) as pzf:
124 if args.compresslevel is not None:
125 pzf.compresslevel = args.compresslevel
126 pzf.writepy(args.sysconfig_data)
127 for entry in sorted(args.srcdir_lib.iterdir()):
128 entry = entry.resolve()
129 if entry.name == "__pycache__":
130 continue
131 if entry.name.endswith(".py") or entry.is_dir():
132 # writepy() writes .pyc files (bytecode).
133 pzf.writepy(entry, filterfunc=filterfunc)
134
135
136 def detect_extension_modules(args: argparse.Namespace) -> dict[str, bool]:
137 modules = {}
138
139 # disabled by Modules/Setup.local ?
140 with open(args.buildroot / "Makefile") as f:
141 for line in f:
142 if line.startswith("MODDISABLED_NAMES="):
143 disabled = line.split("=", 1)[1].strip().split()
144 for modname in disabled:
145 modules[modname] = False
146 break
147
148 # disabled by configure?
149 with open(args.sysconfig_data) as f:
150 data = f.read()
151 loc: dict[str, dict[str, str]] = {}
152 exec(data, globals(), loc)
153
154 for key, value in loc["build_time_vars"].items():
155 if not key.startswith("MODULE_") or not key.endswith("_STATE"):
156 continue
157 if value not in {"yes", "disabled", "missing", "n/a"}:
158 raise ValueError(f"Unsupported value '{value}' for {key}")
159
160 modname = key[7:-6].lower()
161 if modname not in modules:
162 modules[modname] = value == "yes"
163 return modules
164
165
166 def path(val: str) -> pathlib.Path:
167 return pathlib.Path(val).absolute()
168
169
170 parser = argparse.ArgumentParser()
171 parser.add_argument(
172 "--buildroot",
173 help="absolute path to build root",
174 default=pathlib.Path(".").absolute(),
175 type=path,
176 )
177 parser.add_argument(
178 "--prefix",
179 help="install prefix",
180 default=pathlib.Path("/usr/local"),
181 type=path,
182 )
183 parser.add_argument(
184 "-o",
185 "--output",
186 help="output file",
187 type=path,
188 )
189
190
191 def main() -> None:
192 args = parser.parse_args()
193
194 relative_prefix = args.prefix.relative_to(pathlib.Path("/"))
195 args.srcdir = SRCDIR
196 args.srcdir_lib = SRCDIR_LIB
197 args.wasm_root = args.buildroot / relative_prefix
198 args.wasm_stdlib = args.wasm_root / WASM_STDLIB
199 args.wasm_dynload = args.wasm_root / WASM_DYNLOAD
200
201 # bpo-17004: zipimport supports only zlib compression.
202 # Emscripten ZIP_STORED + -sLZ4=1 linker flags results in larger file.
203 args.compression = zipfile.ZIP_DEFLATED
204 args.compresslevel = 9
205
206 args.builddir = get_builddir(args)
207 args.sysconfig_data = get_sysconfigdata(args)
208 if not args.sysconfig_data.is_file():
209 raise ValueError(f"sysconfigdata file {args.sysconfig_data} missing.")
210
211 extmods = detect_extension_modules(args)
212 omit_files = list(OMIT_FILES)
213 if sysconfig.get_platform().startswith("emscripten"):
214 omit_files.extend(OMIT_NETWORKING_FILES)
215 for modname, modfiles in OMIT_MODULE_FILES.items():
216 if not extmods.get(modname):
217 omit_files.extend(modfiles)
218
219 args.omit_files_absolute = {
220 (args.srcdir_lib / name).resolve() for name in omit_files
221 }
222
223 # Empty, unused directory for dynamic libs, but required for site initialization.
224 args.wasm_dynload.mkdir(parents=True, exist_ok=True)
225 marker = args.wasm_dynload / ".empty"
226 marker.touch()
227 # The rest of stdlib that's useful in a WASM context.
228 create_stdlib_zip(args)
229 size = round(args.output.stat().st_size / 1024**2, 2)
230 parser.exit(0, f"Created {args.output} ({size} MiB)\n")
231
232
233 if __name__ == "__main__":
234 main()