Debian Codegraph @

Tools/wasm/wasm_build.py

raw
1 #!/usr/bin/env python3
2 """Build script for Python on WebAssembly platforms.
3
4 $ ./Tools/wasm/wasm_builder.py emscripten-browser build repl
5 $ ./Tools/wasm/wasm_builder.py emscripten-node-dl build test
6 $ ./Tools/wasm/wasm_builder.py wasi build test
7
8 Primary build targets are "emscripten-node-dl" (NodeJS, dynamic linking),
9 "emscripten-browser", and "wasi".
10
11 Emscripten builds require a recent Emscripten SDK. The tools looks for an
12 activated EMSDK environment (". /path/to/emsdk_env.sh"). System packages
13 (Debian, Homebrew) are not supported.
14
15 WASI builds require WASI SDK and wasmtime. The tool looks for 'WASI_SDK_PATH'
16 and falls back to /opt/wasi-sdk.
17
18 The 'build' Python interpreter must be rebuilt every time Python's byte code
19 changes.
20
21 ./Tools/wasm/wasm_builder.py --clean build build
22
23 """
24
25 import argparse
26 import dataclasses
27 import enum
28 import logging
29 import os
30 import pathlib
31 import re
32 import shlex
33 import shutil
34 import socket
35 import subprocess
36 import sys
37 import sysconfig
38 import tempfile
39 import time
40 import warnings
41 import webbrowser
42 from collections.abc import Callable, Iterable
43
44 # for Python 3.8
45 from typing import (
46 Any,
47 cast,
48 )
49
50 logger = logging.getLogger("wasm_build")
51
52 SRCDIR = pathlib.Path(__file__).parent.parent.parent.absolute()
53 WASMTOOLS = SRCDIR / "Tools" / "wasm"
54 BUILDDIR = SRCDIR / "builddir"
55 CONFIGURE = SRCDIR / "configure"
56 SETUP_LOCAL = SRCDIR / "Modules" / "Setup.local"
57
58 HAS_CCACHE = shutil.which("ccache") is not None
59
60 # path to WASI-SDK root
61 WASI_SDK_PATH = pathlib.Path(os.environ.get("WASI_SDK_PATH", "/opt/wasi-sdk"))
62
63 # path to Emscripten SDK config file.
64 # auto-detect's EMSDK in /opt/emsdk without ". emsdk_env.sh".
65 EM_CONFIG = pathlib.Path(
66 os.environ.setdefault("EM_CONFIG", "/opt/emsdk/.emscripten")
67 )
68 EMSDK_MIN_VERSION = (3, 1, 19)
69 EMSDK_BROKEN_VERSION = {
70 (3, 1, 14): "https://github.com/emscripten-core/emscripten/issues/17338",
71 (3, 1, 16): "https://github.com/emscripten-core/emscripten/issues/17393",
72 (3, 1, 20): "https://github.com/emscripten-core/emscripten/issues/17720",
73 }
74 _MISSING = pathlib.Path("MISSING")
75
76 WASM_WEBSERVER = WASMTOOLS / "wasm_webserver.py"
77
78 CLEAN_SRCDIR = f"""
79 Builds require a clean source directory. Please use a clean checkout or
80 run "make clean -C '{SRCDIR}'".
81 """
82
83 INSTALL_NATIVE = """
84 Builds require a C compiler (gcc, clang), make, pkg-config, and development
85 headers for dependencies like zlib.
86
87 Debian/Ubuntu: sudo apt install build-essential git curl pkg-config zlib1g-dev
88 Fedora/CentOS: sudo dnf install gcc make git-core curl pkgconfig zlib-devel
89 """
90
91 INSTALL_EMSDK = """
92 wasm32-emscripten builds need Emscripten SDK. Please follow instructions at
93 https://emscripten.org/docs/getting_started/downloads.html how to install
94 Emscripten and how to activate the SDK with "emsdk_env.sh".
95
96 git clone https://github.com/emscripten-core/emsdk.git /path/to/emsdk
97 cd /path/to/emsdk
98 ./emsdk install latest
99 ./emsdk activate latest
100 source /path/to/emsdk_env.sh
101 """
102
103 INSTALL_WASI_SDK = """
104 wasm32-wasi builds need WASI SDK. Please fetch the latest SDK from
105 https://github.com/WebAssembly/wasi-sdk/releases and install it to
106 "/opt/wasi-sdk". Alternatively you can install the SDK in a different location
107 and point the environment variable WASI_SDK_PATH to the root directory
108 of the SDK. The SDK is available for Linux x86_64, macOS x86_64, and MinGW.
109 """
110
111 INSTALL_WASMTIME = """
112 wasm32-wasi tests require wasmtime on PATH. Please follow instructions at
113 https://wasmtime.dev/ to install wasmtime.
114 """
115
116
117 def parse_emconfig(
118 emconfig: pathlib.Path = EM_CONFIG,
119 ) -> tuple[pathlib.Path, pathlib.Path]:
120 """Parse EM_CONFIG file and lookup EMSCRIPTEN_ROOT and NODE_JS.
121
122 The ".emscripten" config file is a Python snippet that uses "EM_CONFIG"
123 environment variable. EMSCRIPTEN_ROOT is the "upstream/emscripten"
124 subdirectory with tools like "emconfigure".
125 """
126 if not emconfig.exists():
127 return _MISSING, _MISSING
128 with open(emconfig, encoding="utf-8") as f:
129 code = f.read()
130 # EM_CONFIG file is a Python snippet
131 local: dict[str, Any] = {}
132 exec(code, globals(), local)
133 emscripten_root = pathlib.Path(local["EMSCRIPTEN_ROOT"])
134 node_js = pathlib.Path(local["NODE_JS"])
135 return emscripten_root, node_js
136
137
138 EMSCRIPTEN_ROOT, NODE_JS = parse_emconfig()
139
140
141 def read_python_version(configure: pathlib.Path = CONFIGURE) -> str:
142 """Read PACKAGE_VERSION from configure script
143
144 configure and configure.ac are the canonical source for major and
145 minor version number.
146 """
147 version_re = re.compile(r"^PACKAGE_VERSION='(\d\.\d+)'")
148 with configure.open(encoding="utf-8") as f:
149 for line in f:
150 mo = version_re.match(line)
151 if mo:
152 return mo.group(1)
153 raise ValueError(f"PACKAGE_VERSION not found in {configure}")
154
155
156 PYTHON_VERSION = read_python_version()
157
158
159 class ConditionError(ValueError):
160 def __init__(self, info: str, text: str) -> None:
161 self.info = info
162 self.text = text
163
164 def __str__(self) -> str:
165 return f"{type(self).__name__}: '{self.info}'\n{self.text}"
166
167
168 class MissingDependency(ConditionError):
169 pass
170
171
172 class DirtySourceDirectory(ConditionError):
173 pass
174
175
176 @dataclasses.dataclass
177 class Platform:
178 """Platform-specific settings
179
180 - CONFIG_SITE override
181 - configure wrapper (e.g. emconfigure)
182 - make wrapper (e.g. emmake)
183 - additional environment variables
184 - check function to verify SDK
185 """
186
187 name: str
188 pythonexe: str
189 config_site: pathlib.PurePath | None
190 configure_wrapper: pathlib.Path | None
191 make_wrapper: pathlib.PurePath | None
192 environ: dict[str, Any]
193 check: Callable[[], None]
194 # Used for build_emports().
195 ports: pathlib.PurePath | None
196 cc: pathlib.PurePath | None
197
198 def getenv(self, profile: "BuildProfile") -> dict[str, Any]:
199 return self.environ.copy()
200
201
202 def _check_clean_src() -> None:
203 candidates = [
204 SRCDIR / "Programs" / "python.o",
205 SRCDIR / "Python" / "frozen_modules" / "importlib._bootstrap.h",
206 ]
207 for candidate in candidates:
208 if candidate.exists():
209 raise DirtySourceDirectory(os.fspath(candidate), CLEAN_SRCDIR)
210
211
212 def _check_native() -> None:
213 if not any(shutil.which(cc) for cc in ["cc", "gcc", "clang"]):
214 raise MissingDependency("cc", INSTALL_NATIVE)
215 if not shutil.which("make"):
216 raise MissingDependency("make", INSTALL_NATIVE)
217 if sys.platform == "linux":
218 # skip pkg-config check on macOS
219 if not shutil.which("pkg-config"):
220 raise MissingDependency("pkg-config", INSTALL_NATIVE)
221 # zlib is needed to create zip files
222 for devel in ["zlib"]:
223 try:
224 subprocess.check_call(["pkg-config", "--exists", devel])
225 except subprocess.CalledProcessError:
226 raise MissingDependency(devel, INSTALL_NATIVE) from None
227 _check_clean_src()
228
229
230 NATIVE = Platform(
231 "native",
232 # macOS has python.exe
233 pythonexe=sysconfig.get_config_var("BUILDPYTHON") or "python",
234 config_site=None,
235 configure_wrapper=None,
236 ports=None,
237 cc=None,
238 make_wrapper=None,
239 environ={},
240 check=_check_native,
241 )
242
243
244 def _check_emscripten() -> None:
245 if EMSCRIPTEN_ROOT is _MISSING:
246 raise MissingDependency("Emscripten SDK EM_CONFIG", INSTALL_EMSDK)
247 # sanity check
248 emconfigure = EMSCRIPTEN.configure_wrapper
249 if emconfigure is not None and not emconfigure.exists():
250 raise MissingDependency(os.fspath(emconfigure), INSTALL_EMSDK)
251 # version check
252 version_txt = EMSCRIPTEN_ROOT / "emscripten-version.txt"
253 if not version_txt.exists():
254 raise MissingDependency(os.fspath(version_txt), INSTALL_EMSDK)
255 with open(version_txt) as f:
256 version = f.read().strip().strip('"')
257 if version.endswith("-git"):
258 # git / upstream / tot-upstream installation
259 version = version[:-4]
260 version_tuple = cast(
261 tuple[int, int, int], tuple(int(v) for v in version.split("."))
262 )
263 if version_tuple < EMSDK_MIN_VERSION:
264 raise ConditionError(
265 os.fspath(version_txt),
266 f"Emscripten SDK {version} in '{EMSCRIPTEN_ROOT}' is older than "
267 "minimum required version "
268 f"{'.'.join(str(v) for v in EMSDK_MIN_VERSION)}.",
269 )
270 broken = EMSDK_BROKEN_VERSION.get(version_tuple)
271 if broken is not None:
272 raise ConditionError(
273 os.fspath(version_txt),
274 (
275 f"Emscripten SDK {version} in '{EMSCRIPTEN_ROOT}' has known "
276 f"bugs, see {broken}."
277 ),
278 )
279 if os.environ.get("PKG_CONFIG_PATH"):
280 warnings.warn(
281 "PKG_CONFIG_PATH is set and not empty. emconfigure overrides "
282 "this environment variable. Use EM_PKG_CONFIG_PATH instead."
283 )
284 _check_clean_src()
285
286
287 EMSCRIPTEN = Platform(
288 "emscripten",
289 pythonexe="python.js",
290 config_site=WASMTOOLS / "config.site-wasm32-emscripten",
291 configure_wrapper=EMSCRIPTEN_ROOT / "emconfigure",
292 ports=EMSCRIPTEN_ROOT / "embuilder",
293 cc=EMSCRIPTEN_ROOT / "emcc",
294 make_wrapper=EMSCRIPTEN_ROOT / "emmake",
295 environ={
296 # workaround for https://github.com/emscripten-core/emscripten/issues/17635
297 "TZ": "UTC",
298 "EM_COMPILER_WRAPPER": "ccache" if HAS_CCACHE else None,
299 "PATH": [EMSCRIPTEN_ROOT, os.environ["PATH"]],
300 },
301 check=_check_emscripten,
302 )
303
304
305 def _check_wasi() -> None:
306 wasm_ld = WASI_SDK_PATH / "bin" / "wasm-ld"
307 if not wasm_ld.exists():
308 raise MissingDependency(os.fspath(wasm_ld), INSTALL_WASI_SDK)
309 wasmtime = shutil.which("wasmtime")
310 if wasmtime is None:
311 raise MissingDependency("wasmtime", INSTALL_WASMTIME)
312 _check_clean_src()
313
314
315 WASI = Platform(
316 "wasi",
317 pythonexe="python.wasm",
318 config_site=WASMTOOLS / "config.site-wasm32-wasi",
319 configure_wrapper=WASMTOOLS / "wasi-env",
320 ports=None,
321 cc=WASI_SDK_PATH / "bin" / "clang",
322 make_wrapper=None,
323 environ={
324 "WASI_SDK_PATH": WASI_SDK_PATH,
325 # workaround for https://github.com/python/cpython/issues/95952
326 "HOSTRUNNER": (
327 "wasmtime run "
328 "--wasm max-wasm-stack=16777216 "
329 "--wasi preview2 "
330 "--dir {srcdir}::/ "
331 "--env PYTHONPATH=/{relbuilddir}/build/lib.wasi-wasm32-{version}:/Lib"
332 ),
333 "PATH": [WASI_SDK_PATH / "bin", os.environ["PATH"]],
334 },
335 check=_check_wasi,
336 )
337
338
339 class Host(enum.Enum):
340 """Target host triplet"""
341
342 wasm32_emscripten = "wasm32-unknown-emscripten"
343 wasm64_emscripten = "wasm64-unknown-emscripten"
344 wasm32_wasi = "wasm32-unknown-wasi"
345 wasm64_wasi = "wasm64-unknown-wasi"
346 # current platform
347 build = sysconfig.get_config_var("BUILD_GNU_TYPE")
348
349 @property
350 def platform(self) -> Platform:
351 if self.is_emscripten:
352 return EMSCRIPTEN
353 elif self.is_wasi:
354 return WASI
355 else:
356 return NATIVE
357
358 @property
359 def is_emscripten(self) -> bool:
360 cls = type(self)
361 return self in {cls.wasm32_emscripten, cls.wasm64_emscripten}
362
363 @property
364 def is_wasi(self) -> bool:
365 cls = type(self)
366 return self in {cls.wasm32_wasi, cls.wasm64_wasi}
367
368 def get_extra_paths(self) -> Iterable[pathlib.PurePath]:
369 """Host-specific os.environ["PATH"] entries.
370
371 Emscripten's Node version 14.x works well for wasm32-emscripten.
372 wasm64-emscripten requires more recent v8 version, e.g. node 16.x.
373 Attempt to use system's node command.
374 """
375 cls = type(self)
376 if self == cls.wasm32_emscripten:
377 return [NODE_JS.parent]
378 elif self == cls.wasm64_emscripten:
379 # TODO: look for recent node
380 return []
381 else:
382 return []
383
384 @property
385 def emport_args(self) -> list[str]:
386 """Host-specific port args (Emscripten)."""
387 cls = type(self)
388 if self is cls.wasm64_emscripten:
389 return ["-sMEMORY64=1"]
390 elif self is cls.wasm32_emscripten:
391 return ["-sMEMORY64=0"]
392 else:
393 return []
394
395 @property
396 def embuilder_args(self) -> list[str]:
397 """Host-specific embuilder args (Emscripten)."""
398 cls = type(self)
399 if self is cls.wasm64_emscripten:
400 return ["--wasm64"]
401 else:
402 return []
403
404
405 class EmscriptenTarget(enum.Enum):
406 """Emscripten-specific targets (--with-emscripten-target)"""
407
408 browser = "browser"
409 browser_debug = "browser-debug"
410 node = "node"
411 node_debug = "node-debug"
412
413 @property
414 def is_browser(self) -> bool:
415 cls = type(self)
416 return self in {cls.browser, cls.browser_debug}
417
418 @property
419 def emport_args(self) -> list[str]:
420 """Target-specific port args."""
421 cls = type(self)
422 if self in {cls.browser_debug, cls.node_debug}:
423 # some libs come in debug and non-debug builds
424 return ["-O0"]
425 else:
426 return ["-O2"]
427
428
429 class SupportLevel(enum.Enum):
430 supported = "tier 3, supported"
431 working = "working, unsupported"
432 experimental = "experimental, may be broken"
433 broken = "broken / unavailable"
434
435 def __bool__(self) -> bool:
436 cls = type(self)
437 return self in {cls.supported, cls.working}
438
439
440 @dataclasses.dataclass
441 class BuildProfile:
442 name: str
443 support_level: SupportLevel
444 host: Host
445 target: EmscriptenTarget | None = None
446 dynamic_linking: bool | None = None
447 pthreads: bool | None = None
448 default_testopts: str = "-j2"
449
450 @property
451 def is_browser(self) -> bool:
452 """Is this a browser build?"""
453 return self.target is not None and self.target.is_browser
454
455 @property
456 def builddir(self) -> pathlib.Path:
457 """Path to build directory"""
458 return BUILDDIR / self.name
459
460 @property
461 def python_cmd(self) -> pathlib.Path:
462 """Path to python executable"""
463 return self.builddir / self.host.platform.pythonexe
464
465 @property
466 def makefile(self) -> pathlib.Path:
467 """Path to Makefile"""
468 return self.builddir / "Makefile"
469
470 @property
471 def configure_cmd(self) -> list[str]:
472 """Generate configure command"""
473 # use relative path, so WASI tests can find lib prefix.
474 # pathlib.Path.relative_to() does not work here.
475 configure = os.path.relpath(CONFIGURE, self.builddir)
476 cmd = [configure, "-C"]
477 platform = self.host.platform
478 if platform.configure_wrapper:
479 cmd.insert(0, os.fspath(platform.configure_wrapper))
480
481 cmd.append(f"--host={self.host.value}")
482 cmd.append(f"--build={Host.build.value}")
483
484 if self.target is not None:
485 assert self.host.is_emscripten
486 cmd.append(f"--with-emscripten-target={self.target.value}")
487
488 if self.dynamic_linking is not None:
489 assert self.host.is_emscripten
490 opt = "enable" if self.dynamic_linking else "disable"
491 cmd.append(f"--{opt}-wasm-dynamic-linking")
492
493 if self.pthreads is not None:
494 opt = "enable" if self.pthreads else "disable"
495 cmd.append(f"--{opt}-wasm-pthreads")
496
497 if self.host != Host.build:
498 cmd.append(f"--with-build-python={BUILD.python_cmd}")
499
500 if platform.config_site is not None:
501 cmd.append(f"CONFIG_SITE={platform.config_site}")
502
503 return cmd
504
505 @property
506 def make_cmd(self) -> list[str]:
507 """Generate make command"""
508 cmd = ["make"]
509 platform = self.host.platform
510 if platform.make_wrapper:
511 cmd.insert(0, os.fspath(platform.make_wrapper))
512 return cmd
513
514 def getenv(self) -> dict[str, Any]:
515 """Generate environ dict for platform"""
516 env = os.environ.copy()
517 if hasattr(os, "process_cpu_count"):
518 cpu_count = os.process_cpu_count()
519 else:
520 cpu_count = os.cpu_count()
521 env.setdefault("MAKEFLAGS", f"-j{cpu_count}")
522 platenv = self.host.platform.getenv(self)
523 for key, value in platenv.items():
524 if value is None:
525 env.pop(key, None)
526 elif key == "PATH":
527 # list of path items, prefix with extra paths
528 new_path: list[pathlib.PurePath] = []
529 new_path.extend(self.host.get_extra_paths())
530 new_path.extend(value)
531 env[key] = os.pathsep.join(os.fspath(p) for p in new_path)
532 elif isinstance(value, str):
533 env[key] = value.format(
534 relbuilddir=self.builddir.relative_to(SRCDIR),
535 srcdir=SRCDIR,
536 version=PYTHON_VERSION,
537 )
538 else:
539 env[key] = value
540 return env
541
542 def _run_cmd(
543 self,
544 cmd: Iterable[str],
545 args: Iterable[str] = (),
546 cwd: pathlib.Path | None = None,
547 ) -> int:
548 cmd = list(cmd)
549 cmd.extend(args)
550 if cwd is None:
551 cwd = self.builddir
552 logger.info('Running "%s" in "%s"', shlex.join(cmd), cwd)
553 return subprocess.check_call(
554 cmd,
555 cwd=os.fspath(cwd),
556 env=self.getenv(),
557 )
558
559 def _check_execute(self) -> None:
560 if self.is_browser:
561 raise ValueError(f"Cannot execute on {self.target}")
562
563 def run_build(self, *args: str) -> None:
564 """Run configure (if necessary) and make"""
565 if not self.makefile.exists():
566 logger.info("Makefile not found, running configure")
567 self.run_configure(*args)
568 self.run_make("all", *args)
569
570 def run_configure(self, *args: str) -> int:
571 """Run configure script to generate Makefile"""
572 os.makedirs(self.builddir, exist_ok=True)
573 return self._run_cmd(self.configure_cmd, args)
574
575 def run_make(self, *args: str) -> int:
576 """Run make (defaults to build all)"""
577 return self._run_cmd(self.make_cmd, args)
578
579 def run_pythoninfo(self, *args: str) -> int:
580 """Run 'make pythoninfo'"""
581 self._check_execute()
582 return self.run_make("pythoninfo", *args)
583
584 def run_test(self, target: str, testopts: str | None = None) -> int:
585 """Run buildbottests"""
586 self._check_execute()
587 if testopts is None:
588 testopts = self.default_testopts
589 return self.run_make(target, f"TESTOPTS={testopts}")
590
591 def run_py(self, *args: str) -> int:
592 """Run Python with hostrunner"""
593 self._check_execute()
594 return self.run_make(
595 "--eval",
596 f"run: all; $(HOSTRUNNER) ./$(PYTHON) {shlex.join(args)}",
597 "run",
598 )
599
600 def run_browser(self, bind: str = "127.0.0.1", port: int = 8000) -> None:
601 """Run WASM webserver and open build in browser"""
602 relbuilddir = self.builddir.relative_to(SRCDIR)
603 url = f"http://{bind}:{port}/{relbuilddir}/python.html"
604 args = [
605 sys.executable,
606 os.fspath(WASM_WEBSERVER),
607 "--bind",
608 bind,
609 "--port",
610 str(port),
611 ]
612 srv = subprocess.Popen(args, cwd=SRCDIR)
613 # wait for server
614 end = time.monotonic() + 3.0
615 while time.monotonic() < end and srv.returncode is None:
616 try:
617 with socket.create_connection((bind, port), timeout=0.1) as _:
618 pass
619 except OSError:
620 time.sleep(0.01)
621 else:
622 break
623
624 webbrowser.open(url)
625
626 try:
627 srv.wait()
628 except KeyboardInterrupt:
629 pass
630
631 def clean(self, all: bool = False) -> None:
632 """Clean build directory"""
633 if all:
634 if self.builddir.exists():
635 shutil.rmtree(self.builddir)
636 elif self.makefile.exists():
637 self.run_make("clean")
638
639 def build_emports(self, force: bool = False) -> None:
640 """Pre-build emscripten ports."""
641 platform = self.host.platform
642 if platform.ports is None or platform.cc is None:
643 raise ValueError("Need ports and CC command")
644
645 embuilder_cmd = [os.fspath(platform.ports)]
646 embuilder_cmd.extend(self.host.embuilder_args)
647 if force:
648 embuilder_cmd.append("--force")
649
650 ports_cmd = [os.fspath(platform.cc)]
651 ports_cmd.extend(self.host.emport_args)
652 if self.target:
653 ports_cmd.extend(self.target.emport_args)
654
655 if self.dynamic_linking:
656 # Trigger PIC build.
657 ports_cmd.append("-sMAIN_MODULE")
658 embuilder_cmd.append("--pic")
659
660 if self.pthreads:
661 # Trigger multi-threaded build.
662 ports_cmd.append("-sUSE_PTHREADS")
663
664 # Pre-build libbz2, libsqlite3, libz, and some system libs.
665 ports_cmd.extend(["-sUSE_ZLIB", "-sUSE_BZIP2", "-sUSE_SQLITE3"])
666 # Multi-threaded sqlite3 has different suffix
667 embuilder_cmd.extend([
668 "build",
669 "bzip2",
670 "sqlite3-mt" if self.pthreads else "sqlite3",
671 "zlib",
672 ])
673
674 self._run_cmd(embuilder_cmd, cwd=SRCDIR)
675
676 with tempfile.TemporaryDirectory(suffix="-py-emport") as tmpdir:
677 tmppath = pathlib.Path(tmpdir)
678 main_c = tmppath / "main.c"
679 main_js = tmppath / "main.js"
680 with main_c.open("w") as f:
681 f.write("int main(void) { return 0; }\n")
682 args = [
683 os.fspath(main_c),
684 "-o",
685 os.fspath(main_js),
686 ]
687 self._run_cmd(ports_cmd, args, cwd=tmppath)
688
689
690 # native build (build Python)
691 BUILD = BuildProfile(
692 "build",
693 support_level=SupportLevel.working,
694 host=Host.build,
695 )
696
697 _profiles = [
698 BUILD,
699 # wasm32-emscripten
700 BuildProfile(
701 "emscripten-browser",
702 support_level=SupportLevel.supported,
703 host=Host.wasm32_emscripten,
704 target=EmscriptenTarget.browser,
705 dynamic_linking=True,
706 ),
707 BuildProfile(
708 "emscripten-browser-debug",
709 support_level=SupportLevel.working,
710 host=Host.wasm32_emscripten,
711 target=EmscriptenTarget.browser_debug,
712 dynamic_linking=True,
713 ),
714 BuildProfile(
715 "emscripten-node-dl",
716 support_level=SupportLevel.supported,
717 host=Host.wasm32_emscripten,
718 target=EmscriptenTarget.node,
719 dynamic_linking=True,
720 ),
721 BuildProfile(
722 "emscripten-node-dl-debug",
723 support_level=SupportLevel.working,
724 host=Host.wasm32_emscripten,
725 target=EmscriptenTarget.node_debug,
726 dynamic_linking=True,
727 ),
728 BuildProfile(
729 "emscripten-node-pthreads",
730 support_level=SupportLevel.supported,
731 host=Host.wasm32_emscripten,
732 target=EmscriptenTarget.node,
733 pthreads=True,
734 ),
735 BuildProfile(
736 "emscripten-node-pthreads-debug",
737 support_level=SupportLevel.working,
738 host=Host.wasm32_emscripten,
739 target=EmscriptenTarget.node_debug,
740 pthreads=True,
741 ),
742 # Emscripten build with both pthreads and dynamic linking is crashing.
743 BuildProfile(
744 "emscripten-node-dl-pthreads-debug",
745 support_level=SupportLevel.broken,
746 host=Host.wasm32_emscripten,
747 target=EmscriptenTarget.node_debug,
748 dynamic_linking=True,
749 pthreads=True,
750 ),
751 # wasm64-emscripten (requires Emscripten >= 3.1.21)
752 BuildProfile(
753 "wasm64-emscripten-node-debug",
754 support_level=SupportLevel.experimental,
755 host=Host.wasm64_emscripten,
756 target=EmscriptenTarget.node_debug,
757 # MEMORY64 is not compatible with dynamic linking
758 dynamic_linking=False,
759 pthreads=False,
760 ),
761 # wasm32-wasi
762 BuildProfile(
763 "wasi",
764 support_level=SupportLevel.supported,
765 host=Host.wasm32_wasi,
766 ),
767 # wasm32-wasi-threads
768 BuildProfile(
769 "wasi-threads",
770 support_level=SupportLevel.experimental,
771 host=Host.wasm32_wasi,
772 pthreads=True,
773 ),
774 # no SDK available yet
775 # BuildProfile(
776 # "wasm64-wasi",
777 # support_level=SupportLevel.broken,
778 # host=Host.wasm64_wasi,
779 # ),
780 ]
781
782 PROFILES = {p.name: p for p in _profiles}
783
784 parser = argparse.ArgumentParser(
785 "wasm_build.py",
786 description=__doc__,
787 formatter_class=argparse.RawTextHelpFormatter,
788 )
789
790 parser.add_argument(
791 "--clean",
792 "-c",
793 help="Clean build directories first",
794 action="store_true",
795 )
796
797 parser.add_argument(
798 "--verbose",
799 "-v",
800 help="Verbose logging",
801 action="store_true",
802 )
803
804 parser.add_argument(
805 "--silent",
806 help="Run configure and make in silent mode",
807 action="store_true",
808 )
809
810 parser.add_argument(
811 "--testopts",
812 help=(
813 "Additional test options for 'test' and 'hostrunnertest', e.g. "
814 "--testopts='-v test_os'."
815 ),
816 default=None,
817 )
818
819 # Don't list broken and experimental variants in help
820 platforms_choices = [p.name for p in _profiles] + ["cleanall"]
821 platforms_help = [p.name for p in _profiles if p.support_level] + ["cleanall"]
822 parser.add_argument(
823 "platform",
824 metavar="PLATFORM",
825 help=f"Build platform: {', '.join(platforms_help)}",
826 choices=platforms_choices,
827 )
828
829 ops = {
830 "build": "auto build (build 'build' Python, emports, configure, compile)",
831 "configure": "run ./configure",
832 "compile": "run 'make all'",
833 "pythoninfo": "run 'make pythoninfo'",
834 "test": "run 'make buildbottest TESTOPTS=...' (supports parallel tests)",
835 "hostrunnertest": "run 'make hostrunnertest TESTOPTS=...'",
836 "repl": "start interactive REPL / webserver + browser session",
837 "clean": "run 'make clean'",
838 "cleanall": "remove all build directories",
839 "emports": "build Emscripten port with embuilder (only Emscripten)",
840 }
841 ops_help = "\n".join(f"{op:16s} {help}" for op, help in ops.items())
842 parser.add_argument(
843 "ops",
844 metavar="OP",
845 help=f"operation (default: build)\n\n{ops_help}",
846 choices=tuple(ops),
847 default="build",
848 nargs="*",
849 )
850
851
852 def main() -> None:
853 args = parser.parse_args()
854 logging.basicConfig(
855 level=logging.INFO if args.verbose else logging.ERROR,
856 format="%(message)s",
857 )
858
859 if args.platform == "cleanall":
860 for builder in PROFILES.values():
861 builder.clean(all=True)
862 parser.exit(0)
863
864 # additional configure and make args
865 cm_args = ("--silent",) if args.silent else ()
866
867 # nargs=* with default quirk
868 if args.ops == "build":
869 args.ops = ["build"]
870
871 builder = PROFILES[args.platform]
872 try:
873 builder.host.platform.check()
874 except ConditionError as e:
875 parser.error(str(e))
876
877 if args.clean:
878 builder.clean(all=False)
879
880 # hack for WASI
881 if builder.host.is_wasi and not SETUP_LOCAL.exists():
882 SETUP_LOCAL.touch()
883
884 # auto-build
885 if "build" in args.ops:
886 # check and create build Python
887 if builder is not BUILD:
888 logger.info("Auto-building 'build' Python.")
889 try:
890 BUILD.host.platform.check()
891 except ConditionError as e:
892 parser.error(str(e))
893 if args.clean:
894 BUILD.clean(all=False)
895 BUILD.run_build(*cm_args)
896 # build Emscripten ports with embuilder
897 if builder.host.is_emscripten and "emports" not in args.ops:
898 builder.build_emports()
899
900 for op in args.ops:
901 logger.info("\n*** %s %s", args.platform, op)
902 if op == "build":
903 builder.run_build(*cm_args)
904 elif op == "configure":
905 builder.run_configure(*cm_args)
906 elif op == "compile":
907 builder.run_make("all", *cm_args)
908 elif op == "pythoninfo":
909 builder.run_pythoninfo(*cm_args)
910 elif op == "repl":
911 if builder.is_browser:
912 builder.run_browser()
913 else:
914 builder.run_py()
915 elif op == "test":
916 builder.run_test("buildbottest", testopts=args.testopts)
917 elif op == "hostrunnertest":
918 builder.run_test("hostrunnertest", testopts=args.testopts)
919 elif op == "clean":
920 builder.clean(all=False)
921 elif op == "cleanall":
922 builder.clean(all=True)
923 elif op == "emports":
924 builder.build_emports(force=args.clean)
925 else:
926 raise ValueError(op)
927
928 print(builder.builddir)
929 parser.exit(0)
930
931
932 if __name__ == "__main__":
933 main()