Debian Codegraph @

Tools/wasm/wasi/__main__.py

raw
1 #!/usr/bin/env python3
2
3 import argparse
4 import contextlib
5 import functools
6 import os
7
8 try:
9 from os import process_cpu_count as cpu_count
10 except ImportError:
11 from os import cpu_count
12 import pathlib
13 import shutil
14 import subprocess
15 import sys
16 import sysconfig
17 import tempfile
18
19 CHECKOUT = pathlib.Path(__file__).parent.parent.parent.parent
20 assert (CHECKOUT / "configure").is_file(), (
21 "Please update the location of the file"
22 )
23
24 CROSS_BUILD_DIR = CHECKOUT / "cross-build"
25 # Build platform can also be found via `config.guess`.
26 BUILD_DIR = CROSS_BUILD_DIR / sysconfig.get_config_var("BUILD_GNU_TYPE")
27
28 LOCAL_SETUP = CHECKOUT / "Modules" / "Setup.local"
29 LOCAL_SETUP_MARKER = (
30 b"# Generated by Tools/wasm/wasi .\n"
31 b"# Required to statically build extension modules."
32 )
33
34 WASI_SDK_VERSION = 24
35
36 WASMTIME_VAR_NAME = "WASMTIME"
37 WASMTIME_HOST_RUNNER_VAR = f"{{{WASMTIME_VAR_NAME}}}"
38
39
40 def separator():
41 """Print a separator line across the terminal width."""
42 try:
43 tput_output = subprocess.check_output(
44 ["tput", "cols"], encoding="utf-8"
45 )
46 except subprocess.CalledProcessError:
47 terminal_width = 80
48 else:
49 terminal_width = int(tput_output.strip())
50 print("โŽฏ" * terminal_width)
51
52
53 def log(emoji, message, *, spacing=None):
54 """Print a notification with an emoji.
55
56 If 'spacing' is None, calculate the spacing based on the number of code points
57 in the emoji as terminals "eat" a space when the emoji has multiple code points.
58 """
59 if spacing is None:
60 spacing = " " if len(emoji) == 1 else " "
61 print("".join([emoji, spacing, message]))
62
63
64 def updated_env(updates={}):
65 """Create a new dict representing the environment to use.
66
67 The changes made to the execution environment are printed out.
68 """
69 env_defaults = {}
70 # https://reproducible-builds.org/docs/source-date-epoch/
71 git_epoch_cmd = ["git", "log", "-1", "--pretty=%ct"]
72 try:
73 epoch = subprocess.check_output(
74 git_epoch_cmd, encoding="utf-8"
75 ).strip()
76 env_defaults["SOURCE_DATE_EPOCH"] = epoch
77 except subprocess.CalledProcessError:
78 pass # Might be building from a tarball.
79 # This layering lets SOURCE_DATE_EPOCH from os.environ takes precedence.
80 environment = env_defaults | os.environ | updates
81
82 env_diff = {}
83 for key, value in environment.items():
84 if os.environ.get(key) != value:
85 env_diff[key] = value
86
87 env_vars = (
88 f"\n {key}={item}" for key, item in sorted(env_diff.items())
89 )
90 log("๐ŸŒŽ", f"Environment changes:{''.join(env_vars)}")
91
92 return environment
93
94
95 def subdir(working_dir, *, clean_ok=False):
96 """Decorator to change to a working directory."""
97
98 def decorator(func):
99 @functools.wraps(func)
100 def wrapper(context):
101 nonlocal working_dir
102
103 if callable(working_dir):
104 working_dir = working_dir(context)
105 separator()
106 log("๐Ÿ“", os.fsdecode(working_dir))
107 if (
108 clean_ok
109 and getattr(context, "clean", False)
110 and working_dir.exists()
111 ):
112 log("๐Ÿšฎ", "Deleting directory (--clean)...")
113 shutil.rmtree(working_dir)
114
115 working_dir.mkdir(parents=True, exist_ok=True)
116
117 with contextlib.chdir(working_dir):
118 return func(context, working_dir)
119
120 return wrapper
121
122 return decorator
123
124
125 def call(command, *, context=None, quiet=False, logdir=None, **kwargs):
126 """Execute a command.
127
128 If 'quiet' is true, then redirect stdout and stderr to a temporary file.
129 """
130 if context is not None:
131 quiet = context.quiet
132 logdir = context.logdir
133 elif quiet and logdir is None:
134 raise ValueError("When quiet is True, logdir must be specified")
135
136 log("โฏ", " ".join(map(str, command)), spacing=" ")
137 if not quiet:
138 stdout = None
139 stderr = None
140 else:
141 stdout = tempfile.NamedTemporaryFile(
142 "w",
143 encoding="utf-8",
144 delete=False,
145 dir=logdir,
146 prefix="cpython-wasi-",
147 suffix=".log",
148 )
149 stderr = subprocess.STDOUT
150 log("๐Ÿ“", f"Logging output to {stdout.name} (--quiet)...")
151
152 subprocess.check_call(command, **kwargs, stdout=stdout, stderr=stderr)
153
154
155 def build_python_path():
156 """The path to the build Python binary."""
157 binary = BUILD_DIR / "python"
158 if not binary.is_file():
159 binary = binary.with_suffix(".exe")
160 if not binary.is_file():
161 raise FileNotFoundError(
162 f"Unable to find `python(.exe)` in {BUILD_DIR}"
163 )
164
165 return binary
166
167
168 def build_python_is_pydebug():
169 """Find out if the build Python is a pydebug build."""
170 test = "import sys, test.support; sys.exit(test.support.Py_DEBUG)"
171 result = subprocess.run(
172 [build_python_path(), "-c", test],
173 capture_output=True,
174 )
175 return bool(result.returncode)
176
177
178 @subdir(BUILD_DIR, clean_ok=True)
179 def configure_build_python(context, working_dir):
180 """Configure the build/host Python."""
181 if LOCAL_SETUP.exists():
182 if LOCAL_SETUP.read_bytes() == LOCAL_SETUP_MARKER:
183 log("๐Ÿ‘", f"{LOCAL_SETUP} exists ...")
184 else:
185 log("โš ๏ธ", f"{LOCAL_SETUP} exists, but has unexpected contents")
186 else:
187 log("๐Ÿ“", f"Creating {LOCAL_SETUP} ...")
188 LOCAL_SETUP.write_bytes(LOCAL_SETUP_MARKER)
189
190 configure = [os.path.relpath(CHECKOUT / "configure", working_dir)]
191 if context.args:
192 configure.extend(context.args)
193
194 call(configure, context=context)
195
196
197 @subdir(BUILD_DIR)
198 def make_build_python(context, working_dir):
199 """Make/build the build Python."""
200 call(["make", "--jobs", str(cpu_count()), "all"], context=context)
201
202 binary = build_python_path()
203 cmd = [
204 binary,
205 "-c",
206 "import sys; "
207 "print(f'{sys.version_info.major}.{sys.version_info.minor}')",
208 ]
209 version = subprocess.check_output(cmd, encoding="utf-8").strip()
210
211 log("๐ŸŽ‰", f"{binary} {version}")
212
213
214 def find_wasi_sdk():
215 """Find the path to the WASI SDK."""
216 wasi_sdk_path = None
217
218 if wasi_sdk_path_env_var := os.environ.get("WASI_SDK_PATH"):
219 wasi_sdk_path = pathlib.Path(wasi_sdk_path_env_var)
220 else:
221 opt_path = pathlib.Path("/opt")
222 # WASI SDK versions have a ``.0`` suffix, but it's a constant; the WASI SDK team
223 # has said they don't plan to ever do a point release and all of their Git tags
224 # lack the ``.0`` suffix.
225 # Starting with WASI SDK 23, the tarballs went from containing a directory named
226 # ``wasi-sdk-{WASI_SDK_VERSION}.0`` to e.g.
227 # ``wasi-sdk-{WASI_SDK_VERSION}.0-x86_64-linux``.
228 potential_sdks = [
229 path
230 for path in opt_path.glob(f"wasi-sdk-{WASI_SDK_VERSION}.0*")
231 if path.is_dir()
232 ]
233 if len(potential_sdks) == 1:
234 wasi_sdk_path = potential_sdks[0]
235 elif (default_path := opt_path / "wasi-sdk").is_dir():
236 wasi_sdk_path = default_path
237
238 # Starting with WASI SDK 25, a VERSION file is included in the root
239 # of the SDK directory that we can read to warn folks when they are using
240 # an unsupported version.
241 if wasi_sdk_path and (version_file := wasi_sdk_path / "VERSION").is_file():
242 version_details = version_file.read_text(encoding="utf-8")
243 found_version = version_details.splitlines()[0]
244 # Make sure there's a trailing dot to avoid false positives if somehow the
245 # supported version is a prefix of the found version (e.g. `25` and `2567`).
246 if not found_version.startswith(f"{WASI_SDK_VERSION}."):
247 major_version = found_version.partition(".")[0]
248 log(
249 "โš ๏ธ",
250 f" Found WASI SDK {major_version}, "
251 f"but WASI SDK {WASI_SDK_VERSION} is the supported version",
252 )
253
254 return wasi_sdk_path
255
256
257 def wasi_sdk_env(context):
258 """Calculate environment variables for building with wasi-sdk."""
259 wasi_sdk_path = context.wasi_sdk_path
260 sysroot = wasi_sdk_path / "share" / "wasi-sysroot"
261 env = {
262 "CC": "clang",
263 "CPP": "clang-cpp",
264 "CXX": "clang++",
265 "AR": "llvm-ar",
266 "RANLIB": "ranlib",
267 }
268
269 for env_var, binary_name in list(env.items()):
270 env[env_var] = os.fsdecode(wasi_sdk_path / "bin" / binary_name)
271
272 if not wasi_sdk_path.name.startswith("wasi-sdk"):
273 for compiler in ["CC", "CPP", "CXX"]:
274 env[compiler] += f" --sysroot={sysroot}"
275
276 env["PKG_CONFIG_PATH"] = ""
277 env["PKG_CONFIG_LIBDIR"] = os.pathsep.join(
278 map(
279 os.fsdecode,
280 [sysroot / "lib" / "pkgconfig", sysroot / "share" / "pkgconfig"],
281 )
282 )
283 env["PKG_CONFIG_SYSROOT_DIR"] = os.fsdecode(sysroot)
284
285 env["WASI_SDK_PATH"] = os.fsdecode(wasi_sdk_path)
286 env["WASI_SYSROOT"] = os.fsdecode(sysroot)
287
288 env["PATH"] = os.pathsep.join([
289 os.fsdecode(wasi_sdk_path / "bin"),
290 os.environ["PATH"],
291 ])
292
293 return env
294
295
296 @subdir(lambda context: CROSS_BUILD_DIR / context.host_triple, clean_ok=True)
297 def configure_wasi_python(context, working_dir):
298 """Configure the WASI/host build."""
299 if not context.wasi_sdk_path or not context.wasi_sdk_path.exists():
300 raise ValueError(
301 "WASI-SDK not found; "
302 "download from "
303 "https://github.com/WebAssembly/wasi-sdk and/or "
304 "specify via $WASI_SDK_PATH or --wasi-sdk"
305 )
306
307 config_site = os.fsdecode(
308 CHECKOUT / "Tools" / "wasm" / "wasi" / "config.site-wasm32-wasi"
309 )
310
311 wasi_build_dir = working_dir.relative_to(CHECKOUT)
312
313 python_build_dir = BUILD_DIR / "build"
314 lib_dirs = list(python_build_dir.glob("lib.*"))
315 assert len(lib_dirs) == 1, (
316 f"Expected a single lib.* directory in {python_build_dir}"
317 )
318 lib_dir = os.fsdecode(lib_dirs[0])
319 python_version = lib_dir.rpartition("-")[-1]
320 sysconfig_data_dir = (
321 f"{wasi_build_dir}/build/lib.wasi-wasm32-{python_version}"
322 )
323
324 # Use PYTHONPATH to include sysconfig data which must be anchored to the
325 # WASI guest's `/` directory.
326 args = {
327 "GUEST_DIR": "/",
328 "HOST_DIR": CHECKOUT,
329 "ENV_VAR_NAME": "PYTHONPATH",
330 "ENV_VAR_VALUE": f"/{sysconfig_data_dir}",
331 "PYTHON_WASM": working_dir / "python.wasm",
332 }
333 # Check dynamically for wasmtime in case it was specified manually via
334 # `--host-runner`.
335 if WASMTIME_HOST_RUNNER_VAR in context.host_runner:
336 if wasmtime := shutil.which("wasmtime"):
337 args[WASMTIME_VAR_NAME] = wasmtime
338 else:
339 raise FileNotFoundError(
340 "wasmtime not found; download from "
341 "https://github.com/bytecodealliance/wasmtime"
342 )
343 host_runner = context.host_runner.format_map(args)
344 env_additions = {"CONFIG_SITE": config_site, "HOSTRUNNER": host_runner}
345 build_python = os.fsdecode(build_python_path())
346 # The path to `configure` MUST be relative, else `python.wasm` is unable
347 # to find the stdlib due to Python not recognizing that it's being
348 # executed from within a checkout.
349 configure = [
350 os.path.relpath(CHECKOUT / "configure", working_dir),
351 f"--host={context.host_triple}",
352 f"--build={BUILD_DIR.name}",
353 f"--with-build-python={build_python}",
354 ]
355 if build_python_is_pydebug():
356 configure.append("--with-pydebug")
357 if context.args:
358 configure.extend(context.args)
359 call(
360 configure,
361 env=updated_env(env_additions | wasi_sdk_env(context)),
362 context=context,
363 )
364
365 python_wasm = working_dir / "python.wasm"
366 exec_script = working_dir / "python.sh"
367 with exec_script.open("w", encoding="utf-8") as file:
368 file.write(f'#!/bin/sh\nexec {host_runner} {python_wasm} "$@"\n')
369 exec_script.chmod(0o755)
370 log("๐Ÿƒ", f"Created {exec_script} (--host-runner)... ")
371 sys.stdout.flush()
372
373
374 @subdir(lambda context: CROSS_BUILD_DIR / context.host_triple)
375 def make_wasi_python(context, working_dir):
376 """Run `make` for the WASI/host build."""
377 call(
378 ["make", "--jobs", str(cpu_count()), "all"],
379 env=updated_env(),
380 context=context,
381 )
382
383 exec_script = working_dir / "python.sh"
384 call([exec_script, "--version"], quiet=False)
385 log(
386 "๐ŸŽ‰",
387 f"Use `{exec_script.relative_to(context.init_dir)}` "
388 "to run CPython w/ the WASI host specified by --host-runner",
389 )
390
391
392 def clean_contents(context):
393 """Delete all files created by this script."""
394 if CROSS_BUILD_DIR.exists():
395 log("๐Ÿงน", f"Deleting {CROSS_BUILD_DIR} ...")
396 shutil.rmtree(CROSS_BUILD_DIR)
397
398 if LOCAL_SETUP.exists():
399 if LOCAL_SETUP.read_bytes() == LOCAL_SETUP_MARKER:
400 log("๐Ÿงน", f"Deleting generated {LOCAL_SETUP} ...")
401
402
403 def build_steps(*steps):
404 """Construct a command from other steps."""
405
406 def builder(context):
407 for step in steps:
408 step(context)
409
410 return builder
411
412
413 def main():
414 default_host_triple = "wasm32-wasip1"
415 default_wasi_sdk = find_wasi_sdk()
416 default_host_runner = (
417 f"{WASMTIME_HOST_RUNNER_VAR} run "
418 # Make sure the stack size will work for a pydebug
419 # build.
420 # Use 16 MiB stack.
421 "--wasm max-wasm-stack=16777216 "
422 # Enable thread support; causes use of preview1.
423 # "--wasm threads=y --wasi threads=y "
424 # Map the checkout to / to load the stdlib from /Lib.
425 "--dir {HOST_DIR}::{GUEST_DIR} "
426 # Set PYTHONPATH to the sysconfig data.
427 "--env {ENV_VAR_NAME}={ENV_VAR_VALUE}"
428 )
429 default_logdir = pathlib.Path(tempfile.gettempdir())
430
431 parser = argparse.ArgumentParser()
432 subcommands = parser.add_subparsers(dest="subcommand")
433 build = subcommands.add_parser("build", help="Build everything")
434 configure_build = subcommands.add_parser(
435 "configure-build-python", help="Run `configure` for the build Python"
436 )
437 make_build = subcommands.add_parser(
438 "make-build-python", help="Run `make` for the build Python"
439 )
440 build_python = subcommands.add_parser(
441 "build-python", help="Build the build Python"
442 )
443 configure_host = subcommands.add_parser(
444 "configure-host",
445 help="Run `configure` for the "
446 "host/WASI (pydebug builds "
447 "are inferred from the build "
448 "Python)",
449 )
450 make_host = subcommands.add_parser(
451 "make-host", help="Run `make` for the host/WASI"
452 )
453 build_host = subcommands.add_parser(
454 "build-host", help="Build the host/WASI Python"
455 )
456 subcommands.add_parser(
457 "clean", help="Delete files and directories created by this script"
458 )
459 for subcommand in (
460 build,
461 configure_build,
462 make_build,
463 build_python,
464 configure_host,
465 make_host,
466 build_host,
467 ):
468 subcommand.add_argument(
469 "--quiet",
470 action="store_true",
471 default=False,
472 dest="quiet",
473 help="Redirect output from subprocesses to a log file",
474 )
475 subcommand.add_argument(
476 "--logdir",
477 type=pathlib.Path,
478 default=default_logdir,
479 help=f"Directory to store log files; defaults to {default_logdir}",
480 )
481 for subcommand in (
482 configure_build,
483 configure_host,
484 build_python,
485 build_host,
486 ):
487 subcommand.add_argument(
488 "--clean",
489 action="store_true",
490 default=False,
491 dest="clean",
492 help="Delete any relevant directories before building",
493 )
494 for subcommand in (
495 build,
496 configure_build,
497 configure_host,
498 build_python,
499 build_host,
500 ):
501 subcommand.add_argument(
502 "args", nargs="*", help="Extra arguments to pass to `configure`"
503 )
504 for subcommand in build, configure_host, build_host:
505 subcommand.add_argument(
506 "--wasi-sdk",
507 type=pathlib.Path,
508 dest="wasi_sdk_path",
509 default=default_wasi_sdk,
510 help=f"Path to the WASI SDK; defaults to {default_wasi_sdk}",
511 )
512 subcommand.add_argument(
513 "--host-runner",
514 action="store",
515 default=default_host_runner,
516 dest="host_runner",
517 help="Command template for running the WASI host; defaults to "
518 f"`{default_host_runner}`",
519 )
520 for subcommand in build, configure_host, make_host, build_host:
521 subcommand.add_argument(
522 "--host-triple",
523 action="store",
524 default=default_host_triple,
525 help="The target triple for the WASI host build; "
526 f"defaults to {default_host_triple}",
527 )
528
529 context = parser.parse_args()
530 context.init_dir = pathlib.Path().absolute()
531
532 build_build_python = build_steps(configure_build_python, make_build_python)
533 build_wasi_python = build_steps(configure_wasi_python, make_wasi_python)
534
535 dispatch = {
536 "configure-build-python": configure_build_python,
537 "make-build-python": make_build_python,
538 "build-python": build_build_python,
539 "configure-host": configure_wasi_python,
540 "make-host": make_wasi_python,
541 "build-host": build_wasi_python,
542 "build": build_steps(build_build_python, build_wasi_python),
543 "clean": clean_contents,
544 }
545 dispatch[context.subcommand](context)
546
547
548 if __name__ == "__main__":
549 main()