Debian Codegraph @

Tools/wasm/wasi.py

raw
1 #!/usr/bin/env python3
2
3 import argparse
4 import contextlib
5 import functools
6 import os
7
8 try:
9 from os import process_cpu_count as cpu_count
10 except ImportError:
11 from os import cpu_count
12 import pathlib
13 import shutil
14 import subprocess
15 import sys
16 import sysconfig
17 import tempfile
18
19 CHECKOUT = pathlib.Path(__file__).parent.parent.parent
20
21 CROSS_BUILD_DIR = CHECKOUT / "cross-build"
22 BUILD_DIR = CROSS_BUILD_DIR / "build"
23
24 LOCAL_SETUP = CHECKOUT / "Modules" / "Setup.local"
25 LOCAL_SETUP_MARKER = b"# Generated by Tools/wasm/wasi.py\n"
26
27 WASMTIME_VAR_NAME = "WASMTIME"
28 WASMTIME_HOST_RUNNER_VAR = f"{{{WASMTIME_VAR_NAME}}}"
29
30
31 def updated_env(updates={}):
32 """Create a new dict representing the environment to use.
33
34 The changes made to the execution environment are printed out.
35 """
36 env_defaults = {}
37 # https://reproducible-builds.org/docs/source-date-epoch/
38 git_epoch_cmd = ["git", "log", "-1", "--pretty=%ct"]
39 try:
40 epoch = subprocess.check_output(
41 git_epoch_cmd, encoding="utf-8"
42 ).strip()
43 env_defaults["SOURCE_DATE_EPOCH"] = epoch
44 except subprocess.CalledProcessError:
45 pass # Might be building from a tarball.
46 # This layering lets SOURCE_DATE_EPOCH from os.environ takes precedence.
47 environment = env_defaults | os.environ | updates
48
49 env_diff = {}
50 for key, value in environment.items():
51 if os.environ.get(key) != value:
52 env_diff[key] = value
53
54 print("๐ŸŒŽ Environment changes:")
55 for key in sorted(env_diff.keys()):
56 print(f" {key}={env_diff[key]}")
57
58 return environment
59
60
61 def subdir(working_dir, *, clean_ok=False):
62 """Decorator to change to a working directory."""
63
64 def decorator(func):
65 @functools.wraps(func)
66 def wrapper(context):
67 nonlocal working_dir
68
69 if callable(working_dir):
70 working_dir = working_dir(context)
71 try:
72 tput_output = subprocess.check_output(
73 ["tput", "cols"], encoding="utf-8"
74 )
75 except subprocess.CalledProcessError:
76 terminal_width = 80
77 else:
78 terminal_width = int(tput_output.strip())
79 print("โŽฏ" * terminal_width)
80 print("๐Ÿ“", working_dir)
81 if (
82 clean_ok
83 and getattr(context, "clean", False)
84 and working_dir.exists()
85 ):
86 print("๐Ÿšฎ Deleting directory (--clean)...")
87 shutil.rmtree(working_dir)
88
89 working_dir.mkdir(parents=True, exist_ok=True)
90
91 with contextlib.chdir(working_dir):
92 return func(context, working_dir)
93
94 return wrapper
95
96 return decorator
97
98
99 def call(command, *, quiet, **kwargs):
100 """Execute a command.
101
102 If 'quiet' is true, then redirect stdout and stderr to a temporary file.
103 """
104 print("โฏ", " ".join(map(str, command)))
105 if not quiet:
106 stdout = None
107 stderr = None
108 else:
109 stdout = tempfile.NamedTemporaryFile(
110 "w",
111 encoding="utf-8",
112 delete=False,
113 prefix="cpython-wasi-",
114 suffix=".log",
115 )
116 stderr = subprocess.STDOUT
117 print(f"๐Ÿ“ Logging output to {stdout.name} (--quiet)...")
118
119 subprocess.check_call(command, **kwargs, stdout=stdout, stderr=stderr)
120
121
122 def build_platform():
123 """The name of the build/host platform."""
124 # Can also be found via `config.guess`.`
125 return sysconfig.get_config_var("BUILD_GNU_TYPE")
126
127
128 def build_python_path():
129 """The path to the build Python binary."""
130 binary = BUILD_DIR / "python"
131 if not binary.is_file():
132 binary = binary.with_suffix(".exe")
133 if not binary.is_file():
134 raise FileNotFoundError(
135 f"Unable to find `python(.exe)` in {BUILD_DIR}"
136 )
137
138 return binary
139
140
141 @subdir(BUILD_DIR, clean_ok=True)
142 def configure_build_python(context, working_dir):
143 """Configure the build/host Python."""
144 if LOCAL_SETUP.exists():
145 print(f"๐Ÿ‘ {LOCAL_SETUP} exists ...")
146 else:
147 print(f"๐Ÿ“ Touching {LOCAL_SETUP} ...")
148 LOCAL_SETUP.write_bytes(LOCAL_SETUP_MARKER)
149
150 configure = [os.path.relpath(CHECKOUT / "configure", working_dir)]
151 if context.args:
152 configure.extend(context.args)
153
154 call(configure, quiet=context.quiet)
155
156
157 @subdir(BUILD_DIR)
158 def make_build_python(context, working_dir):
159 """Make/build the build Python."""
160 call(["make", "--jobs", str(cpu_count()), "all"], quiet=context.quiet)
161
162 binary = build_python_path()
163 cmd = [
164 binary,
165 "-c",
166 "import sys; "
167 "print(f'{sys.version_info.major}.{sys.version_info.minor}')",
168 ]
169 version = subprocess.check_output(cmd, encoding="utf-8").strip()
170
171 print(f"๐ŸŽ‰ {binary} {version}")
172
173
174 @subdir(BUILD_DIR)
175 def pythoninfo_build_python(context, working_dir):
176 """Display build info of the build Python."""
177 call(["make", "pythoninfo"], quiet=context.quiet)
178
179
180 def find_wasi_sdk():
181 """Find the path to wasi-sdk."""
182 if wasi_sdk_path := os.environ.get("WASI_SDK_PATH"):
183 return pathlib.Path(wasi_sdk_path)
184 elif (default_path := pathlib.Path("/opt/wasi-sdk")).exists():
185 return default_path
186
187
188 def wasi_sdk_env(context):
189 """Calculate environment variables for building with wasi-sdk."""
190 wasi_sdk_path = context.wasi_sdk_path
191 sysroot = wasi_sdk_path / "share" / "wasi-sysroot"
192 env = {
193 "CC": "clang",
194 "CPP": "clang-cpp",
195 "CXX": "clang++",
196 "AR": "llvm-ar",
197 "RANLIB": "ranlib",
198 }
199
200 for env_var, binary_name in list(env.items()):
201 env[env_var] = os.fsdecode(wasi_sdk_path / "bin" / binary_name)
202
203 if wasi_sdk_path != pathlib.Path("/opt/wasi-sdk"):
204 for compiler in ["CC", "CPP", "CXX"]:
205 env[compiler] += f" --sysroot={sysroot}"
206
207 env["PKG_CONFIG_PATH"] = ""
208 env["PKG_CONFIG_LIBDIR"] = os.pathsep.join(
209 map(
210 os.fsdecode,
211 [sysroot / "lib" / "pkgconfig", sysroot / "share" / "pkgconfig"],
212 )
213 )
214 env["PKG_CONFIG_SYSROOT_DIR"] = os.fsdecode(sysroot)
215
216 env["WASI_SDK_PATH"] = os.fsdecode(wasi_sdk_path)
217 env["WASI_SYSROOT"] = os.fsdecode(sysroot)
218
219 env["PATH"] = os.pathsep.join([
220 os.fsdecode(wasi_sdk_path / "bin"),
221 os.environ["PATH"],
222 ])
223
224 return env
225
226
227 @subdir(lambda context: CROSS_BUILD_DIR / context.host_triple, clean_ok=True)
228 def configure_wasi_python(context, working_dir):
229 """Configure the WASI/host build."""
230 if not context.wasi_sdk_path or not context.wasi_sdk_path.exists():
231 raise ValueError(
232 "WASI-SDK not found; "
233 "download from "
234 "https://github.com/WebAssembly/wasi-sdk and/or "
235 "specify via $WASI_SDK_PATH or --wasi-sdk"
236 )
237
238 config_site = os.fsdecode(
239 CHECKOUT / "Tools" / "wasm" / "config.site-wasm32-wasi"
240 )
241
242 wasi_build_dir = working_dir.relative_to(CHECKOUT)
243
244 python_build_dir = BUILD_DIR / "build"
245 lib_dirs = list(python_build_dir.glob("lib.*"))
246 assert len(lib_dirs) == 1, (
247 f"Expected a single lib.* directory in {python_build_dir}"
248 )
249 lib_dir = os.fsdecode(lib_dirs[0])
250 pydebug = lib_dir.endswith("-pydebug")
251 python_version = lib_dir.removesuffix("-pydebug").rpartition("-")[-1]
252 sysconfig_data = f"{wasi_build_dir}/build/lib.wasi-wasm32-{python_version}"
253 if pydebug:
254 sysconfig_data += "-pydebug"
255
256 # Use PYTHONPATH to include sysconfig data which must be anchored to the
257 # WASI guest's `/` directory.
258 args = {
259 "GUEST_DIR": "/",
260 "HOST_DIR": CHECKOUT,
261 "ENV_VAR_NAME": "PYTHONPATH",
262 "ENV_VAR_VALUE": f"/{sysconfig_data}",
263 "PYTHON_WASM": working_dir / "python.wasm",
264 "ARGV0": wasi_build_dir / "python.wasm",
265 }
266 # Check dynamically for wasmtime in case it was specified manually via
267 # `--host-runner`.
268 if WASMTIME_HOST_RUNNER_VAR in context.host_runner:
269 if wasmtime := shutil.which("wasmtime"):
270 args[WASMTIME_VAR_NAME] = wasmtime
271 else:
272 raise FileNotFoundError(
273 "wasmtime not found; download from "
274 "https://github.com/bytecodealliance/wasmtime"
275 )
276 host_runner = context.host_runner.format_map(args)
277 env_additions = {"CONFIG_SITE": config_site, "HOSTRUNNER": host_runner}
278 build_python = os.fsdecode(build_python_path())
279 # The path to `configure` MUST be relative, else `python.wasm` is unable
280 # to find the stdlib due to Python not recognizing that it's being
281 # executed from within a checkout.
282 configure = [
283 os.path.relpath(CHECKOUT / "configure", working_dir),
284 f"--host={context.host_triple}",
285 f"--build={build_platform()}",
286 f"--with-build-python={build_python}",
287 ]
288 if pydebug:
289 configure.append("--with-pydebug")
290 if context.args:
291 configure.extend(context.args)
292 call(
293 configure,
294 env=updated_env(env_additions | wasi_sdk_env(context)),
295 quiet=context.quiet,
296 )
297
298 python_wasm = working_dir / "python.wasm"
299 exec_script = working_dir / "python.sh"
300 with exec_script.open("w", encoding="utf-8") as file:
301 file.write(f'#!/bin/sh\nexec {host_runner} {python_wasm} "$@"\n')
302 exec_script.chmod(0o755)
303 print(f"๐Ÿƒโ€โ™€๏ธ Created {exec_script} ... ")
304 sys.stdout.flush()
305
306
307 @subdir(lambda context: CROSS_BUILD_DIR / context.host_triple)
308 def make_wasi_python(context, working_dir):
309 """Run `make` for the WASI/host build."""
310 call(
311 ["make", "--jobs", str(cpu_count()), "all"],
312 env=updated_env(),
313 quiet=context.quiet,
314 )
315
316 exec_script = working_dir / "python.sh"
317 subprocess.check_call([exec_script, "--version"])
318
319
320 def build_all(context):
321 """Build everything."""
322 steps = [
323 configure_build_python,
324 make_build_python,
325 pythoninfo_build_python,
326 configure_wasi_python,
327 make_wasi_python,
328 pythoninfo_wasi_python,
329 ]
330 for step in steps:
331 step(context)
332
333
334 @subdir(lambda context: CROSS_BUILD_DIR / context.host_triple)
335 def pythoninfo_wasi_python(context, working_dir):
336 """Display build info of the host/WASI Python."""
337 call(["make", "pythoninfo"], quiet=context.quiet)
338
339
340 def clean_contents(context):
341 """Delete all files created by this script."""
342 if CROSS_BUILD_DIR.exists():
343 print(f"๐Ÿงน Deleting {CROSS_BUILD_DIR} ...")
344 shutil.rmtree(CROSS_BUILD_DIR)
345
346 if LOCAL_SETUP.exists():
347 with LOCAL_SETUP.open("rb") as file:
348 if file.read(len(LOCAL_SETUP_MARKER)) == LOCAL_SETUP_MARKER:
349 print(f"๐Ÿงน Deleting generated {LOCAL_SETUP} ...")
350
351
352 def main():
353 default_host_runner = (
354 f"{WASMTIME_HOST_RUNNER_VAR} run "
355 # Make sure the stack size will work for a pydebug
356 # build.
357 # The 8388608 value comes from `ulimit -s` under Linux
358 # which equates to 8291 KiB.
359 "--wasm max-wasm-stack=8388608 "
360 # Use WASI 0.2 primitives.
361 "--wasi preview2 "
362 # Explicitly set the argv[0] value
363 "--argv0 {ARGV0} "
364 # Enable thread support; causes use of preview1.
365 # "--wasm threads=y --wasi threads=y "
366 # Map the checkout to / to load the stdlib from /Lib.
367 "--dir {HOST_DIR}::{GUEST_DIR} "
368 # Set PYTHONPATH to the sysconfig data.
369 "--env {ENV_VAR_NAME}={ENV_VAR_VALUE}"
370 )
371
372 parser = argparse.ArgumentParser()
373 subcommands = parser.add_subparsers(dest="subcommand")
374 build = subcommands.add_parser("build", help="Build everything")
375 configure_build = subcommands.add_parser(
376 "configure-build-python", help="Run `configure` for the build Python"
377 )
378 make_build = subcommands.add_parser(
379 "make-build-python", help="Run `make` for the build Python"
380 )
381 pythoninfo_build = subcommands.add_parser(
382 "pythoninfo-build", help="Display build info of the build Python"
383 )
384 configure_host = subcommands.add_parser(
385 "configure-host",
386 help="Run `configure` for the "
387 "host/WASI (pydebug builds "
388 "are inferred from the build "
389 "Python)",
390 )
391 make_host = subcommands.add_parser(
392 "make-host", help="Run `make` for the host/WASI"
393 )
394 pythoninfo_host = subcommands.add_parser(
395 "pythoninfo-host", help="Display build info of the host/WASI Python"
396 )
397 subcommands.add_parser(
398 "clean", help="Delete files and directories created by this script"
399 )
400 for subcommand in (
401 build,
402 configure_build,
403 make_build,
404 pythoninfo_build,
405 configure_host,
406 make_host,
407 pythoninfo_host,
408 ):
409 subcommand.add_argument(
410 "--quiet",
411 action="store_true",
412 default=False,
413 dest="quiet",
414 help="Redirect output from subprocesses to a log file",
415 )
416 for subcommand in configure_build, configure_host:
417 subcommand.add_argument(
418 "--clean",
419 action="store_true",
420 default=False,
421 dest="clean",
422 help="Delete any relevant directories before building",
423 )
424 for subcommand in build, configure_build, configure_host:
425 subcommand.add_argument(
426 "args", nargs="*", help="Extra arguments to pass to `configure`"
427 )
428 for subcommand in build, configure_host:
429 subcommand.add_argument(
430 "--wasi-sdk",
431 type=pathlib.Path,
432 dest="wasi_sdk_path",
433 default=find_wasi_sdk(),
434 help="Path to wasi-sdk; defaults to "
435 "$WASI_SDK_PATH or /opt/wasi-sdk",
436 )
437 subcommand.add_argument(
438 "--host-runner",
439 action="store",
440 default=default_host_runner,
441 dest="host_runner",
442 help="Command template for running the WASI host "
443 "(default designed for wasmtime 14 or newer: "
444 f"`{default_host_runner}`)",
445 )
446 for subcommand in (
447 build,
448 configure_host,
449 make_host,
450 pythoninfo_host,
451 ):
452 subcommand.add_argument(
453 "--host-triple",
454 action="store",
455 default="wasm32-wasip1",
456 help="The target triple for the WASI host build",
457 )
458
459 context = parser.parse_args()
460
461 dispatch = {
462 "configure-build-python": configure_build_python,
463 "make-build-python": make_build_python,
464 "pythoninfo-build": pythoninfo_build_python,
465 "configure-host": configure_wasi_python,
466 "make-host": make_wasi_python,
467 "pythoninfo-host": pythoninfo_wasi_python,
468 "build": build_all,
469 "clean": clean_contents,
470 None: lambda args: parser.print_help(),
471 }
472 dispatch[context.subcommand](context)
473
474
475 if __name__ == "__main__":
476 main()