| 1 |
#!/usr/bin/env python3 |
| 2 |
|
| 3 |
import argparse |
| 4 |
import contextlib |
| 5 |
import functools |
| 6 |
import os |
| 7 |
|
| 8 |
try: |
| 9 |
from os import process_cpu_count as cpu_count |
| 10 |
except ImportError: |
| 11 |
from os import cpu_count |
| 12 |
import pathlib |
| 13 |
import shutil |
| 14 |
import subprocess |
| 15 |
import sys |
| 16 |
import sysconfig |
| 17 |
import tempfile |
| 18 |
|
| 19 |
CHECKOUT = pathlib.Path(__file__).parent.parent.parent |
| 20 |
|
| 21 |
CROSS_BUILD_DIR = CHECKOUT / "cross-build" |
| 22 |
BUILD_DIR = CROSS_BUILD_DIR / "build" |
| 23 |
|
| 24 |
LOCAL_SETUP = CHECKOUT / "Modules" / "Setup.local" |
| 25 |
LOCAL_SETUP_MARKER = b"# Generated by Tools/wasm/wasi.py\n" |
| 26 |
|
| 27 |
WASMTIME_VAR_NAME = "WASMTIME" |
| 28 |
WASMTIME_HOST_RUNNER_VAR = f"{{{WASMTIME_VAR_NAME}}}" |
| 29 |
|
| 30 |
|
| 31 |
def updated_env(updates={}): |
| 32 |
"""Create a new dict representing the environment to use. |
| 33 |
|
| 34 |
The changes made to the execution environment are printed out. |
| 35 |
""" |
| 36 |
env_defaults = {} |
| 37 |
# https://reproducible-builds.org/docs/source-date-epoch/ |
| 38 |
git_epoch_cmd = ["git", "log", "-1", "--pretty=%ct"] |
| 39 |
try: |
| 40 |
epoch = subprocess.check_output( |
| 41 |
git_epoch_cmd, encoding="utf-8" |
| 42 |
).strip() |
| 43 |
env_defaults["SOURCE_DATE_EPOCH"] = epoch |
| 44 |
except subprocess.CalledProcessError: |
| 45 |
pass # Might be building from a tarball. |
| 46 |
# This layering lets SOURCE_DATE_EPOCH from os.environ takes precedence. |
| 47 |
environment = env_defaults | os.environ | updates |
| 48 |
|
| 49 |
env_diff = {} |
| 50 |
for key, value in environment.items(): |
| 51 |
if os.environ.get(key) != value: |
| 52 |
env_diff[key] = value |
| 53 |
|
| 54 |
print("๐ Environment changes:") |
| 55 |
for key in sorted(env_diff.keys()): |
| 56 |
print(f" {key}={env_diff[key]}") |
| 57 |
|
| 58 |
return environment |
| 59 |
|
| 60 |
|
| 61 |
def subdir(working_dir, *, clean_ok=False): |
| 62 |
"""Decorator to change to a working directory.""" |
| 63 |
|
| 64 |
def decorator(func): |
| 65 |
@functools.wraps(func) |
| 66 |
def wrapper(context): |
| 67 |
nonlocal working_dir |
| 68 |
|
| 69 |
if callable(working_dir): |
| 70 |
working_dir = working_dir(context) |
| 71 |
try: |
| 72 |
tput_output = subprocess.check_output( |
| 73 |
["tput", "cols"], encoding="utf-8" |
| 74 |
) |
| 75 |
except subprocess.CalledProcessError: |
| 76 |
terminal_width = 80 |
| 77 |
else: |
| 78 |
terminal_width = int(tput_output.strip()) |
| 79 |
print("โฏ" * terminal_width) |
| 80 |
print("๐", working_dir) |
| 81 |
if ( |
| 82 |
clean_ok |
| 83 |
and getattr(context, "clean", False) |
| 84 |
and working_dir.exists() |
| 85 |
): |
| 86 |
print("๐ฎ Deleting directory (--clean)...") |
| 87 |
shutil.rmtree(working_dir) |
| 88 |
|
| 89 |
working_dir.mkdir(parents=True, exist_ok=True) |
| 90 |
|
| 91 |
with contextlib.chdir(working_dir): |
| 92 |
return func(context, working_dir) |
| 93 |
|
| 94 |
return wrapper |
| 95 |
|
| 96 |
return decorator |
| 97 |
|
| 98 |
|
| 99 |
def call(command, *, quiet, **kwargs): |
| 100 |
"""Execute a command. |
| 101 |
|
| 102 |
If 'quiet' is true, then redirect stdout and stderr to a temporary file. |
| 103 |
""" |
| 104 |
print("โฏ", " ".join(map(str, command))) |
| 105 |
if not quiet: |
| 106 |
stdout = None |
| 107 |
stderr = None |
| 108 |
else: |
| 109 |
stdout = tempfile.NamedTemporaryFile( |
| 110 |
"w", |
| 111 |
encoding="utf-8", |
| 112 |
delete=False, |
| 113 |
prefix="cpython-wasi-", |
| 114 |
suffix=".log", |
| 115 |
) |
| 116 |
stderr = subprocess.STDOUT |
| 117 |
print(f"๐ Logging output to {stdout.name} (--quiet)...") |
| 118 |
|
| 119 |
subprocess.check_call(command, **kwargs, stdout=stdout, stderr=stderr) |
| 120 |
|
| 121 |
|
| 122 |
def build_platform(): |
| 123 |
"""The name of the build/host platform.""" |
| 124 |
# Can also be found via `config.guess`.` |
| 125 |
return sysconfig.get_config_var("BUILD_GNU_TYPE") |
| 126 |
|
| 127 |
|
| 128 |
def build_python_path(): |
| 129 |
"""The path to the build Python binary.""" |
| 130 |
binary = BUILD_DIR / "python" |
| 131 |
if not binary.is_file(): |
| 132 |
binary = binary.with_suffix(".exe") |
| 133 |
if not binary.is_file(): |
| 134 |
raise FileNotFoundError( |
| 135 |
f"Unable to find `python(.exe)` in {BUILD_DIR}" |
| 136 |
) |
| 137 |
|
| 138 |
return binary |
| 139 |
|
| 140 |
|
| 141 |
@subdir(BUILD_DIR, clean_ok=True) |
| 142 |
def configure_build_python(context, working_dir): |
| 143 |
"""Configure the build/host Python.""" |
| 144 |
if LOCAL_SETUP.exists(): |
| 145 |
print(f"๐ {LOCAL_SETUP} exists ...") |
| 146 |
else: |
| 147 |
print(f"๐ Touching {LOCAL_SETUP} ...") |
| 148 |
LOCAL_SETUP.write_bytes(LOCAL_SETUP_MARKER) |
| 149 |
|
| 150 |
configure = [os.path.relpath(CHECKOUT / "configure", working_dir)] |
| 151 |
if context.args: |
| 152 |
configure.extend(context.args) |
| 153 |
|
| 154 |
call(configure, quiet=context.quiet) |
| 155 |
|
| 156 |
|
| 157 |
@subdir(BUILD_DIR) |
| 158 |
def make_build_python(context, working_dir): |
| 159 |
"""Make/build the build Python.""" |
| 160 |
call(["make", "--jobs", str(cpu_count()), "all"], quiet=context.quiet) |
| 161 |
|
| 162 |
binary = build_python_path() |
| 163 |
cmd = [ |
| 164 |
binary, |
| 165 |
"-c", |
| 166 |
"import sys; " |
| 167 |
"print(f'{sys.version_info.major}.{sys.version_info.minor}')", |
| 168 |
] |
| 169 |
version = subprocess.check_output(cmd, encoding="utf-8").strip() |
| 170 |
|
| 171 |
print(f"๐ {binary} {version}") |
| 172 |
|
| 173 |
|
| 174 |
@subdir(BUILD_DIR) |
| 175 |
def pythoninfo_build_python(context, working_dir): |
| 176 |
"""Display build info of the build Python.""" |
| 177 |
call(["make", "pythoninfo"], quiet=context.quiet) |
| 178 |
|
| 179 |
|
| 180 |
def find_wasi_sdk(): |
| 181 |
"""Find the path to wasi-sdk.""" |
| 182 |
if wasi_sdk_path := os.environ.get("WASI_SDK_PATH"): |
| 183 |
return pathlib.Path(wasi_sdk_path) |
| 184 |
elif (default_path := pathlib.Path("/opt/wasi-sdk")).exists(): |
| 185 |
return default_path |
| 186 |
|
| 187 |
|
| 188 |
def wasi_sdk_env(context): |
| 189 |
"""Calculate environment variables for building with wasi-sdk.""" |
| 190 |
wasi_sdk_path = context.wasi_sdk_path |
| 191 |
sysroot = wasi_sdk_path / "share" / "wasi-sysroot" |
| 192 |
env = { |
| 193 |
"CC": "clang", |
| 194 |
"CPP": "clang-cpp", |
| 195 |
"CXX": "clang++", |
| 196 |
"AR": "llvm-ar", |
| 197 |
"RANLIB": "ranlib", |
| 198 |
} |
| 199 |
|
| 200 |
for env_var, binary_name in list(env.items()): |
| 201 |
env[env_var] = os.fsdecode(wasi_sdk_path / "bin" / binary_name) |
| 202 |
|
| 203 |
if wasi_sdk_path != pathlib.Path("/opt/wasi-sdk"): |
| 204 |
for compiler in ["CC", "CPP", "CXX"]: |
| 205 |
env[compiler] += f" --sysroot={sysroot}" |
| 206 |
|
| 207 |
env["PKG_CONFIG_PATH"] = "" |
| 208 |
env["PKG_CONFIG_LIBDIR"] = os.pathsep.join( |
| 209 |
map( |
| 210 |
os.fsdecode, |
| 211 |
[sysroot / "lib" / "pkgconfig", sysroot / "share" / "pkgconfig"], |
| 212 |
) |
| 213 |
) |
| 214 |
env["PKG_CONFIG_SYSROOT_DIR"] = os.fsdecode(sysroot) |
| 215 |
|
| 216 |
env["WASI_SDK_PATH"] = os.fsdecode(wasi_sdk_path) |
| 217 |
env["WASI_SYSROOT"] = os.fsdecode(sysroot) |
| 218 |
|
| 219 |
env["PATH"] = os.pathsep.join([ |
| 220 |
os.fsdecode(wasi_sdk_path / "bin"), |
| 221 |
os.environ["PATH"], |
| 222 |
]) |
| 223 |
|
| 224 |
return env |
| 225 |
|
| 226 |
|
| 227 |
@subdir(lambda context: CROSS_BUILD_DIR / context.host_triple, clean_ok=True) |
| 228 |
def configure_wasi_python(context, working_dir): |
| 229 |
"""Configure the WASI/host build.""" |
| 230 |
if not context.wasi_sdk_path or not context.wasi_sdk_path.exists(): |
| 231 |
raise ValueError( |
| 232 |
"WASI-SDK not found; " |
| 233 |
"download from " |
| 234 |
"https://github.com/WebAssembly/wasi-sdk and/or " |
| 235 |
"specify via $WASI_SDK_PATH or --wasi-sdk" |
| 236 |
) |
| 237 |
|
| 238 |
config_site = os.fsdecode( |
| 239 |
CHECKOUT / "Tools" / "wasm" / "config.site-wasm32-wasi" |
| 240 |
) |
| 241 |
|
| 242 |
wasi_build_dir = working_dir.relative_to(CHECKOUT) |
| 243 |
|
| 244 |
python_build_dir = BUILD_DIR / "build" |
| 245 |
lib_dirs = list(python_build_dir.glob("lib.*")) |
| 246 |
assert len(lib_dirs) == 1, ( |
| 247 |
f"Expected a single lib.* directory in {python_build_dir}" |
| 248 |
) |
| 249 |
lib_dir = os.fsdecode(lib_dirs[0]) |
| 250 |
pydebug = lib_dir.endswith("-pydebug") |
| 251 |
python_version = lib_dir.removesuffix("-pydebug").rpartition("-")[-1] |
| 252 |
sysconfig_data = f"{wasi_build_dir}/build/lib.wasi-wasm32-{python_version}" |
| 253 |
if pydebug: |
| 254 |
sysconfig_data += "-pydebug" |
| 255 |
|
| 256 |
# Use PYTHONPATH to include sysconfig data which must be anchored to the |
| 257 |
# WASI guest's `/` directory. |
| 258 |
args = { |
| 259 |
"GUEST_DIR": "/", |
| 260 |
"HOST_DIR": CHECKOUT, |
| 261 |
"ENV_VAR_NAME": "PYTHONPATH", |
| 262 |
"ENV_VAR_VALUE": f"/{sysconfig_data}", |
| 263 |
"PYTHON_WASM": working_dir / "python.wasm", |
| 264 |
"ARGV0": wasi_build_dir / "python.wasm", |
| 265 |
} |
| 266 |
# Check dynamically for wasmtime in case it was specified manually via |
| 267 |
# `--host-runner`. |
| 268 |
if WASMTIME_HOST_RUNNER_VAR in context.host_runner: |
| 269 |
if wasmtime := shutil.which("wasmtime"): |
| 270 |
args[WASMTIME_VAR_NAME] = wasmtime |
| 271 |
else: |
| 272 |
raise FileNotFoundError( |
| 273 |
"wasmtime not found; download from " |
| 274 |
"https://github.com/bytecodealliance/wasmtime" |
| 275 |
) |
| 276 |
host_runner = context.host_runner.format_map(args) |
| 277 |
env_additions = {"CONFIG_SITE": config_site, "HOSTRUNNER": host_runner} |
| 278 |
build_python = os.fsdecode(build_python_path()) |
| 279 |
# The path to `configure` MUST be relative, else `python.wasm` is unable |
| 280 |
# to find the stdlib due to Python not recognizing that it's being |
| 281 |
# executed from within a checkout. |
| 282 |
configure = [ |
| 283 |
os.path.relpath(CHECKOUT / "configure", working_dir), |
| 284 |
f"--host={context.host_triple}", |
| 285 |
f"--build={build_platform()}", |
| 286 |
f"--with-build-python={build_python}", |
| 287 |
] |
| 288 |
if pydebug: |
| 289 |
configure.append("--with-pydebug") |
| 290 |
if context.args: |
| 291 |
configure.extend(context.args) |
| 292 |
call( |
| 293 |
configure, |
| 294 |
env=updated_env(env_additions | wasi_sdk_env(context)), |
| 295 |
quiet=context.quiet, |
| 296 |
) |
| 297 |
|
| 298 |
python_wasm = working_dir / "python.wasm" |
| 299 |
exec_script = working_dir / "python.sh" |
| 300 |
with exec_script.open("w", encoding="utf-8") as file: |
| 301 |
file.write(f'#!/bin/sh\nexec {host_runner} {python_wasm} "$@"\n') |
| 302 |
exec_script.chmod(0o755) |
| 303 |
print(f"๐โโ๏ธ Created {exec_script} ... ") |
| 304 |
sys.stdout.flush() |
| 305 |
|
| 306 |
|
| 307 |
@subdir(lambda context: CROSS_BUILD_DIR / context.host_triple) |
| 308 |
def make_wasi_python(context, working_dir): |
| 309 |
"""Run `make` for the WASI/host build.""" |
| 310 |
call( |
| 311 |
["make", "--jobs", str(cpu_count()), "all"], |
| 312 |
env=updated_env(), |
| 313 |
quiet=context.quiet, |
| 314 |
) |
| 315 |
|
| 316 |
exec_script = working_dir / "python.sh" |
| 317 |
subprocess.check_call([exec_script, "--version"]) |
| 318 |
|
| 319 |
|
| 320 |
def build_all(context): |
| 321 |
"""Build everything.""" |
| 322 |
steps = [ |
| 323 |
configure_build_python, |
| 324 |
make_build_python, |
| 325 |
pythoninfo_build_python, |
| 326 |
configure_wasi_python, |
| 327 |
make_wasi_python, |
| 328 |
pythoninfo_wasi_python, |
| 329 |
] |
| 330 |
for step in steps: |
| 331 |
step(context) |
| 332 |
|
| 333 |
|
| 334 |
@subdir(lambda context: CROSS_BUILD_DIR / context.host_triple) |
| 335 |
def pythoninfo_wasi_python(context, working_dir): |
| 336 |
"""Display build info of the host/WASI Python.""" |
| 337 |
call(["make", "pythoninfo"], quiet=context.quiet) |
| 338 |
|
| 339 |
|
| 340 |
def clean_contents(context): |
| 341 |
"""Delete all files created by this script.""" |
| 342 |
if CROSS_BUILD_DIR.exists(): |
| 343 |
print(f"๐งน Deleting {CROSS_BUILD_DIR} ...") |
| 344 |
shutil.rmtree(CROSS_BUILD_DIR) |
| 345 |
|
| 346 |
if LOCAL_SETUP.exists(): |
| 347 |
with LOCAL_SETUP.open("rb") as file: |
| 348 |
if file.read(len(LOCAL_SETUP_MARKER)) == LOCAL_SETUP_MARKER: |
| 349 |
print(f"๐งน Deleting generated {LOCAL_SETUP} ...") |
| 350 |
|
| 351 |
|
| 352 |
def main(): |
| 353 |
default_host_runner = ( |
| 354 |
f"{WASMTIME_HOST_RUNNER_VAR} run " |
| 355 |
# Make sure the stack size will work for a pydebug |
| 356 |
# build. |
| 357 |
# The 8388608 value comes from `ulimit -s` under Linux |
| 358 |
# which equates to 8291 KiB. |
| 359 |
"--wasm max-wasm-stack=8388608 " |
| 360 |
# Use WASI 0.2 primitives. |
| 361 |
"--wasi preview2 " |
| 362 |
# Explicitly set the argv[0] value |
| 363 |
"--argv0 {ARGV0} " |
| 364 |
# Enable thread support; causes use of preview1. |
| 365 |
# "--wasm threads=y --wasi threads=y " |
| 366 |
# Map the checkout to / to load the stdlib from /Lib. |
| 367 |
"--dir {HOST_DIR}::{GUEST_DIR} " |
| 368 |
# Set PYTHONPATH to the sysconfig data. |
| 369 |
"--env {ENV_VAR_NAME}={ENV_VAR_VALUE}" |
| 370 |
) |
| 371 |
|
| 372 |
parser = argparse.ArgumentParser() |
| 373 |
subcommands = parser.add_subparsers(dest="subcommand") |
| 374 |
build = subcommands.add_parser("build", help="Build everything") |
| 375 |
configure_build = subcommands.add_parser( |
| 376 |
"configure-build-python", help="Run `configure` for the build Python" |
| 377 |
) |
| 378 |
make_build = subcommands.add_parser( |
| 379 |
"make-build-python", help="Run `make` for the build Python" |
| 380 |
) |
| 381 |
pythoninfo_build = subcommands.add_parser( |
| 382 |
"pythoninfo-build", help="Display build info of the build Python" |
| 383 |
) |
| 384 |
configure_host = subcommands.add_parser( |
| 385 |
"configure-host", |
| 386 |
help="Run `configure` for the " |
| 387 |
"host/WASI (pydebug builds " |
| 388 |
"are inferred from the build " |
| 389 |
"Python)", |
| 390 |
) |
| 391 |
make_host = subcommands.add_parser( |
| 392 |
"make-host", help="Run `make` for the host/WASI" |
| 393 |
) |
| 394 |
pythoninfo_host = subcommands.add_parser( |
| 395 |
"pythoninfo-host", help="Display build info of the host/WASI Python" |
| 396 |
) |
| 397 |
subcommands.add_parser( |
| 398 |
"clean", help="Delete files and directories created by this script" |
| 399 |
) |
| 400 |
for subcommand in ( |
| 401 |
build, |
| 402 |
configure_build, |
| 403 |
make_build, |
| 404 |
pythoninfo_build, |
| 405 |
configure_host, |
| 406 |
make_host, |
| 407 |
pythoninfo_host, |
| 408 |
): |
| 409 |
subcommand.add_argument( |
| 410 |
"--quiet", |
| 411 |
action="store_true", |
| 412 |
default=False, |
| 413 |
dest="quiet", |
| 414 |
help="Redirect output from subprocesses to a log file", |
| 415 |
) |
| 416 |
for subcommand in configure_build, configure_host: |
| 417 |
subcommand.add_argument( |
| 418 |
"--clean", |
| 419 |
action="store_true", |
| 420 |
default=False, |
| 421 |
dest="clean", |
| 422 |
help="Delete any relevant directories before building", |
| 423 |
) |
| 424 |
for subcommand in build, configure_build, configure_host: |
| 425 |
subcommand.add_argument( |
| 426 |
"args", nargs="*", help="Extra arguments to pass to `configure`" |
| 427 |
) |
| 428 |
for subcommand in build, configure_host: |
| 429 |
subcommand.add_argument( |
| 430 |
"--wasi-sdk", |
| 431 |
type=pathlib.Path, |
| 432 |
dest="wasi_sdk_path", |
| 433 |
default=find_wasi_sdk(), |
| 434 |
help="Path to wasi-sdk; defaults to " |
| 435 |
"$WASI_SDK_PATH or /opt/wasi-sdk", |
| 436 |
) |
| 437 |
subcommand.add_argument( |
| 438 |
"--host-runner", |
| 439 |
action="store", |
| 440 |
default=default_host_runner, |
| 441 |
dest="host_runner", |
| 442 |
help="Command template for running the WASI host " |
| 443 |
"(default designed for wasmtime 14 or newer: " |
| 444 |
f"`{default_host_runner}`)", |
| 445 |
) |
| 446 |
for subcommand in ( |
| 447 |
build, |
| 448 |
configure_host, |
| 449 |
make_host, |
| 450 |
pythoninfo_host, |
| 451 |
): |
| 452 |
subcommand.add_argument( |
| 453 |
"--host-triple", |
| 454 |
action="store", |
| 455 |
default="wasm32-wasip1", |
| 456 |
help="The target triple for the WASI host build", |
| 457 |
) |
| 458 |
|
| 459 |
context = parser.parse_args() |
| 460 |
|
| 461 |
dispatch = { |
| 462 |
"configure-build-python": configure_build_python, |
| 463 |
"make-build-python": make_build_python, |
| 464 |
"pythoninfo-build": pythoninfo_build_python, |
| 465 |
"configure-host": configure_wasi_python, |
| 466 |
"make-host": make_wasi_python, |
| 467 |
"pythoninfo-host": pythoninfo_wasi_python, |
| 468 |
"build": build_all, |
| 469 |
"clean": clean_contents, |
| 470 |
None: lambda args: parser.print_help(), |
| 471 |
} |
| 472 |
dispatch[context.subcommand](context) |
| 473 |
|
| 474 |
|
| 475 |
if __name__ == "__main__": |
| 476 |
main() |