| 1 |
#!/usr/bin/env python3 |
| 2 |
|
| 3 |
import argparse |
| 4 |
import contextlib |
| 5 |
import functools |
| 6 |
import hashlib |
| 7 |
import json |
| 8 |
import os |
| 9 |
import shutil |
| 10 |
import subprocess |
| 11 |
import sys |
| 12 |
import sysconfig |
| 13 |
import tempfile |
| 14 |
from pathlib import Path |
| 15 |
from textwrap import dedent |
| 16 |
from urllib.request import urlopen |
| 17 |
|
| 18 |
import tomllib |
| 19 |
|
| 20 |
try: |
| 21 |
from os import process_cpu_count as cpu_count |
| 22 |
except ImportError: |
| 23 |
from os import cpu_count |
| 24 |
|
| 25 |
|
| 26 |
EMSCRIPTEN_DIR = Path(__file__).parent |
| 27 |
CHECKOUT = EMSCRIPTEN_DIR.parent.parent |
| 28 |
CONFIG_FILE = EMSCRIPTEN_DIR / "config.toml" |
| 29 |
|
| 30 |
DEFAULT_CROSS_BUILD_DIR = CHECKOUT / "cross-build" |
| 31 |
HOST_TRIPLE = "wasm32-emscripten" |
| 32 |
|
| 33 |
|
| 34 |
@functools.cache |
| 35 |
def load_config_toml(): |
| 36 |
with CONFIG_FILE.open("rb") as file: |
| 37 |
return tomllib.load(file) |
| 38 |
|
| 39 |
|
| 40 |
@functools.cache |
| 41 |
def required_emscripten_version(): |
| 42 |
return load_config_toml()["emscripten-version"] |
| 43 |
|
| 44 |
|
| 45 |
@functools.cache |
| 46 |
def emsdk_cache_root(emsdk_cache): |
| 47 |
required_version = required_emscripten_version() |
| 48 |
return Path(emsdk_cache).absolute() / required_version |
| 49 |
|
| 50 |
|
| 51 |
@functools.cache |
| 52 |
def emsdk_activate_path(emsdk_cache): |
| 53 |
return emsdk_cache_root(emsdk_cache) / "emsdk/emsdk_env.sh" |
| 54 |
|
| 55 |
|
| 56 |
def get_build_paths(cross_build_dir=None, emsdk_cache=None): |
| 57 |
"""Compute all build paths from the given cross-build directory.""" |
| 58 |
if cross_build_dir is None: |
| 59 |
cross_build_dir = DEFAULT_CROSS_BUILD_DIR |
| 60 |
cross_build_dir = Path(cross_build_dir).absolute() |
| 61 |
host_triple_dir = cross_build_dir / HOST_TRIPLE |
| 62 |
prefix_dir = host_triple_dir / "prefix" |
| 63 |
if emsdk_cache: |
| 64 |
prefix_dir = emsdk_cache_root(emsdk_cache) / "prefix" |
| 65 |
|
| 66 |
return { |
| 67 |
"cross_build_dir": cross_build_dir, |
| 68 |
"native_build_dir": cross_build_dir / "build", |
| 69 |
"host_triple_dir": host_triple_dir, |
| 70 |
"host_build_dir": host_triple_dir / "build", |
| 71 |
"host_dir": host_triple_dir / "build" / "python", |
| 72 |
"prefix_dir": prefix_dir, |
| 73 |
} |
| 74 |
|
| 75 |
|
| 76 |
LOCAL_SETUP = CHECKOUT / "Modules" / "Setup.local" |
| 77 |
LOCAL_SETUP_MARKER = b"# Generated by Platforms/wasm/emscripten.py\n" |
| 78 |
|
| 79 |
|
| 80 |
@functools.cache |
| 81 |
def validate_emsdk_version(emsdk_cache): |
| 82 |
"""Validate that the emsdk cache contains the required emscripten version.""" |
| 83 |
if emsdk_cache is None: |
| 84 |
print("Build will use EMSDK from current environment.") |
| 85 |
return |
| 86 |
required_version = required_emscripten_version() |
| 87 |
emsdk_env = emsdk_activate_path(emsdk_cache) |
| 88 |
if not emsdk_env.is_file(): |
| 89 |
print( |
| 90 |
f"Required emscripten version {required_version} not found in {emsdk_cache}", |
| 91 |
file=sys.stderr, |
| 92 |
) |
| 93 |
sys.exit(1) |
| 94 |
print(f"โ
Emscripten version {required_version} found in {emsdk_cache}") |
| 95 |
|
| 96 |
|
| 97 |
def parse_env(text): |
| 98 |
result = {} |
| 99 |
for line in text.splitlines(): |
| 100 |
key, val = line.split("=", 1) |
| 101 |
result[key] = val |
| 102 |
return result |
| 103 |
|
| 104 |
|
| 105 |
@functools.cache |
| 106 |
def get_emsdk_environ(emsdk_cache): |
| 107 |
"""Returns os.environ updated by sourcing emsdk_env.sh""" |
| 108 |
if not emsdk_cache: |
| 109 |
return os.environ |
| 110 |
env_text = subprocess.check_output( |
| 111 |
[ |
| 112 |
"bash", |
| 113 |
"-c", |
| 114 |
f"EMSDK_QUIET=1 source {emsdk_activate_path(emsdk_cache)} && env", |
| 115 |
], |
| 116 |
text=True, |
| 117 |
) |
| 118 |
return parse_env(env_text) |
| 119 |
|
| 120 |
|
| 121 |
def updated_env(updates, emsdk_cache): |
| 122 |
"""Create a new dict representing the environment to use. |
| 123 |
|
| 124 |
The changes made to the execution environment are printed out. |
| 125 |
""" |
| 126 |
env_defaults = {} |
| 127 |
# https://reproducible-builds.org/docs/source-date-epoch/ |
| 128 |
git_epoch_cmd = ["git", "log", "-1", "--pretty=%ct"] |
| 129 |
try: |
| 130 |
epoch = subprocess.check_output( |
| 131 |
git_epoch_cmd, encoding="utf-8" |
| 132 |
).strip() |
| 133 |
env_defaults["SOURCE_DATE_EPOCH"] = epoch |
| 134 |
except subprocess.CalledProcessError: |
| 135 |
pass # Might be building from a tarball. |
| 136 |
# This layering lets SOURCE_DATE_EPOCH from os.environ takes precedence. |
| 137 |
environment = env_defaults | get_emsdk_environ(emsdk_cache) | updates |
| 138 |
env_diff = {} |
| 139 |
for key, value in environment.items(): |
| 140 |
if os.environ.get(key) != value: |
| 141 |
env_diff[key] = value |
| 142 |
|
| 143 |
print("๐ Environment changes:") |
| 144 |
for key in sorted(env_diff.keys()): |
| 145 |
print(f" {key}={env_diff[key]}") |
| 146 |
|
| 147 |
return environment |
| 148 |
|
| 149 |
|
| 150 |
def subdir(path_key, *, clean_ok=False): |
| 151 |
"""Decorator to change to a working directory. |
| 152 |
|
| 153 |
path_key is a key into context.build_paths, used to resolve the working |
| 154 |
directory at call time. |
| 155 |
""" |
| 156 |
|
| 157 |
def decorator(func): |
| 158 |
@functools.wraps(func) |
| 159 |
def wrapper(context): |
| 160 |
working_dir = context.build_paths[path_key] |
| 161 |
try: |
| 162 |
tput_output = subprocess.check_output( |
| 163 |
["tput", "cols"], encoding="utf-8" |
| 164 |
) |
| 165 |
terminal_width = int(tput_output.strip()) |
| 166 |
except subprocess.CalledProcessError: |
| 167 |
terminal_width = 80 |
| 168 |
print("โฏ" * terminal_width) |
| 169 |
print("๐", working_dir) |
| 170 |
if ( |
| 171 |
clean_ok |
| 172 |
and getattr(context, "clean", False) |
| 173 |
and working_dir.exists() |
| 174 |
): |
| 175 |
print("๐ฎ Deleting directory (--clean)...") |
| 176 |
shutil.rmtree(working_dir) |
| 177 |
|
| 178 |
working_dir.mkdir(parents=True, exist_ok=True) |
| 179 |
|
| 180 |
with contextlib.chdir(working_dir): |
| 181 |
return func(context, working_dir) |
| 182 |
|
| 183 |
return wrapper |
| 184 |
|
| 185 |
return decorator |
| 186 |
|
| 187 |
|
| 188 |
def call(command, *, quiet, **kwargs): |
| 189 |
"""Execute a command. |
| 190 |
|
| 191 |
If 'quiet' is true, then redirect stdout and stderr to a temporary file. |
| 192 |
""" |
| 193 |
print("โฏ", " ".join(map(str, command))) |
| 194 |
if not quiet: |
| 195 |
stdout = None |
| 196 |
stderr = None |
| 197 |
else: |
| 198 |
stdout = tempfile.NamedTemporaryFile( |
| 199 |
"w", |
| 200 |
encoding="utf-8", |
| 201 |
delete=False, |
| 202 |
prefix="cpython-emscripten-", |
| 203 |
suffix=".log", |
| 204 |
) |
| 205 |
stderr = subprocess.STDOUT |
| 206 |
print(f"๐ Logging output to {stdout.name} (--quiet)...") |
| 207 |
|
| 208 |
subprocess.check_call(command, **kwargs, stdout=stdout, stderr=stderr) |
| 209 |
|
| 210 |
|
| 211 |
def build_platform(): |
| 212 |
"""The name of the build/host platform.""" |
| 213 |
# Can also be found via `config.guess`.` |
| 214 |
return sysconfig.get_config_var("BUILD_GNU_TYPE") |
| 215 |
|
| 216 |
|
| 217 |
def build_python_path(context): |
| 218 |
"""The path to the build Python binary.""" |
| 219 |
native_build_dir = context.build_paths["native_build_dir"] |
| 220 |
binary = native_build_dir / "python" |
| 221 |
if not binary.is_file(): |
| 222 |
binary = binary.with_suffix(".exe") |
| 223 |
if not binary.is_file(): |
| 224 |
raise FileNotFoundError( |
| 225 |
f"Unable to find `python(.exe)` in {native_build_dir}" |
| 226 |
) |
| 227 |
|
| 228 |
return binary |
| 229 |
|
| 230 |
|
| 231 |
def install_emscripten(context): |
| 232 |
emsdk_cache = context.emsdk_cache |
| 233 |
if emsdk_cache is None: |
| 234 |
print("install-emscripten requires --emsdk-cache", file=sys.stderr) |
| 235 |
sys.exit(1) |
| 236 |
version = required_emscripten_version() |
| 237 |
emsdk_target = emsdk_cache_root(emsdk_cache) / "emsdk" |
| 238 |
if emsdk_target.exists(): |
| 239 |
if not context.quiet: |
| 240 |
print(f"Emscripten version {version} already installed") |
| 241 |
return |
| 242 |
if not context.quiet: |
| 243 |
print(f"Installing emscripten version {version}") |
| 244 |
emsdk_target.mkdir(parents=True) |
| 245 |
call( |
| 246 |
[ |
| 247 |
"git", |
| 248 |
"clone", |
| 249 |
"https://github.com/emscripten-core/emsdk.git", |
| 250 |
emsdk_target, |
| 251 |
], |
| 252 |
quiet=context.quiet, |
| 253 |
) |
| 254 |
call([emsdk_target / "emsdk", "install", version], quiet=context.quiet) |
| 255 |
call([emsdk_target / "emsdk", "activate", version], quiet=context.quiet) |
| 256 |
if not context.quiet: |
| 257 |
print(f"Installed emscripten version {version}") |
| 258 |
|
| 259 |
|
| 260 |
@subdir("native_build_dir", clean_ok=True) |
| 261 |
def configure_build_python(context, working_dir): |
| 262 |
"""Configure the build/host Python.""" |
| 263 |
if LOCAL_SETUP.exists(): |
| 264 |
print(f"๐ {LOCAL_SETUP} exists ...") |
| 265 |
else: |
| 266 |
print(f"๐ Touching {LOCAL_SETUP} ...") |
| 267 |
LOCAL_SETUP.write_bytes(LOCAL_SETUP_MARKER) |
| 268 |
|
| 269 |
configure = [os.path.relpath(CHECKOUT / "configure", working_dir)] |
| 270 |
if context.args: |
| 271 |
configure.extend(context.args) |
| 272 |
|
| 273 |
call(configure, quiet=context.quiet) |
| 274 |
|
| 275 |
|
| 276 |
@subdir("native_build_dir") |
| 277 |
def make_build_python(context, working_dir): |
| 278 |
"""Make/build the build Python.""" |
| 279 |
call(["make", "--jobs", str(cpu_count()), "all"], quiet=context.quiet) |
| 280 |
|
| 281 |
binary = build_python_path(context) |
| 282 |
cmd = [ |
| 283 |
binary, |
| 284 |
"-c", |
| 285 |
"import sys; " |
| 286 |
"print(f'{sys.version_info.major}.{sys.version_info.minor}')", |
| 287 |
] |
| 288 |
version = subprocess.check_output(cmd, encoding="utf-8").strip() |
| 289 |
|
| 290 |
print(f"๐ {binary} {version}") |
| 291 |
|
| 292 |
|
| 293 |
@subdir("native_build_dir") |
| 294 |
def pythoninfo_build_python(context, working_dir): |
| 295 |
"""Display build info of the build Python.""" |
| 296 |
call(["make", "pythoninfo"], quiet=context.quiet) |
| 297 |
|
| 298 |
|
| 299 |
def check_shasum(file: str, expected_shasum: str): |
| 300 |
with open(file, "rb") as f: |
| 301 |
digest = hashlib.file_digest(f, "sha256") |
| 302 |
if digest.hexdigest() != expected_shasum: |
| 303 |
raise RuntimeError(f"Unexpected shasum for {file}") |
| 304 |
|
| 305 |
|
| 306 |
def download_and_unpack(working_dir: Path, url: str, expected_shasum: str): |
| 307 |
with tempfile.NamedTemporaryFile( |
| 308 |
suffix=".tar.gz", delete_on_close=False |
| 309 |
) as tmp_file: |
| 310 |
with urlopen(url) as response: |
| 311 |
shutil.copyfileobj(response, tmp_file) |
| 312 |
tmp_file.close() |
| 313 |
check_shasum(tmp_file.name, expected_shasum) |
| 314 |
shutil.unpack_archive(tmp_file.name, working_dir) |
| 315 |
|
| 316 |
|
| 317 |
def should_build_library(prefix, name, config, quiet): |
| 318 |
cached_config = prefix / (name + ".json") |
| 319 |
if not cached_config.exists(): |
| 320 |
if not quiet: |
| 321 |
print( |
| 322 |
f"No cached build of {name} version {config['version']} found, building" |
| 323 |
) |
| 324 |
return True |
| 325 |
|
| 326 |
try: |
| 327 |
with cached_config.open("rb") as f: |
| 328 |
cached_config = json.load(f) |
| 329 |
except json.JSONDecodeError: |
| 330 |
if not quiet: |
| 331 |
print(f"Cached data for {name} invalid, rebuilding") |
| 332 |
return True |
| 333 |
if config == cached_config: |
| 334 |
if not quiet: |
| 335 |
print( |
| 336 |
f"Found cached build of {name} version {config['version']}, not rebuilding" |
| 337 |
) |
| 338 |
return False |
| 339 |
|
| 340 |
if not quiet: |
| 341 |
print( |
| 342 |
f"Found cached build of {name} version {config['version']} but it's out of date, rebuilding" |
| 343 |
) |
| 344 |
return True |
| 345 |
|
| 346 |
|
| 347 |
def write_library_config(prefix, name, config, quiet): |
| 348 |
cached_config = prefix / (name + ".json") |
| 349 |
with cached_config.open("w") as f: |
| 350 |
json.dump(config, f) |
| 351 |
if not quiet: |
| 352 |
print(f"Succeded building {name}, wrote config to {cached_config}") |
| 353 |
|
| 354 |
|
| 355 |
@subdir("host_build_dir", clean_ok=True) |
| 356 |
def make_emscripten_libffi(context, working_dir): |
| 357 |
validate_emsdk_version(context.emsdk_cache) |
| 358 |
prefix = context.build_paths["prefix_dir"] |
| 359 |
libffi_config = load_config_toml()["dependencies"]["libffi"] |
| 360 |
with open(EMSCRIPTEN_DIR / "make_libffi.sh", "rb") as f: |
| 361 |
libffi_config["make_libffi_shasum"] = hashlib.file_digest(f, "sha256").hexdigest() |
| 362 |
if not should_build_library( |
| 363 |
prefix, "libffi", libffi_config, context.quiet |
| 364 |
): |
| 365 |
return |
| 366 |
|
| 367 |
if context.check_up_to_date: |
| 368 |
print("libffi out of date, expected to be up to date", file=sys.stderr) |
| 369 |
sys.exit(1) |
| 370 |
|
| 371 |
url = libffi_config["url"] |
| 372 |
version = libffi_config["version"] |
| 373 |
shasum = libffi_config["shasum"] |
| 374 |
libffi_dir = working_dir / f"libffi-{version}" |
| 375 |
shutil.rmtree(libffi_dir, ignore_errors=True) |
| 376 |
download_and_unpack( |
| 377 |
working_dir, |
| 378 |
url.format(version=version), |
| 379 |
shasum, |
| 380 |
) |
| 381 |
call( |
| 382 |
[EMSCRIPTEN_DIR / "make_libffi.sh"], |
| 383 |
env=updated_env({"PREFIX": prefix}, context.emsdk_cache), |
| 384 |
cwd=libffi_dir, |
| 385 |
quiet=context.quiet, |
| 386 |
) |
| 387 |
write_library_config(prefix, "libffi", libffi_config, context.quiet) |
| 388 |
|
| 389 |
|
| 390 |
@subdir("host_build_dir", clean_ok=True) |
| 391 |
def make_mpdec(context, working_dir): |
| 392 |
validate_emsdk_version(context.emsdk_cache) |
| 393 |
prefix = context.build_paths["prefix_dir"] |
| 394 |
mpdec_config = load_config_toml()["dependencies"]["mpdec"] |
| 395 |
if not should_build_library(prefix, "mpdec", mpdec_config, context.quiet): |
| 396 |
return |
| 397 |
|
| 398 |
if context.check_up_to_date: |
| 399 |
print("libmpdec out of date, expected to be up to date", file=sys.stderr) |
| 400 |
sys.exit(1) |
| 401 |
|
| 402 |
url = mpdec_config["url"] |
| 403 |
version = mpdec_config["version"] |
| 404 |
shasum = mpdec_config["shasum"] |
| 405 |
mpdec_dir = working_dir / f"mpdecimal-{version}" |
| 406 |
shutil.rmtree(mpdec_dir, ignore_errors=True) |
| 407 |
download_and_unpack( |
| 408 |
working_dir, |
| 409 |
url.format(version=version), |
| 410 |
shasum, |
| 411 |
) |
| 412 |
call( |
| 413 |
[ |
| 414 |
"emconfigure", |
| 415 |
mpdec_dir / "configure", |
| 416 |
"CFLAGS=-fPIC", |
| 417 |
"--prefix", |
| 418 |
prefix, |
| 419 |
"--disable-shared", |
| 420 |
], |
| 421 |
cwd=mpdec_dir, |
| 422 |
quiet=context.quiet, |
| 423 |
env=updated_env({}, context.emsdk_cache), |
| 424 |
) |
| 425 |
call( |
| 426 |
["make", "install"], |
| 427 |
cwd=mpdec_dir, |
| 428 |
quiet=context.quiet, |
| 429 |
) |
| 430 |
write_library_config(prefix, "mpdec", mpdec_config, context.quiet) |
| 431 |
|
| 432 |
|
| 433 |
def make_dependencies(context): |
| 434 |
make_emscripten_libffi(context) |
| 435 |
make_mpdec(context) |
| 436 |
|
| 437 |
|
| 438 |
def calculate_node_path(): |
| 439 |
node_version = os.environ.get("PYTHON_NODE_VERSION", None) |
| 440 |
if node_version is None: |
| 441 |
node_version = load_config_toml()["node-version"] |
| 442 |
|
| 443 |
subprocess.run( |
| 444 |
[ |
| 445 |
"bash", |
| 446 |
"-c", |
| 447 |
f"source ~/.nvm/nvm.sh && nvm install {node_version}", |
| 448 |
], |
| 449 |
check=True, |
| 450 |
) |
| 451 |
|
| 452 |
res = subprocess.run( |
| 453 |
[ |
| 454 |
"bash", |
| 455 |
"-c", |
| 456 |
f"source ~/.nvm/nvm.sh && nvm which {node_version}", |
| 457 |
], |
| 458 |
text=True, |
| 459 |
capture_output=True, |
| 460 |
check=True, |
| 461 |
) |
| 462 |
return res.stdout.strip() |
| 463 |
|
| 464 |
|
| 465 |
@subdir("host_dir", clean_ok=True) |
| 466 |
def configure_emscripten_python(context, working_dir): |
| 467 |
"""Configure the emscripten/host build.""" |
| 468 |
validate_emsdk_version(context.emsdk_cache) |
| 469 |
host_runner = context.host_runner |
| 470 |
if host_runner is None: |
| 471 |
host_runner = calculate_node_path() |
| 472 |
|
| 473 |
paths = context.build_paths |
| 474 |
config_site = os.fsdecode(EMSCRIPTEN_DIR / "config.site-wasm32-emscripten") |
| 475 |
|
| 476 |
emscripten_build_dir = working_dir.relative_to(CHECKOUT) |
| 477 |
|
| 478 |
python_build_dir = paths["native_build_dir"] / "build" |
| 479 |
lib_dirs = list(python_build_dir.glob("lib.*")) |
| 480 |
assert len(lib_dirs) == 1, ( |
| 481 |
f"Expected a single lib.* directory in {python_build_dir}" |
| 482 |
) |
| 483 |
lib_dir = os.fsdecode(lib_dirs[0]) |
| 484 |
pydebug = lib_dir.endswith("-pydebug") |
| 485 |
python_version = lib_dir.removesuffix("-pydebug").rpartition("-")[-1] |
| 486 |
sysconfig_data = ( |
| 487 |
f"{emscripten_build_dir}/build/lib.emscripten-wasm32-{python_version}" |
| 488 |
) |
| 489 |
if pydebug: |
| 490 |
sysconfig_data += "-pydebug" |
| 491 |
pkg_config_path_dir = (paths["prefix_dir"] / "lib/pkgconfig/").resolve() |
| 492 |
env_additions = { |
| 493 |
"CONFIG_SITE": config_site, |
| 494 |
"HOSTRUNNER": host_runner, |
| 495 |
"EM_PKG_CONFIG_PATH": str(pkg_config_path_dir), |
| 496 |
} |
| 497 |
build_python = os.fsdecode(build_python_path(context)) |
| 498 |
configure = [ |
| 499 |
"emconfigure", |
| 500 |
os.path.relpath(CHECKOUT / "configure", working_dir), |
| 501 |
"CFLAGS=-DPY_CALL_TRAMPOLINE -sUSE_BZIP2", |
| 502 |
"PKG_CONFIG=pkg-config", |
| 503 |
f"--host={HOST_TRIPLE}", |
| 504 |
f"--build={build_platform()}", |
| 505 |
f"--with-build-python={build_python}", |
| 506 |
"--without-pymalloc", |
| 507 |
"--disable-shared", |
| 508 |
"--disable-ipv6", |
| 509 |
"--enable-big-digits=30", |
| 510 |
"--enable-wasm-dynamic-linking", |
| 511 |
f"--prefix={paths['prefix_dir']}", |
| 512 |
] |
| 513 |
if pydebug: |
| 514 |
configure.append("--with-pydebug") |
| 515 |
if context.args: |
| 516 |
configure.extend(context.args) |
| 517 |
call( |
| 518 |
configure, |
| 519 |
env=updated_env(env_additions, context.emsdk_cache), |
| 520 |
quiet=context.quiet, |
| 521 |
) |
| 522 |
|
| 523 |
shutil.copy( |
| 524 |
EMSCRIPTEN_DIR / "node_entry.mjs", working_dir / "node_entry.mjs" |
| 525 |
) |
| 526 |
|
| 527 |
shutil.copy( |
| 528 |
EMSCRIPTEN_DIR / "streams.mjs", working_dir / "streams.mjs" |
| 529 |
) |
| 530 |
|
| 531 |
node_entry = working_dir / "node_entry.mjs" |
| 532 |
exec_script = working_dir / "python.sh" |
| 533 |
exec_script.write_text( |
| 534 |
dedent( |
| 535 |
f"""\ |
| 536 |
#!/bin/sh |
| 537 |
|
| 538 |
# Macs come with FreeBSD coreutils which doesn't have the -s option |
| 539 |
# so feature detect and work around it. |
| 540 |
if which grealpath > /dev/null 2>&1; then |
| 541 |
# It has brew installed gnu core utils, use that |
| 542 |
REALPATH="grealpath -s" |
| 543 |
elif which realpath > /dev/null 2>&1 && realpath --version > /dev/null 2>&1 && realpath --version | grep GNU > /dev/null 2>&1; then |
| 544 |
# realpath points to GNU realpath so use it. |
| 545 |
REALPATH="realpath -s" |
| 546 |
else |
| 547 |
# Shim for macs without GNU coreutils |
| 548 |
abs_path () {{ |
| 549 |
echo "$(cd "$(dirname "$1")" || exit; pwd)/$(basename "$1")" |
| 550 |
}} |
| 551 |
REALPATH=abs_path |
| 552 |
fi |
| 553 |
|
| 554 |
# Before node 24, --experimental-wasm-jspi uses different API, |
| 555 |
# After node 24 JSPI is on by default. |
| 556 |
ARGS=$({host_runner} -e "$(cat <<"EOF" |
| 557 |
const major_version = Number(process.version.split(".")[0].slice(1)); |
| 558 |
if (major_version === 24) {{ |
| 559 |
process.stdout.write("--experimental-wasm-jspi"); |
| 560 |
}} |
| 561 |
EOF |
| 562 |
)") |
| 563 |
|
| 564 |
# We compute our own path, not following symlinks and pass it in so that |
| 565 |
# node_entry.mjs can set sys.executable correctly. |
| 566 |
# Intentionally allow word splitting on NODEFLAGS. |
| 567 |
exec {host_runner} $NODEFLAGS $ARGS {node_entry} --this-program="$($REALPATH "$0")" "$@" |
| 568 |
""" |
| 569 |
) |
| 570 |
) |
| 571 |
exec_script.chmod(0o755) |
| 572 |
print(f"๐โโ๏ธ Created {exec_script} ... ") |
| 573 |
sys.stdout.flush() |
| 574 |
|
| 575 |
|
| 576 |
@subdir("host_dir") |
| 577 |
def make_emscripten_python(context, working_dir): |
| 578 |
"""Run `make` for the emscripten/host build.""" |
| 579 |
call( |
| 580 |
["make", "--jobs", str(cpu_count()), "all"], |
| 581 |
env=updated_env({}, context.emsdk_cache), |
| 582 |
quiet=context.quiet, |
| 583 |
) |
| 584 |
|
| 585 |
exec_script = working_dir / "python.sh" |
| 586 |
subprocess.check_call([exec_script, "--version"]) |
| 587 |
|
| 588 |
|
| 589 |
def run_emscripten_python(context, args=None): |
| 590 |
"""Run the built emscripten Python.""" |
| 591 |
host_dir = context.build_paths["host_dir"] |
| 592 |
exec_script = host_dir / "python.sh" |
| 593 |
if not exec_script.is_file(): |
| 594 |
print("Emscripten not built", file=sys.stderr) |
| 595 |
sys.exit(1) |
| 596 |
|
| 597 |
if args is None: |
| 598 |
args = context.args |
| 599 |
# Strip the "--" separator if present |
| 600 |
if args and args[0] == "--": |
| 601 |
args = args[1:] |
| 602 |
|
| 603 |
if context.test: |
| 604 |
args = load_config_toml()["test-args"] + args |
| 605 |
elif context.pythoninfo: |
| 606 |
args = load_config_toml()["pythoninfo-args"] + args |
| 607 |
|
| 608 |
os.execv(str(exec_script), [str(exec_script), *args]) |
| 609 |
|
| 610 |
|
| 611 |
def pythoninfo_emscripten_python(context): |
| 612 |
"""Display build info of the host/Emscripten Python.""" |
| 613 |
run_emscripten_python(context, ["-m", "test.pythoninfo"]) |
| 614 |
|
| 615 |
|
| 616 |
def build_target(context): |
| 617 |
"""Build one or more targets.""" |
| 618 |
steps = [] |
| 619 |
if context.target in {"build", "all"}: |
| 620 |
steps.extend([ |
| 621 |
configure_build_python, |
| 622 |
make_build_python, |
| 623 |
pythoninfo_build_python, |
| 624 |
]) |
| 625 |
if context.target in {"host", "all"}: |
| 626 |
steps.extend([ |
| 627 |
make_emscripten_libffi, |
| 628 |
make_mpdec, |
| 629 |
configure_emscripten_python, |
| 630 |
make_emscripten_python, |
| 631 |
pythoninfo_emscripten_python, |
| 632 |
]) |
| 633 |
|
| 634 |
for step in steps: |
| 635 |
step(context) |
| 636 |
|
| 637 |
|
| 638 |
def clean_contents(context): |
| 639 |
"""Delete all files created by this script.""" |
| 640 |
if context.target in {"all", "build"}: |
| 641 |
build_dir = context.build_paths["native_build_dir"] |
| 642 |
if build_dir.exists(): |
| 643 |
print(f"๐งน Deleting {build_dir} ...") |
| 644 |
shutil.rmtree(build_dir) |
| 645 |
|
| 646 |
if context.target in {"all", "host"}: |
| 647 |
host_triple_dir = context.build_paths["host_triple_dir"] |
| 648 |
if host_triple_dir.exists(): |
| 649 |
print(f"๐งน Deleting {host_triple_dir} ...") |
| 650 |
shutil.rmtree(host_triple_dir) |
| 651 |
|
| 652 |
if LOCAL_SETUP.exists(): |
| 653 |
with LOCAL_SETUP.open("rb") as file: |
| 654 |
if file.read(len(LOCAL_SETUP_MARKER)) == LOCAL_SETUP_MARKER: |
| 655 |
print(f"๐งน Deleting generated {LOCAL_SETUP} ...") |
| 656 |
|
| 657 |
|
| 658 |
def add_cross_build_dir_option(subcommand): |
| 659 |
subcommand.add_argument( |
| 660 |
"--cross-build-dir", |
| 661 |
action="store", |
| 662 |
default=os.environ.get("CROSS_BUILD_DIR"), |
| 663 |
dest="cross_build_dir", |
| 664 |
help=( |
| 665 |
"Path to the cross-build directory " |
| 666 |
f"(default: {DEFAULT_CROSS_BUILD_DIR}). " |
| 667 |
"Can also be set with the CROSS_BUILD_DIR environment variable." |
| 668 |
), |
| 669 |
) |
| 670 |
|
| 671 |
|
| 672 |
def main(): |
| 673 |
parser = argparse.ArgumentParser() |
| 674 |
subcommands = parser.add_subparsers(dest="subcommand") |
| 675 |
|
| 676 |
install_emscripten_cmd = subcommands.add_parser( |
| 677 |
"install-emscripten", |
| 678 |
help="Install the appropriate version of Emscripten", |
| 679 |
) |
| 680 |
|
| 681 |
build = subcommands.add_parser("build", help="Build everything") |
| 682 |
build.add_argument( |
| 683 |
"target", |
| 684 |
nargs="?", |
| 685 |
default="all", |
| 686 |
choices=["all", "host", "build"], |
| 687 |
help=( |
| 688 |
"What should be built. 'build' for just the build platform, or " |
| 689 |
"'host' for the host platform, or 'all' for both. Defaults to 'all'." |
| 690 |
), |
| 691 |
) |
| 692 |
|
| 693 |
configure_build = subcommands.add_parser( |
| 694 |
"configure-build-python", help="Run `configure` for the build Python" |
| 695 |
) |
| 696 |
|
| 697 |
make_mpdec_cmd = subcommands.add_parser( |
| 698 |
"make-mpdec", |
| 699 |
help="Clone mpdec repo, configure and build it for emscripten", |
| 700 |
) |
| 701 |
|
| 702 |
make_libffi_cmd = subcommands.add_parser( |
| 703 |
"make-libffi", |
| 704 |
help="Clone libffi repo, configure and build it for emscripten", |
| 705 |
) |
| 706 |
|
| 707 |
make_dependencies_cmd = subcommands.add_parser( |
| 708 |
"make-dependencies", |
| 709 |
help="Build all static library dependencies", |
| 710 |
) |
| 711 |
|
| 712 |
for cmd in [make_mpdec_cmd, make_libffi_cmd, make_dependencies_cmd]: |
| 713 |
cmd.add_argument( |
| 714 |
"--check-up-to-date", |
| 715 |
action="store_true", |
| 716 |
default=False, |
| 717 |
help=("If passed, will fail if dependency is out of date"), |
| 718 |
) |
| 719 |
|
| 720 |
make_build = subcommands.add_parser( |
| 721 |
"make-build-python", help="Run `make` for the build Python" |
| 722 |
) |
| 723 |
|
| 724 |
pythoninfo_build = subcommands.add_parser( |
| 725 |
"pythoninfo-build", help="Display build info of the build Python" |
| 726 |
) |
| 727 |
|
| 728 |
configure_host = subcommands.add_parser( |
| 729 |
"configure-host", |
| 730 |
help=( |
| 731 |
"Run `configure` for the host/emscripten " |
| 732 |
"(pydebug builds are inferred from the build Python)" |
| 733 |
), |
| 734 |
) |
| 735 |
|
| 736 |
make_host = subcommands.add_parser( |
| 737 |
"make-host", help="Run `make` for the host/emscripten" |
| 738 |
) |
| 739 |
|
| 740 |
pythoninfo_host = subcommands.add_parser( |
| 741 |
"pythoninfo-host", help="Display build info of the host/Emscripten Python" |
| 742 |
) |
| 743 |
|
| 744 |
run = subcommands.add_parser( |
| 745 |
"run", |
| 746 |
help="Run the built emscripten Python", |
| 747 |
) |
| 748 |
run.add_argument( |
| 749 |
"--test", |
| 750 |
action="store_true", |
| 751 |
default=False, |
| 752 |
help=( |
| 753 |
"Add the default test arguments to the beginning of the command. " |
| 754 |
"Default arguments loaded from Platforms/emscripten/config.toml" |
| 755 |
), |
| 756 |
) |
| 757 |
run.add_argument( |
| 758 |
"--pythoninfo", |
| 759 |
action="store_true", |
| 760 |
default=False, |
| 761 |
help="Run -m test.pythoninfo", |
| 762 |
) |
| 763 |
run.add_argument( |
| 764 |
"args", |
| 765 |
nargs=argparse.REMAINDER, |
| 766 |
help=( |
| 767 |
"Arguments to pass to the emscripten Python " |
| 768 |
"(use '--' to separate from run options)" |
| 769 |
), |
| 770 |
) |
| 771 |
add_cross_build_dir_option(run) |
| 772 |
|
| 773 |
clean = subcommands.add_parser( |
| 774 |
"clean", help="Delete files and directories created by this script" |
| 775 |
) |
| 776 |
clean.add_argument( |
| 777 |
"target", |
| 778 |
nargs="?", |
| 779 |
default="host", |
| 780 |
choices=["all", "host", "build"], |
| 781 |
help=( |
| 782 |
"What should be cleaned. 'build' for just the build platform, or " |
| 783 |
"'host' for the host platform, or 'all' for both. Defaults to 'host'." |
| 784 |
), |
| 785 |
) |
| 786 |
|
| 787 |
for subcommand in ( |
| 788 |
install_emscripten_cmd, |
| 789 |
build, |
| 790 |
configure_build, |
| 791 |
make_libffi_cmd, |
| 792 |
make_mpdec_cmd, |
| 793 |
make_dependencies_cmd, |
| 794 |
make_build, |
| 795 |
pythoninfo_build, |
| 796 |
configure_host, |
| 797 |
make_host, |
| 798 |
pythoninfo_host, |
| 799 |
clean, |
| 800 |
): |
| 801 |
subcommand.add_argument( |
| 802 |
"--quiet", |
| 803 |
action="store_true", |
| 804 |
default="QUIET" in os.environ, |
| 805 |
dest="quiet", |
| 806 |
help=( |
| 807 |
"Redirect output from subprocesses to a log file. " |
| 808 |
"Can also be set with the QUIET environment variable." |
| 809 |
), |
| 810 |
) |
| 811 |
add_cross_build_dir_option(subcommand) |
| 812 |
subcommand.add_argument( |
| 813 |
"--emsdk-cache", |
| 814 |
action="store", |
| 815 |
default=os.environ.get("EMSDK_CACHE"), |
| 816 |
dest="emsdk_cache", |
| 817 |
help=( |
| 818 |
"Path to emsdk cache directory. If provided, validates that " |
| 819 |
"the required emscripten version is installed. " |
| 820 |
"Can also be set with the EMSDK_CACHE environment variable." |
| 821 |
), |
| 822 |
) |
| 823 |
|
| 824 |
for subcommand in configure_build, configure_host: |
| 825 |
subcommand.add_argument( |
| 826 |
"--clean", |
| 827 |
action="store_true", |
| 828 |
default=False, |
| 829 |
dest="clean", |
| 830 |
help="Delete any relevant directories before building", |
| 831 |
) |
| 832 |
|
| 833 |
for subcommand in build, configure_build, configure_host: |
| 834 |
subcommand.add_argument( |
| 835 |
"args", nargs="*", help="Extra arguments to pass to `configure`" |
| 836 |
) |
| 837 |
|
| 838 |
for subcommand in build, configure_host: |
| 839 |
subcommand.add_argument( |
| 840 |
"--host-runner", |
| 841 |
action="store", |
| 842 |
default=None, |
| 843 |
dest="host_runner", |
| 844 |
help="Command template for running the emscripten host " |
| 845 |
"(default: use nvm to install the node version specified in config.toml)", |
| 846 |
) |
| 847 |
|
| 848 |
context = parser.parse_args() |
| 849 |
context.emsdk_cache = getattr(context, "emsdk_cache", None) |
| 850 |
context.cross_build_dir = getattr(context, "cross_build_dir", None) |
| 851 |
context.check_up_to_date = getattr(context, "check_up_to_date", False) |
| 852 |
|
| 853 |
if context.emsdk_cache: |
| 854 |
context.emsdk_cache = Path(context.emsdk_cache).absolute() |
| 855 |
|
| 856 |
context.build_paths = get_build_paths( |
| 857 |
context.cross_build_dir, context.emsdk_cache |
| 858 |
) |
| 859 |
|
| 860 |
dispatch = { |
| 861 |
"install-emscripten": install_emscripten, |
| 862 |
"make-libffi": make_emscripten_libffi, |
| 863 |
"make-mpdec": make_mpdec, |
| 864 |
"make-dependencies": make_dependencies, |
| 865 |
"configure-build-python": configure_build_python, |
| 866 |
"make-build-python": make_build_python, |
| 867 |
"pythoninfo-build": pythoninfo_build_python, |
| 868 |
"configure-host": configure_emscripten_python, |
| 869 |
"make-host": make_emscripten_python, |
| 870 |
"pythoninfo-host": pythoninfo_emscripten_python, |
| 871 |
"build": build_target, |
| 872 |
"run": run_emscripten_python, |
| 873 |
"clean": clean_contents, |
| 874 |
} |
| 875 |
|
| 876 |
if not context.subcommand: |
| 877 |
# No command provided, display help and exit |
| 878 |
print( |
| 879 |
"Expected one of", |
| 880 |
", ".join(sorted(dispatch.keys())), |
| 881 |
file=sys.stderr, |
| 882 |
) |
| 883 |
parser.print_help(sys.stderr) |
| 884 |
sys.exit(1) |
| 885 |
dispatch[context.subcommand](context) |
| 886 |
|
| 887 |
|
| 888 |
if __name__ == "__main__": |
| 889 |
main() |