Debian Codegraph @

Apple/__main__.py

raw
1 #!/usr/bin/env python3
2 ##########################################################################
3 # Apple XCframework build script
4 #
5 # This script simplifies the process of configuring, compiling and packaging an
6 # XCframework for an Apple platform.
7 #
8 # At present, it only supports iOS, but it has been constructed so that it
9 # could be used on any Apple platform.
10 #
11 # The simplest entry point is:
12 #
13 # $ python Apple ci iOS
14 #
15 # which will:
16 # * Clean any pre-existing build artefacts
17 # * Configure and make a Python that can be used for the build
18 # * Configure and make a Python for each supported iOS architecture and ABI
19 # * Combine the outputs of the builds from the previous step into a single
20 # XCframework, merging binaries into a "fat" binary if necessary
21 # * Clone a copy of the testbed, configured to use the XCframework
22 # * Construct a tarball containing the release artefacts
23 # * Run the test suite using the generated XCframework.
24 #
25 # This is the complete sequence that would be needed in CI to build and test
26 # a candidate release artefact.
27 #
28 # Each individual step can be invoked individually - there are commands to
29 # clean, configure-build, make-build, configure-host, make-host, package, and
30 # test.
31 #
32 # There is also a build command that can be used to combine the configure and
33 # make steps for the build Python, an individual host, all hosts, or all
34 # builds.
35 ##########################################################################
36 from __future__ import annotations
37
38 import argparse
39 import os
40 import platform
41 import re
42 import shlex
43 import shutil
44 import signal
45 import subprocess
46 import sys
47 import sysconfig
48 import time
49 from collections.abc import Callable, Sequence
50 from contextlib import contextmanager
51 from datetime import datetime, timezone
52 from os.path import basename, relpath
53 from pathlib import Path
54 from subprocess import CalledProcessError
55
56 EnvironmentT = dict[str, str]
57 ArgsT = Sequence[str | Path]
58
59 SCRIPT_NAME = Path(__file__).name
60 PYTHON_DIR = Path(__file__).resolve().parent.parent
61
62 CROSS_BUILD_DIR = PYTHON_DIR / "cross-build"
63
64 HOSTS: dict[str, dict[str, dict[str, str]]] = {
65 # Structure of this data:
66 # * Platform identifier
67 # * an XCframework slice that must exist for that platform
68 # * a host triple: the multiarch spec for that host
69 "iOS": {
70 "ios-arm64": {
71 "arm64-apple-ios": "arm64-iphoneos",
72 },
73 "ios-arm64_x86_64-simulator": {
74 "arm64-apple-ios-simulator": "arm64-iphonesimulator",
75 "x86_64-apple-ios-simulator": "x86_64-iphonesimulator",
76 },
77 },
78 }
79
80
81 def subdir(name: str, create: bool = False) -> Path:
82 """Ensure that a cross-build directory for the given name exists."""
83 path = CROSS_BUILD_DIR / name
84 if not path.exists():
85 if not create:
86 sys.exit(
87 f"{path} does not exist. Create it by running the appropriate "
88 f"`configure` subcommand of {SCRIPT_NAME}."
89 )
90 else:
91 path.mkdir(parents=True)
92 return path
93
94
95 def run(
96 command: ArgsT,
97 *,
98 host: str | None = None,
99 env: EnvironmentT | None = None,
100 log: bool | None = True,
101 **kwargs,
102 ) -> subprocess.CompletedProcess:
103 """Run a command in an Apple development environment.
104
105 Optionally logs the executed command to the console.
106 """
107 kwargs.setdefault("check", True)
108 if env is None:
109 env = os.environ.copy()
110
111 if host:
112 host_env = apple_env(host)
113 print_env(host_env)
114 env.update(host_env)
115
116 if log:
117 print(">", join_command(command))
118 return subprocess.run(command, env=env, **kwargs)
119
120
121 def join_command(args: str | Path | ArgsT) -> str:
122 """Format a command so it can be copied into a shell.
123
124 Similar to `shlex.join`, but also accepts arguments which are Paths, or a
125 single string/Path outside of a list.
126 """
127 if isinstance(args, (str, Path)):
128 return str(args)
129 else:
130 return shlex.join(map(str, args))
131
132
133 def print_env(env: EnvironmentT) -> None:
134 """Format the environment so it can be pasted into a shell."""
135 for key, value in sorted(env.items()):
136 print(f"export {key}={shlex.quote(value)}")
137
138
139 def apple_env(host: str) -> EnvironmentT:
140 """Construct an Apple development environment for the given host."""
141 env = {
142 "PATH": ":".join([
143 str(PYTHON_DIR / "Apple/iOS/Resources/bin"),
144 str(subdir(host) / "prefix"),
145 "/usr/bin",
146 "/bin",
147 "/usr/sbin",
148 "/sbin",
149 "/Library/Apple/usr/bin",
150 ]),
151 }
152
153 return env
154
155
156 def delete_path(name: str) -> None:
157 """Delete the named cross-build directory, if it exists."""
158 path = CROSS_BUILD_DIR / name
159 if path.exists():
160 print(f"Deleting {path} ...")
161 shutil.rmtree(path)
162
163
164 def all_host_triples(platform: str) -> list[str]:
165 """Return all host triples for the given platform.
166
167 The host triples are the platform definitions used as input to configure
168 (e.g., "arm64-apple-ios-simulator").
169 """
170 triples = []
171 for slice_name, slice_parts in HOSTS[platform].items():
172 triples.extend(list(slice_parts))
173 return triples
174
175
176 def clean(context: argparse.Namespace, target: str | None = None) -> None:
177 """The implementation of the "clean" command."""
178 if target is None:
179 target = context.host
180
181 # If we're explicitly targeting the build, there's no platform or
182 # distribution artefacts. If we're cleaning tests, we keep all built
183 # artefacts. Otherwise, the built artefacts must be dirty, so we remove
184 # them.
185 if target not in {"build", "test"}:
186 paths = ["dist", context.platform] + list(HOSTS[context.platform])
187 else:
188 paths = []
189
190 if target in {"all", "build"}:
191 paths.append("build")
192
193 if target in {"all", "hosts"}:
194 paths.extend(all_host_triples(context.platform))
195 elif target not in {"build", "test", "package"}:
196 paths.append(target)
197
198 if target in {"all", "hosts", "test"}:
199 paths.extend([
200 path.name
201 for path in CROSS_BUILD_DIR.glob(f"{context.platform}-testbed.*")
202 ])
203
204 for path in paths:
205 delete_path(path)
206
207
208 def build_python_path() -> Path:
209 """The path to the build Python binary."""
210 build_dir = subdir("build")
211 binary = build_dir / "python"
212 if not binary.is_file():
213 binary = binary.with_suffix(".exe")
214 if not binary.is_file():
215 raise FileNotFoundError(
216 f"Unable to find `python(.exe)` in {build_dir}"
217 )
218
219 return binary
220
221
222 @contextmanager
223 def group(text: str):
224 """A context manager that outputs a log marker around a section of a build.
225
226 If running in a GitHub Actions environment, the GitHub syntax for
227 collapsible log sections is used.
228 """
229 if "GITHUB_ACTIONS" in os.environ:
230 print(f"::group::{text}")
231 else:
232 print(f"===== {text} " + "=" * (70 - len(text)))
233
234 yield
235
236 if "GITHUB_ACTIONS" in os.environ:
237 print("::endgroup::")
238 else:
239 print()
240
241
242 @contextmanager
243 def cwd(subdir: Path):
244 """A context manager that sets the current working directory."""
245 orig = os.getcwd()
246 os.chdir(subdir)
247 yield
248 os.chdir(orig)
249
250
251 def configure_build_python(context: argparse.Namespace) -> None:
252 """The implementation of the "configure-build" command."""
253 if context.clean:
254 clean(context, "build")
255
256 with (
257 group("Configuring build Python"),
258 cwd(subdir("build", create=True)),
259 ):
260 command = [relpath(PYTHON_DIR / "configure")]
261 if context.args:
262 command.extend(context.args)
263 run(command)
264
265
266 def make_build_python(context: argparse.Namespace) -> None:
267 """The implementation of the "make-build" command."""
268 with (
269 group("Compiling build Python"),
270 cwd(subdir("build")),
271 ):
272 run(["make", "-j", str(os.cpu_count())])
273
274
275 def pythoninfo_build_python(context: argparse.Namespace) -> None:
276 """The implementation of the "pythoninfo-build" command."""
277 with (
278 group("Display build info of the build Python"),
279 cwd(subdir("build")),
280 ):
281 run(["make", "pythoninfo"])
282
283
284 def apple_target(host: str) -> str:
285 """Return the Apple platform identifier for a given host triple."""
286 for _, platform_slices in HOSTS.items():
287 for slice_name, slice_parts in platform_slices.items():
288 for host_triple, multiarch in slice_parts.items():
289 if host == host_triple:
290 return ".".join(multiarch.split("-")[::-1])
291
292 raise KeyError(host)
293
294
295 def apple_multiarch(host: str) -> str:
296 """Return the multiarch descriptor for a given host triple."""
297 for _, platform_slices in HOSTS.items():
298 for slice_name, slice_parts in platform_slices.items():
299 for host_triple, multiarch in slice_parts.items():
300 if host == host_triple:
301 return multiarch
302
303 raise KeyError(host)
304
305
306 def unpack_deps(
307 platform: str,
308 host: str,
309 prefix_dir: Path,
310 cache_dir: Path,
311 ) -> None:
312 """Unpack binary dependencies into a provided directory.
313
314 Downloads binaries if they aren't already present. Downloads will be stored
315 in provided cache directory.
316
317 On iOS, as a safety mechanism, any dynamic libraries will be purged from
318 the unpacked dependencies.
319 """
320 # To create new builds of these dependencies, usually all that's necessary
321 # is to push a tag to the cpython-apple-source-deps repository, and GitHub
322 # Actions will do the rest.
323 #
324 # If you're a member of the Python core team, and you'd like to be able to
325 # push these tags yourself, please contact Malcolm Smith or Russell
326 # Keith-Magee.
327 deps_url = "https://github.com/beeware/cpython-apple-source-deps/releases/download"
328 for name_ver in [
329 "BZip2-1.0.8-2",
330 "libFFI-3.4.7-2",
331 "OpenSSL-3.5.7-1",
332 "XZ-5.6.4-2",
333 "mpdecimal-4.0.0-2",
334 "zstd-1.5.7-1",
335 ]:
336 filename = f"{name_ver.lower()}-{apple_target(host)}.tar.gz"
337 archive_path = download(
338 f"{deps_url}/{name_ver}/{filename}",
339 target_dir=cache_dir,
340 )
341 shutil.unpack_archive(archive_path, prefix_dir)
342
343 # Dynamic libraries will be preferentially linked over static;
344 # On iOS, ensure that no dylibs are available in the prefix folder.
345 if platform == "iOS":
346 for dylib in prefix_dir.glob("**/*.dylib"):
347 dylib.unlink()
348
349
350 def download(url: str, target_dir: Path) -> Path:
351 """Download the specified URL into the given directory.
352
353 :return: The path to the downloaded archive.
354 """
355 target_path = Path(target_dir).resolve()
356 target_path.mkdir(exist_ok=True, parents=True)
357
358 out_path = target_path / basename(url)
359 if not Path(out_path).is_file():
360 run([
361 "curl",
362 "-Lf",
363 "--retry",
364 "5",
365 "--retry-all-errors",
366 "-o",
367 out_path,
368 url,
369 ])
370 else:
371 print(f"Using cached version of {basename(url)}")
372 return out_path
373
374
375 def configure_host_python(
376 context: argparse.Namespace,
377 host: str | None = None,
378 ) -> None:
379 """The implementation of the "configure-host" command."""
380 if host is None:
381 host = context.host
382
383 if context.clean:
384 clean(context, host)
385
386 host_dir = subdir(host, create=True)
387 prefix_dir = host_dir / "prefix"
388
389 with group(f"Downloading dependencies ({host})"):
390 if not prefix_dir.exists():
391 prefix_dir.mkdir()
392 cache_dir = (
393 Path(context.cache_dir).resolve()
394 if context.cache_dir
395 else CROSS_BUILD_DIR / "downloads"
396 )
397 unpack_deps(context.platform, host, prefix_dir, cache_dir)
398 else:
399 print("Dependencies already installed")
400
401 with (
402 group(f"Configuring host Python ({host})"),
403 cwd(host_dir),
404 ):
405 command = [
406 # Basic cross-compiling configuration
407 relpath(PYTHON_DIR / "configure"),
408 f"--host={host}",
409 f"--build={sysconfig.get_config_var('BUILD_GNU_TYPE')}",
410 f"--with-build-python={build_python_path()}",
411 "--with-system-libmpdec",
412 "--enable-framework",
413 # Dependent libraries.
414 f"--with-openssl={prefix_dir}",
415 f"LIBLZMA_CFLAGS=-I{prefix_dir}/include",
416 f"LIBLZMA_LIBS=-L{prefix_dir}/lib -llzma",
417 f"LIBFFI_CFLAGS=-I{prefix_dir}/include",
418 f"LIBFFI_LIBS=-L{prefix_dir}/lib -lffi",
419 f"LIBMPDEC_CFLAGS=-I{prefix_dir}/include",
420 f"LIBMPDEC_LIBS=-L{prefix_dir}/lib -lmpdec",
421 f"LIBZSTD_CFLAGS=-I{prefix_dir}/include",
422 f"LIBZSTD_LIBS=-L{prefix_dir}/lib -lzstd",
423 ]
424
425 if context.args:
426 command.extend(context.args)
427 run(command, host=host)
428
429
430 def make_host_python(
431 context: argparse.Namespace,
432 host: str | None = None,
433 ) -> None:
434 """The implementation of the "make-host" command."""
435 if host is None:
436 host = context.host
437
438 with (
439 group(f"Compiling host Python ({host})"),
440 cwd(subdir(host)),
441 ):
442 run(["make", "-j", str(os.cpu_count())], host=host)
443 run(["make", "install"], host=host)
444
445
446 def framework_path(host_triple: str, multiarch: str) -> Path:
447 """The path to a built single-architecture framework product.
448
449 :param host_triple: The host triple (e.g., arm64-apple-ios-simulator)
450 :param multiarch: The multiarch identifier (e.g., arm64-simulator)
451 """
452 return CROSS_BUILD_DIR / f"{host_triple}/Apple/iOS/Frameworks/{multiarch}"
453
454
455 def package_version(prefix_path: Path) -> str:
456 """Extract the Python version being built from patchlevel.h."""
457 for path in prefix_path.glob("**/patchlevel.h"):
458 text = path.read_text(encoding="utf-8")
459 if match := re.search(
460 r'\n\s*#define\s+PY_VERSION\s+"(.+)"\s*\n', text
461 ):
462 version = match[1]
463 # If not building against a tagged commit, add a timestamp to the
464 # version. Follow the PyPA version number rules, as this will make
465 # it easier to process with other tools. The version will have a
466 # `+` suffix once any official release has been made; a freshly
467 # forked main branch will have a version of 3.X.0a0.
468 if version.endswith("a0"):
469 version += "+"
470 if version.endswith("+"):
471 version += datetime.now(timezone.utc).strftime("%Y%m%d.%H%M%S")
472
473 return version
474
475 sys.exit("Unable to determine Python version being packaged.")
476
477
478 def lib_platform_files(dirname, names):
479 """A file filter that ignores platform-specific files in lib."""
480 path = Path(dirname)
481 if (
482 path.parts[-3] == "lib"
483 and path.parts[-2].startswith("python")
484 and path.parts[-1] == "lib-dynload"
485 ):
486 return names
487 elif path.parts[-2] == "lib" and path.parts[-1].startswith("python"):
488 ignored_names = {
489 name
490 for name in names
491 if (
492 name.startswith("_sysconfigdata_")
493 or name.startswith("_sysconfig_vars_")
494 or name == "build-details.json"
495 )
496 }
497 elif path.parts[-1] == "lib":
498 ignored_names = {
499 name
500 for name in names
501 if name.startswith("libpython") and name.endswith(".dylib")
502 }
503 else:
504 ignored_names = set()
505
506 return ignored_names
507
508
509 def lib_non_platform_files(dirname, names):
510 """A file filter that ignores anything *except* platform-specific files
511 in the lib directory.
512 """
513 path = Path(dirname)
514 if path.parts[-2] == "lib" and path.parts[-1].startswith("python"):
515 return (
516 set(names) - lib_platform_files(dirname, names) - {"lib-dynload"}
517 )
518 else:
519 return set()
520
521
522 def create_xcframework(platform: str) -> str:
523 """Build an XCframework from the component parts for the platform.
524
525 :return: The version number of the Python version that was packaged.
526 """
527 package_path = CROSS_BUILD_DIR / platform
528 try:
529 package_path.mkdir()
530 except FileExistsError:
531 raise RuntimeError(
532 f"{platform} XCframework already exists; do you need to run "
533 "with --clean?"
534 ) from None
535
536 frameworks = []
537 # Merge Frameworks for each component SDK. If there's only one architecture
538 # for the SDK, we can use the compiled Python.framework as-is. However, if
539 # there's more than architecture, we need to merge the individual built
540 # frameworks into a merged "fat" framework.
541 for slice_name, slice_parts in HOSTS[platform].items():
542 # Some parts are the same across all slices, so we use can any of the
543 # host frameworks as the source for the merged version. Use the first
544 # one on the list, as it's as representative as any other.
545 first_host_triple, first_multiarch = next(iter(slice_parts.items()))
546 first_framework = (
547 framework_path(first_host_triple, first_multiarch)
548 / "Python.framework"
549 )
550
551 if len(slice_parts) == 1:
552 # The first framework is the only framework, so copy it.
553 print(f"Copying framework for {slice_name}...")
554 frameworks.append(first_framework)
555 else:
556 print(f"Merging framework for {slice_name}...")
557 slice_path = CROSS_BUILD_DIR / slice_name
558 slice_framework = slice_path / "Python.framework"
559 slice_framework.mkdir(exist_ok=True, parents=True)
560
561 # Copy the Info.plist
562 shutil.copy(
563 first_framework / "Info.plist",
564 slice_framework / "Info.plist",
565 )
566
567 # Copy the headers
568 shutil.copytree(
569 first_framework / "Headers",
570 slice_framework / "Headers",
571 )
572
573 # Create the "fat" library binary for the slice
574 run(
575 ["lipo", "-create", "-output", slice_framework / "Python"]
576 + [
577 (
578 framework_path(host_triple, multiarch)
579 / "Python.framework/Python"
580 )
581 for host_triple, multiarch in slice_parts.items()
582 ]
583 )
584
585 # Add this merged slice to the list to be added to the XCframework
586 frameworks.append(slice_framework)
587
588 print()
589 print("Build XCframework...")
590 cmd = [
591 "xcodebuild",
592 "-create-xcframework",
593 "-output",
594 package_path / "Python.xcframework",
595 ]
596 for framework in frameworks:
597 cmd.extend(["-framework", framework])
598
599 run(cmd)
600
601 # Extract the package version from the merged framework
602 version = package_version(package_path / "Python.xcframework")
603 version_tag = ".".join(version.split(".")[:2])
604
605 # On non-macOS platforms, each framework in XCframework only contains the
606 # headers, libPython, plus an Info.plist. Other resources like the standard
607 # library and binary shims aren't allowed to live in framework; they need
608 # to be copied in separately.
609 print()
610 print("Copy additional resources...")
611 has_common_stdlib = False
612 for slice_name, slice_parts in HOSTS[platform].items():
613 # Some parts are the same across all slices, so we can any of the
614 # host frameworks as the source for the merged version.
615 first_host_triple, first_multiarch = next(iter(slice_parts.items()))
616 first_path = framework_path(first_host_triple, first_multiarch)
617 first_framework = first_path / "Python.framework"
618
619 slice_path = package_path / f"Python.xcframework/{slice_name}"
620 slice_framework = slice_path / "Python.framework"
621
622 # Copy the binary helpers
623 print(f" - {slice_name} binaries")
624 shutil.copytree(first_path / "bin", slice_path / "bin")
625
626 # Copy the include path (a symlink to the framework headers)
627 print(f" - {slice_name} include files")
628 shutil.copytree(
629 first_path / "include",
630 slice_path / "include",
631 symlinks=True,
632 )
633
634 # Copy in the cross-architecture pyconfig.h
635 shutil.copy(
636 PYTHON_DIR / f"Apple/{platform}/Resources/pyconfig.h",
637 slice_framework / "Headers/pyconfig.h",
638 )
639
640 print(f" - {slice_name} shared library")
641 # Create a simlink for the fat library
642 shared_lib = slice_path / f"lib/libpython{version_tag}.dylib"
643 shared_lib.parent.mkdir()
644 shared_lib.symlink_to("../Python.framework/Python")
645
646 print(f" - {slice_name} architecture-specific files")
647 for host_triple, multiarch in slice_parts.items():
648 print(f" - {multiarch} standard library")
649 arch, _ = multiarch.split("-", 1)
650
651 if not has_common_stdlib:
652 print(" - using this architecture as the common stdlib")
653 shutil.copytree(
654 framework_path(host_triple, multiarch) / "lib",
655 package_path / "Python.xcframework/lib",
656 ignore=lib_platform_files,
657 symlinks=True,
658 )
659 has_common_stdlib = True
660
661 shutil.copytree(
662 framework_path(host_triple, multiarch) / "lib",
663 slice_path / f"lib-{arch}",
664 ignore=lib_non_platform_files,
665 symlinks=True,
666 )
667
668 # Copy the host's pyconfig.h to an architecture-specific name.
669 arch = multiarch.split("-")[0]
670 host_path = (
671 CROSS_BUILD_DIR
672 / host_triple
673 / "Apple/iOS/Frameworks"
674 / multiarch
675 )
676 host_framework = host_path / "Python.framework"
677 shutil.copy(
678 host_framework / "Headers/pyconfig.h",
679 slice_framework / f"Headers/pyconfig-{arch}.h",
680 )
681
682 # Apple identifies certain libraries as "security risks"; if you
683 # statically link those libraries into a Framework, you become
684 # responsible for providing a privacy manifest for that framework.
685 xcprivacy_file = {
686 "OpenSSL": subdir(host_triple)
687 / "prefix/share/OpenSSL.xcprivacy"
688 }
689 print(f" - {multiarch} xcprivacy files")
690 for module, lib in [
691 ("_hashlib", "OpenSSL"),
692 ("_ssl", "OpenSSL"),
693 ]:
694 shutil.copy(
695 xcprivacy_file[lib],
696 slice_path
697 / f"lib-{arch}/python{version_tag}"
698 / f"lib-dynload/{module}.xcprivacy",
699 )
700
701 print(" - build tools")
702 shutil.copytree(
703 PYTHON_DIR / "Apple/testbed/Python.xcframework/build",
704 package_path / "Python.xcframework/build",
705 )
706
707 return version
708
709
710 def package(context: argparse.Namespace) -> None:
711 """The implementation of the "package" command."""
712 if context.clean:
713 clean(context, "package")
714
715 with group("Building package"):
716 # Create an XCframework
717 version = create_xcframework(context.platform)
718
719 # Clone testbed
720 print()
721 run([
722 sys.executable,
723 "Apple/testbed",
724 "clone",
725 "--platform",
726 context.platform,
727 "--framework",
728 CROSS_BUILD_DIR / context.platform / "Python.xcframework",
729 CROSS_BUILD_DIR / context.platform / "testbed",
730 ])
731
732 # Build the final archive
733 archive_name = (
734 CROSS_BUILD_DIR
735 / "dist"
736 / f"python-{version}-{context.platform}-XCframework"
737 )
738
739 print()
740 print("Create package archive...")
741 shutil.make_archive(
742 str(CROSS_BUILD_DIR / archive_name),
743 format="gztar",
744 root_dir=CROSS_BUILD_DIR / context.platform,
745 base_dir=".",
746 )
747 print()
748 print(f"{archive_name.relative_to(PYTHON_DIR)}.tar.gz created.")
749
750
751 def build(context: argparse.Namespace, host: str | None = None) -> None:
752 """The implementation of the "build" command."""
753 if host is None:
754 host = context.host
755
756 if context.clean:
757 clean(context, host)
758
759 if host in {"all", "build"}:
760 for step in [
761 configure_build_python,
762 make_build_python,
763 pythoninfo_build_python,
764 ]:
765 step(context)
766
767 if host == "build":
768 hosts = []
769 elif host in {"all", "hosts"}:
770 hosts = all_host_triples(context.platform)
771 else:
772 hosts = [host]
773
774 for step_host in hosts:
775 for step in [
776 configure_host_python,
777 make_host_python,
778 ]:
779 step(context, host=step_host)
780
781 if host == "all":
782 package(context)
783
784
785 def test(context: argparse.Namespace, host: str | None = None) -> None: # noqa: PT028
786 """The implementation of the "test" command."""
787 if host is None:
788 host = context.host
789
790 if context.clean:
791 clean(context, "test")
792
793 with group(f"Test {'XCframework' if host in {'all', 'hosts'} else host}"):
794 timestamp = str(time.time_ns())[:-6]
795 testbed_dir = (
796 CROSS_BUILD_DIR / f"{context.platform}-testbed.{timestamp}"
797 )
798 if host in {"all", "hosts"}:
799 framework_path = (
800 CROSS_BUILD_DIR / context.platform / "Python.xcframework"
801 )
802 else:
803 build_arch = platform.machine()
804 host_arch = host.split("-")[0]
805
806 if not host.endswith("-simulator"):
807 print("Skipping test suite non-simulator build.")
808 return
809 elif build_arch != host_arch:
810 print(
811 f"Skipping test suite for an {host_arch} build "
812 f"on an {build_arch} machine."
813 )
814 return
815 else:
816 framework_path = (
817 CROSS_BUILD_DIR
818 / host
819 / f"Apple/{context.platform}"
820 / f"Frameworks/{apple_multiarch(host)}"
821 )
822
823 run([
824 sys.executable,
825 "Apple/testbed",
826 "clone",
827 "--platform",
828 context.platform,
829 "--framework",
830 framework_path,
831 testbed_dir,
832 ])
833
834 run(
835 [
836 sys.executable,
837 testbed_dir,
838 "run",
839 "--verbose",
840 ]
841 + (
842 ["--simulator", str(context.simulator)]
843 if context.simulator
844 else []
845 )
846 + [
847 "--",
848 "test",
849 f"--{context.ci_mode or 'fast'}-ci",
850 "--single-process",
851 "--no-randomize",
852 # Timeout handling requires subprocesses; explicitly setting
853 # the timeout to -1 disables the faulthandler.
854 "--timeout=-1",
855 # Adding Python options requires the use of a subprocess to
856 # start a new Python interpreter.
857 "--dont-add-python-opts",
858 ]
859 )
860
861
862 def apple_sim_host(platform_name: str) -> str:
863 """Determine the native simulator target for this platform."""
864 for _, slice_parts in HOSTS[platform_name].items():
865 for host_triple in slice_parts:
866 parts = host_triple.split("-")
867 if parts[0] == platform.machine() and parts[-1] == "simulator":
868 return host_triple
869
870 raise KeyError(platform_name)
871
872
873 def ci(context: argparse.Namespace) -> None:
874 """The implementation of the "ci" command.
875
876 In "Fast" mode, this compiles the build python, and the simulator for the
877 build machine's architecture; and runs the test suite with `--fast-ci`
878 configuration.
879
880 In "Slow" mode, it compiles the build python, plus all candidate
881 architectures (both device and simulator); then runs the test suite with
882 `--slow-ci` configuration.
883 """
884 clean(context, "all")
885 if context.ci_mode == "slow":
886 # In slow mode, build and test the full XCframework
887 build(context, host="all")
888 test(context, host="all")
889 else:
890 # In fast mode, just build the simulator platform.
891 sim_host = apple_sim_host(context.platform)
892 build(context, host="build")
893 build(context, host=sim_host)
894 test(context, host=sim_host)
895
896
897 def parse_args() -> argparse.Namespace:
898 parser = argparse.ArgumentParser(
899 description=(
900 "A tool for managing the build, package and test process of "
901 "CPython on Apple platforms."
902 ),
903 )
904 parser.suggest_on_error = True
905 subcommands = parser.add_subparsers(dest="subcommand", required=True)
906
907 clean = subcommands.add_parser(
908 "clean",
909 help="Delete all build directories",
910 )
911
912 configure_build = subcommands.add_parser(
913 "configure-build", help="Run `configure` for the build Python"
914 )
915 make_build = subcommands.add_parser(
916 "make-build", help="Run `make` for the build Python"
917 )
918 pythoninfo_build = subcommands.add_parser(
919 "pythoninfo-build", help="Display build info of the build Python"
920 )
921 configure_host = subcommands.add_parser(
922 "configure-host",
923 help="Run `configure` for a specific platform and target",
924 )
925 make_host = subcommands.add_parser(
926 "make-host",
927 help="Run `make` for a specific platform and target",
928 )
929 package = subcommands.add_parser(
930 "package",
931 help="Create a release package for the platform",
932 )
933 build = subcommands.add_parser(
934 "build",
935 help="Build all platform targets and create the XCframework",
936 )
937 test = subcommands.add_parser(
938 "test",
939 help="Run the testbed for a specific platform",
940 )
941 ci = subcommands.add_parser(
942 "ci",
943 help="Run build, package, and test",
944 )
945
946 # platform argument
947 for cmd in [clean, configure_host, make_host, package, build, test, ci]:
948 cmd.add_argument(
949 "platform",
950 choices=HOSTS.keys(),
951 help="The target platform to build",
952 )
953
954 # host triple argument
955 for cmd in [configure_host, make_host]:
956 cmd.add_argument(
957 "host",
958 help="The host triple to build (e.g., arm64-apple-ios-simulator)",
959 )
960 # optional host triple argument
961 for cmd in [clean, build, test]:
962 cmd.add_argument(
963 "host",
964 nargs="?",
965 default="all",
966 help=(
967 "The host triple to build (e.g., arm64-apple-ios-simulator), "
968 "or 'build' for just the build platform, or 'hosts' for all "
969 "host platforms, or 'all' for the build platform and all "
970 "hosts. Defaults to 'all'"
971 ),
972 )
973
974 # --cross-build-dir argument
975 for cmd in [
976 clean,
977 configure_build,
978 make_build,
979 pythoninfo_build,
980 configure_host,
981 make_host,
982 build,
983 package,
984 test,
985 ci,
986 ]:
987 cmd.add_argument(
988 "--cross-build-dir",
989 action="store",
990 default=os.environ.get("CROSS_BUILD_DIR"),
991 dest="cross_build_dir",
992 type=Path,
993 help=(
994 "Path to the cross-build directory "
995 f"(default: {CROSS_BUILD_DIR}). Can also be set "
996 "with the CROSS_BUILD_DIR environment variable."
997 ),
998 )
999
1000 # --clean option
1001 for cmd in [configure_build, configure_host, build, package, test, ci]:
1002 cmd.add_argument(
1003 "--clean",
1004 action="store_true",
1005 default=False,
1006 dest="clean",
1007 help="Delete the relevant build directories first",
1008 )
1009
1010 # --cache-dir option
1011 for cmd in [configure_host, build, ci]:
1012 cmd.add_argument(
1013 "--cache-dir",
1014 default=os.environ.get("CACHE_DIR"),
1015 help="The directory to store cached downloads.",
1016 )
1017
1018 # --simulator option
1019 for cmd in [test, ci]:
1020 cmd.add_argument(
1021 "--simulator",
1022 help=(
1023 "The name of the simulator to use (eg: 'iPhone 16e'). "
1024 "Defaults to the most recently released 'entry level' "
1025 "iPhone device. Device architecture and OS version can also "
1026 "be specified; e.g., "
1027 "`--simulator 'iPhone 16 Pro,arch=arm64,OS=26.0'` would "
1028 "run on an ARM64 iPhone 16 Pro simulator running iOS 26.0."
1029 ),
1030 )
1031 group = cmd.add_mutually_exclusive_group()
1032 group.add_argument(
1033 "--fast-ci",
1034 action="store_const",
1035 dest="ci_mode",
1036 const="fast",
1037 help="Add test arguments for GitHub Actions",
1038 )
1039 group.add_argument(
1040 "--slow-ci",
1041 action="store_const",
1042 dest="ci_mode",
1043 const="slow",
1044 help="Add test arguments for buildbots",
1045 )
1046
1047 for subcommand in [configure_build, configure_host, build, ci]:
1048 subcommand.add_argument(
1049 "args", nargs="*", help="Extra arguments to pass to `configure`"
1050 )
1051
1052 return parser.parse_args()
1053
1054
1055 def print_called_process_error(e: subprocess.CalledProcessError) -> None:
1056 for stream_name in ["stdout", "stderr"]:
1057 content = getattr(e, stream_name)
1058 stream = getattr(sys, stream_name)
1059 if content:
1060 stream.write(content)
1061 if not content.endswith("\n"):
1062 stream.write("\n")
1063
1064 # shlex uses single quotes, so we surround the command with double quotes.
1065 print(
1066 f'Command "{join_command(e.cmd)}" returned exit status {e.returncode}'
1067 )
1068
1069
1070 def main() -> None:
1071 # Handle SIGTERM the same way as SIGINT. This ensures that if we're
1072 # terminated by the buildbot worker, we'll make an attempt to clean up our
1073 # subprocesses.
1074 def signal_handler(*args):
1075 os.kill(os.getpid(), signal.SIGINT)
1076
1077 signal.signal(signal.SIGTERM, signal_handler)
1078
1079 # Process command line arguments
1080 context = parse_args()
1081
1082 # Set the CROSS_BUILD_DIR if an argument was provided
1083 if context.cross_build_dir:
1084 global CROSS_BUILD_DIR
1085 CROSS_BUILD_DIR = context.cross_build_dir.resolve()
1086
1087 dispatch: dict[str, Callable] = {
1088 "clean": clean,
1089 "configure-build": configure_build_python,
1090 "make-build": make_build_python,
1091 "pythoninfo-build": pythoninfo_build_python,
1092 "configure-host": configure_host_python,
1093 "make-host": make_host_python,
1094 "package": package,
1095 "build": build,
1096 "test": test,
1097 "ci": ci,
1098 }
1099
1100 try:
1101 dispatch[context.subcommand](context)
1102 except CalledProcessError as e:
1103 print()
1104 print_called_process_error(e)
1105 sys.exit(1)
1106 except RuntimeError as e:
1107 print()
1108 print(e)
1109 sys.exit(2)
1110
1111
1112 if __name__ == "__main__":
1113 # Under the buildbot, stdout is not a TTY, but we must still flush after
1114 # every line to make sure our output appears in the correct order relative
1115 # to the output of our subprocesses.
1116 for stream in [sys.stdout, sys.stderr]:
1117 stream.reconfigure(line_buffering=True)
1118
1119 main()