Debian Codegraph @

Android/android.py

raw
1 #!/usr/bin/env python3
2
3 import asyncio
4 import argparse
5 import json
6 import os
7 import platform
8 import re
9 import shlex
10 import shutil
11 import signal
12 import subprocess
13 import sys
14 import sysconfig
15 from asyncio import wait_for
16 from contextlib import asynccontextmanager
17 from datetime import datetime, timezone
18 from enum import IntEnum, auto
19 from glob import glob
20 from os.path import abspath, basename, relpath
21 from pathlib import Path
22 from subprocess import CalledProcessError
23 from tempfile import TemporaryDirectory
24
25
26 SCRIPT_NAME = Path(__file__).name
27 ANDROID_DIR = Path(__file__).resolve().parent
28 PYTHON_DIR = ANDROID_DIR.parent
29 in_source_tree = (
30 ANDROID_DIR.name == "Android" and (PYTHON_DIR / "pyconfig.h.in").exists()
31 )
32
33 ENV_SCRIPT = ANDROID_DIR / "android-env.sh"
34 TESTBED_DIR = ANDROID_DIR / "testbed"
35 CROSS_BUILD_DIR = PYTHON_DIR / "cross-build"
36
37 HOSTS = [
38 "aarch64-linux-android",
39 "arm-linux-androideabi",
40 "i686-linux-android",
41 "x86_64-linux-android",
42 ]
43 APP_ID = "org.python.testbed"
44 DECODE_ARGS = ("UTF-8", "backslashreplace")
45
46
47 try:
48 android_home = Path(os.environ['ANDROID_HOME'])
49 except KeyError:
50 sys.exit("The ANDROID_HOME environment variable is required.")
51
52 adb = Path(
53 f"{android_home}/platform-tools/adb"
54 + (".exe" if os.name == "nt" else "")
55 )
56
57 gradlew = Path(
58 f"{TESTBED_DIR}/gradlew"
59 + (".bat" if os.name == "nt" else "")
60 )
61
62 # Whether we've seen any output from Python yet.
63 python_started = False
64
65 # Buffer for verbose output which will be displayed only if a test fails and
66 # there has been no output from Python.
67 hidden_output = []
68
69
70 # Based on android/log.h in the NDK.
71 class LogPriority(IntEnum):
72 UNKNOWN = 0
73 DEFAULT = auto()
74 VERBOSE = auto()
75 DEBUG = auto()
76 INFO = auto()
77 WARN = auto()
78 ERROR = auto()
79 FATAL = auto()
80 SILENT = auto()
81
82
83 def log_verbose(context, line, stream=sys.stdout):
84 if context.verbose:
85 stream.write(line)
86 else:
87 hidden_output.append((stream, line))
88
89
90 def delete_glob(pattern):
91 # Path.glob doesn't accept non-relative patterns.
92 for path in glob(str(pattern)):
93 path = Path(path)
94 print(f"Deleting {path} ...")
95 if path.is_dir() and not path.is_symlink():
96 shutil.rmtree(path)
97 else:
98 path.unlink()
99
100
101 def subdir(*parts, create=False):
102 path = CROSS_BUILD_DIR.joinpath(*parts)
103 if not path.exists():
104 if not create:
105 sys.exit(
106 f"{path} does not exist. Create it by running the appropriate "
107 f"`configure` subcommand of {SCRIPT_NAME}.")
108 else:
109 path.mkdir(parents=True)
110 return path
111
112
113 def run(command, *, host=None, env=None, log=True, **kwargs):
114 kwargs.setdefault("check", True)
115 if env is None:
116 env = os.environ.copy()
117
118 if host:
119 host_env = android_env(host)
120 print_env(host_env)
121 env.update(host_env)
122
123 if log:
124 print(">", join_command(command))
125 return subprocess.run(command, env=env, **kwargs)
126
127
128 # Format a command so it can be copied into a shell. Like shlex.join, but also
129 # accepts arguments which are Paths, or a single string/Path outside of a list.
130 def join_command(args):
131 if isinstance(args, (str, Path)):
132 return str(args)
133 else:
134 return shlex.join(map(str, args))
135
136
137 # Format the environment so it can be pasted into a shell.
138 def print_env(env):
139 for key, value in sorted(env.items()):
140 print(f"export {key}={shlex.quote(value)}")
141
142
143 def android_env(host):
144 if host:
145 prefix = subdir(host) / "prefix"
146 else:
147 prefix = ANDROID_DIR / "prefix"
148 sysconfig_files = prefix.glob("lib/python*/_sysconfigdata__android_*.py")
149 sysconfig_filename = next(sysconfig_files).name
150 host = re.fullmatch(r"_sysconfigdata__android_(.+).py", sysconfig_filename)[1]
151
152 env_output = subprocess.run(
153 f"set -eu; "
154 f"HOST={host}; "
155 f"PREFIX={prefix}; "
156 f". {ENV_SCRIPT}; "
157 f"export",
158 check=True, shell=True, stdout=subprocess.PIPE, encoding='utf-8',
159 ).stdout
160
161 env = {}
162 for line in env_output.splitlines():
163 # We don't require every line to match, as there may be some other
164 # output from installing the NDK.
165 if match := re.search(
166 "^(declare -x |export )?(\\w+)=['\"]?(.*?)['\"]?$", line
167 ):
168 key, value = match[2], match[3]
169 if os.environ.get(key) != value:
170 env[key] = value
171
172 if not env:
173 raise ValueError(f"Found no variables in {ENV_SCRIPT.name} output:\n"
174 + env_output)
175 return env
176
177
178 def build_python_path():
179 """The path to the build Python binary."""
180 build_dir = subdir("build")
181 binary = build_dir / "python"
182 if not binary.is_file():
183 binary = binary.with_suffix(".exe")
184 if not binary.is_file():
185 raise FileNotFoundError("Unable to find `python(.exe)` in "
186 f"{build_dir}")
187
188 return binary
189
190
191 def configure_build_python(context):
192 if context.clean:
193 clean("build")
194 os.chdir(subdir("build", create=True))
195
196 command = [relpath(PYTHON_DIR / "configure")]
197 if context.args:
198 command.extend(context.args)
199 run(command)
200
201
202 def make_build_python(context):
203 os.chdir(subdir("build"))
204 run(["make", "-j", str(os.cpu_count())])
205
206
207 def pythoninfo_build_python(context):
208 os.chdir(subdir("build"))
209 run(["make", "pythoninfo"])
210
211
212 # To create new builds of these dependencies, usually all that's necessary is to
213 # push a tag to the cpython-android-source-deps repository, and GitHub Actions
214 # will do the rest.
215 #
216 # If you're a member of the Python core team, and you'd like to be able to push
217 # these tags yourself, please contact Malcolm Smith or Russell Keith-Magee.
218 def unpack_deps(host, prefix_dir, cache_dir):
219 os.chdir(prefix_dir)
220 deps_url = "https://github.com/beeware/cpython-android-source-deps/releases/download"
221 for name_ver in [
222 "bzip2-1.0.8-3",
223 "libffi-3.4.4-3",
224 "openssl-3.0.21-0",
225 "sqlite-3.50.4-0",
226 "xz-5.4.6-1"
227 ]:
228 filename = f"{name_ver}-{host}.tar.gz"
229 out_path = download(f"{deps_url}/{name_ver}/{filename}", cache_dir)
230 shutil.unpack_archive(out_path)
231
232
233 def download(url, cache_dir):
234 out_path = cache_dir / basename(url)
235 cache_dir.mkdir(parents=True, exist_ok=True)
236 if not out_path.is_file():
237 run(["curl", "-Lf", "--retry", "5", "--retry-all-errors", "-o", out_path, url])
238 else:
239 print(f"Using cached version of {basename(url)}")
240 return out_path
241
242
243 def configure_host_python(context, host=None):
244 if host is None:
245 host = context.host
246 if context.clean:
247 clean(host)
248
249 host_dir = subdir(host, create=True)
250 prefix_dir = host_dir / "prefix"
251 if not prefix_dir.exists():
252 prefix_dir.mkdir()
253 cache_dir = (
254 Path(context.cache_dir).resolve()
255 if context.cache_dir
256 else CROSS_BUILD_DIR / "downloads"
257 )
258 unpack_deps(host, prefix_dir, cache_dir)
259
260 os.chdir(host_dir)
261 command = [
262 # Basic cross-compiling configuration
263 relpath(PYTHON_DIR / "configure"),
264 f"--host={host}",
265 f"--build={sysconfig.get_config_var('BUILD_GNU_TYPE')}",
266 f"--with-build-python={build_python_path()}",
267 "--without-ensurepip",
268
269 # Android always uses a shared libpython.
270 "--enable-shared",
271 "--without-static-libpython",
272
273 # Dependent libraries. The others are found using pkg-config: see
274 # android-env.sh.
275 f"--with-openssl={prefix_dir}",
276 ]
277
278 if context.args:
279 command.extend(context.args)
280 run(command, host=host)
281
282
283 def make_host_python(context, host=None):
284 if host is None:
285 host = context.host
286 # The CFLAGS and LDFLAGS set in android-env include the prefix dir, so
287 # delete any previous Python installation to prevent it being used during
288 # the build.
289 host_dir = subdir(host)
290 prefix_dir = host_dir / "prefix"
291 for pattern in ("include/python*", "lib/libpython*", "lib/python*"):
292 delete_glob(f"{prefix_dir}/{pattern}")
293
294 # The Android environment variables were already captured in the Makefile by
295 # `configure`, and passing them again when running `make` may cause some
296 # flags to be duplicated. So we don't use the `host` argument here.
297 os.chdir(host_dir)
298 run(["make", "-j", str(os.cpu_count())])
299
300 # The `make install` output is very verbose and rarely useful, so
301 # suppress it by default.
302 run(
303 ["make", "install", f"prefix={prefix_dir}"],
304 capture_output=not context.verbose,
305 )
306
307
308 def build_targets(context):
309 if context.target in {"all", "build"}:
310 configure_build_python(context)
311 make_build_python(context)
312 pythoninfo_build_python(context)
313
314 for host in HOSTS:
315 if context.target in {"all", "hosts", host}:
316 configure_host_python(context, host)
317 make_host_python(context, host)
318
319
320 def clean(host):
321 delete_glob(CROSS_BUILD_DIR / host)
322
323
324 def clean_targets(context):
325 if context.target in {"all", "build"}:
326 clean("build")
327
328 for host in HOSTS:
329 if context.target in {"all", "hosts", host}:
330 clean(host)
331
332
333 def setup_ci():
334 if "GITHUB_ACTIONS" in os.environ:
335 # Enable emulator hardware acceleration
336 # (https://github.blog/changelog/2024-04-02-github-actions-hardware-accelerated-android-virtualization-now-available/).
337 if platform.system() == "Linux":
338 run(
339 ["sudo", "tee", "/etc/udev/rules.d/99-kvm4all.rules"],
340 input='KERNEL=="kvm", GROUP="kvm", MODE="0666", OPTIONS+="static_node=kvm"\n',
341 text=True,
342 )
343 run(["sudo", "udevadm", "control", "--reload-rules"])
344 run(["sudo", "udevadm", "trigger", "--name-match=kvm"])
345
346 # Free up disk space by deleting unused versions of the NDK
347 # (https://github.com/freakboy3742/pyspamsum/pull/108).
348 for line in ENV_SCRIPT.read_text().splitlines():
349 if match := re.fullmatch(r"ndk_version=(.+)", line):
350 ndk_version = match[1]
351 break
352 else:
353 raise ValueError(f"Failed to find NDK version in {ENV_SCRIPT.name}")
354
355 for item in (android_home / "ndk").iterdir():
356 if item.name[0].isdigit() and item.name != ndk_version:
357 delete_glob(item)
358
359
360 def setup_sdk():
361 sdkmanager = android_home / (
362 "cmdline-tools/latest/bin/sdkmanager"
363 + (".bat" if os.name == "nt" else "")
364 )
365
366 # Gradle will fail if it needs to install an SDK package whose license
367 # hasn't been accepted, so pre-accept all licenses.
368 if not all((android_home / "licenses" / path).exists() for path in [
369 "android-sdk-arm-dbt-license", "android-sdk-license"
370 ]):
371 run(
372 [sdkmanager, "--licenses"],
373 text=True,
374 capture_output=True,
375 input="y\n" * 100,
376 )
377
378 # Gradle may install this automatically, but we can't rely on that because
379 # we need to run adb within the logcat task.
380 if not adb.exists():
381 run([sdkmanager, "platform-tools"])
382
383
384 # To avoid distributing compiled artifacts without corresponding source code,
385 # the Gradle wrapper is not included in the CPython repository. Instead, we
386 # extract it from the Gradle GitHub repository.
387 def setup_testbed():
388 paths = ["gradlew", "gradlew.bat", "gradle/wrapper/gradle-wrapper.jar"]
389 if all((TESTBED_DIR / path).exists() for path in paths):
390 return
391
392 # Any version of the wrapper can run any reasonably close version of Gradle, so this
393 # doesn't need to match the Gradle version used for the build, which is specified in
394 # testbed/gradle/wrapper/gradle-wrapper.properties.
395 version = "9.5.0"
396
397 for path in paths:
398 out_path = TESTBED_DIR / path
399 out_path.parent.mkdir(exist_ok=True)
400 download(
401 f"https://raw.githubusercontent.com/gradle/gradle/v{version}/{path}",
402 out_path.parent,
403 )
404 os.chmod(out_path, 0o755)
405
406
407 # Work around a bug involving sys.exit and TaskGroups
408 # (https://github.com/python/cpython/issues/101515).
409 def exit(*args):
410 raise MySystemExit(*args)
411
412
413 class MySystemExit(Exception):
414 pass
415
416
417 # The `test` subcommand runs all subprocesses through this context manager so
418 # that no matter what happens, they can always be cancelled from another task,
419 # and they will always be cleaned up on exit.
420 @asynccontextmanager
421 async def async_process(*args, **kwargs):
422 process = await asyncio.create_subprocess_exec(*args, **kwargs)
423 try:
424 yield process
425 finally:
426 if process.returncode is None:
427 # Allow a reasonably long time for Gradle to clean itself up,
428 # because we don't want stale emulators left behind.
429 timeout = 10
430 process.terminate()
431 try:
432 await wait_for(process.wait(), timeout)
433 except TimeoutError:
434 print(
435 f"Command {args} did not terminate after {timeout} seconds "
436 f" - sending SIGKILL"
437 )
438 process.kill()
439
440 # Even after killing the process we must still wait for it,
441 # otherwise we'll get the warning "Exception ignored in __del__".
442 await wait_for(process.wait(), timeout=1)
443
444
445 async def async_check_output(*args, **kwargs):
446 async with async_process(
447 *args, stdout=subprocess.PIPE, stderr=subprocess.PIPE, **kwargs
448 ) as process:
449 stdout, stderr = await process.communicate()
450 if process.returncode == 0:
451 return stdout.decode(*DECODE_ARGS)
452 else:
453 raise CalledProcessError(
454 process.returncode, args,
455 stdout.decode(*DECODE_ARGS), stderr.decode(*DECODE_ARGS)
456 )
457
458
459 # Return a list of the serial numbers of connected devices. Emulators will have
460 # serials of the form "emulator-5678".
461 async def list_devices():
462 serials = []
463 header_found = False
464
465 lines = (await async_check_output(adb, "devices")).splitlines()
466 for line in lines:
467 # Ignore blank lines, and all lines before the header.
468 line = line.strip()
469 if line == "List of devices attached":
470 header_found = True
471 elif header_found and line:
472 try:
473 serial, status = line.split()
474 except ValueError:
475 raise ValueError(f"failed to parse {line!r}")
476 if status == "device":
477 serials.append(serial)
478
479 if not header_found:
480 raise ValueError(f"failed to parse {lines}")
481 return serials
482
483
484 async def find_device(context, initial_devices):
485 if context.managed:
486 print("Waiting for managed device - this may take several minutes")
487 while True:
488 new_devices = set(await list_devices()).difference(initial_devices)
489 if len(new_devices) == 0:
490 await asyncio.sleep(1)
491 elif len(new_devices) == 1:
492 serial = new_devices.pop()
493 print(f"Serial: {serial}")
494 return serial
495 else:
496 exit(f"Found more than one new device: {new_devices}")
497 else:
498 return context.connected
499
500
501 # An older version of this script in #121595 filtered the logs by UID instead.
502 # But logcat can't filter by UID until API level 31. If we ever switch back to
503 # filtering by UID, we'll also have to filter by time so we only show messages
504 # produced after the initial call to `stop_app`.
505 #
506 # We're more likely to miss the PID because it's shorter-lived, so there's a
507 # workaround in PythonSuite.kt to stop it being *too* short-lived.
508 async def find_pid(serial):
509 print("Waiting for app to start - this may take several minutes")
510 shown_error = False
511 while True:
512 try:
513 # `pidof` requires API level 24 or higher. The level 23 emulator
514 # includes it, but it doesn't work (it returns all processes).
515 pid = (await async_check_output(
516 adb, "-s", serial, "shell", "pidof", "-s", APP_ID
517 )).strip()
518 except CalledProcessError as e:
519 # If the app isn't running yet, pidof gives no output. So if there
520 # is output, there must have been some other error. However, this
521 # sometimes happens transiently, especially when running a managed
522 # emulator for the first time, so don't make it fatal.
523 if (e.stdout or e.stderr) and not shown_error:
524 print_called_process_error(e)
525 print("This may be transient, so continuing to wait")
526 shown_error = True
527 else:
528 # Some older devices (e.g. Nexus 4) return zero even when no process
529 # was found, so check whether we actually got any output.
530 if pid:
531 print(f"PID: {pid}")
532 return pid
533
534 # Loop fairly rapidly to avoid missing a short-lived process.
535 await asyncio.sleep(0.2)
536
537
538 async def logcat_task(context, initial_devices):
539 # Gradle may need to do some large downloads of libraries and emulator
540 # images. This will happen during find_device in --managed mode, or find_pid
541 # in --connected mode.
542 startup_timeout = 600
543 serial = await wait_for(find_device(context, initial_devices), startup_timeout)
544 pid = await wait_for(find_pid(serial), startup_timeout)
545
546 # `--pid` requires API level 24 or higher.
547 #
548 # `--binary` mode is used in order to detect which messages end with a
549 # newline, which most of the other modes don't indicate (except `--format
550 # long`). For example, every time pytest runs a test, it prints a "." and
551 # flushes the stream. Each "." becomes a separate log message, but we should
552 # show them all on the same line.
553 args = [adb, "-s", serial, "logcat", "--pid", pid, "--binary"]
554 logcat_started = False
555 async with async_process(
556 *args, stdout=subprocess.PIPE, stderr=None
557 ) as process:
558 while True:
559 try:
560 priority, tag, message = await read_logcat(process.stdout)
561 logcat_started = True
562 except asyncio.IncompleteReadError:
563 break
564
565 # Exclude high-volume messages which are rarely useful.
566 if context.verbose < 2 and "from python test_syslog" in message:
567 continue
568
569 # Put high-level messages on stderr so they're highlighted in the
570 # buildbot logs. This will include Python's own stderr.
571 stream = sys.stderr if priority >= LogPriority.WARN else sys.stdout
572
573 # The app's stdout and stderr should be passed through transparently
574 # to our own corresponding streams.
575 if tag in ["python.stdout", "python.stderr"]:
576 global python_started
577 python_started = True
578 stream.write(message)
579 stream.flush()
580 else:
581 # Non-Python messages add a lot of noise, but they may
582 # sometimes help explain a failure. Format them in the same way
583 # as `logcat --format tag`.
584 formatted = f"{priority.name[0]}/{tag}: {message}"
585 if not formatted.endswith("\n"):
586 formatted += "\n"
587 log_verbose(context, formatted, stream)
588
589 # If the device disconnects while logcat is running, which always
590 # happens in --managed mode, some versions of adb return non-zero.
591 # Distinguish this from a logcat startup error by checking whether we've
592 # received any logcat messages yet.
593 status = await wait_for(process.wait(), timeout=1)
594 if status != 0 and not logcat_started:
595 raise CalledProcessError(status, args)
596
597
598 # Read one binary log message from the given StreamReader. The message format is
599 # described at https://android.stackexchange.com/a/74660. All supported versions
600 # of Android use format version 2 or later.
601 async def read_logcat(stream):
602 async def read_bytes(size):
603 return await stream.readexactly(size)
604
605 async def read_int(size):
606 return int.from_bytes(await read_bytes(size), "little")
607
608 payload_len = await read_int(2)
609 if payload_len < 2:
610 # 1 byte for priority, 1 byte for null terminator of tag.
611 raise ValueError(f"payload length {payload_len} is too short")
612
613 header_len = await read_int(2)
614 if header_len < 4:
615 raise ValueError(f"header length {header_len} is too short")
616 await read_bytes(header_len - 4) # Ignore other header fields.
617
618 priority_int = await read_int(1)
619 try:
620 priority = LogPriority(priority_int)
621 except ValueError:
622 priority = LogPriority.UNKNOWN
623
624 payload = await read_bytes(payload_len - 1)
625 payload_fields = payload.split(b"\0")
626 if len(payload_fields) < 2:
627 raise ValueError(
628 f"payload {payload!r} does not contain at least 2 "
629 f"null-separated fields"
630 )
631 tag, message, *_ = [
632 field.decode(*DECODE_ARGS) for field in payload_fields
633 ]
634 return priority, tag, message
635
636
637 def stop_app(serial):
638 run([adb, "-s", serial, "shell", "am", "force-stop", APP_ID], log=False)
639
640
641 async def gradle_task(context):
642 env = os.environ.copy()
643 if context.managed:
644 task_prefix = context.managed
645 else:
646 task_prefix = "connected"
647 env["ANDROID_SERIAL"] = context.connected
648
649 # Ensure that CROSS_BUILD_DIR is in the Gradle environment, regardless
650 # of whether it was set by environment variable or `--cross-build-dir`.
651 env["CROSS_BUILD_DIR"] = CROSS_BUILD_DIR
652
653 if context.ci_mode:
654 context.args[0:0] = [
655 # See _add_ci_python_opts in libregrtest/main.py.
656 "-W", "error", "-bb", "-E",
657
658 # Randomization is disabled because order-dependent failures are
659 # much less likely to pass on a rerun in single-process mode.
660 "-m", "test",
661 f"--{context.ci_mode}-ci", "--single-process", "--no-randomize"
662 ]
663
664 if not any(arg in context.args for arg in ["-c", "-m"]):
665 context.args[0:0] = ["-m", "test"]
666
667 args = [
668 gradlew, "--console", "plain", f"{task_prefix}DebugAndroidTest",
669 ] + [
670 f"-P{name}={value}"
671 for name, value in [
672 ("python.sitePackages", context.site_packages),
673 ("python.cwd", context.cwd),
674 (
675 "android.testInstrumentationRunnerArguments.pythonArgs",
676 json.dumps(context.args),
677 ),
678 ]
679 if value
680 ]
681 if context.verbose >= 2:
682 args.append("--info")
683 log_verbose(context, f"> {join_command(args)}\n")
684
685 try:
686 async with async_process(
687 *args, cwd=TESTBED_DIR, env=env,
688 stdout=subprocess.PIPE, stderr=subprocess.STDOUT,
689 ) as process:
690 while line := (await process.stdout.readline()).decode(*DECODE_ARGS):
691 # Gradle may take several minutes to install SDK packages, so
692 # it's worth showing those messages even in non-verbose mode.
693 if line.startswith('Preparing "Install'):
694 sys.stdout.write(line)
695 else:
696 log_verbose(context, line)
697
698 status = await wait_for(process.wait(), timeout=1)
699 if status == 0:
700 exit(0)
701 else:
702 raise CalledProcessError(status, args)
703 finally:
704 # Gradle does not stop the tests when interrupted.
705 if context.connected:
706 stop_app(context.connected)
707
708
709 async def run_testbed(context):
710 setup_ci()
711 setup_sdk()
712 setup_testbed()
713
714 if context.managed:
715 # In this mode, Gradle will create a device with an unpredictable name.
716 # So we save a list of the running devices before starting Gradle, and
717 # find_device then waits for a new device to appear.
718 initial_devices = await list_devices()
719 else:
720 # In case the previous shutdown was unclean, make sure the app isn't
721 # running, otherwise we might show logs from a previous run. This is
722 # unnecessary in --managed mode, because Gradle creates a new emulator
723 # every time.
724 stop_app(context.connected)
725 initial_devices = None
726
727 try:
728 async with asyncio.TaskGroup() as tg:
729 tg.create_task(logcat_task(context, initial_devices))
730 tg.create_task(gradle_task(context))
731 except* MySystemExit as e:
732 raise SystemExit(*e.exceptions[0].args) from None
733 except* CalledProcessError as e:
734 # If Python produced no output, then the user probably wants to see the
735 # verbose output to explain why the test failed.
736 if not python_started:
737 for stream, line in hidden_output:
738 stream.write(line)
739
740 # Extract it from the ExceptionGroup so it can be handled by `main`.
741 raise e.exceptions[0]
742
743
744 def package_version(prefix_dir):
745 patchlevel_glob = f"{prefix_dir}/include/python*/patchlevel.h"
746 patchlevel_paths = glob(patchlevel_glob)
747 if len(patchlevel_paths) != 1:
748 sys.exit(f"{patchlevel_glob} matched {len(patchlevel_paths)} paths.")
749
750 for line in open(patchlevel_paths[0]):
751 if match := re.fullmatch(r'\s*#define\s+PY_VERSION\s+"(.+)"\s*', line):
752 version = match[1]
753 break
754 else:
755 sys.exit(f"Failed to find Python version in {patchlevel_paths[0]}.")
756
757 # If not building against a tagged commit, add a timestamp to the version.
758 # Follow the PyPA version number rules, as this will make it easier to
759 # process with other tools.
760 if version.endswith("+"):
761 version += datetime.now(timezone.utc).strftime("%Y%m%d.%H%M%S")
762
763 return version
764
765
766 def package(context):
767 prefix_dir = subdir(context.host, "prefix")
768 version = package_version(prefix_dir)
769
770 with TemporaryDirectory(prefix=SCRIPT_NAME) as temp_dir:
771 temp_dir = Path(temp_dir)
772
773 # Include all tracked files from the Android directory.
774 for line in run(
775 ["git", "ls-files"],
776 cwd=ANDROID_DIR, capture_output=True, text=True, log=False,
777 ).stdout.splitlines():
778 src = ANDROID_DIR / line
779 dst = temp_dir / line
780 dst.parent.mkdir(parents=True, exist_ok=True)
781 shutil.copy2(src, dst, follow_symlinks=False)
782
783 # Include anything from the prefix directory which could be useful
784 # either for embedding Python in an app, or building third-party
785 # packages against it.
786 for rel_dir, patterns in [
787 ("include", ["openssl*", "python*", "sqlite*"]),
788 ("lib", ["engines-3", "libcrypto*.so", "libpython*", "libsqlite*",
789 "libssl*.so", "ossl-modules", "python*"]),
790 ("lib/pkgconfig", ["*crypto*", "*ssl*", "*python*", "*sqlite*"]),
791 ]:
792 for pattern in patterns:
793 for src in glob(f"{prefix_dir}/{rel_dir}/{pattern}"):
794 dst = temp_dir / relpath(src, prefix_dir.parent)
795 dst.parent.mkdir(parents=True, exist_ok=True)
796 if Path(src).is_dir():
797 shutil.copytree(
798 src, dst, symlinks=True,
799 ignore=lambda *args: ["__pycache__"]
800 )
801 else:
802 shutil.copy2(src, dst, follow_symlinks=False)
803
804 # Strip debug information.
805 if not context.debug:
806 so_files = glob(f"{temp_dir}/**/*.so", recursive=True)
807 run([android_env(context.host)["STRIP"], *so_files], log=False)
808
809 dist_dir = subdir(context.host, "dist", create=True)
810 package_path = shutil.make_archive(
811 f"{dist_dir}/python-{version}-{context.host}", "gztar", temp_dir
812 )
813 print(f"Wrote {package_path}")
814 return package_path
815
816
817 def ci(context):
818 for step in [
819 configure_build_python,
820 make_build_python,
821 pythoninfo_build_python,
822 configure_host_python,
823 make_host_python,
824 package,
825 ]:
826 caption = (
827 step.__name__.replace("_", " ")
828 .capitalize()
829 .replace("python", "Python")
830 )
831 print(f"::group::{caption}")
832 result = step(context)
833 if step is package:
834 package_path = result
835 print("::endgroup::")
836
837 if (
838 "GITHUB_ACTIONS" in os.environ
839 and (platform.system(), platform.machine()) != ("Linux", "x86_64")
840 ):
841 print(
842 "Skipping tests: GitHub Actions does not support the Android "
843 "emulator on this platform."
844 )
845 else:
846 with TemporaryDirectory(prefix=SCRIPT_NAME) as temp_dir:
847 print("::group::Tests")
848
849 # Prove the package is self-contained by using it to run the tests.
850 shutil.unpack_archive(package_path, temp_dir)
851 launcher_args = [
852 "--managed", "maxVersion", "-v", f"--{context.ci_mode}-ci"
853 ]
854 run(
855 ["./android.py", "test", *launcher_args],
856 cwd=temp_dir
857 )
858 print("::endgroup::")
859
860
861 def env(context):
862 print_env(android_env(getattr(context, "host", None)))
863
864
865 # Handle SIGTERM the same way as SIGINT. This ensures that if we're terminated
866 # by the buildbot worker, we'll make an attempt to clean up our subprocesses.
867 def install_signal_handler():
868 def signal_handler(*args):
869 os.kill(os.getpid(), signal.SIGINT)
870
871 signal.signal(signal.SIGTERM, signal_handler)
872
873
874 def parse_args():
875 parser = argparse.ArgumentParser()
876 subcommands = parser.add_subparsers(dest="subcommand", required=True)
877
878 def add_parser(*args, **kwargs):
879 parser = subcommands.add_parser(*args, **kwargs)
880 parser.add_argument(
881 "--cross-build-dir",
882 action="store",
883 default=os.environ.get("CROSS_BUILD_DIR"),
884 dest="cross_build_dir",
885 type=Path,
886 help=(
887 "Path to the cross-build directory "
888 f"(default: {CROSS_BUILD_DIR}). Can also be set "
889 "with the CROSS_BUILD_DIR environment variable."
890 ),
891 )
892 parser.add_argument(
893 "-v", "--verbose", action="count", default=0,
894 help="Show verbose output. Use twice to be even more verbose.")
895 return parser
896
897 # Subcommands
898 build = add_parser(
899 "build",
900 help="Run configure and make for the selected target"
901 )
902 configure_build = add_parser(
903 "configure-build", help="Run `configure` for the build Python")
904 add_parser(
905 "make-build", help="Run `make` for the build Python")
906 add_parser(
907 "pythoninfo-build", help="Display build info of the build Python")
908 configure_host = add_parser(
909 "configure-host", help="Run `configure` for Android")
910 make_host = add_parser(
911 "make-host", help="Run `make` for Android")
912
913 clean = add_parser(
914 "clean",
915 help="Delete build directories for the selected target"
916 )
917
918 test = add_parser("test", help="Run the testbed app")
919 package = add_parser("package", help="Make a release package")
920 ci = add_parser("ci", help="Run build, package and test")
921 env = add_parser("env", help="Print environment variables")
922
923 # Common arguments
924 # --cache-dir option
925 for cmd in [configure_host, build, ci]:
926 cmd.add_argument(
927 "--cache-dir",
928 default=os.environ.get("CACHE_DIR"),
929 help="The directory to store cached downloads.",
930 )
931
932 # --clean option
933 for subcommand in [build, configure_build, configure_host, ci]:
934 subcommand.add_argument(
935 "--clean", action="store_true", default=False, dest="clean",
936 help="Delete the relevant build directories first")
937
938 # Allow "all", "build" and "hosts" targets for some commands
939 for subcommand in [clean, build]:
940 subcommand.add_argument(
941 "target",
942 nargs="?",
943 default="all",
944 choices=["all", "build", "hosts"] + HOSTS,
945 help=(
946 "The host triplet (e.g., aarch64-linux-android), "
947 "or 'build' for just the build platform, or 'hosts' for all "
948 "host platforms, or 'all' for the build platform and all "
949 "hosts. Defaults to 'all'"
950 ),
951 )
952
953 host_commands = [configure_host, make_host, package, ci]
954 if in_source_tree:
955 host_commands.append(env)
956 for subcommand in host_commands:
957 subcommand.add_argument(
958 "host", metavar="HOST", choices=HOSTS,
959 help="Host triplet: choices=[%(choices)s]")
960
961 for subcommand in [build, configure_build, configure_host, ci]:
962 subcommand.add_argument("args", nargs="*",
963 help="Extra arguments to pass to `configure`")
964
965 # Test arguments
966 device_group = test.add_mutually_exclusive_group(required=True)
967 device_group.add_argument(
968 "--connected", metavar="SERIAL", help="Run on a connected device. "
969 "Connect it yourself, then get its serial from `adb devices`.")
970 device_group.add_argument(
971 "--managed", metavar="NAME", help="Run on a Gradle-managed device. "
972 "These are defined in `managedDevices` in testbed/app/build.gradle.kts.")
973
974 test.add_argument(
975 "--site-packages", metavar="DIR", type=abspath,
976 help="Directory to copy as the app's site-packages.")
977 test.add_argument(
978 "--cwd", metavar="DIR", type=abspath,
979 help="Directory to copy as the app's working directory.")
980 test.add_argument(
981 "args", nargs="*", help=f"Python command-line arguments. "
982 f"Separate them from {SCRIPT_NAME}'s own arguments with `--`. "
983 f"If neither -c nor -m are included, `-m test` will be prepended, "
984 f"which will run Python's own test suite.")
985
986 # Package arguments.
987 for subcommand in [package, ci]:
988 subcommand.add_argument(
989 "-g", action="store_true", default=False, dest="debug",
990 help="Include debug information in package")
991
992 # CI arguments
993 for subcommand in [test, ci]:
994 group = subcommand.add_mutually_exclusive_group(required=subcommand is ci)
995 group.add_argument(
996 "--fast-ci", action="store_const", dest="ci_mode", const="fast",
997 help="Add test arguments for GitHub Actions")
998 group.add_argument(
999 "--slow-ci", action="store_const", dest="ci_mode", const="slow",
1000 help="Add test arguments for buildbots")
1001
1002 return parser.parse_args()
1003
1004
1005 def main():
1006 install_signal_handler()
1007
1008 # Under the buildbot, stdout is not a TTY, but we must still flush after
1009 # every line to make sure our output appears in the correct order relative
1010 # to the output of our subprocesses.
1011 for stream in [sys.stdout, sys.stderr]:
1012 stream.reconfigure(line_buffering=True)
1013
1014 context = parse_args()
1015
1016 # Set the CROSS_BUILD_DIR if an argument was provided
1017 if context.cross_build_dir:
1018 global CROSS_BUILD_DIR
1019 CROSS_BUILD_DIR = context.cross_build_dir.resolve()
1020
1021 dispatch = {
1022 "configure-build": configure_build_python,
1023 "make-build": make_build_python,
1024 "pythoninfo-build": pythoninfo_build_python,
1025 "configure-host": configure_host_python,
1026 "make-host": make_host_python,
1027 "build": build_targets,
1028 "clean": clean_targets,
1029 "test": run_testbed,
1030 "package": package,
1031 "ci": ci,
1032 "env": env,
1033 }
1034
1035 try:
1036 result = dispatch[context.subcommand](context)
1037 if asyncio.iscoroutine(result):
1038 asyncio.run(result)
1039 except CalledProcessError as e:
1040 print_called_process_error(e)
1041 sys.exit(1)
1042
1043
1044 def print_called_process_error(e):
1045 for stream_name in ["stdout", "stderr"]:
1046 content = getattr(e, stream_name)
1047 if isinstance(content, bytes):
1048 content = content.decode(*DECODE_ARGS)
1049 stream = getattr(sys, stream_name)
1050 if content:
1051 stream.write(content)
1052 if not content.endswith("\n"):
1053 stream.write("\n")
1054
1055 # shlex uses single quotes, so we surround the command with double quotes.
1056 print(
1057 f'Command "{join_command(e.cmd)}" returned exit status {e.returncode}'
1058 )
1059
1060
1061 if __name__ == "__main__":
1062 main()