Debian Codegraph @

Apple/testbed/__main__.py

raw
1 import argparse
2 import json
3 import os
4 import re
5 import shlex
6 import shutil
7 import subprocess
8 import sys
9 from pathlib import Path
10
11 TEST_SLICES = {
12 "iOS": "ios-arm64_x86_64-simulator",
13 }
14
15 DECODE_ARGS = ("UTF-8", "backslashreplace")
16
17 # The system log prefixes each line:
18 # 2025-01-17 16:14:29.093742+0800 iOSTestbed[23987:1fd393b4] ...
19 # 2025-01-17 16:14:29.093742+0800 iOSTestbed[23987:1fd393b4] ...
20
21 LOG_PREFIX_REGEX = re.compile(
22 r"^\d{4}-\d{2}-\d{2}" # YYYY-MM-DD
23 r"\s+\d+:\d{2}:\d{2}\.\d+\+\d{4}" # HH:MM:SS.ssssss+ZZZZ
24 r"\s+iOSTestbed\[\d+:\w+\] " # Process/thread ID
25 )
26
27
28 # Select a simulator device to use.
29 def select_simulator_device(platform):
30 # List the testing simulators, in JSON format
31 raw_json = subprocess.check_output(["xcrun", "simctl", "list", "-j"])
32 json_data = json.loads(raw_json)
33
34 if platform == "iOS":
35 # Any iOS device will do; we'll look for "SE" devices - but the name
36 # isn't consistent over time. Older Xcode versions will use "iPhone SE
37 # (Nth generation)"; As of 2025, they've started using "iPhone 16e".
38 #
39 # When Xcode is updated after a new release, new devices will be
40 # available and old ones will be dropped from the set available on the
41 # latest iOS version. Select the one with the highest minimum runtime
42 # version - this is an indicator of the "newest" released device, which
43 # should always be supported on the "most recent" iOS version.
44 se_simulators = sorted(
45 (devicetype["minRuntimeVersion"], devicetype["name"])
46 for devicetype in json_data["devicetypes"]
47 if devicetype["productFamily"] == "iPhone"
48 and (
49 (
50 "iPhone " in devicetype["name"]
51 and devicetype["name"].endswith("e")
52 )
53 or "iPhone SE " in devicetype["name"]
54 )
55 )
56 simulator = se_simulators[-1][1]
57 else:
58 raise ValueError(f"Unknown platform {platform}")
59
60 return simulator
61
62
63 def xcode_test(location: Path, platform: str, simulator: str, verbose: bool):
64 # Build and run the test suite on the named simulator.
65 args = [
66 "-project",
67 str(location / f"{platform}Testbed.xcodeproj"),
68 "-scheme",
69 f"{platform}Testbed",
70 "-destination",
71 f"platform={platform} Simulator,name={simulator}",
72 "-derivedDataPath",
73 str(location / "DerivedData"),
74 ]
75 verbosity_args = [] if verbose else ["-quiet"]
76
77 print("Building test project...")
78 subprocess.run(
79 ["xcodebuild", "build-for-testing"] + args + verbosity_args,
80 check=True,
81 )
82
83 # Any environment variable prefixed with TEST_RUNNER_ is exposed into the
84 # test runner environment. There are some variables (like those identifying
85 # CI platforms) that can be useful to have access to.
86 test_env = os.environ.copy()
87 if "GITHUB_ACTIONS" in os.environ:
88 test_env["TEST_RUNNER_GITHUB_ACTIONS"] = os.environ["GITHUB_ACTIONS"]
89
90 print("Running test project...")
91 # Test execution *can't* be run -quiet; verbose mode
92 # is how we see the output of the test output.
93 process = subprocess.Popen(
94 ["xcodebuild", "test-without-building"] + args,
95 stdout=subprocess.PIPE,
96 stderr=subprocess.STDOUT,
97 env=test_env,
98 )
99 while line := (process.stdout.readline()).decode(*DECODE_ARGS):
100 # Strip the timestamp/process prefix from each log line
101 line = LOG_PREFIX_REGEX.sub("", line)
102 sys.stdout.write(line)
103 sys.stdout.flush()
104
105 status = process.wait(timeout=5)
106 exit(status)
107
108
109 def copy(src, tgt):
110 """An all-purpose copy.
111
112 If src is a file, it is copied. If src is a symlink, it is copied *as a
113 symlink*. If src is a directory, the full tree is duplicated, with symlinks
114 being preserved.
115 """
116 if src.is_file() or src.is_symlink():
117 shutil.copyfile(src, tgt, follow_symlinks=False)
118 else:
119 shutil.copytree(src, tgt, symlinks=True)
120
121
122 def clone_testbed(
123 source: Path,
124 target: Path,
125 framework: Path,
126 platform: str,
127 apps: list[Path],
128 ) -> None:
129 if target.exists():
130 print(f"{target} already exists; aborting without creating project.")
131 sys.exit(10)
132
133 if framework is None:
134 if not (
135 source / "Python.xcframework" / TEST_SLICES[platform] / "bin"
136 ).is_dir():
137 print(
138 f"The testbed being cloned ({source}) does not contain "
139 "a framework with slices. Re-run with --framework"
140 )
141 sys.exit(11)
142 else:
143 if not framework.is_dir():
144 print(f"{framework} does not exist.")
145 sys.exit(12)
146 elif not (
147 framework.suffix == ".xcframework"
148 or (framework / "Python.framework").is_dir()
149 ):
150 print(
151 f"{framework} is not an XCframework, "
152 f"or a simulator slice of a framework build."
153 )
154 sys.exit(13)
155
156 print("Cloning testbed project:")
157 print(f" Cloning {source}...", end="")
158 # Only copy the files for the platform being cloned plus the files common
159 # to all platforms. The XCframework will be copied later, if needed.
160 target.mkdir(parents=True)
161
162 for name in [
163 "__main__.py",
164 "TestbedTests",
165 "Testbed.lldbinit",
166 f"{platform}Testbed",
167 f"{platform}Testbed.xcodeproj",
168 f"{platform}Testbed.xctestplan",
169 ]:
170 copy(source / name, target / name)
171
172 print(" done")
173
174 orig_xc_framework_path = source / "Python.xcframework"
175 xc_framework_path = target / "Python.xcframework"
176 test_framework_path = xc_framework_path / TEST_SLICES[platform]
177 if framework is not None:
178 if framework.suffix == ".xcframework":
179 print(" Installing XCFramework...", end="")
180 xc_framework_path.symlink_to(
181 framework.relative_to(xc_framework_path.parent, walk_up=True)
182 )
183 print(" done")
184 else:
185 print(" Installing simulator framework...", end="")
186 # We're only installing a slice of a framework; we need
187 # to do a full tree copy to make sure we don't damage
188 # symlinked content.
189 shutil.copytree(orig_xc_framework_path, xc_framework_path)
190 if test_framework_path.is_dir():
191 shutil.rmtree(test_framework_path)
192 else:
193 test_framework_path.unlink(missing_ok=True)
194 test_framework_path.symlink_to(
195 framework.relative_to(test_framework_path.parent, walk_up=True)
196 )
197 print(" done")
198 else:
199 copy(orig_xc_framework_path, xc_framework_path)
200
201 if (
202 xc_framework_path.is_symlink()
203 and not xc_framework_path.readlink().is_absolute()
204 ):
205 # XCFramework is a relative symlink. Rewrite the symlink relative
206 # to the new location.
207 print(" Rewriting symlink to XCframework...", end="")
208 resolved_xc_framework_path = (
209 source / xc_framework_path.readlink()
210 ).resolve()
211 xc_framework_path.unlink()
212 xc_framework_path.symlink_to(
213 resolved_xc_framework_path.relative_to(
214 xc_framework_path.parent, walk_up=True
215 )
216 )
217 print(" done")
218 elif (
219 test_framework_path.is_symlink()
220 and not test_framework_path.readlink().is_absolute()
221 ):
222 print(" Rewriting symlink to simulator framework...", end="")
223 # Simulator framework is a relative symlink. Rewrite the symlink
224 # relative to the new location.
225 orig_test_framework_path = (
226 source / "Python.XCframework" / test_framework_path.readlink()
227 ).resolve()
228 test_framework_path.unlink()
229 test_framework_path.symlink_to(
230 orig_test_framework_path.relative_to(
231 test_framework_path.parent, walk_up=True
232 )
233 )
234 print(" done")
235 else:
236 print(" Using pre-existing Python framework.")
237
238 for app_src in apps:
239 print(f" Installing app {app_src.name!r}...", end="")
240 app_target = target / f"Testbed/app/{app_src.name}"
241 if app_target.is_dir():
242 shutil.rmtree(app_target)
243 shutil.copytree(app_src, app_target)
244 print(" done")
245
246 print(f"Successfully cloned testbed: {target.resolve()}")
247
248
249 def update_test_plan(testbed_path, platform, args):
250 # Modify the test plan to use the requested test arguments.
251 test_plan_path = testbed_path / f"{platform}Testbed.xctestplan"
252 with test_plan_path.open("r", encoding="utf-8") as f:
253 test_plan = json.load(f)
254
255 test_plan["defaultOptions"]["commandLineArgumentEntries"] = [
256 {"argument": shlex.quote(arg)} for arg in args
257 ]
258
259 with test_plan_path.open("w", encoding="utf-8") as f:
260 json.dump(test_plan, f, indent=2)
261
262
263 def run_testbed(
264 platform: str,
265 simulator: str | None,
266 args: list[str],
267 verbose: bool = False,
268 ):
269 location = Path(__file__).parent
270 print("Updating test plan...", end="")
271 update_test_plan(location, platform, args)
272 print(" done.")
273
274 # xcodebuild doesn't guarantee that the CoreSimulatorService daemon is
275 # running prior to using a simulator, but calling `xcrun simctl list` does.
276 # Determining the default simulator that *would* be used is a cheap action;
277 # so use that as a way to ensure that the CoreSimulatorService daemon is
278 # running.
279 default_simulator = select_simulator_device(platform)
280 if simulator is None:
281 simulator = default_simulator
282 print(f"Running test on {simulator}")
283
284 xcode_test(
285 location,
286 platform=platform,
287 simulator=simulator,
288 verbose=verbose,
289 )
290
291
292 def main():
293 # Look for directories like `iOSTestbed` as an indicator of the platforms
294 # that the testbed folder supports. The original source testbed can support
295 # many platforms, but when cloned, only one platform is preserved.
296 available_platforms = [
297 platform
298 for platform in ["iOS"]
299 if (Path(__file__).parent / f"{platform}Testbed").is_dir()
300 ]
301
302 parser = argparse.ArgumentParser(
303 description=(
304 "Manages the process of testing an Apple Python project "
305 "through Xcode."
306 ),
307 )
308
309 subcommands = parser.add_subparsers(dest="subcommand")
310 clone = subcommands.add_parser(
311 "clone",
312 description=(
313 "Clone the testbed project, copying in a Python framework and"
314 "any specified application code."
315 ),
316 help="Clone a testbed project to a new location.",
317 )
318 clone.add_argument(
319 "--framework",
320 help=(
321 "The location of the XCFramework (or simulator-only slice of an "
322 "XCFramework) to use when running the testbed"
323 ),
324 )
325 clone.add_argument(
326 "--platform",
327 dest="platform",
328 choices=available_platforms,
329 default=available_platforms[0],
330 help=f"The platform to target (default: {available_platforms[0]})",
331 )
332 clone.add_argument(
333 "--app",
334 dest="apps",
335 action="append",
336 default=[],
337 help="The location of any code to include in the testbed project",
338 )
339 clone.add_argument(
340 "location",
341 help="The path where the testbed will be cloned.",
342 )
343
344 run = subcommands.add_parser(
345 "run",
346 usage=(
347 "%(prog)s [-h] [--simulator SIMULATOR] -- "
348 "<test arg> [<test arg> ...]"
349 ),
350 description=(
351 "Run a testbed project. The arguments provided after `--` will be "
352 "passed to the running iOS process as if they were arguments to "
353 "`python -m`."
354 ),
355 help="Run a testbed project",
356 )
357 run.add_argument(
358 "--platform",
359 dest="platform",
360 choices=available_platforms,
361 default=available_platforms[0],
362 help=f"The platform to target (default: {available_platforms[0]})",
363 )
364 run.add_argument(
365 "--simulator",
366 help=(
367 "The name of the simulator to use (eg: 'iPhone 16e'). Defaults to "
368 "the most recently released 'entry level' iPhone device. Device "
369 "architecture and OS version can also be specified; e.g., "
370 "`--simulator 'iPhone 16 Pro,arch=arm64,OS=26.0'` would run on "
371 "an ARM64 iPhone 16 Pro simulator running iOS 26.0."
372 ),
373 )
374 run.add_argument(
375 "-v",
376 "--verbose",
377 action="store_true",
378 help="Enable verbose output",
379 )
380
381 try:
382 pos = sys.argv.index("--")
383 testbed_args = sys.argv[1:pos]
384 test_args = sys.argv[pos + 1 :]
385 except ValueError:
386 testbed_args = sys.argv[1:]
387 test_args = []
388
389 context = parser.parse_args(testbed_args)
390
391 if context.subcommand == "clone":
392 clone_testbed(
393 source=Path(__file__).parent.resolve(),
394 target=Path(context.location).resolve(),
395 framework=Path(context.framework).resolve()
396 if context.framework
397 else None,
398 platform=context.platform,
399 apps=[Path(app) for app in context.apps],
400 )
401 elif context.subcommand == "run":
402 if test_args:
403 if not (
404 Path(__file__).parent
405 / "Python.xcframework"
406 / TEST_SLICES[context.platform]
407 / "bin"
408 ).is_dir():
409 print(
410 "Testbed does not contain a compiled Python framework. "
411 f"Use `python {sys.argv[0]} clone ...` to create a "
412 "runnable clone of this testbed."
413 )
414 sys.exit(20)
415
416 run_testbed(
417 platform=context.platform,
418 simulator=context.simulator,
419 verbose=context.verbose,
420 args=test_args,
421 )
422 else:
423 print(
424 "Must specify test arguments "
425 f"(e.g., {sys.argv[0]} run -- test)"
426 )
427 print()
428 parser.print_help(sys.stderr)
429 sys.exit(21)
430 else:
431 parser.print_help(sys.stderr)
432 sys.exit(1)
433
434
435 if __name__ == "__main__":
436 # Under the buildbot, stdout is not a TTY, but we must still flush after
437 # every line to make sure our output appears in the correct order relative
438 # to the output of our subprocesses.
439 for stream in [sys.stdout, sys.stderr]:
440 stream.reconfigure(line_buffering=True)
441 main()