Debian Codegraph @

vmtest/manage.py

raw
1 # Copyright (c) Meta Platforms, Inc. and affiliates.
2 # SPDX-License-Identifier: LGPL-2.1-or-later
3
4 import argparse
5 import asyncio
6 import itertools
7 import logging
8 import os
9 from pathlib import Path
10 import re
11 import sys
12 from typing import (
13 AsyncIterator,
14 Dict,
15 List,
16 Mapping,
17 NamedTuple,
18 Optional,
19 Sequence,
20 Tuple,
21 Union,
22 )
23
24 import aiohttp
25 import uritemplate
26
27 from util import KernelVersion
28 from vmtest.asynciosubprocess import check_call, check_output
29 from vmtest.config import (
30 ARCHITECTURES,
31 KERNEL_FLAVORS,
32 Architecture,
33 Compiler,
34 KernelFlavor,
35 kconfig_localversion,
36 )
37 from vmtest.download import VMTEST_GITHUB_RELEASE, Downloader, available_kernel_releases
38 from vmtest.githubapi import AioGitHubApi
39 from vmtest.kbuild import KBuild, apply_patches
40
41 logger = logging.getLogger(__name__)
42
43 # [inclusive, exclusive) ranges of kernel versions to ignore when building
44 # latest releases of each version.
45 IGNORE_KERNEL_RANGES = (
46 (KernelVersion("~"), KernelVersion("4.9")),
47 (KernelVersion("4.10~"), KernelVersion("4.14")),
48 (KernelVersion("4.15~"), KernelVersion("4.19")),
49 (KernelVersion("4.20~"), KernelVersion("5.4")),
50 (KernelVersion("5.5~"), KernelVersion("5.10")),
51 )
52
53 # Use the GitHub mirrors rather than the official kernel.org repositories since
54 # this script usually runs in GitHub Actions.
55 LINUX_GIT_URL = "https://github.com/torvalds/linux.git"
56 STABLE_LINUX_GIT_URL = "https://github.com/gregkh/linux.git"
57
58
59 async def get_latest_kernel_tags() -> List[str]:
60 mainline_refs, stable_refs = await asyncio.gather(
61 check_output("git", "ls-remote", "--tags", "--refs", LINUX_GIT_URL),
62 check_output("git", "ls-remote", "--tags", "--refs", STABLE_LINUX_GIT_URL),
63 )
64 latest: Dict[str, KernelVersion] = {}
65 for match in itertools.chain(
66 re.finditer(
67 r"^[a-f0-9]+\s+refs/tags/v([0-9]+\.[0-9]+)(-rc[0-9]+)?$",
68 mainline_refs.decode(),
69 re.M,
70 ),
71 re.finditer(
72 r"^[a-f0-9]+\s+refs/tags/v([0-9]+\.[0-9]+)(\.[0-9]+)$",
73 stable_refs.decode(),
74 re.M,
75 ),
76 ):
77 version = KernelVersion(match.group(1) + (match.group(2) or ""))
78 for start_version, end_version in IGNORE_KERNEL_RANGES:
79 if start_version <= version < end_version:
80 break
81 else:
82 latest[match.group(1)] = max(version, latest.get(match.group(1), version))
83 return ["v" + str(version) for version in sorted(latest.values(), reverse=True)]
84
85
86 async def fetch_kernel_tags(kernel_dir: Path, kernel_tags: Sequence[str]) -> None:
87 if not kernel_dir.exists():
88 logger.info("creating kernel repository in %s", kernel_dir)
89 await check_call("git", "init", "-q", str(kernel_dir))
90
91 mainline_tags = []
92 stable_tags = []
93 for tag in kernel_tags:
94 if re.fullmatch(r"v[0-9]+\.[0-9]+\.[0-9]+", tag):
95 stable_tags.append(tag)
96 else:
97 mainline_tags.append(tag)
98
99 for name, url, tags in (
100 ("mainline", LINUX_GIT_URL, mainline_tags),
101 ("stable", STABLE_LINUX_GIT_URL, stable_tags),
102 ):
103 if tags:
104 logger.info("fetching %s kernel tags: %s", name, ", ".join(tags))
105 await check_call(
106 "git",
107 "-C",
108 str(kernel_dir),
109 "fetch",
110 "--depth",
111 "1",
112 url,
113 *(f"refs/tags/{tag}:refs/tags/{tag}" for tag in tags),
114 )
115
116
117 async def build_kernels(
118 kernel_dir: Path,
119 build_dir: Path,
120 kernel_revs: Sequence[
121 Tuple[str, Sequence[Tuple[Architecture, Sequence[KernelFlavor]]]]
122 ],
123 compilers: Mapping[str, Compiler],
124 keep_builds: bool,
125 ) -> AsyncIterator[Path]:
126 build_dir.mkdir(parents=True, exist_ok=True)
127 for rev, arches in kernel_revs:
128 logger.info("checking out %s in %s", rev, kernel_dir)
129 await check_call(
130 "git", "-C", str(kernel_dir), "checkout", "--force", "--quiet", rev
131 )
132 await check_call("git", "-C", str(kernel_dir), "clean", "-dqf")
133 await apply_patches(kernel_dir)
134 for arch, flavors in arches:
135 env = {**os.environ, **compilers[arch.name].env()}
136 for flavor in flavors:
137 flavor_rev_build_dir = (
138 build_dir / f"build-{arch.name}-{flavor.name}-{rev}"
139 )
140 with open(
141 build_dir / f"build-{arch.name}-{flavor.name}-{rev}.log", "w"
142 ) as build_log_file:
143 kbuild = KBuild(
144 kernel_dir,
145 flavor_rev_build_dir,
146 arch,
147 flavor,
148 env=env,
149 build_log_file=build_log_file,
150 )
151 await kbuild.build()
152 yield await kbuild.package("tar.zst", build_dir)
153 if not keep_builds:
154 logger.info("deleting %s", flavor_rev_build_dir)
155 # Shell out instead of using, e.g., shutil.rmtree(), to
156 # avoid blocking the main thread and the GIL.
157 await check_call("rm", "-rf", str(flavor_rev_build_dir))
158
159
160 class AssetUploadWork(NamedTuple):
161 upload_url: str
162 path: Union[str, bytes, Path]
163 name: str
164 content_type: str
165
166
167 async def asset_uploader(
168 gh: AioGitHubApi,
169 queue: "asyncio.Queue[Optional[AssetUploadWork]]",
170 ) -> bool:
171 success = True
172 while True:
173 work = await queue.get()
174 if not work:
175 queue.task_done()
176 return success
177 logger.info("uploading %s", work.name)
178 try:
179 with open(work.path, "rb") as f:
180 await gh.upload(
181 uritemplate.expand(work.upload_url, name=work.name),
182 f,
183 work.content_type,
184 )
185 except Exception:
186 logger.exception("uploading %s failed", work.name)
187 success = False
188 else:
189 logger.info("uploaded %s", work.name)
190 finally:
191 queue.task_done()
192
193
194 async def main() -> None:
195 logging.basicConfig(
196 format="%(asctime)s:%(levelname)s:%(name)s:%(message)s", level=logging.INFO
197 )
198
199 parser = argparse.ArgumentParser(
200 description="Build and upload drgn vmtest assets",
201 formatter_class=argparse.ArgumentDefaultsHelpFormatter,
202 )
203 parser.add_argument(
204 "-K",
205 "--latest-kernels",
206 action="store_true",
207 help="build and upload latest supported kernel releases",
208 )
209 parser.add_argument(
210 "-a",
211 "--architecture",
212 dest="architectures",
213 choices=["all", *sorted(ARCHITECTURES)],
214 action="append",
215 help="build architecture; may be given multiple times (default: all)",
216 default=argparse.SUPPRESS,
217 )
218 parser.add_argument(
219 "-f",
220 "--flavor",
221 dest="flavors",
222 choices=["all", *KERNEL_FLAVORS],
223 action="append",
224 help="build flavor; may be given multiple times (default: all)",
225 default=argparse.SUPPRESS,
226 )
227 parser.add_argument(
228 "--no-build",
229 dest="build",
230 action="store_false",
231 help="don't build or upload anything; just log what would be built",
232 )
233 parser.add_argument(
234 "--no-upload",
235 dest="upload",
236 action="store_false",
237 help="build but don't upload anything to GitHub",
238 )
239 parser.add_argument(
240 "--kernel-directory",
241 metavar="DIR",
242 type=Path,
243 help="kernel Git repository directory (created if needed)",
244 default=".",
245 )
246 parser.add_argument(
247 "--build-directory",
248 metavar="DIR",
249 type=Path,
250 help="directory for build artifacts",
251 default=".",
252 )
253 parser.add_argument(
254 "--no-keep-builds",
255 dest="keep_builds",
256 action="store_false",
257 help="delete kernel builds after packaging (default if uploading)",
258 default=argparse.SUPPRESS,
259 )
260 parser.add_argument(
261 "--keep-builds",
262 action="store_true",
263 help="keep kernel builds after packaging (default if not uploading)",
264 default=argparse.SUPPRESS,
265 )
266 parser.add_argument(
267 "--download-directory",
268 metavar="DIR",
269 type=Path,
270 default="build/vmtest",
271 help="directory to download assets to",
272 )
273 args = parser.parse_args()
274
275 if not hasattr(args, "architectures") or "all" in args.architectures:
276 args.architectures = list(ARCHITECTURES.values())
277 else:
278 args.architectures = [
279 arch for arch in ARCHITECTURES.values() if arch.name in args.architectures
280 ]
281 if not hasattr(args, "flavors") or "all" in args.flavors:
282 args.flavors = list(KERNEL_FLAVORS.values())
283 else:
284 args.flavors = [
285 flavor for flavor in KERNEL_FLAVORS.values() if flavor.name in args.flavors
286 ]
287
288 if not args.build:
289 args.upload = False
290 if not hasattr(args, "keep_builds"):
291 args.keep_builds = not args.upload
292
293 async with aiohttp.ClientSession(trust_env=True) as session:
294 GITHUB_TOKEN = os.getenv("GITHUB_TOKEN")
295 if GITHUB_TOKEN is None and args.upload:
296 sys.exit("GITHUB_TOKEN environment variable is not set")
297 gh = AioGitHubApi(session, GITHUB_TOKEN)
298
299 args.download_directory.mkdir(parents=True, exist_ok=True)
300 github_release_coro = gh.get_release_by_tag(
301 *VMTEST_GITHUB_RELEASE,
302 cache=args.download_directory / "github_release.json",
303 )
304 if args.latest_kernels:
305 github_release, latest_kernel_tags = await asyncio.gather(
306 github_release_coro, get_latest_kernel_tags()
307 )
308 else:
309 github_release = await github_release_coro
310
311 kernel_releases = available_kernel_releases(github_release)
312 logger.info("available kernel releases:")
313 for arch in args.architectures:
314 arch_kernel_releases = kernel_releases.get(arch.name, {})
315 logger.info(
316 " %s: %s",
317 arch.name,
318 ", ".join(
319 sorted(arch_kernel_releases, key=KernelVersion, reverse=True)
320 ),
321 )
322
323 to_build = []
324 if args.latest_kernels:
325 logger.info("latest kernel versions: %s", ", ".join(latest_kernel_tags))
326 for tag in latest_kernel_tags:
327 tag_arches_to_build = []
328
329 match = re.fullmatch(r"v([0-9]+\.[0-9]+)(\.[0-9]+)?(-rc\d+)?", tag)
330 assert match
331 version = "".join(
332 [match.group(1), match.group(2) or ".0", match.group(3) or ""]
333 )
334
335 for arch in args.architectures:
336 if not arch.kernel_version_supported(version):
337 continue
338
339 arch_kernel_releases = kernel_releases.get(arch.name, {})
340 tag_arch_flavors_to_build = [
341 flavor
342 for flavor in args.flavors
343 if (version + kconfig_localversion(arch, flavor, version))
344 not in arch_kernel_releases
345 ]
346
347 if tag_arch_flavors_to_build:
348 tag_arches_to_build.append((arch, tag_arch_flavors_to_build))
349 if tag_arches_to_build:
350 to_build.append((tag, tag_arches_to_build))
351
352 if to_build:
353 logger.info("kernel versions to build:")
354 for tag, tag_arches_to_build in to_build:
355 logger.info(
356 " %s (%s)",
357 tag,
358 ", ".join(
359 [
360 f"{arch.name} [{', '.join([flavor.name for flavor in tag_arch_flavors_to_build])}]"
361 for arch, tag_arch_flavors_to_build in tag_arches_to_build
362 ]
363 ),
364 )
365
366 if args.build:
367 downloader = Downloader(args.download_directory)
368 compilers = {
369 arch.name: downloader.download_compiler(
370 downloader.resolve_compiler(arch)
371 )
372 for arch in {
373 arch.name: arch
374 for _, tag_arches_to_build in to_build
375 for arch, _ in tag_arches_to_build
376 }.values()
377 }
378
379 if args.upload:
380 upload_queue: "asyncio.Queue[Optional[AssetUploadWork]]" = (
381 asyncio.Queue()
382 )
383 uploader = asyncio.create_task(asset_uploader(gh, upload_queue))
384
385 await fetch_kernel_tags(
386 args.kernel_directory, [tag for tag, _ in to_build]
387 )
388
389 async for kernel_package in build_kernels(
390 args.kernel_directory,
391 args.build_directory,
392 to_build,
393 compilers,
394 args.keep_builds,
395 ):
396 if args.upload:
397 await upload_queue.put(
398 AssetUploadWork(
399 upload_url=github_release["upload_url"],
400 path=kernel_package,
401 name=kernel_package.name,
402 content_type="application/zstd",
403 )
404 )
405 else:
406 logger.info("would upload %s", kernel_package)
407
408 if args.upload:
409 await upload_queue.put(None)
410 await upload_queue.join()
411 if not await uploader:
412 sys.exit("some uploads failed")
413 else:
414 logger.info("nothing to build")
415
416
417 if __name__ == "__main__":
418 asyncio.run(main())