Debian Codegraph @

vmtest/download.py

raw
1 # Copyright (c) Meta Platforms, Inc. and affiliates.
2 # SPDX-License-Identifier: LGPL-2.1-or-later
3
4 import argparse
5 from contextlib import contextmanager
6 import fnmatch
7 import functools
8 import glob
9 import logging
10 import os
11 from pathlib import Path
12 import queue
13 import re
14 import shutil
15 import subprocess
16 import tempfile
17 import threading
18 from typing import (
19 Any,
20 Callable,
21 Dict,
22 Generator,
23 Iterable,
24 Iterator,
25 List,
26 NamedTuple,
27 Optional,
28 Union,
29 )
30 import urllib.request
31
32 from _drgn_util.platform import NORMALIZED_MACHINE_NAME
33 from util import KernelVersion
34 from vmtest.config import (
35 ARCHITECTURES,
36 HOST_ARCHITECTURE,
37 KERNEL_ORG_COMPILER_VERSION,
38 Architecture,
39 Compiler,
40 Kernel,
41 compiler_name,
42 compiler_url,
43 local_kernel,
44 )
45 from vmtest.githubapi import GitHubApi
46
47 logger = logging.getLogger(__name__)
48
49 VMTEST_GITHUB_RELEASE = ("osandov", "drgn", "vmtest-assets")
50
51
52 GitHubAsset = Dict[str, Any]
53
54
55 def available_kernel_releases(
56 github_release: Dict[str, Any]
57 ) -> Dict[str, Dict[str, GitHubAsset]]:
58 releases: Dict[str, Dict[str, GitHubAsset]] = {}
59 pattern = re.compile(r"kernel-(?P<release>.*)\.(?P<arch>\w+)\.tar\.zst")
60 for asset in github_release["assets"]:
61 match = pattern.fullmatch(asset["name"])
62 if match:
63 try:
64 arch_releases = releases[match.group("arch")]
65 except KeyError:
66 arch_releases = releases[match.group("arch")] = {}
67 arch_releases[match.group("release")] = asset
68 return releases
69
70
71 class DownloadKernel(NamedTuple):
72 arch: Architecture
73 pattern: str
74
75
76 class DownloadCompiler(NamedTuple):
77 target: Architecture
78
79
80 Download = Union[DownloadKernel, DownloadCompiler]
81 Downloaded = Union[Kernel, Compiler]
82
83
84 class DownloadNotFoundError(Exception):
85 pass
86
87
88 _KERNEL_ORG_COMPILER_HOST_NAME = (
89 None
90 if HOST_ARCHITECTURE is None
91 else HOST_ARCHITECTURE.kernel_org_compiler_host_name
92 )
93
94
95 def downloaded_compiler(download_dir: Path, target: Architecture) -> Compiler:
96 if _KERNEL_ORG_COMPILER_HOST_NAME is None:
97 raise DownloadNotFoundError(
98 f"kernel.org compilers are not available for {NORMALIZED_MACHINE_NAME} hosts"
99 )
100 return Compiler(
101 target=target,
102 bin=download_dir
103 / compiler_name(
104 _KERNEL_ORG_COMPILER_HOST_NAME, target.kernel_org_compiler_target_name
105 )
106 / "bin",
107 prefix=target.kernel_org_compiler_target_name + "-",
108 )
109
110
111 class Downloader:
112 def __init__(self, directory: Path) -> None:
113 self._directory = directory
114 self._gh = GitHubApi(os.getenv("GITHUB_TOKEN"))
115 self._cached_kernel_releases: Optional[Dict[str, Dict[str, GitHubAsset]]] = None
116
117 def _available_kernel_releases(self) -> Dict[str, Dict[str, GitHubAsset]]:
118 if self._cached_kernel_releases is None:
119 logger.info("getting available kernel releases")
120 self._directory.mkdir(parents=True, exist_ok=True)
121 self._cached_kernel_releases = available_kernel_releases(
122 self._gh.get_release_by_tag(
123 *VMTEST_GITHUB_RELEASE,
124 cache=self._directory / "github_release.json",
125 ),
126 )
127 return self._cached_kernel_releases
128
129 def resolve_kernel(self, arch: Architecture, pattern: str) -> Kernel:
130 if pattern.startswith(".") or pattern.startswith("/"):
131 return local_kernel(arch, Path(pattern))
132 elif pattern == glob.escape(pattern):
133 release = pattern
134 else:
135 try:
136 release = max(
137 (
138 available
139 for available in self._available_kernel_releases()[arch.name]
140 if fnmatch.fnmatch(available, pattern)
141 ),
142 key=KernelVersion,
143 )
144 except ValueError:
145 raise DownloadNotFoundError(
146 f"no available kernel release matches {pattern!r} on {arch.name}"
147 )
148 else:
149 logger.info(
150 "kernel release pattern %s matches %s on %s",
151 pattern,
152 release,
153 arch.name,
154 )
155 kernel_dir = self._directory / arch.name / ("kernel-" + release)
156 if (
157 not kernel_dir.exists()
158 and release not in self._available_kernel_releases()[arch.name]
159 ):
160 raise DownloadNotFoundError(
161 f"kernel release {release} not found on {arch.name}"
162 )
163 return Kernel(arch, release, kernel_dir)
164
165 def download_kernel(self, kernel: Kernel) -> Kernel:
166 if kernel.path.exists():
167 # As a policy, vmtest assets will never be updated with the same
168 # name. Therefore, if the kernel was previously downloaded, we
169 # don't need to download it again.
170 logger.info(
171 "kernel release %s for %s already downloaded to %s",
172 kernel.release,
173 kernel.arch.name,
174 kernel.path,
175 )
176 return kernel
177
178 url = self._available_kernel_releases()[kernel.arch.name][kernel.release]["url"]
179 logger.info(
180 "downloading kernel release %s for %s to %s from %s",
181 kernel.release,
182 kernel.arch.name,
183 kernel.path,
184 url,
185 )
186 kernel.path.parent.mkdir(parents=True, exist_ok=True)
187 tmp_dir = Path(tempfile.mkdtemp(dir=kernel.path.parent))
188 try:
189 # Don't assume that the available version of tar has zstd support or
190 # the non-standard -I/--use-compress-program option.
191 with subprocess.Popen(
192 ["zstd", "-d", "-", "--stdout"],
193 stdin=subprocess.PIPE,
194 stdout=subprocess.PIPE,
195 ) as zstd_proc, subprocess.Popen(
196 ["tar", "-C", str(tmp_dir), "-x"],
197 stdin=zstd_proc.stdout,
198 ) as tar_proc:
199 assert zstd_proc.stdin is not None
200 try:
201 with self._gh.download(url) as resp:
202 shutil.copyfileobj(resp, zstd_proc.stdin)
203 finally:
204 zstd_proc.stdin.close()
205 if zstd_proc.returncode != 0:
206 raise subprocess.CalledProcessError(
207 zstd_proc.returncode, zstd_proc.args
208 )
209 if tar_proc.returncode != 0:
210 raise subprocess.CalledProcessError(tar_proc.returncode, tar_proc.args)
211 except BaseException:
212 shutil.rmtree(tmp_dir, ignore_errors=True)
213 raise
214 else:
215 tmp_dir.rename(kernel.path)
216 return kernel
217
218 def resolve_compiler(self, target: Architecture) -> Compiler:
219 return downloaded_compiler(self._directory, target)
220
221 def download_compiler(self, compiler: Compiler) -> Compiler:
222 dir = compiler.bin.parent
223 if dir.exists():
224 logger.info(
225 "compiler for %s already downloaded to %s", compiler.target.name, dir
226 )
227 else:
228 assert _KERNEL_ORG_COMPILER_HOST_NAME is not None
229 url = compiler_url(
230 _KERNEL_ORG_COMPILER_HOST_NAME,
231 compiler.target.kernel_org_compiler_target_name,
232 )
233 logger.info(
234 "downloading compiler for %s from %s to %s",
235 compiler.target.name,
236 url,
237 dir,
238 )
239 dir.parent.mkdir(parents=True, exist_ok=True)
240 with tempfile.TemporaryDirectory(dir=dir.parent) as tmp_name:
241 tmp_dir = Path(tmp_name)
242 with subprocess.Popen(
243 ["xz", "--decompress"],
244 stdin=subprocess.PIPE,
245 stdout=subprocess.PIPE,
246 ) as xz_proc, subprocess.Popen(
247 ["tar", "-C", str(tmp_dir), "-x"],
248 stdin=xz_proc.stdout,
249 ) as tar_proc:
250 assert xz_proc.stdin is not None
251 try:
252 with urllib.request.urlopen(url) as resp:
253 shutil.copyfileobj(resp, xz_proc.stdin)
254 finally:
255 xz_proc.stdin.close()
256 if xz_proc.returncode != 0:
257 raise subprocess.CalledProcessError(
258 xz_proc.returncode, xz_proc.args
259 )
260 if tar_proc.returncode != 0:
261 raise subprocess.CalledProcessError(
262 tar_proc.returncode, tar_proc.args
263 )
264 archive_subdir = Path(
265 f"gcc-{KERNEL_ORG_COMPILER_VERSION}-nolibc/{compiler.target.kernel_org_compiler_target_name}"
266 )
267 archive_bin_subdir = archive_subdir / "bin"
268 if not (tmp_dir / archive_bin_subdir).exists():
269 raise FileNotFoundError(
270 f"downloaded archive does not contain {archive_bin_subdir}"
271 )
272 (tmp_dir / archive_subdir).rename(dir)
273 return compiler
274
275
276 def _download_thread(
277 download_dir: Path,
278 downloads: Iterable[Download],
279 q: "queue.Queue[Union[Downloaded, Exception]]",
280 ) -> None:
281 try:
282 downloader = Downloader(download_dir)
283 download_calls: List[Callable[[], Downloaded]] = []
284
285 for download in downloads:
286 if isinstance(download, DownloadKernel):
287 kernel = downloader.resolve_kernel(download.arch, download.pattern)
288 download_calls.append(
289 functools.partial(downloader.download_kernel, kernel)
290 )
291 elif isinstance(download, DownloadCompiler):
292 compiler = downloader.resolve_compiler(download.target)
293 download_calls.append(
294 functools.partial(downloader.download_compiler, compiler)
295 )
296 else:
297 assert False
298
299 for call in download_calls:
300 q.put(call())
301 except Exception as e:
302 q.put(e)
303
304
305 @contextmanager
306 def download_in_thread(
307 download_dir: Path, downloads: Iterable[Download], max_pending_kernels: int = 0
308 ) -> Generator[Iterator[Downloaded], None, None]:
309 q: "queue.Queue[Union[Downloaded, Exception]]" = queue.Queue(
310 maxsize=max_pending_kernels
311 )
312
313 def aux() -> Iterator[Downloaded]:
314 while True:
315 obj = q.get()
316 if isinstance(obj, StopIteration):
317 break
318 elif isinstance(obj, Exception):
319 raise obj
320 yield obj
321
322 thread = None
323 try:
324 thread = threading.Thread(
325 target=_download_thread,
326 args=(download_dir, downloads, q),
327 daemon=True,
328 )
329 thread.start()
330 yield aux()
331 finally:
332 if thread:
333 thread.join()
334
335
336 ARCH_ARGPARSE_METAVAR = f"{{{','.join(ARCHITECTURES)}}}"
337 if HOST_ARCHITECTURE is None:
338 DOWNLOAD_KERNEL_ARGPARSE_METAVAR = f"{ARCH_ARGPARSE_METAVAR}:PATTERN"
339 DEFAULT_ARCH_ARGPARSE_HELP = ""
340 else:
341 DOWNLOAD_KERNEL_ARGPARSE_METAVAR = f"[{ARCH_ARGPARSE_METAVAR}:]PATTERN"
342 DEFAULT_ARCH_ARGPARSE_HELP = f" (default: {HOST_ARCHITECTURE.name})"
343
344
345 def architecture_argparse_type(arg: str) -> Architecture:
346 try:
347 return ARCHITECTURES[arg]
348 except KeyError:
349 raise argparse.ArgumentTypeError(
350 f"architecture must be one of ({', '.join(ARCHITECTURES)})"
351 ) from None
352
353
354 def download_kernel_argparse_type(arg: str) -> DownloadKernel:
355 arch_name, sep, pattern = arg.rpartition(":")
356 if sep:
357 arch = architecture_argparse_type(arch_name)
358 else:
359 if HOST_ARCHITECTURE is None:
360 raise argparse.ArgumentTypeError("architecture is required")
361 arch = HOST_ARCHITECTURE
362 return DownloadKernel(arch, pattern)
363
364
365 def main() -> None:
366 logging.basicConfig(
367 format="%(asctime)s:%(levelname)s:%(name)s:%(message)s", level=logging.INFO
368 )
369
370 parser = argparse.ArgumentParser(
371 description="Download drgn vmtest assets",
372 formatter_class=argparse.ArgumentDefaultsHelpFormatter,
373 )
374 parser.add_argument(
375 "-k",
376 "--kernel",
377 metavar=DOWNLOAD_KERNEL_ARGPARSE_METAVAR,
378 dest="downloads",
379 action="append",
380 type=download_kernel_argparse_type,
381 default=argparse.SUPPRESS,
382 help=f"download latest kernel for given architecture{DEFAULT_ARCH_ARGPARSE_HELP} matching glob pattern; may be given multiple times",
383 )
384 parser.add_argument(
385 "-c",
386 "--compiler",
387 metavar=ARCH_ARGPARSE_METAVAR,
388 dest="downloads",
389 action="append",
390 nargs=1 if HOST_ARCHITECTURE is None else "?",
391 type=lambda arg: DownloadCompiler(architecture_argparse_type(arg)),
392 default=argparse.SUPPRESS,
393 help=f"download compiler for given architecture{DEFAULT_ARCH_ARGPARSE_HELP}; may be given multiple times",
394 )
395 parser.add_argument(
396 "-d",
397 "--download-directory",
398 metavar="DIR",
399 type=Path,
400 default="build/vmtest",
401 help="directory to download assets to",
402 )
403 args = parser.parse_args()
404 if not hasattr(args, "downloads"):
405 args.downloads = []
406 # --compiler with no argument is appended as None. Fix it.
407 for i, download_arg in enumerate(args.downloads):
408 if download_arg is None:
409 assert HOST_ARCHITECTURE is not None
410 args.downloads[i] = DownloadCompiler(HOST_ARCHITECTURE)
411
412 downloader = Downloader(args.download_directory)
413 for download in args.downloads:
414 if isinstance(download, DownloadKernel):
415 kernel = downloader.download_kernel(
416 downloader.resolve_kernel(download.arch, download.pattern)
417 )
418 print(
419 f"kernel: arch={kernel.arch.name} release={kernel.release} path={kernel.path}"
420 )
421 elif isinstance(download, DownloadCompiler):
422 compiler = downloader.download_compiler(
423 downloader.resolve_compiler(download.target)
424 )
425 print(
426 f"compiler: target={compiler.target.name} bin={compiler.bin} prefix={compiler.prefix}"
427 )
428 else:
429 assert False
430
431
432 if __name__ == "__main__":
433 main()