Debian Codegraph @

vmtest/mirror_compilers.py

raw
1 # Copyright (c) Meta Platforms, Inc. and affiliates.
2 # SPDX-License-Identifier: LGPL-2.1-or-later
3
4 import argparse
5 import logging
6 import os
7 from pathlib import Path
8 import shutil
9 import sys
10 import tempfile
11 import urllib.request
12
13 import uritemplate
14
15 from vmtest.config import ARCHITECTURES, KERNEL_ORG_COMPILER_VERSION, compiler_name
16 from vmtest.githubapi import GitHubApi
17
18 logger = logging.getLogger(__name__)
19
20
21 _KERNEL_ORG_COMPILER_URL = "https://mirrors.kernel.org/pub/tools/crosstool/"
22
23
24 def _kernel_org_compiler_url(host_name: str, target_name: str) -> str:
25 return f"{_KERNEL_ORG_COMPILER_URL}files/bin/{host_name}/{KERNEL_ORG_COMPILER_VERSION}/{compiler_name(host_name, target_name)}.tar.xz"
26
27
28 def main() -> None:
29 logging.basicConfig(
30 format="%(asctime)s:%(levelname)s:%(name)s:%(message)s", level=logging.INFO
31 )
32
33 parser = argparse.ArgumentParser(
34 description="mirror compilers from kernel.org to GitHub"
35 )
36 parser.parse_args()
37
38 GITHUB_TOKEN = os.getenv("GITHUB_TOKEN")
39 if GITHUB_TOKEN is None:
40 sys.exit("GITHUB_TOKEN environment variable is not set")
41 gh = GitHubApi(GITHUB_TOKEN)
42
43 host_names = []
44 target_names = []
45 for arch in ARCHITECTURES.values():
46 if arch.kernel_org_compiler_host_name is not None:
47 host_names.append(arch.kernel_org_compiler_host_name)
48 target_names.append(arch.kernel_org_compiler_target_name)
49
50 release = gh.get_release_by_tag("osandov", "drgn", "vmtest-compilers")
51 upload_url = release["upload_url"]
52 available_compilers = {asset["name"] for asset in release["assets"]}
53
54 with tempfile.TemporaryDirectory() as tmp:
55 tmp_dir = Path(tmp)
56
57 for host_name in host_names:
58 for target_name in target_names:
59 url = _kernel_org_compiler_url(host_name, target_name)
60 file_name = url.rpartition("/")[2]
61
62 if file_name in available_compilers:
63 logger.info("%s is already available", file_name)
64 continue
65
66 file_path = tmp_dir / file_name
67
68 with file_path.open("w+b") as f:
69 logger.info("downloading %s", url)
70 with urllib.request.urlopen(url) as resp:
71 shutil.copyfileobj(resp, f)
72 logger.info("downloaded %s", url)
73
74 content_length = f.tell()
75 f.flush()
76 f.seek(0)
77
78 logger.info("uploading %s", file_name)
79 print(uritemplate.expand(upload_url, name=file_name))
80 gh.upload(
81 uritemplate.expand(upload_url, name=file_name),
82 f,
83 "application/x-xz",
84 content_length,
85 )
86 logger.info("uploaded %s", file_name)
87
88 file_path.unlink()
89
90
91 if __name__ == "__main__":
92 main()