| 1 |
# Copyright (c) Meta Platforms, Inc. and affiliates. |
| 2 |
# SPDX-License-Identifier: LGPL-2.1-or-later |
| 3 |
|
| 4 |
import argparse |
| 5 |
import logging |
| 6 |
import os |
| 7 |
from pathlib import Path |
| 8 |
import shutil |
| 9 |
import sys |
| 10 |
import tempfile |
| 11 |
import urllib.request |
| 12 |
|
| 13 |
import uritemplate |
| 14 |
|
| 15 |
from vmtest.config import ARCHITECTURES, KERNEL_ORG_COMPILER_VERSION, compiler_name |
| 16 |
from vmtest.githubapi import GitHubApi |
| 17 |
|
| 18 |
logger = logging.getLogger(__name__) |
| 19 |
|
| 20 |
|
| 21 |
_KERNEL_ORG_COMPILER_URL = "https://mirrors.kernel.org/pub/tools/crosstool/" |
| 22 |
|
| 23 |
|
| 24 |
def _kernel_org_compiler_url(host_name: str, target_name: str) -> str: |
| 25 |
return f"{_KERNEL_ORG_COMPILER_URL}files/bin/{host_name}/{KERNEL_ORG_COMPILER_VERSION}/{compiler_name(host_name, target_name)}.tar.xz" |
| 26 |
|
| 27 |
|
| 28 |
def main() -> None: |
| 29 |
logging.basicConfig( |
| 30 |
format="%(asctime)s:%(levelname)s:%(name)s:%(message)s", level=logging.INFO |
| 31 |
) |
| 32 |
|
| 33 |
parser = argparse.ArgumentParser( |
| 34 |
description="mirror compilers from kernel.org to GitHub" |
| 35 |
) |
| 36 |
parser.parse_args() |
| 37 |
|
| 38 |
GITHUB_TOKEN = os.getenv("GITHUB_TOKEN") |
| 39 |
if GITHUB_TOKEN is None: |
| 40 |
sys.exit("GITHUB_TOKEN environment variable is not set") |
| 41 |
gh = GitHubApi(GITHUB_TOKEN) |
| 42 |
|
| 43 |
host_names = [] |
| 44 |
target_names = [] |
| 45 |
for arch in ARCHITECTURES.values(): |
| 46 |
if arch.kernel_org_compiler_host_name is not None: |
| 47 |
host_names.append(arch.kernel_org_compiler_host_name) |
| 48 |
target_names.append(arch.kernel_org_compiler_target_name) |
| 49 |
|
| 50 |
release = gh.get_release_by_tag("osandov", "drgn", "vmtest-compilers") |
| 51 |
upload_url = release["upload_url"] |
| 52 |
available_compilers = {asset["name"] for asset in release["assets"]} |
| 53 |
|
| 54 |
with tempfile.TemporaryDirectory() as tmp: |
| 55 |
tmp_dir = Path(tmp) |
| 56 |
|
| 57 |
for host_name in host_names: |
| 58 |
for target_name in target_names: |
| 59 |
url = _kernel_org_compiler_url(host_name, target_name) |
| 60 |
file_name = url.rpartition("/")[2] |
| 61 |
|
| 62 |
if file_name in available_compilers: |
| 63 |
logger.info("%s is already available", file_name) |
| 64 |
continue |
| 65 |
|
| 66 |
file_path = tmp_dir / file_name |
| 67 |
|
| 68 |
with file_path.open("w+b") as f: |
| 69 |
logger.info("downloading %s", url) |
| 70 |
with urllib.request.urlopen(url) as resp: |
| 71 |
shutil.copyfileobj(resp, f) |
| 72 |
logger.info("downloaded %s", url) |
| 73 |
|
| 74 |
content_length = f.tell() |
| 75 |
f.flush() |
| 76 |
f.seek(0) |
| 77 |
|
| 78 |
logger.info("uploading %s", file_name) |
| 79 |
print(uritemplate.expand(upload_url, name=file_name)) |
| 80 |
gh.upload( |
| 81 |
uritemplate.expand(upload_url, name=file_name), |
| 82 |
f, |
| 83 |
"application/x-xz", |
| 84 |
content_length, |
| 85 |
) |
| 86 |
logger.info("uploaded %s", file_name) |
| 87 |
|
| 88 |
file_path.unlink() |
| 89 |
|
| 90 |
|
| 91 |
if __name__ == "__main__": |
| 92 |
main() |