| 1 |
# Copyright (c) Meta Platforms, Inc. and affiliates. |
| 2 |
# SPDX-License-Identifier: LGPL-2.1-or-later |
| 3 |
|
| 4 |
import argparse |
| 5 |
import asyncio |
| 6 |
import itertools |
| 7 |
import logging |
| 8 |
import os |
| 9 |
from pathlib import Path |
| 10 |
import re |
| 11 |
import sys |
| 12 |
from typing import ( |
| 13 |
AsyncIterator, |
| 14 |
Dict, |
| 15 |
List, |
| 16 |
Mapping, |
| 17 |
NamedTuple, |
| 18 |
Optional, |
| 19 |
Sequence, |
| 20 |
Tuple, |
| 21 |
Union, |
| 22 |
) |
| 23 |
|
| 24 |
import aiohttp |
| 25 |
import uritemplate |
| 26 |
|
| 27 |
from util import KernelVersion |
| 28 |
from vmtest.asynciosubprocess import check_call, check_output |
| 29 |
from vmtest.config import ( |
| 30 |
ARCHITECTURES, |
| 31 |
KERNEL_FLAVORS, |
| 32 |
Architecture, |
| 33 |
Compiler, |
| 34 |
KernelFlavor, |
| 35 |
kconfig_localversion, |
| 36 |
) |
| 37 |
from vmtest.download import VMTEST_GITHUB_RELEASE, Downloader, available_kernel_releases |
| 38 |
from vmtest.githubapi import AioGitHubApi |
| 39 |
from vmtest.kbuild import KBuild, apply_patches |
| 40 |
|
| 41 |
logger = logging.getLogger(__name__) |
| 42 |
|
| 43 |
# [inclusive, exclusive) ranges of kernel versions to ignore when building |
| 44 |
# latest releases of each version. |
| 45 |
IGNORE_KERNEL_RANGES = ( |
| 46 |
(KernelVersion("~"), KernelVersion("4.9")), |
| 47 |
(KernelVersion("4.10~"), KernelVersion("4.14")), |
| 48 |
(KernelVersion("4.15~"), KernelVersion("4.19")), |
| 49 |
(KernelVersion("4.20~"), KernelVersion("5.4")), |
| 50 |
(KernelVersion("5.5~"), KernelVersion("5.10")), |
| 51 |
) |
| 52 |
|
| 53 |
# Use the GitHub mirrors rather than the official kernel.org repositories since |
| 54 |
# this script usually runs in GitHub Actions. |
| 55 |
LINUX_GIT_URL = "https://github.com/torvalds/linux.git" |
| 56 |
STABLE_LINUX_GIT_URL = "https://github.com/gregkh/linux.git" |
| 57 |
|
| 58 |
|
| 59 |
async def get_latest_kernel_tags() -> List[str]: |
| 60 |
mainline_refs, stable_refs = await asyncio.gather( |
| 61 |
check_output("git", "ls-remote", "--tags", "--refs", LINUX_GIT_URL), |
| 62 |
check_output("git", "ls-remote", "--tags", "--refs", STABLE_LINUX_GIT_URL), |
| 63 |
) |
| 64 |
latest: Dict[str, KernelVersion] = {} |
| 65 |
for match in itertools.chain( |
| 66 |
re.finditer( |
| 67 |
r"^[a-f0-9]+\s+refs/tags/v([0-9]+\.[0-9]+)(-rc[0-9]+)?$", |
| 68 |
mainline_refs.decode(), |
| 69 |
re.M, |
| 70 |
), |
| 71 |
re.finditer( |
| 72 |
r"^[a-f0-9]+\s+refs/tags/v([0-9]+\.[0-9]+)(\.[0-9]+)$", |
| 73 |
stable_refs.decode(), |
| 74 |
re.M, |
| 75 |
), |
| 76 |
): |
| 77 |
version = KernelVersion(match.group(1) + (match.group(2) or "")) |
| 78 |
for start_version, end_version in IGNORE_KERNEL_RANGES: |
| 79 |
if start_version <= version < end_version: |
| 80 |
break |
| 81 |
else: |
| 82 |
latest[match.group(1)] = max(version, latest.get(match.group(1), version)) |
| 83 |
return ["v" + str(version) for version in sorted(latest.values(), reverse=True)] |
| 84 |
|
| 85 |
|
| 86 |
async def fetch_kernel_tags(kernel_dir: Path, kernel_tags: Sequence[str]) -> None: |
| 87 |
if not kernel_dir.exists(): |
| 88 |
logger.info("creating kernel repository in %s", kernel_dir) |
| 89 |
await check_call("git", "init", "-q", str(kernel_dir)) |
| 90 |
|
| 91 |
mainline_tags = [] |
| 92 |
stable_tags = [] |
| 93 |
for tag in kernel_tags: |
| 94 |
if re.fullmatch(r"v[0-9]+\.[0-9]+\.[0-9]+", tag): |
| 95 |
stable_tags.append(tag) |
| 96 |
else: |
| 97 |
mainline_tags.append(tag) |
| 98 |
|
| 99 |
for name, url, tags in ( |
| 100 |
("mainline", LINUX_GIT_URL, mainline_tags), |
| 101 |
("stable", STABLE_LINUX_GIT_URL, stable_tags), |
| 102 |
): |
| 103 |
if tags: |
| 104 |
logger.info("fetching %s kernel tags: %s", name, ", ".join(tags)) |
| 105 |
await check_call( |
| 106 |
"git", |
| 107 |
"-C", |
| 108 |
str(kernel_dir), |
| 109 |
"fetch", |
| 110 |
"--depth", |
| 111 |
"1", |
| 112 |
url, |
| 113 |
*(f"refs/tags/{tag}:refs/tags/{tag}" for tag in tags), |
| 114 |
) |
| 115 |
|
| 116 |
|
| 117 |
async def build_kernels( |
| 118 |
kernel_dir: Path, |
| 119 |
build_dir: Path, |
| 120 |
kernel_revs: Sequence[ |
| 121 |
Tuple[str, Sequence[Tuple[Architecture, Sequence[KernelFlavor]]]] |
| 122 |
], |
| 123 |
compilers: Mapping[str, Compiler], |
| 124 |
keep_builds: bool, |
| 125 |
) -> AsyncIterator[Path]: |
| 126 |
build_dir.mkdir(parents=True, exist_ok=True) |
| 127 |
for rev, arches in kernel_revs: |
| 128 |
logger.info("checking out %s in %s", rev, kernel_dir) |
| 129 |
await check_call( |
| 130 |
"git", "-C", str(kernel_dir), "checkout", "--force", "--quiet", rev |
| 131 |
) |
| 132 |
await check_call("git", "-C", str(kernel_dir), "clean", "-dqf") |
| 133 |
await apply_patches(kernel_dir) |
| 134 |
for arch, flavors in arches: |
| 135 |
env = {**os.environ, **compilers[arch.name].env()} |
| 136 |
for flavor in flavors: |
| 137 |
flavor_rev_build_dir = ( |
| 138 |
build_dir / f"build-{arch.name}-{flavor.name}-{rev}" |
| 139 |
) |
| 140 |
with open( |
| 141 |
build_dir / f"build-{arch.name}-{flavor.name}-{rev}.log", "w" |
| 142 |
) as build_log_file: |
| 143 |
kbuild = KBuild( |
| 144 |
kernel_dir, |
| 145 |
flavor_rev_build_dir, |
| 146 |
arch, |
| 147 |
flavor, |
| 148 |
env=env, |
| 149 |
build_log_file=build_log_file, |
| 150 |
) |
| 151 |
await kbuild.build() |
| 152 |
yield await kbuild.package("tar.zst", build_dir) |
| 153 |
if not keep_builds: |
| 154 |
logger.info("deleting %s", flavor_rev_build_dir) |
| 155 |
# Shell out instead of using, e.g., shutil.rmtree(), to |
| 156 |
# avoid blocking the main thread and the GIL. |
| 157 |
await check_call("rm", "-rf", str(flavor_rev_build_dir)) |
| 158 |
|
| 159 |
|
| 160 |
class AssetUploadWork(NamedTuple): |
| 161 |
upload_url: str |
| 162 |
path: Union[str, bytes, Path] |
| 163 |
name: str |
| 164 |
content_type: str |
| 165 |
|
| 166 |
|
| 167 |
async def asset_uploader( |
| 168 |
gh: AioGitHubApi, |
| 169 |
queue: "asyncio.Queue[Optional[AssetUploadWork]]", |
| 170 |
) -> bool: |
| 171 |
success = True |
| 172 |
while True: |
| 173 |
work = await queue.get() |
| 174 |
if not work: |
| 175 |
queue.task_done() |
| 176 |
return success |
| 177 |
logger.info("uploading %s", work.name) |
| 178 |
try: |
| 179 |
with open(work.path, "rb") as f: |
| 180 |
await gh.upload( |
| 181 |
uritemplate.expand(work.upload_url, name=work.name), |
| 182 |
f, |
| 183 |
work.content_type, |
| 184 |
) |
| 185 |
except Exception: |
| 186 |
logger.exception("uploading %s failed", work.name) |
| 187 |
success = False |
| 188 |
else: |
| 189 |
logger.info("uploaded %s", work.name) |
| 190 |
finally: |
| 191 |
queue.task_done() |
| 192 |
|
| 193 |
|
| 194 |
async def main() -> None: |
| 195 |
logging.basicConfig( |
| 196 |
format="%(asctime)s:%(levelname)s:%(name)s:%(message)s", level=logging.INFO |
| 197 |
) |
| 198 |
|
| 199 |
parser = argparse.ArgumentParser( |
| 200 |
description="Build and upload drgn vmtest assets", |
| 201 |
formatter_class=argparse.ArgumentDefaultsHelpFormatter, |
| 202 |
) |
| 203 |
parser.add_argument( |
| 204 |
"-K", |
| 205 |
"--latest-kernels", |
| 206 |
action="store_true", |
| 207 |
help="build and upload latest supported kernel releases", |
| 208 |
) |
| 209 |
parser.add_argument( |
| 210 |
"-a", |
| 211 |
"--architecture", |
| 212 |
dest="architectures", |
| 213 |
choices=["all", *sorted(ARCHITECTURES)], |
| 214 |
action="append", |
| 215 |
help="build architecture; may be given multiple times (default: all)", |
| 216 |
default=argparse.SUPPRESS, |
| 217 |
) |
| 218 |
parser.add_argument( |
| 219 |
"-f", |
| 220 |
"--flavor", |
| 221 |
dest="flavors", |
| 222 |
choices=["all", *KERNEL_FLAVORS], |
| 223 |
action="append", |
| 224 |
help="build flavor; may be given multiple times (default: all)", |
| 225 |
default=argparse.SUPPRESS, |
| 226 |
) |
| 227 |
parser.add_argument( |
| 228 |
"--no-build", |
| 229 |
dest="build", |
| 230 |
action="store_false", |
| 231 |
help="don't build or upload anything; just log what would be built", |
| 232 |
) |
| 233 |
parser.add_argument( |
| 234 |
"--no-upload", |
| 235 |
dest="upload", |
| 236 |
action="store_false", |
| 237 |
help="build but don't upload anything to GitHub", |
| 238 |
) |
| 239 |
parser.add_argument( |
| 240 |
"--kernel-directory", |
| 241 |
metavar="DIR", |
| 242 |
type=Path, |
| 243 |
help="kernel Git repository directory (created if needed)", |
| 244 |
default=".", |
| 245 |
) |
| 246 |
parser.add_argument( |
| 247 |
"--build-directory", |
| 248 |
metavar="DIR", |
| 249 |
type=Path, |
| 250 |
help="directory for build artifacts", |
| 251 |
default=".", |
| 252 |
) |
| 253 |
parser.add_argument( |
| 254 |
"--no-keep-builds", |
| 255 |
dest="keep_builds", |
| 256 |
action="store_false", |
| 257 |
help="delete kernel builds after packaging (default if uploading)", |
| 258 |
default=argparse.SUPPRESS, |
| 259 |
) |
| 260 |
parser.add_argument( |
| 261 |
"--keep-builds", |
| 262 |
action="store_true", |
| 263 |
help="keep kernel builds after packaging (default if not uploading)", |
| 264 |
default=argparse.SUPPRESS, |
| 265 |
) |
| 266 |
parser.add_argument( |
| 267 |
"--download-directory", |
| 268 |
metavar="DIR", |
| 269 |
type=Path, |
| 270 |
default="build/vmtest", |
| 271 |
help="directory to download assets to", |
| 272 |
) |
| 273 |
args = parser.parse_args() |
| 274 |
|
| 275 |
if not hasattr(args, "architectures") or "all" in args.architectures: |
| 276 |
args.architectures = list(ARCHITECTURES.values()) |
| 277 |
else: |
| 278 |
args.architectures = [ |
| 279 |
arch for arch in ARCHITECTURES.values() if arch.name in args.architectures |
| 280 |
] |
| 281 |
if not hasattr(args, "flavors") or "all" in args.flavors: |
| 282 |
args.flavors = list(KERNEL_FLAVORS.values()) |
| 283 |
else: |
| 284 |
args.flavors = [ |
| 285 |
flavor for flavor in KERNEL_FLAVORS.values() if flavor.name in args.flavors |
| 286 |
] |
| 287 |
|
| 288 |
if not args.build: |
| 289 |
args.upload = False |
| 290 |
if not hasattr(args, "keep_builds"): |
| 291 |
args.keep_builds = not args.upload |
| 292 |
|
| 293 |
async with aiohttp.ClientSession(trust_env=True) as session: |
| 294 |
GITHUB_TOKEN = os.getenv("GITHUB_TOKEN") |
| 295 |
if GITHUB_TOKEN is None and args.upload: |
| 296 |
sys.exit("GITHUB_TOKEN environment variable is not set") |
| 297 |
gh = AioGitHubApi(session, GITHUB_TOKEN) |
| 298 |
|
| 299 |
args.download_directory.mkdir(parents=True, exist_ok=True) |
| 300 |
github_release_coro = gh.get_release_by_tag( |
| 301 |
*VMTEST_GITHUB_RELEASE, |
| 302 |
cache=args.download_directory / "github_release.json", |
| 303 |
) |
| 304 |
if args.latest_kernels: |
| 305 |
github_release, latest_kernel_tags = await asyncio.gather( |
| 306 |
github_release_coro, get_latest_kernel_tags() |
| 307 |
) |
| 308 |
else: |
| 309 |
github_release = await github_release_coro |
| 310 |
|
| 311 |
kernel_releases = available_kernel_releases(github_release) |
| 312 |
logger.info("available kernel releases:") |
| 313 |
for arch in args.architectures: |
| 314 |
arch_kernel_releases = kernel_releases.get(arch.name, {}) |
| 315 |
logger.info( |
| 316 |
" %s: %s", |
| 317 |
arch.name, |
| 318 |
", ".join( |
| 319 |
sorted(arch_kernel_releases, key=KernelVersion, reverse=True) |
| 320 |
), |
| 321 |
) |
| 322 |
|
| 323 |
to_build = [] |
| 324 |
if args.latest_kernels: |
| 325 |
logger.info("latest kernel versions: %s", ", ".join(latest_kernel_tags)) |
| 326 |
for tag in latest_kernel_tags: |
| 327 |
tag_arches_to_build = [] |
| 328 |
|
| 329 |
match = re.fullmatch(r"v([0-9]+\.[0-9]+)(\.[0-9]+)?(-rc\d+)?", tag) |
| 330 |
assert match |
| 331 |
version = "".join( |
| 332 |
[match.group(1), match.group(2) or ".0", match.group(3) or ""] |
| 333 |
) |
| 334 |
|
| 335 |
for arch in args.architectures: |
| 336 |
if not arch.kernel_version_supported(version): |
| 337 |
continue |
| 338 |
|
| 339 |
arch_kernel_releases = kernel_releases.get(arch.name, {}) |
| 340 |
tag_arch_flavors_to_build = [ |
| 341 |
flavor |
| 342 |
for flavor in args.flavors |
| 343 |
if (version + kconfig_localversion(arch, flavor, version)) |
| 344 |
not in arch_kernel_releases |
| 345 |
] |
| 346 |
|
| 347 |
if tag_arch_flavors_to_build: |
| 348 |
tag_arches_to_build.append((arch, tag_arch_flavors_to_build)) |
| 349 |
if tag_arches_to_build: |
| 350 |
to_build.append((tag, tag_arches_to_build)) |
| 351 |
|
| 352 |
if to_build: |
| 353 |
logger.info("kernel versions to build:") |
| 354 |
for tag, tag_arches_to_build in to_build: |
| 355 |
logger.info( |
| 356 |
" %s (%s)", |
| 357 |
tag, |
| 358 |
", ".join( |
| 359 |
[ |
| 360 |
f"{arch.name} [{', '.join([flavor.name for flavor in tag_arch_flavors_to_build])}]" |
| 361 |
for arch, tag_arch_flavors_to_build in tag_arches_to_build |
| 362 |
] |
| 363 |
), |
| 364 |
) |
| 365 |
|
| 366 |
if args.build: |
| 367 |
downloader = Downloader(args.download_directory) |
| 368 |
compilers = { |
| 369 |
arch.name: downloader.download_compiler( |
| 370 |
downloader.resolve_compiler(arch) |
| 371 |
) |
| 372 |
for arch in { |
| 373 |
arch.name: arch |
| 374 |
for _, tag_arches_to_build in to_build |
| 375 |
for arch, _ in tag_arches_to_build |
| 376 |
}.values() |
| 377 |
} |
| 378 |
|
| 379 |
if args.upload: |
| 380 |
upload_queue: "asyncio.Queue[Optional[AssetUploadWork]]" = ( |
| 381 |
asyncio.Queue() |
| 382 |
) |
| 383 |
uploader = asyncio.create_task(asset_uploader(gh, upload_queue)) |
| 384 |
|
| 385 |
await fetch_kernel_tags( |
| 386 |
args.kernel_directory, [tag for tag, _ in to_build] |
| 387 |
) |
| 388 |
|
| 389 |
async for kernel_package in build_kernels( |
| 390 |
args.kernel_directory, |
| 391 |
args.build_directory, |
| 392 |
to_build, |
| 393 |
compilers, |
| 394 |
args.keep_builds, |
| 395 |
): |
| 396 |
if args.upload: |
| 397 |
await upload_queue.put( |
| 398 |
AssetUploadWork( |
| 399 |
upload_url=github_release["upload_url"], |
| 400 |
path=kernel_package, |
| 401 |
name=kernel_package.name, |
| 402 |
content_type="application/zstd", |
| 403 |
) |
| 404 |
) |
| 405 |
else: |
| 406 |
logger.info("would upload %s", kernel_package) |
| 407 |
|
| 408 |
if args.upload: |
| 409 |
await upload_queue.put(None) |
| 410 |
await upload_queue.join() |
| 411 |
if not await uploader: |
| 412 |
sys.exit("some uploads failed") |
| 413 |
else: |
| 414 |
logger.info("nothing to build") |
| 415 |
|
| 416 |
|
| 417 |
if __name__ == "__main__": |
| 418 |
asyncio.run(main()) |