| 1 |
# Copyright (c) Meta Platforms, Inc. and affiliates. |
| 2 |
# SPDX-License-Identifier: GPL-3.0-or-later |
| 3 |
|
| 4 |
import argparse |
| 5 |
import asyncio |
| 6 |
from collections import defaultdict |
| 7 |
import logging |
| 8 |
import operator |
| 9 |
import os |
| 10 |
from pathlib import Path |
| 11 |
import re |
| 12 |
import shutil |
| 13 |
import sys |
| 14 |
from typing import Any, Awaitable, Callable, Dict, Iterable, List, Tuple, TypeVar |
| 15 |
|
| 16 |
import aiohttp |
| 17 |
|
| 18 |
from util import KernelVersion |
| 19 |
from vmtest.download import VMTEST_GITHUB_RELEASE |
| 20 |
from vmtest.githubapi import AioGitHubApi |
| 21 |
|
| 22 |
logger = logging.getLogger(__name__) |
| 23 |
|
| 24 |
|
| 25 |
def _prompt_yes_no(prompt: str, default: bool = True) -> bool: |
| 26 |
prompt += " [Y/n] " if default else " [y/N] " |
| 27 |
sys.stdout.flush() |
| 28 |
sys.stderr.write(prompt) |
| 29 |
sys.stderr.flush() |
| 30 |
answer = input().strip().lower() |
| 31 |
if answer.startswith("y"): |
| 32 |
return True |
| 33 |
elif answer.startswith("n"): |
| 34 |
return False |
| 35 |
else: |
| 36 |
return default |
| 37 |
|
| 38 |
|
| 39 |
def _kernel_asset_sort_key(x: Tuple[str, str, Any]) -> Tuple[KernelVersion, str]: |
| 40 |
release, arch, _ = x |
| 41 |
return KernelVersion(release), arch |
| 42 |
|
| 43 |
|
| 44 |
_AssetT = TypeVar("_AssetT") |
| 45 |
|
| 46 |
|
| 47 |
async def _prune( |
| 48 |
*, |
| 49 |
kernels: Iterable[Tuple[str, str, _AssetT]], |
| 50 |
kmods: Iterable[Tuple[str, str, _AssetT]] = (), |
| 51 |
keep: int, |
| 52 |
asset_name: Callable[[_AssetT], str], |
| 53 |
delete_asset: Callable[[_AssetT], Awaitable[None]], |
| 54 |
) -> int: |
| 55 |
def print_asset_list(assets: List[Tuple[str, str, _AssetT]]) -> None: |
| 56 |
for _, _, asset in assets: |
| 57 |
print(f" {asset_name(asset)}") |
| 58 |
|
| 59 |
grouped_kernels: Dict[Tuple[str, str, str], List[Tuple[str, _AssetT]]] = ( |
| 60 |
defaultdict(list) |
| 61 |
) |
| 62 |
unrecognized_kernels: List[Tuple[str, str, _AssetT]] = [] |
| 63 |
|
| 64 |
for release, arch, asset in kernels: |
| 65 |
match = re.fullmatch( |
| 66 |
r"(?P<major_minor>[0-9]+\.[0-9]+)\.(?P<patch_rc>[0-9]+(?:-rc[0-9]+)?)-vmtest(?P<vmtest>[0-9]+(?:\.[0-9]+)*)(?P<flavor>[^.]+)", |
| 67 |
release, |
| 68 |
) |
| 69 |
if match: |
| 70 |
grouped_kernels[ |
| 71 |
(match.group("major_minor"), match.group("flavor"), arch) |
| 72 |
].append((release, asset)) |
| 73 |
else: |
| 74 |
unrecognized_kernels.append((release, arch, asset)) |
| 75 |
|
| 76 |
kernels_to_keep = {(release, arch) for release, arch, _ in unrecognized_kernels} |
| 77 |
kernels_to_delete: List[Tuple[str, str, _AssetT]] = [] |
| 78 |
for (_, _, arch), kernel_group in grouped_kernels.items(): |
| 79 |
kernel_group.sort(key=lambda x: KernelVersion(x[0]), reverse=True) |
| 80 |
for release, _ in kernel_group[:keep]: |
| 81 |
kernels_to_keep.add((release, arch)) |
| 82 |
kernels_to_delete.extend( |
| 83 |
(release, arch, asset) for release, asset in kernel_group[keep:] |
| 84 |
) |
| 85 |
kernels_to_delete.sort(key=_kernel_asset_sort_key) |
| 86 |
|
| 87 |
kmods_to_delete = [ |
| 88 |
(release, arch, asset) |
| 89 |
for release, arch, asset in kmods |
| 90 |
if (release, arch) not in kernels_to_keep |
| 91 |
] |
| 92 |
kmods_to_delete.sort(key=_kernel_asset_sort_key) |
| 93 |
|
| 94 |
if kernels_to_delete: |
| 95 |
print("Deleting kernels:") |
| 96 |
print_asset_list(kernels_to_delete) |
| 97 |
|
| 98 |
if kmods_to_delete: |
| 99 |
print("Deleting kernel modules:") |
| 100 |
print_asset_list(kmods_to_delete) |
| 101 |
|
| 102 |
if unrecognized_kernels: |
| 103 |
unrecognized_kernels.sort(key=_kernel_asset_sort_key) |
| 104 |
print("Ignoring unrecognized kernels:") |
| 105 |
print_asset_list(unrecognized_kernels) |
| 106 |
|
| 107 |
if not kernels_to_delete and not kmods_to_delete: |
| 108 |
return 0 |
| 109 |
|
| 110 |
if not _prompt_yes_no("Continue?", default=False): |
| 111 |
return 1 |
| 112 |
|
| 113 |
to_delete = kernels_to_delete + kmods_to_delete |
| 114 |
for _, _, asset in to_delete: |
| 115 |
await delete_asset(asset) |
| 116 |
return 0 |
| 117 |
|
| 118 |
|
| 119 |
async def _delete_local_asset(asset: Path) -> None: |
| 120 |
try: |
| 121 |
asset.unlink() |
| 122 |
except IsADirectoryError: |
| 123 |
shutil.rmtree(asset) |
| 124 |
|
| 125 |
|
| 126 |
async def main() -> None: |
| 127 |
parser = argparse.ArgumentParser( |
| 128 |
description="delete old vmtest kernels", |
| 129 |
formatter_class=argparse.ArgumentDefaultsHelpFormatter, |
| 130 |
) |
| 131 |
parser.add_argument( |
| 132 |
"--directory", |
| 133 |
metavar="DIR", |
| 134 |
type=Path, |
| 135 |
default="build/vmtest", |
| 136 |
help="directory for vmtest artifacts", |
| 137 |
) |
| 138 |
group = parser.add_mutually_exclusive_group(required=True) |
| 139 |
group.add_argument( |
| 140 |
"-l", |
| 141 |
"--local", |
| 142 |
action="store_true", |
| 143 |
default=argparse.SUPPRESS, |
| 144 |
help="delete locally downloaded vmtest kernels and their kernel modules", |
| 145 |
) |
| 146 |
group.add_argument( |
| 147 |
"-r", |
| 148 |
"--remote", |
| 149 |
dest="local", |
| 150 |
action="store_false", |
| 151 |
default=argparse.SUPPRESS, |
| 152 |
help="delete uploaded vmtest kernel assets", |
| 153 |
) |
| 154 |
parser.add_argument( |
| 155 |
"-k", "--keep", type=int, default=1, help="number of latest kernels to keep" |
| 156 |
) |
| 157 |
args = parser.parse_args() |
| 158 |
|
| 159 |
if args.keep <= 0: |
| 160 |
sys.exit("keep must be >= 1") |
| 161 |
|
| 162 |
if args.local: |
| 163 |
status = await _prune( |
| 164 |
kernels=( |
| 165 |
(kernel.name[len("kernel-") :], kernel.parent.name, kernel) |
| 166 |
for kernel in args.directory.glob("*/kernel-*") |
| 167 |
), |
| 168 |
kmods=( |
| 169 |
(kmod.name[len("drgn_test-") : -len(".ko")], kmod.parent.name, kmod) |
| 170 |
for kmod in args.directory.glob("*/drgn_test-*.ko") |
| 171 |
), |
| 172 |
keep=args.keep, |
| 173 |
asset_name=str, |
| 174 |
delete_asset=_delete_local_asset, |
| 175 |
) |
| 176 |
else: |
| 177 |
GITHUB_TOKEN = os.getenv("GITHUB_TOKEN") |
| 178 |
if GITHUB_TOKEN is None: |
| 179 |
sys.exit("GITHUB_TOKEN environment variable is not set") |
| 180 |
async with aiohttp.ClientSession(trust_env=True) as session: |
| 181 |
gh = AioGitHubApi(session, GITHUB_TOKEN) |
| 182 |
|
| 183 |
github_release = await gh.get_release_by_tag( |
| 184 |
*VMTEST_GITHUB_RELEASE, cache=args.directory / "github_release.json" |
| 185 |
) |
| 186 |
|
| 187 |
kernels = [] |
| 188 |
for asset in github_release["assets"]: |
| 189 |
match = re.fullmatch( |
| 190 |
r"kernel-(?P<release>.*)\.(?P<arch>[^.]+)\.tar\.zst", asset["name"] |
| 191 |
) |
| 192 |
if match: |
| 193 |
kernels.append((match.group("release"), match.group("arch"), asset)) |
| 194 |
elif asset["name"].startswith("kernel-"): |
| 195 |
logger.warning("ignoring unrecognized kernel: %s", asset["name"]) |
| 196 |
|
| 197 |
async def _delete_remote_asset(asset: Dict[str, Any]) -> None: |
| 198 |
await gh.delete(asset["url"]) |
| 199 |
|
| 200 |
status = await _prune( |
| 201 |
kernels=kernels, |
| 202 |
keep=args.keep, |
| 203 |
asset_name=operator.itemgetter("name"), |
| 204 |
delete_asset=_delete_remote_asset, |
| 205 |
) |
| 206 |
|
| 207 |
sys.exit(status) |
| 208 |
|
| 209 |
|
| 210 |
if __name__ == "__main__": |
| 211 |
asyncio.run(main()) |