Debian Codegraph @

vmtest/prune.py

raw
1 # Copyright (c) Meta Platforms, Inc. and affiliates.
2 # SPDX-License-Identifier: GPL-3.0-or-later
3
4 import argparse
5 import asyncio
6 from collections import defaultdict
7 import logging
8 import operator
9 import os
10 from pathlib import Path
11 import re
12 import shutil
13 import sys
14 from typing import Any, Awaitable, Callable, Dict, Iterable, List, Tuple, TypeVar
15
16 import aiohttp
17
18 from util import KernelVersion
19 from vmtest.download import VMTEST_GITHUB_RELEASE
20 from vmtest.githubapi import AioGitHubApi
21
22 logger = logging.getLogger(__name__)
23
24
25 def _prompt_yes_no(prompt: str, default: bool = True) -> bool:
26 prompt += " [Y/n] " if default else " [y/N] "
27 sys.stdout.flush()
28 sys.stderr.write(prompt)
29 sys.stderr.flush()
30 answer = input().strip().lower()
31 if answer.startswith("y"):
32 return True
33 elif answer.startswith("n"):
34 return False
35 else:
36 return default
37
38
39 def _kernel_asset_sort_key(x: Tuple[str, str, Any]) -> Tuple[KernelVersion, str]:
40 release, arch, _ = x
41 return KernelVersion(release), arch
42
43
44 _AssetT = TypeVar("_AssetT")
45
46
47 async def _prune(
48 *,
49 kernels: Iterable[Tuple[str, str, _AssetT]],
50 kmods: Iterable[Tuple[str, str, _AssetT]] = (),
51 keep: int,
52 asset_name: Callable[[_AssetT], str],
53 delete_asset: Callable[[_AssetT], Awaitable[None]],
54 ) -> int:
55 def print_asset_list(assets: List[Tuple[str, str, _AssetT]]) -> None:
56 for _, _, asset in assets:
57 print(f" {asset_name(asset)}")
58
59 grouped_kernels: Dict[Tuple[str, str, str], List[Tuple[str, _AssetT]]] = (
60 defaultdict(list)
61 )
62 unrecognized_kernels: List[Tuple[str, str, _AssetT]] = []
63
64 for release, arch, asset in kernels:
65 match = re.fullmatch(
66 r"(?P<major_minor>[0-9]+\.[0-9]+)\.(?P<patch_rc>[0-9]+(?:-rc[0-9]+)?)-vmtest(?P<vmtest>[0-9]+(?:\.[0-9]+)*)(?P<flavor>[^.]+)",
67 release,
68 )
69 if match:
70 grouped_kernels[
71 (match.group("major_minor"), match.group("flavor"), arch)
72 ].append((release, asset))
73 else:
74 unrecognized_kernels.append((release, arch, asset))
75
76 kernels_to_keep = {(release, arch) for release, arch, _ in unrecognized_kernels}
77 kernels_to_delete: List[Tuple[str, str, _AssetT]] = []
78 for (_, _, arch), kernel_group in grouped_kernels.items():
79 kernel_group.sort(key=lambda x: KernelVersion(x[0]), reverse=True)
80 for release, _ in kernel_group[:keep]:
81 kernels_to_keep.add((release, arch))
82 kernels_to_delete.extend(
83 (release, arch, asset) for release, asset in kernel_group[keep:]
84 )
85 kernels_to_delete.sort(key=_kernel_asset_sort_key)
86
87 kmods_to_delete = [
88 (release, arch, asset)
89 for release, arch, asset in kmods
90 if (release, arch) not in kernels_to_keep
91 ]
92 kmods_to_delete.sort(key=_kernel_asset_sort_key)
93
94 if kernels_to_delete:
95 print("Deleting kernels:")
96 print_asset_list(kernels_to_delete)
97
98 if kmods_to_delete:
99 print("Deleting kernel modules:")
100 print_asset_list(kmods_to_delete)
101
102 if unrecognized_kernels:
103 unrecognized_kernels.sort(key=_kernel_asset_sort_key)
104 print("Ignoring unrecognized kernels:")
105 print_asset_list(unrecognized_kernels)
106
107 if not kernels_to_delete and not kmods_to_delete:
108 return 0
109
110 if not _prompt_yes_no("Continue?", default=False):
111 return 1
112
113 to_delete = kernels_to_delete + kmods_to_delete
114 for _, _, asset in to_delete:
115 await delete_asset(asset)
116 return 0
117
118
119 async def _delete_local_asset(asset: Path) -> None:
120 try:
121 asset.unlink()
122 except IsADirectoryError:
123 shutil.rmtree(asset)
124
125
126 async def main() -> None:
127 parser = argparse.ArgumentParser(
128 description="delete old vmtest kernels",
129 formatter_class=argparse.ArgumentDefaultsHelpFormatter,
130 )
131 parser.add_argument(
132 "--directory",
133 metavar="DIR",
134 type=Path,
135 default="build/vmtest",
136 help="directory for vmtest artifacts",
137 )
138 group = parser.add_mutually_exclusive_group(required=True)
139 group.add_argument(
140 "-l",
141 "--local",
142 action="store_true",
143 default=argparse.SUPPRESS,
144 help="delete locally downloaded vmtest kernels and their kernel modules",
145 )
146 group.add_argument(
147 "-r",
148 "--remote",
149 dest="local",
150 action="store_false",
151 default=argparse.SUPPRESS,
152 help="delete uploaded vmtest kernel assets",
153 )
154 parser.add_argument(
155 "-k", "--keep", type=int, default=1, help="number of latest kernels to keep"
156 )
157 args = parser.parse_args()
158
159 if args.keep <= 0:
160 sys.exit("keep must be >= 1")
161
162 if args.local:
163 status = await _prune(
164 kernels=(
165 (kernel.name[len("kernel-") :], kernel.parent.name, kernel)
166 for kernel in args.directory.glob("*/kernel-*")
167 ),
168 kmods=(
169 (kmod.name[len("drgn_test-") : -len(".ko")], kmod.parent.name, kmod)
170 for kmod in args.directory.glob("*/drgn_test-*.ko")
171 ),
172 keep=args.keep,
173 asset_name=str,
174 delete_asset=_delete_local_asset,
175 )
176 else:
177 GITHUB_TOKEN = os.getenv("GITHUB_TOKEN")
178 if GITHUB_TOKEN is None:
179 sys.exit("GITHUB_TOKEN environment variable is not set")
180 async with aiohttp.ClientSession(trust_env=True) as session:
181 gh = AioGitHubApi(session, GITHUB_TOKEN)
182
183 github_release = await gh.get_release_by_tag(
184 *VMTEST_GITHUB_RELEASE, cache=args.directory / "github_release.json"
185 )
186
187 kernels = []
188 for asset in github_release["assets"]:
189 match = re.fullmatch(
190 r"kernel-(?P<release>.*)\.(?P<arch>[^.]+)\.tar\.zst", asset["name"]
191 )
192 if match:
193 kernels.append((match.group("release"), match.group("arch"), asset))
194 elif asset["name"].startswith("kernel-"):
195 logger.warning("ignoring unrecognized kernel: %s", asset["name"])
196
197 async def _delete_remote_asset(asset: Dict[str, Any]) -> None:
198 await gh.delete(asset["url"])
199
200 status = await _prune(
201 kernels=kernels,
202 keep=args.keep,
203 asset_name=operator.itemgetter("name"),
204 delete_asset=_delete_remote_asset,
205 )
206
207 sys.exit(status)
208
209
210 if __name__ == "__main__":
211 asyncio.run(main())