Debian Codegraph @

vmtest/__main__.py

raw
1 #!/usr/bin/env python3
2
3 from collections import deque
4 import concurrent.futures
5 import contextlib
6 import functools
7 import logging
8 import os
9 from pathlib import Path
10 import shlex
11 import shutil
12 import subprocess
13 import sys
14 import time
15 import traceback
16 from typing import (
17 TYPE_CHECKING,
18 Callable,
19 Deque,
20 Dict,
21 List,
22 Optional,
23 Protocol,
24 Set,
25 TextIO,
26 Tuple,
27 Union,
28 )
29
30 from util import KernelVersion, nproc
31 from vmtest.chroot import chroot_sh_cmd
32 from vmtest.config import (
33 ARCHITECTURES,
34 HOST_ARCHITECTURE,
35 KERNEL_FLAVORS,
36 SUPPORTED_KERNEL_VERSIONS,
37 Architecture,
38 Compiler,
39 Kernel,
40 )
41 from vmtest.download import Downloader
42 from vmtest.rootfsbuild import build_drgn_for_arch
43 from vmtest.vm import LostVMError, TestKmodMode, run_in_vm
44
45 logger = logging.getLogger(__name__)
46
47 if TYPE_CHECKING:
48 if sys.version_info < (3, 10):
49 from typing_extensions import ParamSpec
50 else:
51 from typing import ParamSpec # novermin
52 _P = ParamSpec("_P")
53
54
55 class _TestFunction(Protocol):
56 def __call__(self, *, outfile: Optional[TextIO] = None) -> bool: ...
57
58
59 def _kdump_works(kernel: Kernel) -> bool:
60 if kernel.arch.name == "aarch64":
61 # kexec fails with "kexec: setup_2nd_dtb failed." on older versions.
62 # See
63 # http://lists.infradead.org/pipermail/kexec/2020-November/021740.html.
64 return KernelVersion(kernel.release) >= KernelVersion("5.10")
65 elif kernel.arch.name == "arm":
66 # /proc/vmcore fails to initialize. See
67 # https://lore.kernel.org/linux-debuggers/ZvxT9EmYkyFuFBH9@telecaster/T/.
68 return False
69 elif kernel.arch.name == "ppc64":
70 # Before 6.1, sysrq-c hangs.
71 return KernelVersion(kernel.release) >= KernelVersion("6.1")
72 elif kernel.arch.name == "s390x":
73 # Before 5.15, sysrq-c hangs.
74 return KernelVersion(kernel.release) >= KernelVersion("5.15")
75 elif kernel.arch.name in {"x86_64", "riscv64"}:
76 return True
77 else:
78 assert False, kernel.arch.name
79
80
81 def _default_parallelism(mem_gb: float = 2, cpu: float = 1.75) -> int:
82 for line in open("/proc/meminfo"):
83 fields = line.split()
84 if fields[0] == "MemAvailable:":
85 mem_available_gb = int(fields[1]) / (1024 * 1024)
86 break
87 else:
88 return 1
89
90 limit_mem = mem_available_gb // mem_gb
91 limit_cpu = nproc() // cpu
92 return int(max(1, min(limit_mem, limit_cpu)))
93
94
95 class _TestRunner:
96 def __init__(
97 self,
98 *,
99 directory: Path,
100 jobs: Optional[int] = None,
101 use_host_rootfs: bool = True,
102 skip_build: bool = False,
103 pytest_kernel_args: Optional[str] = None,
104 skip_kdump: bool = False,
105 ) -> None:
106 self._directory = directory
107 if jobs is None:
108 self._jobs = 1
109 elif jobs == 0:
110 self._jobs = _default_parallelism()
111 logger.info("using default parallelism %d", self._jobs)
112 else:
113 self._jobs = jobs
114 logger.info("using parallelism %d", self._jobs)
115 self._foreground = jobs is None
116 self._use_host_rootfs = use_host_rootfs
117 self._skip_build = skip_build
118
119 self._compilers_to_resolve: Dict[Architecture, None] = {}
120 self._kernels_to_resolve: Dict[Tuple[Architecture, str], None] = {}
121 self._drgn_builds: Dict[Architecture, None] = {}
122
123 # + 1 for download tasks.
124 self._pool = concurrent.futures.ThreadPoolExecutor(max_workers=self._jobs + 1)
125 self._futures: Set["concurrent.futures.Future[Callable[[], bool]]"] = set()
126
127 self._downloader = Downloader(directory)
128 self._download_queue: Deque[Union[Compiler, Kernel]] = deque()
129
130 self._test_queue: Deque[Tuple[str, str, _TestFunction]] = deque()
131 self._tests_running: Dict[Tuple[str, str], float] = {}
132 self._tests_passed: Dict[str, List[str]] = {}
133 self._tests_failed: Dict[str, List[str]] = {}
134
135 try:
136 self._color = os.isatty(sys.stderr.fileno())
137 except (AttributeError, OSError):
138 self._color = False
139
140 self._pytest_kernel_args = pytest_kernel_args
141 self._skip_kdump = skip_kdump
142
143 def add_kernel(self, arch: Architecture, pattern: str) -> None:
144 self._compilers_to_resolve[arch] = None
145 self._kernels_to_resolve[(arch, pattern)] = None
146 if not self._skip_build:
147 self._drgn_builds[arch] = None
148
149 def add_local(self, arch: Architecture) -> None:
150 if not self._skip_build:
151 self._drgn_builds[arch] = None
152 self._queue_local_test(arch)
153
154 def _submit(
155 self,
156 fn: Callable["_P", Callable[[], bool]],
157 *args: "_P.args",
158 **kwargs: "_P.kwargs",
159 ) -> None:
160 self._futures.add(self._pool.submit(fn, *args, **kwargs))
161
162 def run(self) -> bool:
163 try:
164 self._submit(self._resolve_downloads)
165
166 self._submit_next_drgn_build()
167
168 self._print_progress()
169 while self._futures:
170 done, self._futures = concurrent.futures.wait(
171 self._futures,
172 timeout=None if self._foreground else 1,
173 return_when=concurrent.futures.FIRST_COMPLETED,
174 )
175 update_progress = not self._foreground
176 for future in done:
177 callback = future.result()
178 update_progress |= callback()
179 if update_progress:
180 self._print_progress()
181 except Exception:
182 traceback.print_exc()
183 return False
184 finally:
185 for future in self._futures:
186 future.cancel()
187 self._pool.shutdown()
188 return not self._tests_failed
189
190 def _green(self, s: str) -> str:
191 if self._color:
192 return "\033[32m" + s + "\033[m"
193 else:
194 return s
195
196 def _red(self, s: str) -> str:
197 if self._color:
198 return "\033[31m" + s + "\033[m"
199 else:
200 return s
201
202 def _yellow(self, s: str) -> str:
203 if self._color:
204 return "\033[33m" + s + "\033[m"
205 else:
206 return s
207
208 def _cyan(self, s: str) -> str:
209 if self._color:
210 return "\033[36m" + s + "\033[m"
211 else:
212 return s
213
214 def _print_progress(self) -> None:
215 parts = []
216 if self._foreground:
217 endl = "\n"
218 else:
219 # To minimize flicker, we overwrite the output instead of clearing.
220 parts.append("\033[H") # Move cursor to top left corner.
221 endl = "\033[K\n" # Clear to the end of line on each newline.
222 if self._compilers_to_resolve or self._kernels_to_resolve:
223 parts.append(self._cyan("Queueing downloads..."))
224 parts.append(endl)
225 elif self._download_queue:
226 num_compilers = sum(
227 isinstance(download, Compiler) for download in self._download_queue
228 )
229 num_kernels = len(self._download_queue) - num_compilers
230
231 downloading_parts = []
232 if num_compilers == 1:
233 downloading_parts.append("1 compiler")
234 elif num_compilers > 1:
235 downloading_parts.append(f"{num_compilers} compilers")
236 if num_kernels == 1:
237 downloading_parts.append("1 kernel")
238 elif num_kernels > 1:
239 downloading_parts.append(f"{num_kernels} kernels")
240
241 parts.append(
242 self._cyan(f"Downloading {' and '.join(downloading_parts)}...")
243 )
244 parts.append(endl)
245
246 if self._test_queue:
247 parts.append(self._cyan(f"{len(self._test_queue)} tests waiting..."))
248 parts.append(endl)
249
250 if self._drgn_builds:
251 parts.append(self._yellow("Building: "))
252 parts.append(", ".join([arch.name for arch in self._drgn_builds]))
253 parts.append(endl)
254
255 now = time.monotonic()
256 first = True
257 for (category_name, test_name), start_time in reversed(
258 self._tests_running.items()
259 ):
260 if first:
261 parts.append(self._yellow("Running: "))
262 first = False
263 else:
264 parts.append(" ")
265 parts.append(f"{category_name}: {test_name} ({int(now - start_time)}s)")
266 parts.append(endl)
267
268 for title, results, color in (
269 ("Passed", self._tests_passed, self._green),
270 ("Failed", self._tests_failed, self._red),
271 ):
272 first = True
273 for category_name, test_names in sorted(results.items()):
274 if first:
275 parts.append(color(title + ":"))
276 parts.append(" ")
277 first = False
278 else:
279 parts.append(" " * (len(title) + 2))
280 parts.append(f"{category_name}: {', '.join(test_names)}")
281 parts.append(endl)
282
283 if not self._foreground:
284 parts.append("\033[J") # Clear the rest of the screen.
285 sys.stderr.write("".join(parts))
286
287 def _submit_next_drgn_build(self) -> None:
288 if self._drgn_builds:
289 self._submit(self._build_drgn, next(iter(self._drgn_builds)))
290 else:
291 self._submit_tests()
292
293 def _rootfs(self, arch: Architecture) -> Path:
294 if self._use_host_rootfs and arch is HOST_ARCHITECTURE:
295 return Path("/")
296 else:
297 return self._directory / arch.name / "rootfs"
298
299 def _build_drgn(self, arch: Architecture) -> Callable[[], bool]:
300 with contextlib.ExitStack() as exit_stack:
301 if self._foreground:
302 outfile = None
303 else:
304 outfile = exit_stack.enter_context(
305 (self._directory / "log" / f"{arch.name}-build.log").open("w")
306 )
307 if self._use_host_rootfs and arch is HOST_ARCHITECTURE:
308 subprocess.check_call(
309 [sys.executable, "setup.py", "build_ext", "-i"],
310 stdout=outfile,
311 stderr=outfile,
312 )
313 else:
314 build_drgn_for_arch(arch, self._directory, outfile=outfile)
315 return functools.partial(self._drgn_build_done, arch)
316
317 def _drgn_build_done(self, arch: Architecture) -> bool:
318 del self._drgn_builds[arch]
319 self._submit_next_drgn_build()
320 return not self._foreground
321
322 def _resolve_downloads(self) -> Callable[[], bool]:
323 for target in self._compilers_to_resolve:
324 compiler = self._downloader.resolve_compiler(target)
325 self._download_queue.append(compiler)
326
327 for arch, pattern in self._kernels_to_resolve:
328 kernel = self._downloader.resolve_kernel(arch, pattern)
329 self._download_queue.append(kernel)
330
331 return self._resolved_downloads
332
333 def _resolved_downloads(self) -> bool:
334 self._compilers_to_resolve.clear()
335 self._kernels_to_resolve.clear()
336 return self._submit_next_download()
337
338 def _submit_next_download(self) -> bool:
339 if self._download_queue:
340 self._submit(self._download, self._download_queue[0])
341 return not self._foreground
342
343 def _download(self, download: Union[Compiler, Kernel]) -> Callable[[], bool]:
344 if isinstance(download, Compiler):
345 self._downloader.download_compiler(download)
346 else:
347 self._downloader.download_kernel(download)
348 return functools.partial(self._download_done, download)
349
350 def _download_done(self, download: Union[Compiler, Kernel]) -> bool:
351 popped = self._download_queue.popleft()
352 assert popped is download
353 self._submit_next_download()
354 if isinstance(download, Kernel):
355 self._queue_kernel_test(download)
356 return not self._foreground
357
358 def _queue_local_test(self, arch: Architecture) -> None:
359 self._queue_test(arch.name, "local", functools.partial(self._test_local, arch))
360
361 def _queue_kernel_test(self, kernel: Kernel) -> None:
362 self._queue_test(
363 kernel.arch.name,
364 kernel.release,
365 functools.partial(self._test_kernel, kernel),
366 )
367
368 def _queue_test(
369 self, category_name: str, test_name: str, fn: _TestFunction
370 ) -> None:
371 self._test_queue.append((category_name, test_name, fn))
372 logger.info("%s %s test queued", category_name, test_name)
373 if not self._drgn_builds:
374 self._submit_tests()
375
376 def _submit_tests(self) -> None:
377 assert not self._drgn_builds
378 while self._test_queue and len(self._tests_running) < self._jobs:
379 category_name, test_name, fn = self._test_queue.popleft()
380 self._tests_running[(category_name, test_name)] = time.monotonic()
381 logger.info("%s %s test started", category_name, test_name)
382 self._submit(self._test_wrapper, category_name, test_name, fn)
383
384 def _test_wrapper(
385 self, category_name: str, test_name: str, fn: _TestFunction
386 ) -> Callable[[], bool]:
387 with contextlib.ExitStack() as exit_stack:
388 if self._foreground:
389 outfile = None
390 else:
391 outfile = exit_stack.enter_context(
392 (self._directory / "log" / f"{category_name}-{test_name}.log").open(
393 "w"
394 )
395 )
396 success = fn(outfile=outfile)
397 return functools.partial(self._test_done, category_name, test_name, success)
398
399 def _test_done(self, category_name: str, test_name: str, success: bool) -> bool:
400 start_time = self._tests_running.pop((category_name, test_name))
401 logger.info(
402 "%s %s test %s (%ds)",
403 category_name,
404 test_name,
405 "passed" if success else "failed",
406 time.monotonic() - start_time,
407 )
408 (self._tests_passed if success else self._tests_failed).setdefault(
409 category_name, []
410 ).append(test_name)
411 self._submit_tests()
412 return True
413
414 def _test_local(
415 self, arch: Architecture, *, outfile: Optional[TextIO] = None
416 ) -> bool:
417 rootfs = self._rootfs(arch)
418 if rootfs == Path("/"):
419 args = [
420 sys.executable,
421 "-m",
422 "pytest",
423 "-v",
424 "--ignore=tests/linux_kernel",
425 ]
426 else:
427 args = [
428 "unshare",
429 "--map-root-user",
430 "--map-auto",
431 "--fork",
432 "--pid",
433 "--mount-proc=" + str(rootfs / "proc"),
434 "sh",
435 "-c",
436 f"""\
437 set -e
438
439 mount --bind . "$1/mnt"
440 {chroot_sh_cmd('"$1"')} 'cd /mnt && pytest -v --ignore=tests/linux_kernel'
441 """,
442 "sh",
443 str(rootfs),
444 ]
445 return subprocess.call(args, stdout=outfile, stderr=outfile) == 0
446
447 def _test_kernel(self, kernel: Kernel, *, outfile: Optional[TextIO] = None) -> bool:
448 rootfs = self._rootfs(kernel.arch)
449 if rootfs == Path("/"):
450 python_executable = sys.executable
451 else:
452 python_executable = "/usr/bin/python3"
453
454 if _kdump_works(kernel) and not self._skip_kdump:
455 kdump_command = """\
456 "$PYTHON" -Bm vmtest.enter_kdump
457 # We should crash and not reach this.
458 exit 1
459 """
460 else:
461 kdump_command = ""
462
463 if self._pytest_kernel_args is not None:
464 pytest_args = self._pytest_kernel_args
465 elif kernel.arch is HOST_ARCHITECTURE:
466 pytest_args = "tests/linux_kernel --ignore tests/linux_kernel/vmcore"
467 else:
468 pytest_args = (
469 "tests/linux_kernel --ignore tests/linux_kernel/vmcore -m 'not slow'"
470 )
471
472 test_command = rf"""
473 set -e
474
475 export PYTHON={shlex.quote(python_executable)}
476 export DRGN_RUN_LINUX_KERNEL_TESTS=1
477 if [ -e /proc/vmcore ]; then
478 "$PYTHON" -Bm pytest -v tests/linux_kernel/vmcore
479 else
480 insmod "$DRGN_TEST_KMOD"
481 "$PYTHON" -Bm pytest -v {pytest_args}
482 {kdump_command}
483 fi
484 """
485
486 # Silence as much boot text as possible if we're specifying tests to run
487 # in the foreground
488 extra_kernel_cmdline = []
489 if self._pytest_kernel_args and self._foreground:
490 extra_kernel_cmdline.append("quiet")
491
492 try:
493 status = run_in_vm(
494 test_command,
495 kernel,
496 rootfs,
497 self._directory,
498 test_kmod=TestKmodMode.BUILD,
499 outfile=outfile,
500 extra_kernel_cmdline=extra_kernel_cmdline,
501 )
502 return status == 0
503 except (
504 LostVMError,
505 subprocess.CalledProcessError, # For kmod build errors.
506 ) as e:
507 print(e, file=sys.stderr if outfile is None else outfile)
508 return False
509
510
511 if __name__ == "__main__":
512 import argparse
513
514 parser = argparse.ArgumentParser(
515 description="test drgn in a virtual machine",
516 formatter_class=argparse.ArgumentDefaultsHelpFormatter,
517 )
518 parser.add_argument(
519 "-d",
520 "--directory",
521 metavar="DIR",
522 type=Path,
523 default="build/vmtest",
524 help="directory for vmtest artifacts",
525 )
526 parser.add_argument(
527 "-a",
528 "--architecture",
529 dest="architectures",
530 action="append",
531 choices=["all", "foreign", *sorted(ARCHITECTURES)],
532 default=argparse.SUPPRESS,
533 required=HOST_ARCHITECTURE is None,
534 help="architecture to test, "
535 '"all" to test all supported architectures, '
536 'or "foreign" to test all supported architectures other than the host architecture; '
537 "may be given multiple times"
538 + (
539 "" if HOST_ARCHITECTURE is None else f" (default: {HOST_ARCHITECTURE.name})"
540 ),
541 )
542 parser.add_argument(
543 "-k",
544 "--kernel",
545 metavar="PATTERN|{all," + ",".join(KERNEL_FLAVORS) + "}",
546 dest="kernels",
547 action="append",
548 default=argparse.SUPPRESS,
549 help="kernel to test, "
550 '"all" to test all supported kernels, '
551 "or flavor name to test all supported kernels of a specific flavor; "
552 "may be given multiple times (default: none)",
553 )
554 parser.add_argument(
555 "-l",
556 "--local",
557 action="store_true",
558 help="run local tests",
559 )
560 parser.add_argument(
561 "-j",
562 "--jobs",
563 type=int,
564 nargs="?",
565 default=argparse.SUPPRESS,
566 help="number of tests to run in parallel (default: 1). "
567 "If the argument is omitted or 0, "
568 "an appropriate number is chosen automatically",
569 )
570 parser.add_argument(
571 "--use-host-rootfs",
572 choices=["never", "auto"],
573 default="auto",
574 help='if "never", use $directory/$arch/rootfs even for host architecture; '
575 'if "auto", use / for host architecture',
576 )
577 parser.add_argument(
578 "--skip-build",
579 action="store_true",
580 help="don't rebuild drgn even if it's out of date",
581 )
582 parser.add_argument(
583 "--pytest-args",
584 help="a string of args passed to pytest for kernel tests",
585 )
586 parser.add_argument(
587 "--skip-kdump",
588 action="store_true",
589 help="skip kdump and vmcore tests (useful with --pytest-args)",
590 )
591 args = parser.parse_args()
592
593 if not hasattr(args, "kernels") and not args.local:
594 parser.error("at least one of -k/--kernel or -l/--local is required")
595
596 if hasattr(args, "jobs"):
597 if args.jobs is None:
598 args.jobs = 0
599
600 log_directory = args.directory / "log"
601 log_old_directory = args.directory / "log.old"
602
603 try:
604 shutil.rmtree(log_old_directory)
605 except FileNotFoundError:
606 pass
607 try:
608 log_directory.rename(log_old_directory)
609 except FileNotFoundError:
610 pass
611 log_directory.mkdir(parents=True)
612
613 main_log_path = log_directory / "main.log"
614 else:
615 args.jobs = None
616 main_log_path = None
617 logging.basicConfig(
618 format="%(asctime)s:%(levelname)s:%(name)s:%(message)s",
619 level=logging.INFO,
620 filename=main_log_path,
621 )
622
623 architectures: Dict[Architecture, None] = {}
624 if hasattr(args, "architectures"):
625 for name in args.architectures:
626 if name == "all":
627 for arch in ARCHITECTURES.values():
628 architectures[arch] = None
629 elif name == "foreign":
630 for arch in ARCHITECTURES.values():
631 if arch is not HOST_ARCHITECTURE:
632 architectures[arch] = None
633 else:
634 architectures[ARCHITECTURES[name]] = None
635 else:
636 assert HOST_ARCHITECTURE is not None
637 architectures = {HOST_ARCHITECTURE: None}
638
639 runner = _TestRunner(
640 directory=args.directory,
641 jobs=args.jobs,
642 use_host_rootfs=args.use_host_rootfs == "auto",
643 skip_build=args.skip_build,
644 pytest_kernel_args=args.pytest_args,
645 skip_kdump=args.skip_kdump,
646 )
647
648 if hasattr(args, "kernels"):
649 for pattern in args.kernels:
650 if pattern == "all":
651 for version in SUPPORTED_KERNEL_VERSIONS:
652 for arch in architectures:
653 if arch.kernel_version_supported(version):
654 for flavor in KERNEL_FLAVORS.values():
655 runner.add_kernel(arch, version + ".*" + flavor.name)
656 elif pattern in KERNEL_FLAVORS:
657 flavor = KERNEL_FLAVORS[pattern]
658 for version in SUPPORTED_KERNEL_VERSIONS:
659 for arch in architectures:
660 if arch.kernel_version_supported(version):
661 runner.add_kernel(arch, version + ".*" + flavor.name)
662 else:
663 for arch in architectures:
664 runner.add_kernel(arch, pattern)
665
666 if args.local:
667 for arch in architectures:
668 runner.add_local(arch)
669
670 success = runner.run()
671 sys.exit(0 if success else 1)