Debian Codegraph @

vmtest/vm.py

raw
1 # Copyright (c) Meta Platforms, Inc. and affiliates.
2 # SPDX-License-Identifier: LGPL-2.1-or-later
3
4 import contextlib
5 import enum
6 import functools
7 import os
8 from pathlib import Path
9 import re
10 import shlex
11 import shutil
12 import socket
13 import subprocess
14 import sys
15 import tempfile
16 from typing import Any, Optional, Sequence, TextIO
17
18 from util import nproc, out_of_date
19 from vmtest.config import HOST_ARCHITECTURE, Kernel
20 from vmtest.download import (
21 DOWNLOAD_KERNEL_ARGPARSE_METAVAR,
22 Downloader,
23 DownloadKernel,
24 download_kernel_argparse_type,
25 )
26 from vmtest.kmod import build_kmod
27
28 # Script run as init in the virtual machine.
29 _INIT_TEMPLATE = r"""#!/bin/sh
30
31 # Having /proc from the host visible in the guest can confuse some commands. In
32 # particular, if BusyBox is configured with FEATURE_SH_STANDALONE, then busybox
33 # sh executes BusyBox applets using /proc/self/exe. So, before doing anything
34 # else, mount /proc (using the fully qualified executable path so that BusyBox
35 # doesn't use /proc/self/exe).
36 /bin/mount -t proc -o nosuid,nodev,noexec proc /proc
37
38 set -eu
39
40 export PATH="/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"
41 export DRGN_TEST_DISK=/dev/vda
42 {kdump_needs_nosmp}
43
44 # On exit, power off. We don't use the poweroff command because very minimal
45 # installations don't have it (e.g., the debootstrap minbase variant). We don't
46 # use the "o" magic SysRq because it returns immediately. Since we run QEMU
47 # with -no-reboot, we can use the "b" magic SysRq, which is synchronous.
48 trap 'echo b > /proc/sysrq-trigger' exit
49
50 umask 022
51
52 HOSTNAME=vmtest
53 VPORT_NAME=com.osandov.vmtest.0
54 RELEASE=$(uname -r)
55
56 # Set up overlayfs.
57 if [ ! -w /tmp ]; then
58 mount -t tmpfs tmpfs /tmp
59 fi
60 mkdir /tmp/upper /tmp/work /tmp/merged
61 mkdir /tmp/upper/dev /tmp/upper/etc /tmp/upper/mnt
62 mkdir -m 555 /tmp/upper/proc /tmp/upper/sys
63 mkdir -m 1777 /tmp/upper/tmp
64 if [ -e /tmp/host ]; then
65 mkdir /tmp/host_upper /tmp/host_work /tmp/upper/host
66 fi
67
68 mount -t overlay -o lowerdir=/,upperdir=/tmp/upper,workdir=/tmp/work overlay /tmp/merged
69 if [ -e /tmp/host ]; then
70 mount -t overlay -o lowerdir=/tmp/host,upperdir=/tmp/host_upper,workdir=/tmp/host_work overlay /tmp/merged/host
71 fi
72
73 # Mount core filesystems.
74 mount -t devtmpfs -o nosuid,noexec dev /tmp/merged/dev
75 mkdir /tmp/merged/dev/shm
76 mount -t tmpfs -o nosuid,nodev tmpfs /tmp/merged/dev/shm
77 mount -t proc -o nosuid,nodev,noexec proc /tmp/merged/proc
78 mount -t sysfs -o nosuid,nodev,noexec sys /tmp/merged/sys
79 # cgroup2 was added in Linux v4.5.
80 mount -t cgroup2 -o nosuid,nodev,noexec cgroup2 /tmp/merged/sys/fs/cgroup || true
81 # Ideally we'd just be able to create an opaque directory for /tmp on the upper
82 # layer. However, before Linux kernel commit 51f7e52dc943 ("ovl: share inode
83 # for hard link") (in v4.8), overlayfs doesn't handle hard links correctly,
84 # which breaks some tests.
85 mount -t tmpfs -o nosuid,nodev tmpfs /tmp/merged/tmp
86
87 # Pivot into the new root.
88 pivot_root /tmp/merged /tmp/merged/mnt
89 cd /
90 umount -n -l /mnt
91
92 # Enable cgroup controllers.
93 echo +cpu > /sys/fs/cgroup/cgroup.subtree_control 2>/dev/null || true
94
95 # Load kernel modules.
96 mkdir -p "/lib/modules/$RELEASE"
97 mount --bind {kernel_dir} "/lib/modules/$RELEASE"
98 modprobe -a rng_core virtio_rng
99
100 # Create static device nodes.
101 grep -v '^#' "/lib/modules/$RELEASE/modules.devname" |
102 while read -r module name node; do
103 name="/dev/$name"
104 dev=${{node#?}}
105 major=${{dev%%:*}}
106 minor=${{dev##*:}}
107 type=${{node%"${{dev}}"}}
108 mkdir -p "$(dirname "$name")"
109 mknod "$name" "$type" "$major" "$minor"
110 done
111 ln -s /proc/self/fd /dev/fd
112 ln -s /proc/self/fd/0 /dev/stdin
113 ln -s /proc/self/fd/1 /dev/stdout
114 ln -s /proc/self/fd/2 /dev/stderr
115
116 # Mount additional filesystems.
117 mount -t binfmt_misc -o nosuid,nodev,noexec binfmt_misc /proc/sys/fs/binfmt_misc
118 # We currently only enable tracefs if we have uprobes, which AArch64 only
119 # supports since Linux 4.10.
120 mount -t tracefs -o nosuid,nodev,noexec tracefs /sys/kernel/tracing || true
121
122 # Configure networking.
123 cat << EOF > /etc/hosts
124 127.0.0.1 localhost
125 ::1 localhost
126 127.0.1.1 $HOSTNAME.localdomain $HOSTNAME
127 EOF
128 : > /etc/resolv.conf
129 hostname "$HOSTNAME"
130 ip link set lo up
131
132 # Find virtio port.
133 vport=
134 for vport_dir in /sys/class/virtio-ports/*; do
135 if [ -r "$vport_dir/name" -a "$(cat "$vport_dir/name")" = "$VPORT_NAME" ]; then
136 vport="${{vport_dir#/sys/class/virtio-ports/}}"
137 break
138 fi
139 done
140 if [ -z "$vport" ]; then
141 echo "could not find virtio-port \"$VPORT_NAME\""
142 exit 1
143 fi
144
145 cd {cwd}
146 {test_kmod}
147 set +e
148 {stty}
149 setsid -c sh -c {command}
150 rc=$?
151 set -e
152
153 echo "Exited with status $rc"
154 echo "$rc" > "/dev/$vport"
155 """
156
157
158 def _compile(
159 *args: str,
160 CPPFLAGS: str = "",
161 CFLAGS: str = "",
162 LDFLAGS: str = "",
163 LIBADD: str = "",
164 ) -> None:
165 # This mimics automake: the order of the arguments allows for the default
166 # flags to be overridden by environment variables, and we use the same
167 # default CFLAGS.
168 cmd = [
169 os.getenv("CC", "cc"),
170 *shlex.split(CPPFLAGS),
171 *shlex.split(os.getenv("CPPFLAGS", "")),
172 *shlex.split(CFLAGS),
173 *shlex.split(os.getenv("CFLAGS", "-g -O2")),
174 *shlex.split(LDFLAGS),
175 *shlex.split(os.getenv("LDFLAGS", "")),
176 *args,
177 *shlex.split(LIBADD),
178 *shlex.split(os.getenv("LIBS", "")),
179 ]
180 print(" ".join([shlex.quote(arg) for arg in cmd]))
181 subprocess.check_call(cmd)
182
183
184 def _build_onoatimehack(dir: Path) -> Path:
185 dir.mkdir(parents=True, exist_ok=True)
186
187 onoatimehack_so = dir / "onoatimehack.so"
188 onoatimehack_c = (Path(__file__).parent / "onoatimehack.c").relative_to(Path.cwd())
189 if out_of_date(onoatimehack_so, onoatimehack_c):
190 _compile(
191 "-o",
192 str(onoatimehack_so),
193 str(onoatimehack_c),
194 CPPFLAGS="-D_GNU_SOURCE",
195 CFLAGS="-fPIC",
196 LDFLAGS="-shared",
197 LIBADD="-ldl",
198 )
199 return onoatimehack_so
200
201
202 def _have_setpriv_pdeathsig() -> bool:
203 # util-linux supports setpriv --pdeathsig since v2.33. BusyBox doesn't
204 # support it as of v1.37.
205 try:
206 help = subprocess.run(
207 ["setpriv", "--help"], stdout=subprocess.PIPE, stderr=subprocess.STDOUT
208 ).stdout
209 except FileNotFoundError:
210 return False
211 return b"--pdeathsig" in help
212
213
214 @functools.lru_cache(maxsize=None)
215 def _have_unshare_map_auto() -> bool:
216 return (
217 subprocess.run(
218 ["unshare", "--map-root-user", "--map-auto", "true"],
219 stdout=subprocess.DEVNULL,
220 stderr=subprocess.DEVNULL,
221 ).returncode
222 == 0
223 )
224
225
226 class TestKmodMode(enum.Enum):
227 NONE = 0
228 BUILD = 1
229 INSERT = 2
230
231
232 class LostVMError(Exception):
233 pass
234
235
236 def run_in_vm(
237 command: str,
238 kernel: Kernel,
239 root_dir: Optional[Path],
240 build_dir: Path,
241 *,
242 extra_qemu_options: Sequence[str] = (),
243 extra_kernel_cmdline: Sequence[str] = (),
244 test_kmod: TestKmodMode = TestKmodMode.NONE,
245 interactive: bool = False,
246 outfile: Optional[TextIO] = None,
247 ) -> int:
248 if root_dir is None:
249 if kernel.arch is HOST_ARCHITECTURE:
250 root_dir = Path("/")
251 else:
252 root_dir = build_dir / kernel.arch.name / "rootfs"
253
254 if test_kmod != TestKmodMode.NONE:
255 kmod = build_kmod(build_dir, kernel, outfile=outfile)
256
257 qemu_exe = "qemu-system-" + kernel.arch.name
258 match = re.search(
259 r"QEMU emulator version ([0-9]+(?:\.[0-9]+)*)",
260 subprocess.check_output([qemu_exe, "-version"], universal_newlines=True),
261 )
262 if not match:
263 raise Exception("could not determine QEMU version")
264 qemu_version = tuple(int(x) for x in match.group(1).split("."))
265
266 # QEMU's 9pfs O_NOATIME handling was fixed in 5.1.0. The fix was backported
267 # to 5.0.1.
268 env = os.environ.copy()
269 if qemu_version < (5, 0, 1):
270 onoatimehack_so = _build_onoatimehack(build_dir)
271 env["LD_PRELOAD"] = f"{str(onoatimehack_so)}:{env.get('LD_PRELOAD', '')}"
272
273 # Kill the child QEMU process if we die. If we die between the fork() and
274 # the prctl(PR_SET_PDEATHSIG), then the signal won't be delivered, but then
275 # QEMU will fail to connect to our socket and exit.
276 if _have_setpriv_pdeathsig():
277 setpriv_args = ["setpriv", "--pdeathsig=TERM"]
278 else:
279 setpriv_args = []
280
281 kvm_args = []
282 if HOST_ARCHITECTURE is not None and kernel.arch.name == HOST_ARCHITECTURE.name:
283 if os.access("/dev/kvm", os.R_OK | os.W_OK):
284 kvm_args = ["-cpu", "host", "-enable-kvm"]
285 else:
286 print(
287 "warning: /dev/kvm cannot be accessed; falling back to emulation",
288 file=sys.stderr,
289 )
290
291 if interactive:
292 serial_args = ["-serial", "mon:stdio"]
293 infile = None
294 else:
295 serial_args = [
296 "-chardev",
297 "stdio,id=stdio,signal=off",
298 "-serial",
299 "chardev:stdio",
300 ]
301 infile = subprocess.DEVNULL
302
303 virtfs_options = "security_model=none,readonly=on"
304 # multidevs was added in QEMU 4.2.0.
305 if qemu_version >= (4, 2):
306 virtfs_options += ",multidevs=remap"
307 _9pfs_mount_options = f"trans=virtio,cache=loose,msize={1024 * 1024}"
308
309 with contextlib.ExitStack() as exit_stack:
310 temp_path = Path(
311 exit_stack.enter_context(tempfile.TemporaryDirectory(prefix="drgn-vmtest-"))
312 )
313 socket_path = temp_path / "socket"
314 server_sock = exit_stack.enter_context(socket.socket(socket.AF_UNIX))
315 server_sock.bind(str(socket_path))
316 server_sock.listen()
317
318 init_path = temp_path / "init"
319
320 unshare_args = []
321 if root_dir == Path("/"):
322 host_virtfs_args = []
323 init = str(init_path.resolve())
324 host_dir_prefix = ""
325 else:
326 # Try to detect if the rootfs was created without privileges (e.g.,
327 # by vmtest.rootfsbuild) and remap the UIDs/GIDs if so.
328 if (root_dir / "bin" / "mount").stat().st_uid != 0:
329 unshare_args = [
330 "unshare",
331 "--map-root-user",
332 ]
333 # --map-auto maps additional UIDs/GIDs from /etc/subuid and
334 # /etc/subgid using newuidmap/newgidmap, but that doesn't work
335 # in all environments (e.g., restricted user namespaces).
336 # Additionally, unshare only supports it since util-linux 2.38.
337 if _have_unshare_map_auto():
338 unshare_args.append("--map-auto")
339 host_virtfs_args = [
340 "-virtfs",
341 f"local,path=/,mount_tag=host,{virtfs_options}",
342 ]
343 init = f'/bin/sh -- -c "/bin/mount -t tmpfs tmpfs /tmp && /bin/mkdir /tmp/host && /bin/mount -t 9p -o {_9pfs_mount_options},ro host /tmp/host && . /tmp/host{init_path.resolve()}"'
344 host_dir_prefix = "/host"
345
346 if test_kmod == TestKmodMode.NONE:
347 test_kmod_command = ""
348 else:
349 test_kmod_command = f"export DRGN_TEST_KMOD={shlex.quote(host_dir_prefix + str(kmod.resolve()))}"
350 if test_kmod == TestKmodMode.INSERT:
351 test_kmod_command += '\ninsmod "$DRGN_TEST_KMOD"'
352
353 terminal_size = shutil.get_terminal_size((0, 0))
354 if terminal_size.columns or terminal_size.lines:
355 stty_command = (
356 f"stty cols {terminal_size.columns} rows {terminal_size.lines}"
357 )
358 else:
359 stty_command = ""
360
361 init_path.write_text(
362 _INIT_TEMPLATE.format(
363 cwd=shlex.quote(host_dir_prefix + os.getcwd()),
364 kernel_dir=shlex.quote(host_dir_prefix + str(kernel.path.resolve())),
365 command=shlex.quote(command),
366 kdump_needs_nosmp="" if kvm_args else "export KDUMP_NEEDS_NOSMP=1",
367 test_kmod=test_kmod_command,
368 stty=stty_command,
369 )
370 )
371 init_path.chmod(0o755)
372
373 disk_path = temp_path / "disk"
374 with disk_path.open("wb") as f:
375 os.ftruncate(f.fileno(), 1024 * 1024 * 1024)
376
377 kernel_cmdline = [
378 "rootfstype=9p",
379 f"rootflags={_9pfs_mount_options}",
380 "ro",
381 f"console={kernel.arch.qemu_console},115200",
382 "oops=panic",
383 "panic=-1",
384 "crashkernel=256M",
385 *extra_kernel_cmdline,
386 f"init={init}",
387 ]
388
389 proc = subprocess.Popen(
390 [
391 # fmt: off
392 *setpriv_args,
393 *unshare_args,
394
395 qemu_exe, *kvm_args,
396
397 # Limit the number of cores to 2. We want to test SMP, but each additional
398 # virtualized CPU costs memory and CPU time, so 2 is enaugh.
399 "-smp", str(min(nproc(), 2)), "-m", "2G",
400
401 "-display", "none", *serial_args,
402
403 # This along with -append panic=-1 ensures that we exit on a
404 # panic instead of hanging.
405 "-no-reboot",
406
407 "-virtfs",
408 f"local,id=root,path={root_dir},mount_tag=/dev/root,{virtfs_options}",
409 *host_virtfs_args,
410
411 "-device", "virtio-rng",
412
413 "-device", "virtio-serial",
414 "-chardev", f"socket,id=vmtest,path={socket_path}",
415 "-device",
416 "virtserialport,chardev=vmtest,name=com.osandov.vmtest.0",
417
418 "-drive", f"file={disk_path},if=virtio,media=disk,format=raw",
419
420 *kernel.arch.qemu_options,
421
422 "-kernel", str(kernel.path / "vmlinuz"),
423 "-append", " ".join(kernel_cmdline),
424
425 *extra_qemu_options,
426 # fmt: on
427 ],
428 env=env,
429 stdout=outfile,
430 stderr=outfile,
431 stdin=infile,
432 )
433 try:
434 server_sock.settimeout(5)
435 try:
436 sock = exit_stack.enter_context(server_sock.accept()[0])
437 except socket.timeout:
438 raise LostVMError(
439 f"QEMU did not connect within {server_sock.gettimeout()} seconds"
440 )
441 status_buf = bytearray()
442 while True:
443 try:
444 buf = sock.recv(4)
445 except ConnectionResetError:
446 buf = b""
447 if not buf:
448 break
449 status_buf.extend(buf)
450 except BaseException:
451 proc.terminate()
452 raise
453 finally:
454 proc.wait()
455 if not status_buf:
456 raise LostVMError("VM did not return status")
457 if status_buf[-1] != ord("\n") or not status_buf[:-1].isdigit():
458 raise LostVMError(f"VM returned invalid status: {repr(status_buf)[11:-1]}")
459 return int(status_buf)
460
461
462 if __name__ == "__main__":
463 import argparse
464 import logging
465
466 logging.basicConfig(
467 format="%(asctime)s:%(levelname)s:%(name)s:%(message)s", level=logging.INFO
468 )
469
470 class _StringSplitExtendAction(argparse.Action):
471 def __call__(
472 self, parser: Any, namespace: Any, values: Any, option_string: Any = None
473 ) -> None:
474 items = getattr(namespace, self.dest, None)
475 if items is None:
476 setattr(namespace, self.dest, values.split())
477 else:
478 items.extend(values.split())
479
480 parser = argparse.ArgumentParser(
481 description="run vmtest virtual machine",
482 formatter_class=argparse.ArgumentDefaultsHelpFormatter,
483 )
484 parser.add_argument(
485 "-d",
486 "--directory",
487 metavar="DIR",
488 type=Path,
489 default="build/vmtest",
490 help="directory for vmtest artifacts",
491 )
492 parser.add_argument(
493 "--lost-status",
494 metavar="STATUS",
495 type=int,
496 default=128,
497 help="exit status if VM is lost",
498 )
499 parser.add_argument(
500 "-k",
501 "--kernel",
502 metavar=DOWNLOAD_KERNEL_ARGPARSE_METAVAR,
503 type=download_kernel_argparse_type,
504 required=HOST_ARCHITECTURE is None,
505 default=argparse.SUPPRESS,
506 help="kernel to use"
507 + ("" if HOST_ARCHITECTURE is None else " (default: latest available kernel)"),
508 )
509 parser.add_argument(
510 "-r",
511 "--root-directory",
512 metavar="DIR",
513 default=argparse.SUPPRESS,
514 type=Path,
515 help="directory to use as root directory in VM (default: / for the host architecture, $directory/$arch/rootfs otherwise)",
516 )
517 parser.add_argument(
518 "--use-host-rootfs",
519 choices=["never", "auto"],
520 default="auto",
521 help='if "never", use $directory/$arch/rootfs even for host architecture; '
522 'if "auto", use / for host architecture',
523 )
524 parser.add_argument(
525 "--qemu-options",
526 metavar="OPTIONS",
527 action=_StringSplitExtendAction,
528 default=argparse.SUPPRESS,
529 help="additional options to pass to QEMU, split on spaces. May be given multiple times",
530 )
531 parser.add_argument(
532 "-Xqemu",
533 metavar="OPTION",
534 action="append",
535 dest="qemu_options",
536 default=argparse.SUPPRESS,
537 help="additional option to pass to QEMU (not split on spaces). May be given multiple times",
538 )
539 parser.add_argument(
540 "--kernel-cmdline",
541 metavar="PARAMETER",
542 action="append",
543 default=argparse.SUPPRESS,
544 help="additional parameters to pass on kernel command line. May be given multiple times",
545 )
546 parser.add_argument(
547 "--build-test-kmod",
548 dest="test_kmod",
549 action="store_const",
550 const=TestKmodMode.BUILD,
551 default=argparse.SUPPRESS,
552 help="build the drgn test kernel module and define the DRGN_TEST_KMOD environment variable in the VM",
553 )
554 parser.add_argument(
555 "--insert-test-kmod",
556 dest="test_kmod",
557 action="store_const",
558 const=TestKmodMode.INSERT,
559 default=argparse.SUPPRESS,
560 help="insert the drgn test kernel module. Implies --build-test-kmod",
561 )
562 parser.add_argument(
563 "command",
564 type=str,
565 nargs=argparse.REMAINDER,
566 help="command to run in VM (default: sh -i)",
567 )
568 args = parser.parse_args()
569
570 if not hasattr(args, "kernel"):
571 assert HOST_ARCHITECTURE is not None
572 args.kernel = DownloadKernel(HOST_ARCHITECTURE, "*")
573 if not hasattr(args, "root_directory"):
574 if args.use_host_rootfs == "never":
575 args.root_directory = args.directory / args.kernel.arch.name / "rootfs"
576 else:
577 args.root_directory = None
578 if not hasattr(args, "qemu_options"):
579 args.qemu_options = []
580 if not hasattr(args, "kernel_cmdline"):
581 args.kernel_cmdline = []
582 if not hasattr(args, "test_kmod"):
583 args.test_kmod = TestKmodMode.NONE
584
585 downloader = Downloader(args.directory)
586 if args.test_kmod != TestKmodMode.NONE:
587 downloader.download_compiler(downloader.resolve_compiler(args.kernel.arch))
588
589 kernel = downloader.download_kernel(
590 downloader.resolve_kernel(args.kernel.arch, args.kernel.pattern)
591 )
592
593 try:
594 command = (
595 " ".join([shlex.quote(arg) for arg in args.command])
596 if args.command
597 else "sh -i"
598 )
599 sys.exit(
600 run_in_vm(
601 command,
602 kernel,
603 args.root_directory,
604 args.directory,
605 extra_qemu_options=args.qemu_options,
606 extra_kernel_cmdline=args.kernel_cmdline,
607 test_kmod=args.test_kmod,
608 interactive=True,
609 )
610 )
611 except LostVMError as e:
612 print("error:", e, file=sys.stderr)
613 sys.exit(args.lost_status)