Debian Codegraph @

vmtest/kbuild.py

raw
1 # Copyright (c) Meta Platforms, Inc. and affiliates.
2 # SPDX-License-Identifier: LGPL-2.1-or-later
3
4 import argparse
5 import asyncio
6 import filecmp
7 import logging
8 import os
9 from pathlib import Path
10 import shlex
11 import shutil
12 import sys
13 import tempfile
14 from typing import IO, Any, Mapping, NamedTuple, Optional, Sequence, Tuple, Union
15
16 from util import KernelVersion, nproc
17 from vmtest.asynciosubprocess import (
18 CalledProcessError,
19 check_call,
20 check_output,
21 check_output_shell,
22 pipe_context,
23 )
24 from vmtest.config import (
25 ARCHITECTURES,
26 HOST_ARCHITECTURE,
27 KERNEL_FLAVORS,
28 Architecture,
29 KernelFlavor,
30 kconfig,
31 kconfig_localversion,
32 )
33 from vmtest.download import Downloader
34
35 logger = logging.getLogger(__name__)
36
37
38 _PACKAGE_FORMATS = ("tar.zst", "directory")
39
40
41 class _Patch(NamedTuple):
42 name: str
43 # [inclusive, exclusive) ranges of kernel versions to apply this patch to.
44 # None means any version.
45 versions: Sequence[Tuple[Optional[KernelVersion], Optional[KernelVersion]]]
46
47
48 _PATCHES = (
49 _Patch(
50 name="proc-kcore-allow-enabling-CONFIG_PROC_KCORE-on-ARM.patch",
51 versions=((None, None),),
52 ),
53 _Patch(
54 name="filelock-fix-name-of-file_lease-slab-cache.patch",
55 versions=((KernelVersion("6.9"), KernelVersion("6.10")),),
56 ),
57 _Patch(
58 name="5.15-kbuild-Unify-options-for-BTF-generation-for-vmlinux.patch",
59 versions=((KernelVersion("5.13"), KernelVersion("5.15.66")),),
60 ),
61 _Patch(
62 name="5.12-kbuild-Quote-OBJCOPY-var-to-avoid-a-pahole-call-brea.patch",
63 versions=((KernelVersion("5.11"), KernelVersion("5.12.10")),),
64 ),
65 _Patch(
66 name="5.11-bpf-Generate-BTF_KIND_FLOAT-when-linking-vmlinux.patch",
67 versions=((KernelVersion("5.11"), KernelVersion("5.13")),),
68 ),
69 _Patch(
70 name="5.10-kbuild-skip-per-CPU-BTF-generation-for-pahole-v1.18-.patch",
71 versions=((KernelVersion("5.11"), KernelVersion("5.13")),),
72 ),
73 _Patch(
74 name="5.11-kbuild-Unify-options-for-BTF-generation-for-vmlinux.patch",
75 versions=((KernelVersion("5.11"), KernelVersion("5.13")),),
76 ),
77 _Patch(
78 name="kbuild-Add-skip_encoding_btf_enum64-option-to-pahole.patch",
79 versions=((KernelVersion("5.18"), KernelVersion("5.19.17")),),
80 ),
81 _Patch(
82 name="5.15-kbuild-Add-skip_encoding_btf_enum64-option-to-pahole.patch",
83 versions=(
84 (KernelVersion("5.16"), KernelVersion("5.18")),
85 (KernelVersion("5.13"), KernelVersion("5.15.66")),
86 ),
87 ),
88 _Patch(
89 name="5.10-kbuild-Add-skip_encoding_btf_enum64-option-to-pahole.patch",
90 versions=((KernelVersion("5.11"), KernelVersion("5.13")),),
91 ),
92 _Patch(
93 name="s390-mm-make-memory_block_size_bytes-available-for-M.patch",
94 versions=((KernelVersion("4.3"), KernelVersion("4.11")),),
95 ),
96 _Patch(
97 name="libsubcmd-Fix-use-after-free-for-realloc-.-0.patch",
98 versions=(
99 (KernelVersion("5.16"), KernelVersion("5.16.11")),
100 (KernelVersion("5.11"), KernelVersion("5.15.25")),
101 (KernelVersion("5.5"), KernelVersion("5.10.102")),
102 (KernelVersion("4.20"), KernelVersion("5.4.181")),
103 (KernelVersion("4.15"), KernelVersion("4.19.231")),
104 (KernelVersion("4.10"), KernelVersion("4.14.268")),
105 (KernelVersion("4.5"), KernelVersion("4.9.303")),
106 ),
107 ),
108 _Patch(
109 name="sched-work-around-mystery-QEMU-hang.patch",
110 versions=((KernelVersion("5.11"), KernelVersion("5.14")),),
111 ),
112 _Patch(
113 name="5.4-arm64-build-Remove-.eh_frame-sections-due-to-unwind-.patch",
114 versions=((KernelVersion("5.4"), KernelVersion("5.5")),),
115 ),
116 _Patch(
117 name="4.19-arm64-build-Remove-.eh_frame-sections-due-to-unwind-.patch",
118 versions=((KernelVersion("4.19.15"), KernelVersion("4.20")),),
119 ),
120 _Patch(
121 name="4.14-arm64-build-Remove-.eh_frame-sections-due-to-unwind-.patch",
122 versions=((KernelVersion("4.14.93"), KernelVersion("4.15")),),
123 ),
124 _Patch(
125 name="4.9-arm64-build-Remove-.eh_frame-sections-due-to-unwind-.patch",
126 versions=((KernelVersion("4.9"), KernelVersion("4.10")),),
127 ),
128 _Patch(
129 name="powerpc-pseries-Fix-hcall-tracepoints-with-JUMP_LABE.patch",
130 versions=((KernelVersion("6.2"), KernelVersion("6.4")),),
131 ),
132 _Patch(
133 name="s390-kernel-emit-CFI-data-in-.debug_frame-and-discar.patch",
134 versions=((None, KernelVersion("4.15")),),
135 ),
136 _Patch(
137 name="s390-crash-fix-proc-vmcore-reads.patch",
138 versions=((KernelVersion("5.18"), KernelVersion("6.0")),),
139 ),
140 _Patch(
141 name="lib-raid6-add-option-to-skip-algo-benchmarking.patch",
142 versions=((None, KernelVersion("5.0")),),
143 ),
144 _Patch(
145 name="5.18-Revert-Makefile-link-with-z-noexecstack-no-warn-rwx-.patch",
146 versions=((KernelVersion("5.18.18"), KernelVersion("5.19")),),
147 ),
148 # We could backport this further, but we currently only need it between
149 # Linux kernel commits 50428fdc53ba ("powerpc: Add a ppc_inst_as_str()
150 # helper") (in v5.9) and 2a83afe72a2b ("powerpc/64: Drop
151 # ppc_inst_as_str()") (in v6.0).
152 _Patch(
153 name="gcc-12-disable-Wdangling-pointer-warning-for-now.patch",
154 versions=((KernelVersion("5.16"), KernelVersion("5.18.6")),),
155 ),
156 _Patch(
157 name="5.10-gcc-12-disable-Wdangling-pointer-warning-for-now.patch",
158 versions=(
159 (KernelVersion("5.11"), KernelVersion("5.15.49")),
160 (KernelVersion("5.9"), KernelVersion("5.10.183")),
161 ),
162 ),
163 _Patch(
164 name="5.17-page_pool-enable-CONFIG_PAGE_POOL-by-default.patch",
165 versions=((KernelVersion("5.5"), KernelVersion("5.18")),),
166 ),
167 _Patch(
168 name="5.4-page_pool-enable-CONFIG_PAGE_POOL-by-default.patch",
169 versions=((KernelVersion("4.18"), KernelVersion("5.5")),),
170 ),
171 _Patch(
172 name="kernel-reboot-emergency_restart-Set-correct-system_s.patch",
173 versions=(
174 (KernelVersion("6.6"), KernelVersion("6.6.3")),
175 (KernelVersion("6.2"), KernelVersion("6.5.13")),
176 (KernelVersion("5.16"), KernelVersion("6.1.64")),
177 (KernelVersion("5.11"), KernelVersion("5.15.140")),
178 (KernelVersion("5.5"), KernelVersion("5.10.202")),
179 (None, KernelVersion("5.4.262")),
180 ),
181 ),
182 _Patch(
183 name="kbuild-Only-add-fno-var-tracking-assignments-for-old.patch",
184 versions=((KernelVersion("5.1"), KernelVersion("5.10")),),
185 ),
186 _Patch(
187 name="4.19-kbuild-Only-add-fno-var-tracking-assignments-for-old.patch",
188 versions=((None, KernelVersion("5.1")),),
189 ),
190 _Patch(
191 name="6.17-arm64-map-_text-_stext-virtual-address-range-non-exe.patch",
192 versions=(
193 (KernelVersion("6.13"), KernelVersion("6.17.4")),
194 (KernelVersion("6.9"), KernelVersion("6.12.54")),
195 ),
196 ),
197 _Patch(
198 name="6.8-arm64-map-_text-_stext-virtual-address-range-non-exe.patch",
199 versions=(
200 (KernelVersion("6.1"), KernelVersion("6.2")),
201 (KernelVersion("6.4"), KernelVersion("6.9")),
202 ),
203 ),
204 _Patch(
205 name="6.3-arm64-map-_text-_stext-virtual-address-range-non-exe.patch",
206 versions=(
207 (KernelVersion("5.14"), KernelVersion("6.1")),
208 (KernelVersion("6.2"), KernelVersion("6.4")),
209 ),
210 ),
211 _Patch(
212 name="5.13-arm64-map-_text-_stext-virtual-address-range-non-exe.patch",
213 versions=((KernelVersion("5.13"), KernelVersion("5.14")),),
214 ),
215 _Patch(
216 name="5.12-arm64-map-_text-_stext-virtual-address-range-non-exe.patch",
217 versions=((KernelVersion("5.11"), KernelVersion("5.13")),),
218 ),
219 _Patch(
220 name="4.9-ARM-8933-1-replace-Sun-Solaris-style-flag-on-section.patch",
221 versions=((None, KernelVersion("4.14")),),
222 ),
223 _Patch(
224 name="4.14-ARM-8933-1-replace-Sun-Solaris-style-flag-on-section.patch",
225 versions=(
226 (KernelVersion("4.14"), KernelVersion("4.14.329")),
227 (KernelVersion("4.15"), KernelVersion("4.19.298")),
228 (KernelVersion("4.20"), KernelVersion("5.4.203")),
229 ),
230 ),
231 _Patch(
232 name="powerpc-boot-Remove-leading-zero-in-label-in-udelay.patch",
233 versions=(
234 (None, KernelVersion("5.4.299")),
235 (KernelVersion("5.5"), KernelVersion("5.10")),
236 ),
237 ),
238 _Patch(
239 name="riscv-fix-vmemmap-and-vmalloc-offsets-in-proc-kcore.patch",
240 versions=((KernelVersion("6.11"), None),),
241 ),
242 _Patch(
243 name="6.10-riscv-fix-vmemmap-and-vmalloc-offsets-in-proc-kcore.patch",
244 versions=(
245 (KernelVersion("6.8"), KernelVersion("6.11")),
246 (KernelVersion("6.6.51"), KernelVersion("6.7")),
247 ),
248 ),
249 _Patch(
250 name="6.7-riscv-fix-vmemmap-and-vmalloc-offsets-in-proc-kcore.patch",
251 versions=(
252 (KernelVersion("6.7"), KernelVersion("6.8")),
253 (KernelVersion("5.17"), KernelVersion("6.6.51")),
254 ),
255 ),
256 _Patch(
257 name="5.16-riscv-fix-vmemmap-and-vmalloc-offsets-in-proc-kcore.patch",
258 versions=((KernelVersion("4.15"), KernelVersion("5.17")),),
259 ),
260 )
261
262
263 async def apply_patches(kernel_dir: Path) -> None:
264 patch_dir = Path(__file__).parent / "patches"
265 version = KernelVersion(
266 (await check_output("make", "-s", "kernelversion", cwd=kernel_dir))
267 .decode()
268 .strip()
269 )
270 logger.info("applying patches for kernel version %s", version)
271 any_applied = False
272 for patch in _PATCHES:
273 for min_version, max_version in patch.versions:
274 if (min_version is None or min_version <= version) and (
275 max_version is None or version < max_version
276 ):
277 break
278 else:
279 continue
280 logger.info("applying %s", patch.name)
281 any_applied = True
282 proc = await asyncio.create_subprocess_exec(
283 "git",
284 "apply",
285 str(patch_dir / patch.name),
286 cwd=kernel_dir,
287 stderr=asyncio.subprocess.PIPE,
288 )
289 assert proc.stderr is not None # for mypy
290 stderr = await proc.stderr.read()
291 if await proc.wait() != 0:
292 try:
293 await check_call(
294 "git",
295 "apply",
296 "--reverse",
297 "--check",
298 str(patch_dir / patch.name),
299 cwd=kernel_dir,
300 stderr=asyncio.subprocess.DEVNULL,
301 )
302 except CalledProcessError:
303 sys.stderr.buffer.write(stderr)
304 sys.stderr.buffer.flush()
305 raise
306 logger.info("already applied")
307 if not any_applied:
308 logger.info("no patches")
309
310
311 class KBuild:
312 def __init__(
313 self,
314 kernel_dir: Path,
315 build_dir: Path,
316 arch: Architecture,
317 flavor: KernelFlavor,
318 env: Optional[Mapping[str, str]] = None,
319 build_log_file: Union[int, IO[Any], None] = None,
320 ) -> None:
321 self._build_dir = build_dir
322 self._kernel_dir = kernel_dir
323 self._flavor = flavor
324 self._arch = arch
325 self._env = env
326 self._build_stdout = build_log_file
327 self._build_stderr = (
328 None if build_log_file is None else asyncio.subprocess.STDOUT
329 )
330 self._cached_make_args: Optional[Sequence[str]] = None
331 self._cached_kernel_release: Optional[str] = None
332
333 async def _prepare_make(self) -> Sequence[str]:
334 if self._cached_make_args is None:
335 self._build_dir.mkdir(parents=True, exist_ok=True)
336
337 debug_prefix_map = []
338 # GCC uses the "logical" working directory, i.e., the PWD
339 # environment variable, when it can. See
340 # https://gcc.gnu.org/git/?p=gcc.git;a=blob;f=libiberty/getpwd.c;hb=HEAD.
341 # Map both the canonical and logical paths.
342 build_dir_real = self._build_dir.resolve()
343 debug_prefix_map.append(str(build_dir_real) + "=.")
344 build_dir_logical = (
345 await check_output_shell(
346 f"cd {shlex.quote(str(self._build_dir))}; pwd -L",
347 )
348 ).decode()[:-1]
349 if build_dir_logical != str(build_dir_real):
350 debug_prefix_map.append(build_dir_logical + "=.")
351
352 # Before Linux kernel commit 25b146c5b8ce ("kbuild: allow Kbuild to
353 # start from any directory") (in v5.2), O= forces the source
354 # directory to be absolute. Since Linux kernel commit 95fd3f87bfbe
355 # ("kbuild: add a flag to force absolute path for srctree") (in
356 # v5.3), KBUILD_ABS_SRCTREE=1 does the same. This means that except
357 # for v5.2, which we don't support, the source directory will
358 # always be absolute, and we don't need to worry about mapping it
359 # from a relative path.
360 kernel_dir_real = self._kernel_dir.resolve()
361 if kernel_dir_real != build_dir_real:
362 debug_prefix_map.append(str(kernel_dir_real) + "/=./")
363
364 cflags = " ".join(["-fdebug-prefix-map=" + map for map in debug_prefix_map])
365
366 make_args = [
367 "-C",
368 str(self._kernel_dir),
369 "ARCH=" + str(self._arch.kernel_arch),
370 "O=" + str(build_dir_real),
371 "KBUILD_ABS_SRCTREE=1",
372 "KBUILD_BUILD_USER=drgn",
373 "KBUILD_BUILD_HOST=drgn",
374 "HOSTCFLAGS=" + cflags,
375 "KAFLAGS=" + cflags,
376 "KCFLAGS=" + cflags,
377 ]
378
379 version = (
380 (
381 await check_output(
382 "make", *make_args, "-s", "kernelversion", env=self._env
383 )
384 )
385 .decode()
386 .strip()
387 )
388 make_args.append(
389 "LOCALVERSION="
390 + kconfig_localversion(self._arch, self._flavor, version)
391 )
392 make_args.append("-j")
393 make_args.append(str(nproc()))
394
395 self._cached_make_args = make_args
396 return self._cached_make_args
397
398 async def _kernel_release(self) -> str:
399 if self._cached_kernel_release is None:
400 # Must call _prepare_make() first.
401 assert self._cached_make_args is not None
402 self._cached_kernel_release = (
403 (
404 await check_output(
405 "make",
406 *self._cached_make_args,
407 "-s",
408 "kernelrelease",
409 env=self._env,
410 )
411 )
412 .decode()
413 .strip()
414 )
415 return self._cached_kernel_release
416
417 async def build(self) -> None:
418 logger.info(
419 "building %s %s kernel in %s",
420 self._arch.name,
421 self._flavor.name,
422 self._build_dir,
423 )
424 build_log_file_name = getattr(self._build_stdout, "name", None)
425 if build_log_file_name is not None:
426 logger.info("build logs in %s", build_log_file_name)
427
428 make_args = await self._prepare_make()
429
430 config = self._build_dir / ".config"
431 tmp_config = self._build_dir / ".config.vmtest.tmp"
432
433 tmp_config.write_text(kconfig(self._arch, self._flavor))
434 await check_call(
435 "make",
436 *make_args,
437 "KCONFIG_CONFIG=" + tmp_config.name,
438 "olddefconfig",
439 stdout=self._build_stdout,
440 stderr=self._build_stderr,
441 env=self._env,
442 )
443 try:
444 equal = filecmp.cmp(config, tmp_config)
445 if not equal:
446 logger.info("kernel configuration changed")
447 except FileNotFoundError:
448 equal = False
449 logger.info("no previous kernel configuration")
450 if equal:
451 logger.info("kernel configuration did not change")
452 tmp_config.unlink()
453 else:
454 tmp_config.rename(config)
455
456 kernel_release = await self._kernel_release()
457 logger.info("kernel release is %s", kernel_release)
458 await check_call(
459 "make",
460 *make_args,
461 "all",
462 stdout=self._build_stdout,
463 stderr=self._build_stderr,
464 env=self._env,
465 )
466 logger.info(
467 "built kernel %s for %s in %s",
468 kernel_release,
469 self._arch.name,
470 self._build_dir,
471 )
472
473 def _copy_module_build(self, modules_build_dir: Path) -> None:
474 logger.info("copying module build files")
475
476 # Files and directories (as glob patterns) required for external module
477 # builds. This list was determined through trial and error.
478 files = [
479 ".config",
480 "Module.symvers",
481 f"arch/{self._arch.kernel_srcarch}/Makefile",
482 f"arch/{self._arch.kernel_srcarch}/kernel/module.lds",
483 "scripts/Kbuild.include",
484 "scripts/Makefile*",
485 "scripts/basic/fixdep",
486 "scripts/check-local-export",
487 "scripts/gcc-goto.sh",
488 "scripts/gcc-version.sh",
489 "scripts/gen-btf.sh",
490 "scripts/ld-version.sh",
491 "scripts/mod/modpost",
492 "scripts/module-common.c",
493 "scripts/module-common.lds",
494 "scripts/module.lds",
495 "scripts/modules-check.sh",
496 "scripts/pahole-flags.sh",
497 "scripts/pahole-version.sh",
498 "scripts/subarch.include",
499 "tools/bpf/resolve_btfids/resolve_btfids",
500 "tools/objtool/objtool",
501 ]
502 if self._arch.name == "aarch64":
503 # Before Linux kernel commits bca8f17f57bd ("arm64: Get rid of
504 # asm/opcodes.h") and 2fbadc3002c5 ("arm/arm64: xen: Move shared
505 # architecture headers to include/xen/arm" (both in v4.10), AArch64
506 # included these files from 32-bit Arm.
507 files.extend(
508 (
509 "arch/arm/include/asm/opcodes.h",
510 "arch/arm/include/asm/xen/hypercall.h",
511 "arch/arm/include/asm/xen/hypervisor.h",
512 "arch/arm/include/asm/xen/interface.h",
513 "arch/arm/include/asm/xen/page-coherent.h",
514 "arch/arm/include/asm/xen/page.h",
515 )
516 )
517 # Before Linux kernel commit efe0160cfd40 ("powerpc/64: Linker
518 # on-demand sfpr functions for modules") (in v4.13), this must be
519 # available to link into modules.
520 if self._arch.name == "ppc64":
521 files.append("arch/powerpc/lib/crtsavres.o")
522 directories = (
523 f"arch/{self._arch.kernel_srcarch}/include",
524 "include",
525 )
526
527 # Copy from the source and build directories.
528 src_dirs = [self._kernel_dir]
529 if not self._build_dir.samefile(self._kernel_dir):
530 src_dirs.append(self._build_dir)
531 # The top-level Makefile is a special case because we only want the one
532 # from the source directory; the one in the build directory is a stub.
533 shutil.copy2(
534 self._kernel_dir / "Makefile",
535 modules_build_dir / "Makefile",
536 follow_symlinks=False,
537 )
538 for glob in files:
539 for src_dir in src_dirs:
540 for src_path in src_dir.glob(glob):
541 dst_path = modules_build_dir / src_path.relative_to(src_dir)
542 dst_path.parent.mkdir(parents=True, exist_ok=True)
543 shutil.copy2(src_path, dst_path, follow_symlinks=False)
544 for glob in directories:
545 for src_dir in src_dirs:
546 for src_path in src_dir.glob(glob):
547 dst_path = modules_build_dir / src_path.relative_to(src_dir)
548 dst_path.parent.mkdir(parents=True, exist_ok=True)
549 shutil.copytree(
550 src_path,
551 dst_path,
552 ignore=shutil.ignore_patterns("*.cmd"),
553 symlinks=True,
554 dirs_exist_ok=True,
555 )
556
557 async def _test_external_module_build(self, modules_build_dir: Path) -> None:
558 logger.info("testing external module build")
559
560 with tempfile.TemporaryDirectory(
561 prefix="test_module.", dir=self._build_dir
562 ) as tmp_name:
563 test_module_dir = Path(tmp_name)
564 (test_module_dir / "test.c").write_text(
565 r"""
566 #include <linux/module.h>
567
568 static int __init test_init(void)
569 {
570 return 0;
571 }
572
573 module_init(test_init);
574
575 MODULE_LICENSE("GPL");
576 MODULE_DESCRIPTION("Module for testing build");
577 """
578 )
579 (test_module_dir / "Makefile").write_text("obj-m := test.o\n")
580 # Execute make and look for errors in its output. It's not
581 # enough to check its exit status because some of the build
582 # scripts limp along (possibly incorrectly) even if they're
583 # missing some files.
584 cmd = (
585 "make",
586 "ARCH=" + str(self._arch.kernel_arch),
587 "-C",
588 modules_build_dir,
589 f"M={test_module_dir.resolve()}",
590 )
591 proc = await asyncio.create_subprocess_exec(
592 *cmd,
593 stdout=asyncio.subprocess.PIPE,
594 stderr=asyncio.subprocess.PIPE,
595 env=self._env,
596 )
597 assert proc.stdout is not None # for mypy
598 assert proc.stderr is not None # for mypy
599 try:
600 stdout_task: Optional[asyncio.Task[bytes]] = asyncio.create_task(
601 proc.stdout.readline()
602 )
603 stderr_task: Optional[asyncio.Task[bytes]] = asyncio.create_task(
604 proc.stderr.readline()
605 )
606 error = False
607 while stdout_task is not None or stderr_task is not None:
608 aws = []
609 if stdout_task is not None:
610 aws.append(stdout_task)
611 if stderr_task is not None:
612 aws.append(stderr_task)
613 done, pending = await asyncio.wait(
614 aws, return_when=asyncio.FIRST_COMPLETED
615 )
616 for task in done:
617 line = task.result()
618 if b"No such file or directory" in line:
619 error = True
620 if task is stdout_task:
621 sys.stdout.buffer.write(line)
622 sys.stdout.buffer.flush()
623 stdout_task = (
624 asyncio.create_task(proc.stdout.readline())
625 if line
626 else None
627 )
628 else:
629 sys.stderr.buffer.write(line)
630 sys.stderr.buffer.flush()
631 stderr_task = (
632 asyncio.create_task(proc.stderr.readline())
633 if line
634 else None
635 )
636 finally:
637 returncode = await proc.wait()
638 if returncode != 0:
639 raise CalledProcessError(returncode, cmd)
640 if error:
641 raise Exception(f"Command {cmd} output contained error")
642
643 async def package(self, format: str, output_dir: Path) -> Path:
644 if format not in _PACKAGE_FORMATS:
645 raise ValueError("unknown package format")
646
647 make_args = await self._prepare_make()
648 kernel_release = await self._kernel_release()
649
650 extension = "" if format == "directory" else ("." + format)
651 package = output_dir / f"kernel-{kernel_release}.{self._arch.name}{extension}"
652
653 logger.info(
654 "packaging kernel %s for %s from %s to %s",
655 kernel_release,
656 self._arch.name,
657 self._build_dir,
658 package,
659 )
660
661 if self._arch.kernel_srcarch == "riscv":
662 # `qemu-system-riscv64 -kernel` apparently can't boot a compressed
663 # kernel. Since Linux kernel commit e79dfcbfb902 ("riscv: make
664 # image compression configurable") (in v6.10),
665 # CONFIG_KERNEL_UNCOMPRESSED=y makes `make image_name` output the
666 # uncompressed image. Before that, we need a special case.
667 image_name = "Image"
668 else:
669 image_name = (
670 (
671 await check_output(
672 "make", *make_args, "-s", "image_name", env=self._env
673 )
674 )
675 .decode()
676 .strip()
677 )
678
679 package.parent.mkdir(parents=True, exist_ok=True)
680 with tempfile.TemporaryDirectory(
681 prefix=package.name + ".tmp.", dir=package.parent
682 ) as tmp_name:
683 install_dir = Path(tmp_name)
684 modules_dir = install_dir / "lib" / "modules" / kernel_release
685
686 logger.info("installing modules")
687 await check_call(
688 "make",
689 *make_args,
690 "INSTALL_MOD_PATH=" + str(install_dir.resolve()),
691 "modules_install",
692 stdout=self._build_stdout,
693 stderr=self._build_stderr,
694 env=self._env,
695 )
696
697 # `make modules_install` creates build as a symlink to the absolute
698 # path of the build directory. Delete it and make it an empty
699 # directory for us to populate.
700 modules_build_dir = modules_dir / "build"
701 modules_build_dir.unlink()
702 modules_build_dir.mkdir()
703 # Before Linux kernel commit d8131c2965d5 ("kbuild: remove
704 # $(MODLIB)/source symlink") (in v6.6), source is a symlink to the
705 # absolute path of the source directory. It's not needed.
706 try:
707 (modules_dir / "source").unlink()
708 except FileNotFoundError:
709 pass
710
711 logger.info("copying vmlinux")
712 vmlinux = modules_build_dir / "vmlinux"
713 await check_call(
714 (os.environ if self._env is None else self._env).get(
715 "CROSS_COMPILE", ""
716 )
717 + "objcopy",
718 "--remove-relocations=*",
719 self._build_dir / "vmlinux",
720 str(vmlinux),
721 env=self._env,
722 )
723 vmlinux.chmod(0o644)
724
725 logger.info("copying vmlinuz")
726 vmlinuz = modules_dir / "vmlinuz"
727 try:
728 shutil.copy(self._build_dir / image_name, vmlinuz)
729 except FileNotFoundError:
730 # Before Linux kernel commits 06995804b576 ("arm64: Use full
731 # path in KBUILD_IMAGE definition") and 152e6744ebfc ("arm: Use
732 # full path in KBUILD_IMAGE definition") (in v4.12), image_name
733 # may be relative to the architecture boot directory.
734 shutil.copy(
735 self._build_dir
736 / "arch"
737 / self._arch.kernel_srcarch
738 / "boot"
739 / image_name,
740 vmlinuz,
741 )
742 vmlinuz.chmod(0o644)
743
744 self._copy_module_build(modules_build_dir)
745 await self._test_external_module_build(modules_build_dir)
746
747 if format == "directory":
748 logger.info("renaming directory")
749 try:
750 shutil.rmtree(package)
751 except FileNotFoundError:
752 pass
753 modules_dir.rename(package)
754 else:
755 logger.info("creating tarball")
756 tar_cmd = ("tar", "-C", str(modules_dir), "-c", ".")
757 zstd_cmd = ("zstd", "-T0", "-19", "-q", "-", "-o", str(package), "-f")
758 with pipe_context() as (pipe_r, pipe_w):
759 tar_proc, zstd_proc = await asyncio.gather(
760 asyncio.create_subprocess_exec(*tar_cmd, stdout=pipe_w),
761 asyncio.create_subprocess_exec(*zstd_cmd, stdin=pipe_r),
762 )
763 tar_returncode, zstd_returncode = await asyncio.gather(
764 tar_proc.wait(), zstd_proc.wait()
765 )
766 if tar_returncode != 0:
767 raise CalledProcessError(tar_returncode, tar_cmd)
768 if zstd_returncode != 0:
769 raise CalledProcessError(zstd_returncode, zstd_cmd)
770
771 logger.info(
772 "packaged kernel %s for %s from %s to %s",
773 kernel_release,
774 self._arch.name,
775 self._build_dir,
776 package,
777 )
778 return package
779
780
781 async def main() -> None:
782 logging.basicConfig(
783 format="%(asctime)s:%(levelname)s:%(name)s:%(message)s", level=logging.INFO
784 )
785
786 parser = argparse.ArgumentParser(
787 description="Build a drgn vmtest kernel",
788 formatter_class=argparse.ArgumentDefaultsHelpFormatter,
789 )
790 parser.add_argument(
791 "-k",
792 "--kernel-directory",
793 metavar="DIR",
794 type=Path,
795 help="kernel source tree directory",
796 default=".",
797 )
798 parser.add_argument(
799 "-b",
800 "--build-directory",
801 metavar="DIR",
802 type=Path,
803 help="build output directory",
804 default=".",
805 )
806 parser.add_argument(
807 "-p",
808 "--package",
809 metavar="DIR",
810 type=Path,
811 help="also package the built kernel and place it in DIR",
812 default=argparse.SUPPRESS,
813 )
814 parser.add_argument(
815 "--package-format",
816 choices=_PACKAGE_FORMATS,
817 help='package archive format, or "directory" for an unarchived directory',
818 default=_PACKAGE_FORMATS[0],
819 )
820 parser.add_argument(
821 "--patch",
822 action="store_true",
823 help="apply patches to kernel source tree",
824 )
825 parser.add_argument(
826 "-a",
827 "--architecture",
828 choices=sorted(ARCHITECTURES),
829 help="architecture to build for",
830 default=None if HOST_ARCHITECTURE is None else HOST_ARCHITECTURE.name,
831 required=HOST_ARCHITECTURE is None,
832 )
833 parser.add_argument(
834 "-f",
835 "--flavor",
836 choices=KERNEL_FLAVORS,
837 help="kernel configuration flavor. "
838 + ". ".join(
839 [
840 f"{flavor.name}: {flavor.description}"
841 for flavor in KERNEL_FLAVORS.values()
842 ]
843 ),
844 default=next(iter(KERNEL_FLAVORS)),
845 )
846 default_download_compiler_directory = Path("build/vmtest")
847 parser.add_argument(
848 "--download-compiler",
849 metavar="DIR",
850 nargs="?",
851 default=argparse.SUPPRESS,
852 type=Path,
853 help=f"download a compiler to the given directory ({default_download_compiler_directory} by default) and use it to build",
854 )
855 parser.add_argument(
856 "--dump-kconfig",
857 action="store_true",
858 help="dump kernel configuration file to standard output instead of building",
859 )
860 args = parser.parse_args()
861
862 arch = ARCHITECTURES[args.architecture]
863 flavor = KERNEL_FLAVORS[args.flavor]
864
865 if args.dump_kconfig:
866 sys.stdout.write(kconfig(arch, flavor))
867 return
868
869 if hasattr(args, "download_compiler"):
870 if args.download_compiler is None:
871 args.download_compiler = default_download_compiler_directory
872 downloader = Downloader(args.download_compiler)
873 downloaded = downloader.download_compiler(downloader.resolve_compiler(arch))
874 env = {**os.environ, **downloaded.env()}
875 else:
876 env = None
877
878 if args.patch:
879 await apply_patches(args.kernel_directory)
880
881 kbuild = KBuild(args.kernel_directory, args.build_directory, arch, flavor, env)
882 await kbuild.build()
883 if hasattr(args, "package"):
884 await kbuild.package(args.package_format, args.package)
885
886
887 if __name__ == "__main__":
888 asyncio.run(main())