| 1 |
#!/usr/bin/env python3 |
| 2 |
# SPDX-License-Identifier: GPL-2.0 |
| 3 |
# Copyright (C) 2025 Mauro Carvalho Chehab <[email protected]> |
| 4 |
# |
| 5 |
# pylint: disable=R0902, R0912, R0913, R0914, R0915, R0917, C0103 |
| 6 |
# |
| 7 |
# Converted from docs Makefile and parallel-wrapper.sh, both under |
| 8 |
# GPLv2, copyrighted since 2008 by the following authors: |
| 9 |
# |
| 10 |
# Akira Yokosawa <[email protected]> |
| 11 |
# Arnd Bergmann <[email protected]> |
| 12 |
# Breno Leitao <[email protected]> |
| 13 |
# Carlos Bilbao <[email protected]> |
| 14 |
# Dave Young <[email protected]> |
| 15 |
# Donald Hunter <[email protected]> |
| 16 |
# Geert Uytterhoeven <[email protected]> |
| 17 |
# Jani Nikula <[email protected]> |
| 18 |
# Jan Stancek <[email protected]> |
| 19 |
# Jonathan Corbet <[email protected]> |
| 20 |
# Joshua Clayton <[email protected]> |
| 21 |
# Kees Cook <[email protected]> |
| 22 |
# Linus Torvalds <[email protected]> |
| 23 |
# Magnus Damm <[email protected]> |
| 24 |
# Masahiro Yamada <[email protected]> |
| 25 |
# Mauro Carvalho Chehab <[email protected]> |
| 26 |
# Maxim Cournoyer <[email protected]> |
| 27 |
# Peter Foley <[email protected]> |
| 28 |
# Randy Dunlap <[email protected]> |
| 29 |
# Rob Herring <[email protected]> |
| 30 |
# Shuah Khan <[email protected]> |
| 31 |
# Thorsten Blum <[email protected]> |
| 32 |
# Tomas Winkler <[email protected]> |
| 33 |
|
| 34 |
|
| 35 |
""" |
| 36 |
Sphinx build wrapper that handles Kernel-specific business rules: |
| 37 |
|
| 38 |
- it gets the Kernel build environment vars; |
| 39 |
- it determines what's the best parallelism; |
| 40 |
- it handles SPHINXDIRS |
| 41 |
|
| 42 |
This tool ensures that MIN_PYTHON_VERSION is satisfied. If version is |
| 43 |
below that, it seeks for a new Python version. If found, it re-runs using |
| 44 |
the newer version. |
| 45 |
""" |
| 46 |
|
| 47 |
import argparse |
| 48 |
import locale |
| 49 |
import os |
| 50 |
import re |
| 51 |
import shlex |
| 52 |
import shutil |
| 53 |
import subprocess |
| 54 |
import sys |
| 55 |
|
| 56 |
from concurrent import futures |
| 57 |
from glob import glob |
| 58 |
|
| 59 |
|
| 60 |
LIB_DIR = "../lib/python" |
| 61 |
SRC_DIR = os.path.dirname(os.path.realpath(__file__)) |
| 62 |
|
| 63 |
sys.path.insert(0, os.path.join(SRC_DIR, LIB_DIR)) |
| 64 |
|
| 65 |
from kdoc.python_version import PythonVersion |
| 66 |
from kdoc.latex_fonts import LatexFontChecker |
| 67 |
from jobserver import JobserverExec # pylint: disable=C0413,C0411,E0401 |
| 68 |
|
| 69 |
# |
| 70 |
# Some constants |
| 71 |
# |
| 72 |
VENV_DEFAULT = "sphinx_latest" |
| 73 |
MIN_PYTHON_VERSION = PythonVersion("3.7").version |
| 74 |
PAPER = ["", "a4", "letter"] |
| 75 |
|
| 76 |
TARGETS = { |
| 77 |
"cleandocs": { "builder": "clean" }, |
| 78 |
"linkcheckdocs": { "builder": "linkcheck" }, |
| 79 |
"htmldocs": { "builder": "html" }, |
| 80 |
"epubdocs": { "builder": "epub", "out_dir": "epub" }, |
| 81 |
"texinfodocs": { "builder": "texinfo", "out_dir": "texinfo" }, |
| 82 |
"infodocs": { "builder": "texinfo", "out_dir": "texinfo" }, |
| 83 |
"mandocs": { "builder": "man", "out_dir": "man" }, |
| 84 |
"latexdocs": { "builder": "latex", "out_dir": "latex" }, |
| 85 |
"pdfdocs": { "builder": "latex", "out_dir": "latex" }, |
| 86 |
"xmldocs": { "builder": "xml", "out_dir": "xml" }, |
| 87 |
} |
| 88 |
|
| 89 |
|
| 90 |
# |
| 91 |
# SphinxBuilder class |
| 92 |
# |
| 93 |
|
| 94 |
class SphinxBuilder: |
| 95 |
""" |
| 96 |
Handles a sphinx-build target, adding needed arguments to build |
| 97 |
with the Kernel. |
| 98 |
""" |
| 99 |
|
| 100 |
def get_path(self, path, use_cwd=False, abs_path=False): |
| 101 |
""" |
| 102 |
Ancillary routine to handle patches the right way, as shell does. |
| 103 |
|
| 104 |
It first expands "~" and "~user". Then, if patch is not absolute, |
| 105 |
join self.srctree. Finally, if requested, convert to abspath. |
| 106 |
""" |
| 107 |
|
| 108 |
path = os.path.expanduser(path) |
| 109 |
if not path.startswith("/"): |
| 110 |
if use_cwd: |
| 111 |
base = os.getcwd() |
| 112 |
else: |
| 113 |
base = self.srctree |
| 114 |
|
| 115 |
path = os.path.join(base, path) |
| 116 |
|
| 117 |
if abs_path: |
| 118 |
return os.path.abspath(path) |
| 119 |
|
| 120 |
return path |
| 121 |
|
| 122 |
def check_rust(self, sphinxdirs): |
| 123 |
""" |
| 124 |
Checks if Rust is enabled |
| 125 |
""" |
| 126 |
config = os.path.join(self.srctree, ".config") |
| 127 |
|
| 128 |
if not {'.', 'rust'}.intersection(sphinxdirs): |
| 129 |
return False |
| 130 |
|
| 131 |
if not os.path.isfile(config): |
| 132 |
return False |
| 133 |
|
| 134 |
re_rust = re.compile(r"CONFIG_RUST=(m|y)") |
| 135 |
|
| 136 |
try: |
| 137 |
with open(config, "r", encoding="utf-8") as fp: |
| 138 |
for line in fp: |
| 139 |
if re_rust.match(line): |
| 140 |
return True |
| 141 |
|
| 142 |
except OSError as e: |
| 143 |
print(f"Failed to open {config}", file=sys.stderr) |
| 144 |
return False |
| 145 |
|
| 146 |
return False |
| 147 |
|
| 148 |
def get_sphinx_extra_opts(self, n_jobs): |
| 149 |
""" |
| 150 |
Get the number of jobs to be used for docs build passed via command |
| 151 |
line and desired sphinx verbosity. |
| 152 |
|
| 153 |
The number of jobs can be on different places: |
| 154 |
|
| 155 |
1) It can be passed via "-j" argument; |
| 156 |
2) The SPHINXOPTS="-j8" env var may have "-j"; |
| 157 |
3) if called via GNU make, -j specifies the desired number of jobs. |
| 158 |
with GNU makefile, this number is available via POSIX jobserver; |
| 159 |
4) if none of the above is available, it should default to "-jauto", |
| 160 |
and let sphinx decide the best value. |
| 161 |
""" |
| 162 |
|
| 163 |
# |
| 164 |
# SPHINXOPTS env var, if used, contains extra arguments to be used |
| 165 |
# by sphinx-build time. Among them, it may contain sphinx verbosity |
| 166 |
# and desired number of parallel jobs. |
| 167 |
# |
| 168 |
parser = argparse.ArgumentParser() |
| 169 |
parser.add_argument('-j', '--jobs', type=int) |
| 170 |
parser.add_argument('-q', '--quiet', action='store_true') |
| 171 |
parser.add_argument('-v', '--verbose', default=0, action='count') |
| 172 |
|
| 173 |
# |
| 174 |
# Other sphinx-build arguments go as-is, so place them |
| 175 |
# at self.sphinxopts, using shell parser |
| 176 |
# |
| 177 |
sphinxopts = shlex.split(os.environ.get("SPHINXOPTS", "")) |
| 178 |
|
| 179 |
# |
| 180 |
# Build a list of sphinx args, honoring verbosity here if specified |
| 181 |
# |
| 182 |
|
| 183 |
sphinx_args, self.sphinxopts = parser.parse_known_args(sphinxopts) |
| 184 |
|
| 185 |
verbose = sphinx_args.verbose |
| 186 |
if self.verbose: |
| 187 |
verbose += 1 |
| 188 |
|
| 189 |
if sphinx_args.quiet is True: |
| 190 |
verbose = 0 |
| 191 |
|
| 192 |
# |
| 193 |
# If the user explicitly sets "-j" at command line, use it. |
| 194 |
# Otherwise, pick it from SPHINXOPTS args |
| 195 |
# |
| 196 |
if n_jobs: |
| 197 |
self.n_jobs = n_jobs |
| 198 |
elif sphinx_args.jobs: |
| 199 |
self.n_jobs = sphinx_args.jobs |
| 200 |
else: |
| 201 |
self.n_jobs = None |
| 202 |
|
| 203 |
if verbose < 1: |
| 204 |
self.sphinxopts += ["-q"] |
| 205 |
else: |
| 206 |
for i in range(1, sphinx_args.verbose): |
| 207 |
self.sphinxopts += ["-v"] |
| 208 |
|
| 209 |
def __init__(self, builddir, venv=None, verbose=False, n_jobs=None, |
| 210 |
interactive=None): |
| 211 |
"""Initialize internal variables""" |
| 212 |
self.venv = venv |
| 213 |
self.verbose = None |
| 214 |
|
| 215 |
# |
| 216 |
# Normal variables passed from Kernel's makefile |
| 217 |
# |
| 218 |
self.kernelversion = os.environ.get("KERNELVERSION", "unknown") |
| 219 |
self.kernelrelease = os.environ.get("KERNELRELEASE", "unknown") |
| 220 |
self.pdflatex = os.environ.get("PDFLATEX", "xelatex") |
| 221 |
|
| 222 |
# |
| 223 |
# Kernel main Makefile defines a PYTHON3 variable whose default is |
| 224 |
# "python3". When set to a different value, it allows running a |
| 225 |
# diferent version than the default official python3 package. |
| 226 |
# Several distros package python3xx-sphinx packages with newer |
| 227 |
# versions of Python and sphinx-build. |
| 228 |
# |
| 229 |
# Honor such variable different than default |
| 230 |
# |
| 231 |
self.python = os.environ.get("PYTHON3") |
| 232 |
if self.python == "python3": |
| 233 |
self.python = None |
| 234 |
|
| 235 |
if not interactive: |
| 236 |
self.latexopts = os.environ.get("LATEXOPTS", "-interaction=batchmode -no-shell-escape") |
| 237 |
else: |
| 238 |
self.latexopts = os.environ.get("LATEXOPTS", "") |
| 239 |
|
| 240 |
if not verbose: |
| 241 |
verbose = bool(os.environ.get("KBUILD_VERBOSE", "") != "") |
| 242 |
|
| 243 |
if verbose is not None: |
| 244 |
self.verbose = verbose |
| 245 |
|
| 246 |
# |
| 247 |
# Source tree directory. This needs to be at os.environ, as |
| 248 |
# Sphinx extensions use it |
| 249 |
# |
| 250 |
self.srctree = os.environ.get("srctree") |
| 251 |
if not self.srctree: |
| 252 |
self.srctree = "." |
| 253 |
os.environ["srctree"] = self.srctree |
| 254 |
|
| 255 |
# |
| 256 |
# Now that we can expand srctree, get other directories as well |
| 257 |
# |
| 258 |
self.sphinxbuild = os.environ.get("SPHINXBUILD", "sphinx-build") |
| 259 |
self.kerneldoc = self.get_path(os.environ.get("KERNELDOC", |
| 260 |
"tools/docs/kernel-doc")) |
| 261 |
self.builddir = self.get_path(builddir, use_cwd=True, abs_path=True) |
| 262 |
|
| 263 |
# |
| 264 |
# Get directory locations for LaTeX build toolchain |
| 265 |
# |
| 266 |
self.pdflatex_cmd = shutil.which(self.pdflatex) |
| 267 |
self.latexmk_cmd = shutil.which("latexmk") |
| 268 |
|
| 269 |
self.env = os.environ.copy() |
| 270 |
|
| 271 |
self.get_sphinx_extra_opts(n_jobs) |
| 272 |
|
| 273 |
# |
| 274 |
# If venv command line argument is specified, run Sphinx from venv |
| 275 |
# |
| 276 |
if venv: |
| 277 |
bin_dir = os.path.join(venv, "bin") |
| 278 |
if not os.path.isfile(os.path.join(bin_dir, "activate")): |
| 279 |
sys.exit(f"Venv {venv} not found.") |
| 280 |
|
| 281 |
# "activate" virtual env |
| 282 |
self.env["PATH"] = bin_dir + ":" + self.env["PATH"] |
| 283 |
self.env["VIRTUAL_ENV"] = venv |
| 284 |
if "PYTHONHOME" in self.env: |
| 285 |
del self.env["PYTHONHOME"] |
| 286 |
print(f"Setting venv to {venv}") |
| 287 |
|
| 288 |
def run_sphinx(self, sphinx_build, build_args, *args, **pwargs): |
| 289 |
""" |
| 290 |
Executes sphinx-build using current python3 command. |
| 291 |
|
| 292 |
When calling via GNU make, POSIX jobserver is used to tell how |
| 293 |
many jobs are still available from a job pool. claim all remaining |
| 294 |
jobs, as we don't want sphinx-build to run in parallel with other |
| 295 |
jobs. |
| 296 |
|
| 297 |
Despite that, the user may actually force a different value than |
| 298 |
the number of available jobs via command line. |
| 299 |
|
| 300 |
The "with" logic here is used to ensure that the claimed jobs will |
| 301 |
be freed once subprocess finishes |
| 302 |
""" |
| 303 |
|
| 304 |
with JobserverExec() as jobserver: |
| 305 |
if jobserver.claim: |
| 306 |
# |
| 307 |
# when GNU make is used, claim available jobs from jobserver |
| 308 |
# |
| 309 |
n_jobs = str(jobserver.claim) |
| 310 |
else: |
| 311 |
# |
| 312 |
# Otherwise, let sphinx decide by default |
| 313 |
# |
| 314 |
n_jobs = "auto" |
| 315 |
|
| 316 |
# |
| 317 |
# If explicitly requested via command line, override default |
| 318 |
# |
| 319 |
if self.n_jobs: |
| 320 |
n_jobs = str(self.n_jobs) |
| 321 |
|
| 322 |
# |
| 323 |
# We can't simply call python3 sphinx-build, as OpenSUSE |
| 324 |
# Tumbleweed uses an ELF binary file (/usr/bin/alts) to switch |
| 325 |
# between different versions of sphinx-build. So, only call it |
| 326 |
# prepending "python3.xx" when PYTHON3 variable is not default. |
| 327 |
# |
| 328 |
if self.python: |
| 329 |
cmd = [self.python] |
| 330 |
else: |
| 331 |
cmd = [] |
| 332 |
|
| 333 |
cmd += [sphinx_build] |
| 334 |
cmd += [f"-j{n_jobs}"] |
| 335 |
cmd += build_args |
| 336 |
cmd += self.sphinxopts |
| 337 |
|
| 338 |
if self.verbose: |
| 339 |
print(" ".join(cmd)) |
| 340 |
|
| 341 |
return subprocess.call(cmd, *args, **pwargs) |
| 342 |
|
| 343 |
def handle_html(self, css, output_dir): |
| 344 |
""" |
| 345 |
Extra steps for HTML and epub output. |
| 346 |
|
| 347 |
For such targets, we need to ensure that CSS will be properly |
| 348 |
copied to the output _static directory |
| 349 |
""" |
| 350 |
|
| 351 |
if css: |
| 352 |
css = os.path.expanduser(css) |
| 353 |
if not css.startswith("/"): |
| 354 |
css = os.path.join(self.srctree, css) |
| 355 |
|
| 356 |
static_dir = os.path.join(output_dir, "_static") |
| 357 |
os.makedirs(static_dir, exist_ok=True) |
| 358 |
|
| 359 |
try: |
| 360 |
shutil.copy2(css, static_dir) |
| 361 |
except (OSError, IOError) as e: |
| 362 |
print(f"Warning: Failed to copy CSS: {e}", file=sys.stderr) |
| 363 |
|
| 364 |
def build_pdf_file(self, latex_cmd, from_dir, path): |
| 365 |
"""Builds a single pdf file using latex_cmd""" |
| 366 |
try: |
| 367 |
subprocess.run(latex_cmd + [path], |
| 368 |
cwd=from_dir, check=True, env=self.env) |
| 369 |
|
| 370 |
return True |
| 371 |
except subprocess.CalledProcessError: |
| 372 |
return False |
| 373 |
|
| 374 |
def pdf_parallel_build(self, tex_suffix, latex_cmd, tex_files, n_jobs): |
| 375 |
"""Build PDF files in parallel if possible""" |
| 376 |
builds = {} |
| 377 |
build_failed = False |
| 378 |
max_len = 0 |
| 379 |
has_tex = False |
| 380 |
|
| 381 |
# |
| 382 |
# LaTeX PDF error code is almost useless for us: |
| 383 |
# any warning makes it non-zero. For kernel doc builds it always return |
| 384 |
# non-zero even when build succeeds. So, let's do the best next thing: |
| 385 |
# Ignore build errors. At the end, check if all PDF files were built, |
| 386 |
# printing a summary with the built ones and returning 0 if all of |
| 387 |
# them were actually built. |
| 388 |
# |
| 389 |
with futures.ThreadPoolExecutor(max_workers=n_jobs) as executor: |
| 390 |
jobs = {} |
| 391 |
|
| 392 |
for from_dir, pdf_dir, entry in tex_files: |
| 393 |
name = entry.name |
| 394 |
|
| 395 |
if not name.endswith(tex_suffix): |
| 396 |
continue |
| 397 |
|
| 398 |
name = name[:-len(tex_suffix)] |
| 399 |
has_tex = True |
| 400 |
|
| 401 |
future = executor.submit(self.build_pdf_file, latex_cmd, |
| 402 |
from_dir, entry.path) |
| 403 |
jobs[future] = (from_dir, pdf_dir, name) |
| 404 |
|
| 405 |
for future in futures.as_completed(jobs): |
| 406 |
from_dir, pdf_dir, name = jobs[future] |
| 407 |
|
| 408 |
pdf_name = name + ".pdf" |
| 409 |
pdf_from = os.path.join(from_dir, pdf_name) |
| 410 |
pdf_to = os.path.join(pdf_dir, pdf_name) |
| 411 |
out_name = os.path.relpath(pdf_to, self.builddir) |
| 412 |
max_len = max(max_len, len(out_name)) |
| 413 |
|
| 414 |
try: |
| 415 |
success = future.result() |
| 416 |
|
| 417 |
if success and os.path.exists(pdf_from): |
| 418 |
os.rename(pdf_from, pdf_to) |
| 419 |
|
| 420 |
# |
| 421 |
# if verbose, get the name of built PDF file |
| 422 |
# |
| 423 |
if self.verbose: |
| 424 |
builds[out_name] = "SUCCESS" |
| 425 |
else: |
| 426 |
builds[out_name] = "FAILED" |
| 427 |
build_failed = True |
| 428 |
except futures.Error as e: |
| 429 |
builds[out_name] = f"FAILED ({repr(e)})" |
| 430 |
build_failed = True |
| 431 |
|
| 432 |
# |
| 433 |
# Handle case where no .tex files were found |
| 434 |
# |
| 435 |
if not has_tex: |
| 436 |
out_name = "LaTeX files" |
| 437 |
max_len = max(max_len, len(out_name)) |
| 438 |
builds[out_name] = "FAILED: no .tex files were generated" |
| 439 |
build_failed = True |
| 440 |
|
| 441 |
return builds, build_failed, max_len |
| 442 |
|
| 443 |
def handle_pdf(self, output_dirs, deny_vf): |
| 444 |
""" |
| 445 |
Extra steps for PDF output. |
| 446 |
|
| 447 |
As PDF is handled via a LaTeX output, after building the .tex file, |
| 448 |
a new build is needed to create the PDF output from the latex |
| 449 |
directory. |
| 450 |
""" |
| 451 |
builds = {} |
| 452 |
max_len = 0 |
| 453 |
tex_suffix = ".tex" |
| 454 |
tex_files = [] |
| 455 |
|
| 456 |
# |
| 457 |
# Since early 2024, Fedora and openSUSE tumbleweed have started |
| 458 |
# deploying variable-font format of "Noto CJK", causing LaTeX |
| 459 |
# to break with CJK. Work around it, by denying the variable font |
| 460 |
# usage during xelatex build by passing the location of a config |
| 461 |
# file with a deny list. |
| 462 |
# |
| 463 |
# See tools/docs/lib/latex_fonts.py for more details. |
| 464 |
# |
| 465 |
if deny_vf: |
| 466 |
deny_vf = os.path.expanduser(deny_vf) |
| 467 |
if os.path.isdir(deny_vf): |
| 468 |
self.env["XDG_CONFIG_HOME"] = deny_vf |
| 469 |
|
| 470 |
for from_dir in output_dirs: |
| 471 |
pdf_dir = os.path.join(from_dir, "../pdf") |
| 472 |
os.makedirs(pdf_dir, exist_ok=True) |
| 473 |
|
| 474 |
if self.latexmk_cmd: |
| 475 |
latex_cmd = [self.latexmk_cmd, f"-{self.pdflatex}"] |
| 476 |
else: |
| 477 |
latex_cmd = [self.pdflatex] |
| 478 |
|
| 479 |
latex_cmd.extend(shlex.split(self.latexopts)) |
| 480 |
|
| 481 |
# Get a list of tex files to process |
| 482 |
with os.scandir(from_dir) as it: |
| 483 |
for entry in it: |
| 484 |
if entry.name.endswith(tex_suffix): |
| 485 |
tex_files.append((from_dir, pdf_dir, entry)) |
| 486 |
|
| 487 |
# |
| 488 |
# When using make, this won't be used, as the number of jobs comes |
| 489 |
# from POSIX jobserver. So, this covers the case where build comes |
| 490 |
# from command line. On such case, serialize by default, except if |
| 491 |
# the user explicitly sets the number of jobs. |
| 492 |
# |
| 493 |
n_jobs = 1 |
| 494 |
|
| 495 |
# n_jobs is either an integer or "auto". Only use it if it is a number |
| 496 |
if self.n_jobs: |
| 497 |
try: |
| 498 |
n_jobs = int(self.n_jobs) |
| 499 |
except ValueError: |
| 500 |
pass |
| 501 |
|
| 502 |
# |
| 503 |
# When using make, jobserver.claim is the number of jobs that were |
| 504 |
# used with "-j" and that aren't used by other make targets |
| 505 |
# |
| 506 |
with JobserverExec() as jobserver: |
| 507 |
n_jobs = 1 |
| 508 |
|
| 509 |
# |
| 510 |
# Handle the case when a parameter is passed via command line, |
| 511 |
# using it as default, if jobserver doesn't claim anything |
| 512 |
# |
| 513 |
if self.n_jobs: |
| 514 |
try: |
| 515 |
n_jobs = int(self.n_jobs) |
| 516 |
except ValueError: |
| 517 |
pass |
| 518 |
|
| 519 |
if jobserver.claim: |
| 520 |
n_jobs = jobserver.claim |
| 521 |
|
| 522 |
builds, build_failed, max_len = self.pdf_parallel_build(tex_suffix, |
| 523 |
latex_cmd, |
| 524 |
tex_files, |
| 525 |
n_jobs) |
| 526 |
|
| 527 |
# |
| 528 |
# In verbose mode, print a summary with the build results per file. |
| 529 |
# Otherwise, print a single line with all failures, if any. |
| 530 |
# On both cases, return code 1 indicates build failures, |
| 531 |
# |
| 532 |
if self.verbose: |
| 533 |
msg = "Summary" |
| 534 |
msg += "\n" + "=" * len(msg) |
| 535 |
print() |
| 536 |
print(msg) |
| 537 |
|
| 538 |
for pdf_name, pdf_file in builds.items(): |
| 539 |
print(f"{pdf_name:<{max_len}}: {pdf_file}") |
| 540 |
|
| 541 |
print() |
| 542 |
if build_failed: |
| 543 |
msg = LatexFontChecker().check() |
| 544 |
if msg: |
| 545 |
print(msg) |
| 546 |
|
| 547 |
sys.exit("Error: not all PDF files were created.") |
| 548 |
|
| 549 |
elif build_failed: |
| 550 |
n_failures = len(builds) |
| 551 |
failures = ", ".join(builds.keys()) |
| 552 |
|
| 553 |
msg = LatexFontChecker().check() |
| 554 |
if msg: |
| 555 |
print(msg) |
| 556 |
|
| 557 |
sys.exit(f"Error: Can't build {n_failures} PDF file(s): {failures}") |
| 558 |
|
| 559 |
def handle_info(self, output_dirs): |
| 560 |
""" |
| 561 |
Extra steps for Info output. |
| 562 |
|
| 563 |
For texinfo generation, an additional make is needed from the |
| 564 |
texinfo directory. |
| 565 |
""" |
| 566 |
|
| 567 |
for output_dir in output_dirs: |
| 568 |
try: |
| 569 |
subprocess.run(["make", "info"], cwd=output_dir, check=True) |
| 570 |
except subprocess.CalledProcessError as e: |
| 571 |
sys.exit(f"Error generating info docs: {e}") |
| 572 |
|
| 573 |
def handle_man(self, kerneldoc, docs_dir, src_dir, output_dir): |
| 574 |
""" |
| 575 |
Create man pages from kernel-doc output |
| 576 |
""" |
| 577 |
|
| 578 |
re_kernel_doc = re.compile(r"^\.\.\s+kernel-doc::\s*(\S+)") |
| 579 |
re_man = re.compile(r'^\.TH "[^"]*" (\d+) "([^"]*)"') |
| 580 |
|
| 581 |
if docs_dir == src_dir: |
| 582 |
# |
| 583 |
# Pick the entire set of kernel-doc markups from the entire tree |
| 584 |
# |
| 585 |
kdoc_files = set([self.srctree]) |
| 586 |
else: |
| 587 |
kdoc_files = set() |
| 588 |
|
| 589 |
for fname in glob(os.path.join(src_dir, "**"), recursive=True): |
| 590 |
if os.path.isfile(fname) and fname.endswith(".rst"): |
| 591 |
with open(fname, "r", encoding="utf-8") as in_fp: |
| 592 |
data = in_fp.read() |
| 593 |
|
| 594 |
for line in data.split("\n"): |
| 595 |
match = re_kernel_doc.match(line) |
| 596 |
if match: |
| 597 |
if os.path.isfile(match.group(1)): |
| 598 |
kdoc_files.add(match.group(1)) |
| 599 |
|
| 600 |
if not kdoc_files: |
| 601 |
sys.exit(f"Directory {src_dir} doesn't contain kernel-doc tags") |
| 602 |
|
| 603 |
cmd = [ kerneldoc, "-m" ] + sorted(kdoc_files) |
| 604 |
try: |
| 605 |
if self.verbose: |
| 606 |
print(" ".join(cmd)) |
| 607 |
|
| 608 |
result = subprocess.run(cmd, stdout=subprocess.PIPE, text= True) |
| 609 |
|
| 610 |
if result.returncode: |
| 611 |
print(f"Warning: kernel-doc returned {result.returncode} warnings") |
| 612 |
|
| 613 |
except (OSError, ValueError, subprocess.SubprocessError) as e: |
| 614 |
sys.exit(f"Failed to create man pages for {src_dir}: {repr(e)}") |
| 615 |
|
| 616 |
fp = None |
| 617 |
try: |
| 618 |
for line in result.stdout.split("\n"): |
| 619 |
match = re_man.match(line) |
| 620 |
if not match: |
| 621 |
if fp: |
| 622 |
fp.write(line + '\n') |
| 623 |
continue |
| 624 |
|
| 625 |
if fp: |
| 626 |
fp.close() |
| 627 |
|
| 628 |
fname = f"{output_dir}/{match.group(2)}.{match.group(1)}" |
| 629 |
|
| 630 |
if self.verbose: |
| 631 |
print(f"Creating {fname}") |
| 632 |
fp = open(fname, "w", encoding="utf-8") |
| 633 |
fp.write(line + '\n') |
| 634 |
finally: |
| 635 |
if fp: |
| 636 |
fp.close() |
| 637 |
|
| 638 |
def cleandocs(self, builder): # pylint: disable=W0613 |
| 639 |
"""Remove documentation output directory""" |
| 640 |
shutil.rmtree(self.builddir, ignore_errors=True) |
| 641 |
|
| 642 |
def build(self, target, sphinxdirs=None, |
| 643 |
theme=None, css=None, paper=None, deny_vf=None, |
| 644 |
skip_sphinx=False): |
| 645 |
""" |
| 646 |
Build documentation using Sphinx. This is the core function of this |
| 647 |
module. It prepares all arguments required by sphinx-build. |
| 648 |
""" |
| 649 |
|
| 650 |
builder = TARGETS[target]["builder"] |
| 651 |
out_dir = TARGETS[target].get("out_dir", "") |
| 652 |
|
| 653 |
# |
| 654 |
# Cleandocs doesn't require sphinx-build |
| 655 |
# |
| 656 |
if target == "cleandocs": |
| 657 |
self.cleandocs(builder) |
| 658 |
return |
| 659 |
|
| 660 |
if theme: |
| 661 |
os.environ["DOCS_THEME"] = theme |
| 662 |
|
| 663 |
# |
| 664 |
# Other targets require sphinx-build, so check if it exists |
| 665 |
# |
| 666 |
if not skip_sphinx: |
| 667 |
sphinxbuild = shutil.which(self.sphinxbuild, path=self.env["PATH"]) |
| 668 |
if not sphinxbuild and target != "mandocs": |
| 669 |
sys.exit(f"Error: {self.sphinxbuild} not found in PATH.\n") |
| 670 |
|
| 671 |
if target == "pdfdocs": |
| 672 |
if not self.pdflatex_cmd and not self.latexmk_cmd: |
| 673 |
sys.exit("Error: pdflatex or latexmk required for PDF generation") |
| 674 |
|
| 675 |
docs_dir = os.path.abspath(os.path.join(self.srctree, "Documentation")) |
| 676 |
|
| 677 |
# |
| 678 |
# Fill in base arguments for Sphinx build |
| 679 |
# |
| 680 |
kerneldoc = self.kerneldoc |
| 681 |
if kerneldoc.startswith(self.srctree): |
| 682 |
kerneldoc = os.path.relpath(kerneldoc, self.srctree) |
| 683 |
|
| 684 |
if not sphinxdirs: |
| 685 |
sphinxdirs = os.environ.get("SPHINXDIRS", ".") |
| 686 |
|
| 687 |
# |
| 688 |
# sphinxdirs can be a list or a whitespace-separated string |
| 689 |
# |
| 690 |
sphinxdirs_list = [] |
| 691 |
for sphinxdir in sphinxdirs: |
| 692 |
if isinstance(sphinxdir, list): |
| 693 |
sphinxdirs_list += sphinxdir |
| 694 |
else: |
| 695 |
sphinxdirs_list += sphinxdir.split() |
| 696 |
|
| 697 |
args = [ "-b", builder, "-c", docs_dir ] |
| 698 |
|
| 699 |
if builder == "latex": |
| 700 |
if not paper: |
| 701 |
paper = PAPER[1] |
| 702 |
|
| 703 |
args.extend(["-D", f"latex_elements.papersize={paper}paper"]) |
| 704 |
|
| 705 |
rustdoc = self.check_rust(sphinxdirs_list) |
| 706 |
if rustdoc: |
| 707 |
args.extend(["-t", "rustdoc"]) |
| 708 |
|
| 709 |
# |
| 710 |
# The sphinx-build tool has a bug: internally, it tries to set |
| 711 |
# locale with locale.setlocale(locale.LC_ALL, ''). This causes a |
| 712 |
# crash if language is not set. Detect and fix it. |
| 713 |
# |
| 714 |
try: |
| 715 |
locale.setlocale(locale.LC_ALL, '') |
| 716 |
except locale.Error: |
| 717 |
self.env["LC_ALL"] = "C" |
| 718 |
|
| 719 |
# |
| 720 |
# Step 1: Build each directory in separate. |
| 721 |
# |
| 722 |
# This is not the best way of handling it, as cross-references between |
| 723 |
# them will be broken, but this is what we've been doing since |
| 724 |
# the beginning. |
| 725 |
# |
| 726 |
output_dirs = [] |
| 727 |
for sphinxdir in sphinxdirs_list: |
| 728 |
src_dir = os.path.join(docs_dir, sphinxdir) |
| 729 |
doctree_dir = os.path.join(self.builddir, ".doctrees") |
| 730 |
output_dir = os.path.join(self.builddir, sphinxdir, out_dir) |
| 731 |
|
| 732 |
# |
| 733 |
# Make directory names canonical |
| 734 |
# |
| 735 |
src_dir = os.path.normpath(src_dir) |
| 736 |
doctree_dir = os.path.normpath(doctree_dir) |
| 737 |
output_dir = os.path.normpath(output_dir) |
| 738 |
|
| 739 |
os.makedirs(doctree_dir, exist_ok=True) |
| 740 |
os.makedirs(output_dir, exist_ok=True) |
| 741 |
|
| 742 |
output_dirs.append(output_dir) |
| 743 |
|
| 744 |
build_args = args + [ |
| 745 |
"-d", doctree_dir, |
| 746 |
"-D", f"version={self.kernelversion}", |
| 747 |
"-D", f"release={self.kernelrelease}", |
| 748 |
"-D", f"kerneldoc_srctree={self.srctree}", |
| 749 |
src_dir, |
| 750 |
output_dir, |
| 751 |
] |
| 752 |
|
| 753 |
if target == "mandocs": |
| 754 |
self.handle_man(kerneldoc, docs_dir, src_dir, output_dir) |
| 755 |
elif not skip_sphinx: |
| 756 |
try: |
| 757 |
result = self.run_sphinx(sphinxbuild, build_args, |
| 758 |
env=self.env) |
| 759 |
|
| 760 |
if result: |
| 761 |
sys.exit(f"Build failed: return code: {result}") |
| 762 |
|
| 763 |
except (OSError, ValueError, subprocess.SubprocessError) as e: |
| 764 |
sys.exit(f"Build failed: {repr(e)}") |
| 765 |
|
| 766 |
# |
| 767 |
# Ensure that each html/epub output will have needed static files |
| 768 |
# |
| 769 |
if target in ["htmldocs", "epubdocs"]: |
| 770 |
self.handle_html(css, output_dir) |
| 771 |
|
| 772 |
# |
| 773 |
# Step 2: Some targets (PDF and info) require an extra step once |
| 774 |
# sphinx-build finishes |
| 775 |
# |
| 776 |
if target == "pdfdocs": |
| 777 |
self.handle_pdf(output_dirs, deny_vf) |
| 778 |
elif target == "infodocs": |
| 779 |
self.handle_info(output_dirs) |
| 780 |
|
| 781 |
if rustdoc and target in ["htmldocs", "epubdocs"]: |
| 782 |
print("Building rust docs") |
| 783 |
if "MAKE" in self.env: |
| 784 |
cmd = [self.env["MAKE"]] |
| 785 |
else: |
| 786 |
cmd = ["make", "LLVM=1"] |
| 787 |
|
| 788 |
cmd += [ "rustdoc"] |
| 789 |
if self.verbose: |
| 790 |
print(" ".join(cmd)) |
| 791 |
|
| 792 |
try: |
| 793 |
subprocess.run(cmd, check=True) |
| 794 |
except subprocess.CalledProcessError as e: |
| 795 |
print(f"Ignored errors when building rustdoc: {e}. Is RUST enabled?", |
| 796 |
file=sys.stderr) |
| 797 |
|
| 798 |
def jobs_type(value): |
| 799 |
""" |
| 800 |
Handle valid values for -j. Accepts Sphinx "-jauto", plus a number |
| 801 |
equal or bigger than one. |
| 802 |
""" |
| 803 |
if value is None: |
| 804 |
return None |
| 805 |
|
| 806 |
if value.lower() == 'auto': |
| 807 |
return value.lower() |
| 808 |
|
| 809 |
try: |
| 810 |
if int(value) >= 1: |
| 811 |
return value |
| 812 |
|
| 813 |
raise argparse.ArgumentTypeError(f"Minimum jobs is 1, got {value}") |
| 814 |
except ValueError: |
| 815 |
raise argparse.ArgumentTypeError(f"Must be 'auto' or positive integer, got {value}") # pylint: disable=W0707 |
| 816 |
|
| 817 |
EPILOG=""" |
| 818 |
Besides the command line arguments, several environment variables affect its |
| 819 |
default behavior, meant to be used when called via Kernel Makefile: |
| 820 |
|
| 821 |
- KERNELVERSION: Kernel major version |
| 822 |
- KERNELRELEASE: Kernel release |
| 823 |
- KBUILD_VERBOSE: Contains the value of "make V=[0|1] variable. |
| 824 |
When V=0 (KBUILD_VERBOSE=0), sets verbose level to "-q". |
| 825 |
- SPHINXBUILD: Documentation build tool (default: "sphinx-build"). |
| 826 |
- SPHINXOPTS: Extra options pased to SPHINXBUILD |
| 827 |
(default: "-j auto" and "-q" if KBUILD_VERBOSE=0). |
| 828 |
The "-v" flag can be used to increase verbosity. |
| 829 |
If V=0, the first "-v" will drop "-q". |
| 830 |
- PYTHON3: Python command to run SPHINXBUILD |
| 831 |
- PDFLATEX: LaTeX PDF engine. (default: "xelatex") |
| 832 |
- LATEXOPTS: Optional set of command line arguments to the LaTeX engine |
| 833 |
- srctree: Location of the Kernel root directory (default: "."). |
| 834 |
|
| 835 |
""" |
| 836 |
|
| 837 |
def main(): |
| 838 |
""" |
| 839 |
Main function. The only mandatory argument is the target. If not |
| 840 |
specified, the other arguments will use default values if not |
| 841 |
specified at os.environ. |
| 842 |
""" |
| 843 |
parser = argparse.ArgumentParser(formatter_class=argparse.RawTextHelpFormatter, |
| 844 |
description=__doc__, |
| 845 |
epilog=EPILOG) |
| 846 |
|
| 847 |
parser.add_argument("target", choices=list(TARGETS.keys()), |
| 848 |
help="Documentation target to build") |
| 849 |
parser.add_argument("--sphinxdirs", nargs="+", |
| 850 |
help="Specific directories to build") |
| 851 |
parser.add_argument("--builddir", default="output", |
| 852 |
help="Sphinx configuration file (default: %(default)s)") |
| 853 |
|
| 854 |
parser.add_argument("--theme", help="Sphinx theme to use") |
| 855 |
|
| 856 |
parser.add_argument("--css", help="Custom CSS file for HTML/EPUB") |
| 857 |
|
| 858 |
parser.add_argument("--paper", choices=PAPER, default=PAPER[0], |
| 859 |
help="Paper size for LaTeX/PDF output") |
| 860 |
|
| 861 |
parser.add_argument('--deny-vf', |
| 862 |
help="Configuration to deny variable fonts on pdf builds") |
| 863 |
|
| 864 |
parser.add_argument("-v", "--verbose", action='store_true', |
| 865 |
help="place build in verbose mode") |
| 866 |
|
| 867 |
parser.add_argument('-j', '--jobs', type=jobs_type, |
| 868 |
help="Sets number of jobs to use with sphinx-build(default: auto)") |
| 869 |
|
| 870 |
parser.add_argument('-i', '--interactive', action='store_true', |
| 871 |
help="Change latex default to run in interactive mode") |
| 872 |
|
| 873 |
parser.add_argument('-s', '--skip-sphinx-build', action='store_true', |
| 874 |
help="Skip sphinx-build step") |
| 875 |
|
| 876 |
parser.add_argument("-V", "--venv", nargs='?', const=f'{VENV_DEFAULT}', |
| 877 |
default=None, |
| 878 |
help=f'If used, run Sphinx from a venv dir (default dir: {VENV_DEFAULT})') |
| 879 |
|
| 880 |
args = parser.parse_args() |
| 881 |
|
| 882 |
PythonVersion.check_python(MIN_PYTHON_VERSION, show_alternatives=True, |
| 883 |
bail_out=True) |
| 884 |
|
| 885 |
builder = SphinxBuilder(builddir=args.builddir, venv=args.venv, |
| 886 |
verbose=args.verbose, n_jobs=args.jobs, |
| 887 |
interactive=args.interactive) |
| 888 |
|
| 889 |
builder.build(args.target, sphinxdirs=args.sphinxdirs, |
| 890 |
theme=args.theme, css=args.css, paper=args.paper, |
| 891 |
deny_vf=args.deny_vf, |
| 892 |
skip_sphinx=args.skip_sphinx_build) |
| 893 |
|
| 894 |
if __name__ == "__main__": |
| 895 |
main() |