Debian Codegraph @

Tools/build/check_extension_modules.py

raw
1 """Check extension modules
2
3 The script checks shared and built-in extension modules. It verifies that the
4 modules have been built and that they can be imported successfully. Missing
5 modules and failed imports are reported to the user. Shared extension
6 files are renamed on failed import.
7
8 Module information is parsed from several sources:
9
10 - core modules hard-coded in Modules/config.c.in
11 - Windows-specific modules that are hard-coded in PC/config.c
12 - MODULE_{name}_STATE entries in Makefile (provided through sysconfig)
13 - Various makesetup files:
14 - $(srcdir)/Modules/Setup
15 - Modules/Setup.[local|bootstrap|stdlib] files, which are generated
16 from $(srcdir)/Modules/Setup.*.in files
17
18 See --help for more information
19 """
20
21 from __future__ import annotations
22
23 import _imp
24 import argparse
25 import enum
26 import logging
27 import os
28 import pathlib
29 import re
30 import sys
31 import sysconfig
32 import warnings
33 from collections.abc import Iterable
34 from importlib._bootstrap import ( # type: ignore[attr-defined]
35 _load as bootstrap_load,
36 )
37 from importlib.machinery import (
38 BuiltinImporter,
39 ExtensionFileLoader,
40 ModuleSpec,
41 )
42 from importlib.util import spec_from_file_location, spec_from_loader
43 from typing import NamedTuple
44
45 SRC_DIR = pathlib.Path(__file__).parent.parent.parent
46
47 # core modules, hard-coded in Modules/config.h.in
48 CORE_MODULES = {
49 "_ast",
50 "_imp",
51 "_string",
52 "_tokenize",
53 "_warnings",
54 "builtins",
55 "gc",
56 "marshal",
57 "sys",
58 }
59
60 # Windows-only modules
61 WINDOWS_MODULES = {
62 "_overlapped",
63 "_testconsole",
64 "_winapi",
65 "_wmi",
66 "msvcrt",
67 "nt",
68 "winreg",
69 "winsound",
70 }
71
72
73 logger = logging.getLogger(__name__)
74
75 parser = argparse.ArgumentParser(
76 prog="check_extension_modules",
77 description=__doc__,
78 formatter_class=argparse.RawDescriptionHelpFormatter,
79 )
80
81 parser.add_argument(
82 "--verbose",
83 action="store_true",
84 help="Verbose, report builtin, shared, and unavailable modules",
85 )
86
87 parser.add_argument(
88 "--debug",
89 action="store_true",
90 help="Enable debug logging",
91 )
92
93 parser.add_argument(
94 "--strict",
95 action=argparse.BooleanOptionalAction,
96 help=(
97 "Strict check, fail when a module is missing or fails to import"
98 "(default: no, unless env var PYTHONSTRICTEXTENSIONBUILD is set)"
99 ),
100 default=bool(os.environ.get("PYTHONSTRICTEXTENSIONBUILD")),
101 )
102
103 parser.add_argument(
104 "--cross-compiling",
105 action=argparse.BooleanOptionalAction,
106 help=(
107 "Use cross-compiling checks "
108 "(default: no, unless env var _PYTHON_HOST_PLATFORM is set)."
109 ),
110 default="_PYTHON_HOST_PLATFORM" in os.environ,
111 )
112
113 parser.add_argument(
114 "--list-module-names",
115 action="store_true",
116 help="Print a list of module names to stdout and exit",
117 )
118
119
120 @enum.unique
121 class ModuleState(enum.Enum):
122 # Makefile state "yes"
123 BUILTIN = "builtin"
124 SHARED = "shared"
125
126 DISABLED = "disabled"
127 MISSING = "missing"
128 NA = "n/a"
129 # disabled by Setup / makesetup rule
130 DISABLED_SETUP = "disabled_setup"
131
132 def __bool__(self) -> bool:
133 return self.value in {"builtin", "shared"}
134
135
136 class ModuleInfo(NamedTuple):
137 name: str
138 state: ModuleState
139
140
141 class ModuleChecker:
142 pybuilddir_txt = "pybuilddir.txt"
143
144 setup_files = (
145 # see end of configure.ac
146 pathlib.Path("Modules/Setup.local"),
147 pathlib.Path("Modules/Setup.stdlib"),
148 pathlib.Path("Modules/Setup.bootstrap"),
149 SRC_DIR / "Modules/Setup",
150 )
151
152 def __init__(self, cross_compiling: bool = False, strict: bool = False):
153 self.cross_compiling = cross_compiling
154 self.strict_extensions_build = strict
155 self.ext_suffix = sysconfig.get_config_var("EXT_SUFFIX")
156 self.platform = sysconfig.get_platform()
157 self.builddir = self.get_builddir()
158 self.modules = self.get_modules()
159
160 self.builtin_ok: list[ModuleInfo] = []
161 self.shared_ok: list[ModuleInfo] = []
162 self.failed_on_import: list[ModuleInfo] = []
163 self.missing: list[ModuleInfo] = []
164 self.disabled_configure: list[ModuleInfo] = []
165 self.disabled_setup: list[ModuleInfo] = []
166 self.notavailable: list[ModuleInfo] = []
167
168 def check(self) -> None:
169 if not hasattr(_imp, 'create_dynamic'):
170 logger.warning(
171 ('Dynamic extensions not supported '
172 '(HAVE_DYNAMIC_LOADING not defined)'),
173 )
174 for modinfo in self.modules:
175 logger.debug("Checking '%s' (%s)", modinfo.name, self.get_location(modinfo))
176 if modinfo.state == ModuleState.DISABLED:
177 self.disabled_configure.append(modinfo)
178 elif modinfo.state == ModuleState.DISABLED_SETUP:
179 self.disabled_setup.append(modinfo)
180 elif modinfo.state == ModuleState.MISSING:
181 self.missing.append(modinfo)
182 elif modinfo.state == ModuleState.NA:
183 self.notavailable.append(modinfo)
184 else:
185 try:
186 if self.cross_compiling:
187 self.check_module_cross(modinfo)
188 else:
189 self.check_module_import(modinfo)
190 except (ImportError, FileNotFoundError):
191 self.rename_module(modinfo)
192 self.failed_on_import.append(modinfo)
193 else:
194 if modinfo.state == ModuleState.BUILTIN:
195 self.builtin_ok.append(modinfo)
196 else:
197 assert modinfo.state == ModuleState.SHARED
198 self.shared_ok.append(modinfo)
199
200 def summary(self, *, verbose: bool = False) -> None:
201 longest = max([len(e.name) for e in self.modules], default=0)
202
203 def print_three_column(modinfos: list[ModuleInfo]) -> None:
204 names = [modinfo.name for modinfo in modinfos]
205 names.sort(key=str.lower)
206 # guarantee zip() doesn't drop anything
207 while len(names) % 3:
208 names.append("")
209 for l, m, r in zip(names[::3], names[1::3], names[2::3]): # noqa: E741
210 print("%-*s %-*s %-*s" % (longest, l, longest, m, longest, r))
211
212 if verbose and self.builtin_ok:
213 print("The following *built-in* modules have been successfully built:")
214 print_three_column(self.builtin_ok)
215 print()
216
217 if verbose and self.shared_ok:
218 print("The following *shared* modules have been successfully built:")
219 print_three_column(self.shared_ok)
220 print()
221
222 if self.disabled_configure:
223 print("The following modules are *disabled* in configure script:")
224 print_three_column(self.disabled_configure)
225 print()
226
227 if self.disabled_setup:
228 print("The following modules are *disabled* in Modules/Setup files:")
229 print_three_column(self.disabled_setup)
230 print()
231
232 if verbose and self.notavailable:
233 print(
234 f"The following modules are not available on platform '{self.platform}':"
235 )
236 print_three_column(self.notavailable)
237 print()
238
239 if self.missing:
240 print("The necessary bits to build these optional modules were not found:")
241 print_three_column(self.missing)
242 print("To find the necessary bits, look in configure.ac and config.log.")
243 print()
244
245 if self.failed_on_import:
246 print(
247 "Following modules built successfully "
248 "but were removed because they could not be imported:"
249 )
250 print_three_column(self.failed_on_import)
251 print()
252
253 if any(
254 modinfo.name == "_ssl" for modinfo in self.missing + self.failed_on_import
255 ):
256 print("Could not build the ssl module!")
257 print("Python requires a OpenSSL 1.1.1 or newer")
258 if sysconfig.get_config_var("OPENSSL_LDFLAGS"):
259 print("Custom linker flags may require --with-openssl-rpath=auto")
260 print()
261
262 disabled = len(self.disabled_configure) + len(self.disabled_setup)
263 print(
264 f"Checked {len(self.modules)} modules ("
265 f"{len(self.builtin_ok)} built-in, "
266 f"{len(self.shared_ok)} shared, "
267 f"{len(self.notavailable)} n/a on {self.platform}, "
268 f"{disabled} disabled, "
269 f"{len(self.missing)} missing, "
270 f"{len(self.failed_on_import)} failed on import)"
271 )
272
273 def check_strict_build(self) -> None:
274 """Fail if modules are missing and it's a strict build"""
275 if self.strict_extensions_build and (self.failed_on_import or self.missing):
276 raise RuntimeError("Failed to build some stdlib modules")
277
278 def list_module_names(self, *, all: bool = False) -> set[str]:
279 names = {modinfo.name for modinfo in self.modules}
280 if all:
281 names.update(WINDOWS_MODULES)
282 return names
283
284 def get_builddir(self) -> pathlib.Path:
285 try:
286 with open(self.pybuilddir_txt, encoding="utf-8") as f:
287 builddir = f.read()
288 except FileNotFoundError:
289 logger.error("%s must be run from the top build directory", __file__)
290 raise
291 builddir_path = pathlib.Path(builddir)
292 logger.debug("%s: %s", self.pybuilddir_txt, builddir_path)
293 return builddir_path
294
295 def get_modules(self) -> list[ModuleInfo]:
296 """Get module info from sysconfig and Modules/Setup* files"""
297 seen = set()
298 modules = []
299 # parsing order is important, first entry wins
300 for modinfo in self.get_core_modules():
301 modules.append(modinfo)
302 seen.add(modinfo.name)
303 for setup_file in self.setup_files:
304 for modinfo in self.parse_setup_file(setup_file):
305 if modinfo.name not in seen:
306 modules.append(modinfo)
307 seen.add(modinfo.name)
308 for modinfo in self.get_sysconfig_modules():
309 if modinfo.name not in seen:
310 modules.append(modinfo)
311 seen.add(modinfo.name)
312 logger.debug("Found %i modules in total", len(modules))
313 modules.sort()
314 return modules
315
316 def get_core_modules(self) -> Iterable[ModuleInfo]:
317 """Get hard-coded core modules"""
318 for name in CORE_MODULES:
319 modinfo = ModuleInfo(name, ModuleState.BUILTIN)
320 logger.debug("Found core module %s", modinfo)
321 yield modinfo
322
323 def get_sysconfig_modules(self) -> Iterable[ModuleInfo]:
324 """Get modules defined in Makefile through sysconfig
325
326 MODBUILT_NAMES: modules in *static* block
327 MODSHARED_NAMES: modules in *shared* block
328 MODDISABLED_NAMES: modules in *disabled* block
329 """
330 moddisabled = set(sysconfig.get_config_var("MODDISABLED_NAMES").split())
331 if self.cross_compiling:
332 modbuiltin = set(sysconfig.get_config_var("MODBUILT_NAMES").split())
333 else:
334 modbuiltin = set(sys.builtin_module_names)
335
336 for key, value in sysconfig.get_config_vars().items():
337 if not key.startswith("MODULE_") or not key.endswith("_STATE"):
338 continue
339 if value not in {"yes", "disabled", "missing", "n/a"}:
340 raise ValueError(f"Unsupported value '{value}' for {key}")
341
342 modname = key[7:-6].lower()
343 if modname in moddisabled:
344 # Setup "*disabled*" rule
345 state = ModuleState.DISABLED_SETUP
346 elif value in {"disabled", "missing", "n/a"}:
347 state = ModuleState(value)
348 elif modname in modbuiltin:
349 assert value == "yes"
350 state = ModuleState.BUILTIN
351 else:
352 assert value == "yes"
353 state = ModuleState.SHARED
354
355 modinfo = ModuleInfo(modname, state)
356 logger.debug("Found %s in Makefile", modinfo)
357 yield modinfo
358
359 def parse_setup_file(self, setup_file: pathlib.Path) -> Iterable[ModuleInfo]:
360 """Parse a Modules/Setup file"""
361 assign_var = re.compile(r"^\w+=") # EGG_SPAM=foo
362 # default to static module
363 state = ModuleState.BUILTIN
364 logger.debug("Parsing Setup file %s", setup_file)
365 with open(setup_file, encoding="utf-8") as f:
366 for line in f:
367 line = line.strip()
368 if not line or line.startswith("#") or assign_var.match(line):
369 continue
370 match line.split():
371 case ["*shared*"]:
372 state = ModuleState.SHARED
373 case ["*static*"]:
374 state = ModuleState.BUILTIN
375 case ["*disabled*"]:
376 state = ModuleState.DISABLED
377 case ["*noconfig*"]:
378 continue
379 case [*items]:
380 if state == ModuleState.DISABLED:
381 # *disabled* can disable multiple modules per line
382 for item in items:
383 modinfo = ModuleInfo(item, state)
384 logger.debug("Found %s in %s", modinfo, setup_file)
385 yield modinfo
386 elif state in {ModuleState.SHARED, ModuleState.BUILTIN}:
387 # *shared* and *static*, first item is the name of the module.
388 modinfo = ModuleInfo(items[0], state)
389 logger.debug("Found %s in %s", modinfo, setup_file)
390 yield modinfo
391
392 def get_spec(self, modinfo: ModuleInfo) -> ModuleSpec:
393 """Get ModuleSpec for builtin or extension module"""
394 if modinfo.state == ModuleState.SHARED:
395 mod_location = self.get_location(modinfo)
396 assert mod_location is not None
397 location = os.fspath(mod_location)
398 loader = ExtensionFileLoader(modinfo.name, location)
399 spec = spec_from_file_location(modinfo.name, location, loader=loader)
400 assert spec is not None
401 return spec
402 elif modinfo.state == ModuleState.BUILTIN:
403 spec = spec_from_loader(modinfo.name, loader=BuiltinImporter)
404 assert spec is not None
405 return spec
406 else:
407 raise ValueError(modinfo)
408
409 def get_location(self, modinfo: ModuleInfo) -> pathlib.Path | None:
410 """Get shared library location in build directory"""
411 if modinfo.state == ModuleState.SHARED:
412 return self.builddir / f"{modinfo.name}{self.ext_suffix}"
413 else:
414 return None
415
416 def _check_file(self, modinfo: ModuleInfo, spec: ModuleSpec) -> None:
417 """Check that the module file is present and not empty"""
418 if spec.loader is BuiltinImporter: # type: ignore[comparison-overlap]
419 return
420 try:
421 assert spec.origin is not None
422 st = os.stat(spec.origin)
423 except FileNotFoundError:
424 logger.error("%s (%s) is missing", modinfo.name, spec.origin)
425 raise
426 if not st.st_size:
427 raise ImportError(f"{spec.origin} is an empty file")
428
429 def check_module_import(self, modinfo: ModuleInfo) -> None:
430 """Attempt to import module and report errors"""
431 spec = self.get_spec(modinfo)
432 self._check_file(modinfo, spec)
433 try:
434 with warnings.catch_warnings():
435 # ignore deprecation warning from deprecated modules
436 warnings.simplefilter("ignore", DeprecationWarning)
437 bootstrap_load(spec)
438 except ImportError as e:
439 logger.error("%s failed to import: %s", modinfo.name, e)
440 raise
441 except Exception:
442 if not hasattr(_imp, 'create_dynamic'):
443 logger.warning("Dynamic extension '%s' ignored", modinfo.name)
444 return
445 logger.exception("Importing extension '%s' failed!", modinfo.name)
446 raise
447
448 def check_module_cross(self, modinfo: ModuleInfo) -> None:
449 """Sanity check for cross compiling"""
450 spec = self.get_spec(modinfo)
451 self._check_file(modinfo, spec)
452
453 def rename_module(self, modinfo: ModuleInfo) -> None:
454 """Rename module file"""
455 if modinfo.state == ModuleState.BUILTIN:
456 logger.error("Cannot mark builtin module '%s' as failed!", modinfo.name)
457 return
458
459 failed_name = f"{modinfo.name}_failed{self.ext_suffix}"
460 builddir_path = self.get_location(modinfo)
461 assert builddir_path is not None
462 if builddir_path.is_symlink():
463 symlink = builddir_path
464 module_path = builddir_path.resolve().relative_to(os.getcwd())
465 failed_path = module_path.parent / failed_name
466 else:
467 symlink = None
468 module_path = builddir_path
469 failed_path = self.builddir / failed_name
470
471 # remove old failed file
472 failed_path.unlink(missing_ok=True)
473 # remove symlink
474 if symlink is not None:
475 symlink.unlink(missing_ok=True)
476 # rename shared extension file
477 try:
478 module_path.rename(failed_path)
479 except FileNotFoundError:
480 logger.debug("Shared extension file '%s' does not exist.", module_path)
481 else:
482 logger.debug("Rename '%s' -> '%s'", module_path, failed_path)
483
484
485 def main() -> None:
486 args = parser.parse_args()
487 if args.debug:
488 args.verbose = True
489 logging.basicConfig(
490 level=logging.DEBUG if args.debug else logging.INFO,
491 format="[%(levelname)s] %(message)s",
492 )
493
494 checker = ModuleChecker(
495 cross_compiling=args.cross_compiling,
496 strict=args.strict,
497 )
498 if args.list_module_names:
499 names = checker.list_module_names(all=True)
500 for name in sorted(names):
501 print(name)
502 else:
503 checker.check()
504 checker.summary(verbose=args.verbose)
505 try:
506 checker.check_strict_build()
507 except RuntimeError as e:
508 parser.exit(1, f"\nError: {e}\n")
509
510
511 if __name__ == "__main__":
512 main()