| 1 |
"""Check extension modules |
| 2 |
|
| 3 |
The script checks shared and built-in extension modules. It verifies that the |
| 4 |
modules have been built and that they can be imported successfully. Missing |
| 5 |
modules and failed imports are reported to the user. Shared extension |
| 6 |
files are renamed on failed import. |
| 7 |
|
| 8 |
Module information is parsed from several sources: |
| 9 |
|
| 10 |
- core modules hard-coded in Modules/config.c.in |
| 11 |
- Windows-specific modules that are hard-coded in PC/config.c |
| 12 |
- MODULE_{name}_STATE entries in Makefile (provided through sysconfig) |
| 13 |
- Various makesetup files: |
| 14 |
- $(srcdir)/Modules/Setup |
| 15 |
- Modules/Setup.[local|bootstrap|stdlib] files, which are generated |
| 16 |
from $(srcdir)/Modules/Setup.*.in files |
| 17 |
|
| 18 |
See --help for more information |
| 19 |
""" |
| 20 |
|
| 21 |
from __future__ import annotations |
| 22 |
|
| 23 |
import _imp |
| 24 |
import argparse |
| 25 |
import enum |
| 26 |
import logging |
| 27 |
import os |
| 28 |
import pathlib |
| 29 |
import re |
| 30 |
import sys |
| 31 |
import sysconfig |
| 32 |
import warnings |
| 33 |
from collections.abc import Iterable |
| 34 |
from importlib._bootstrap import ( # type: ignore[attr-defined] |
| 35 |
_load as bootstrap_load, |
| 36 |
) |
| 37 |
from importlib.machinery import ( |
| 38 |
BuiltinImporter, |
| 39 |
ExtensionFileLoader, |
| 40 |
ModuleSpec, |
| 41 |
) |
| 42 |
from importlib.util import spec_from_file_location, spec_from_loader |
| 43 |
from typing import NamedTuple |
| 44 |
|
| 45 |
SRC_DIR = pathlib.Path(__file__).parent.parent.parent |
| 46 |
|
| 47 |
# core modules, hard-coded in Modules/config.h.in |
| 48 |
CORE_MODULES = { |
| 49 |
"_ast", |
| 50 |
"_imp", |
| 51 |
"_string", |
| 52 |
"_tokenize", |
| 53 |
"_warnings", |
| 54 |
"builtins", |
| 55 |
"gc", |
| 56 |
"marshal", |
| 57 |
"sys", |
| 58 |
} |
| 59 |
|
| 60 |
# Windows-only modules |
| 61 |
WINDOWS_MODULES = { |
| 62 |
"_overlapped", |
| 63 |
"_testconsole", |
| 64 |
"_winapi", |
| 65 |
"_wmi", |
| 66 |
"msvcrt", |
| 67 |
"nt", |
| 68 |
"winreg", |
| 69 |
"winsound", |
| 70 |
} |
| 71 |
|
| 72 |
|
| 73 |
logger = logging.getLogger(__name__) |
| 74 |
|
| 75 |
parser = argparse.ArgumentParser( |
| 76 |
prog="check_extension_modules", |
| 77 |
description=__doc__, |
| 78 |
formatter_class=argparse.RawDescriptionHelpFormatter, |
| 79 |
) |
| 80 |
|
| 81 |
parser.add_argument( |
| 82 |
"--verbose", |
| 83 |
action="store_true", |
| 84 |
help="Verbose, report builtin, shared, and unavailable modules", |
| 85 |
) |
| 86 |
|
| 87 |
parser.add_argument( |
| 88 |
"--debug", |
| 89 |
action="store_true", |
| 90 |
help="Enable debug logging", |
| 91 |
) |
| 92 |
|
| 93 |
parser.add_argument( |
| 94 |
"--strict", |
| 95 |
action=argparse.BooleanOptionalAction, |
| 96 |
help=( |
| 97 |
"Strict check, fail when a module is missing or fails to import" |
| 98 |
"(default: no, unless env var PYTHONSTRICTEXTENSIONBUILD is set)" |
| 99 |
), |
| 100 |
default=bool(os.environ.get("PYTHONSTRICTEXTENSIONBUILD")), |
| 101 |
) |
| 102 |
|
| 103 |
parser.add_argument( |
| 104 |
"--cross-compiling", |
| 105 |
action=argparse.BooleanOptionalAction, |
| 106 |
help=( |
| 107 |
"Use cross-compiling checks " |
| 108 |
"(default: no, unless env var _PYTHON_HOST_PLATFORM is set)." |
| 109 |
), |
| 110 |
default="_PYTHON_HOST_PLATFORM" in os.environ, |
| 111 |
) |
| 112 |
|
| 113 |
parser.add_argument( |
| 114 |
"--list-module-names", |
| 115 |
action="store_true", |
| 116 |
help="Print a list of module names to stdout and exit", |
| 117 |
) |
| 118 |
|
| 119 |
|
| 120 |
@enum.unique |
| 121 |
class ModuleState(enum.Enum): |
| 122 |
# Makefile state "yes" |
| 123 |
BUILTIN = "builtin" |
| 124 |
SHARED = "shared" |
| 125 |
|
| 126 |
DISABLED = "disabled" |
| 127 |
MISSING = "missing" |
| 128 |
NA = "n/a" |
| 129 |
# disabled by Setup / makesetup rule |
| 130 |
DISABLED_SETUP = "disabled_setup" |
| 131 |
|
| 132 |
def __bool__(self) -> bool: |
| 133 |
return self.value in {"builtin", "shared"} |
| 134 |
|
| 135 |
|
| 136 |
class ModuleInfo(NamedTuple): |
| 137 |
name: str |
| 138 |
state: ModuleState |
| 139 |
|
| 140 |
|
| 141 |
class ModuleChecker: |
| 142 |
pybuilddir_txt = "pybuilddir.txt" |
| 143 |
|
| 144 |
setup_files = ( |
| 145 |
# see end of configure.ac |
| 146 |
pathlib.Path("Modules/Setup.local"), |
| 147 |
pathlib.Path("Modules/Setup.stdlib"), |
| 148 |
pathlib.Path("Modules/Setup.bootstrap"), |
| 149 |
SRC_DIR / "Modules/Setup", |
| 150 |
) |
| 151 |
|
| 152 |
def __init__(self, cross_compiling: bool = False, strict: bool = False): |
| 153 |
self.cross_compiling = cross_compiling |
| 154 |
self.strict_extensions_build = strict |
| 155 |
self.ext_suffix = sysconfig.get_config_var("EXT_SUFFIX") |
| 156 |
self.platform = sysconfig.get_platform() |
| 157 |
self.builddir = self.get_builddir() |
| 158 |
self.modules = self.get_modules() |
| 159 |
|
| 160 |
self.builtin_ok: list[ModuleInfo] = [] |
| 161 |
self.shared_ok: list[ModuleInfo] = [] |
| 162 |
self.failed_on_import: list[ModuleInfo] = [] |
| 163 |
self.missing: list[ModuleInfo] = [] |
| 164 |
self.disabled_configure: list[ModuleInfo] = [] |
| 165 |
self.disabled_setup: list[ModuleInfo] = [] |
| 166 |
self.notavailable: list[ModuleInfo] = [] |
| 167 |
|
| 168 |
def check(self) -> None: |
| 169 |
if not hasattr(_imp, 'create_dynamic'): |
| 170 |
logger.warning( |
| 171 |
('Dynamic extensions not supported ' |
| 172 |
'(HAVE_DYNAMIC_LOADING not defined)'), |
| 173 |
) |
| 174 |
for modinfo in self.modules: |
| 175 |
logger.debug("Checking '%s' (%s)", modinfo.name, self.get_location(modinfo)) |
| 176 |
if modinfo.state == ModuleState.DISABLED: |
| 177 |
self.disabled_configure.append(modinfo) |
| 178 |
elif modinfo.state == ModuleState.DISABLED_SETUP: |
| 179 |
self.disabled_setup.append(modinfo) |
| 180 |
elif modinfo.state == ModuleState.MISSING: |
| 181 |
self.missing.append(modinfo) |
| 182 |
elif modinfo.state == ModuleState.NA: |
| 183 |
self.notavailable.append(modinfo) |
| 184 |
else: |
| 185 |
try: |
| 186 |
if self.cross_compiling: |
| 187 |
self.check_module_cross(modinfo) |
| 188 |
else: |
| 189 |
self.check_module_import(modinfo) |
| 190 |
except (ImportError, FileNotFoundError): |
| 191 |
self.rename_module(modinfo) |
| 192 |
self.failed_on_import.append(modinfo) |
| 193 |
else: |
| 194 |
if modinfo.state == ModuleState.BUILTIN: |
| 195 |
self.builtin_ok.append(modinfo) |
| 196 |
else: |
| 197 |
assert modinfo.state == ModuleState.SHARED |
| 198 |
self.shared_ok.append(modinfo) |
| 199 |
|
| 200 |
def summary(self, *, verbose: bool = False) -> None: |
| 201 |
longest = max([len(e.name) for e in self.modules], default=0) |
| 202 |
|
| 203 |
def print_three_column(modinfos: list[ModuleInfo]) -> None: |
| 204 |
names = [modinfo.name for modinfo in modinfos] |
| 205 |
names.sort(key=str.lower) |
| 206 |
# guarantee zip() doesn't drop anything |
| 207 |
while len(names) % 3: |
| 208 |
names.append("") |
| 209 |
for l, m, r in zip(names[::3], names[1::3], names[2::3]): # noqa: E741 |
| 210 |
print("%-*s %-*s %-*s" % (longest, l, longest, m, longest, r)) |
| 211 |
|
| 212 |
if verbose and self.builtin_ok: |
| 213 |
print("The following *built-in* modules have been successfully built:") |
| 214 |
print_three_column(self.builtin_ok) |
| 215 |
print() |
| 216 |
|
| 217 |
if verbose and self.shared_ok: |
| 218 |
print("The following *shared* modules have been successfully built:") |
| 219 |
print_three_column(self.shared_ok) |
| 220 |
print() |
| 221 |
|
| 222 |
if self.disabled_configure: |
| 223 |
print("The following modules are *disabled* in configure script:") |
| 224 |
print_three_column(self.disabled_configure) |
| 225 |
print() |
| 226 |
|
| 227 |
if self.disabled_setup: |
| 228 |
print("The following modules are *disabled* in Modules/Setup files:") |
| 229 |
print_three_column(self.disabled_setup) |
| 230 |
print() |
| 231 |
|
| 232 |
if verbose and self.notavailable: |
| 233 |
print( |
| 234 |
f"The following modules are not available on platform '{self.platform}':" |
| 235 |
) |
| 236 |
print_three_column(self.notavailable) |
| 237 |
print() |
| 238 |
|
| 239 |
if self.missing: |
| 240 |
print("The necessary bits to build these optional modules were not found:") |
| 241 |
print_three_column(self.missing) |
| 242 |
print("To find the necessary bits, look in configure.ac and config.log.") |
| 243 |
print() |
| 244 |
|
| 245 |
if self.failed_on_import: |
| 246 |
print( |
| 247 |
"Following modules built successfully " |
| 248 |
"but were removed because they could not be imported:" |
| 249 |
) |
| 250 |
print_three_column(self.failed_on_import) |
| 251 |
print() |
| 252 |
|
| 253 |
if any( |
| 254 |
modinfo.name == "_ssl" for modinfo in self.missing + self.failed_on_import |
| 255 |
): |
| 256 |
print("Could not build the ssl module!") |
| 257 |
print("Python requires a OpenSSL 1.1.1 or newer") |
| 258 |
if sysconfig.get_config_var("OPENSSL_LDFLAGS"): |
| 259 |
print("Custom linker flags may require --with-openssl-rpath=auto") |
| 260 |
print() |
| 261 |
|
| 262 |
disabled = len(self.disabled_configure) + len(self.disabled_setup) |
| 263 |
print( |
| 264 |
f"Checked {len(self.modules)} modules (" |
| 265 |
f"{len(self.builtin_ok)} built-in, " |
| 266 |
f"{len(self.shared_ok)} shared, " |
| 267 |
f"{len(self.notavailable)} n/a on {self.platform}, " |
| 268 |
f"{disabled} disabled, " |
| 269 |
f"{len(self.missing)} missing, " |
| 270 |
f"{len(self.failed_on_import)} failed on import)" |
| 271 |
) |
| 272 |
|
| 273 |
def check_strict_build(self) -> None: |
| 274 |
"""Fail if modules are missing and it's a strict build""" |
| 275 |
if self.strict_extensions_build and (self.failed_on_import or self.missing): |
| 276 |
raise RuntimeError("Failed to build some stdlib modules") |
| 277 |
|
| 278 |
def list_module_names(self, *, all: bool = False) -> set[str]: |
| 279 |
names = {modinfo.name for modinfo in self.modules} |
| 280 |
if all: |
| 281 |
names.update(WINDOWS_MODULES) |
| 282 |
return names |
| 283 |
|
| 284 |
def get_builddir(self) -> pathlib.Path: |
| 285 |
try: |
| 286 |
with open(self.pybuilddir_txt, encoding="utf-8") as f: |
| 287 |
builddir = f.read() |
| 288 |
except FileNotFoundError: |
| 289 |
logger.error("%s must be run from the top build directory", __file__) |
| 290 |
raise |
| 291 |
builddir_path = pathlib.Path(builddir) |
| 292 |
logger.debug("%s: %s", self.pybuilddir_txt, builddir_path) |
| 293 |
return builddir_path |
| 294 |
|
| 295 |
def get_modules(self) -> list[ModuleInfo]: |
| 296 |
"""Get module info from sysconfig and Modules/Setup* files""" |
| 297 |
seen = set() |
| 298 |
modules = [] |
| 299 |
# parsing order is important, first entry wins |
| 300 |
for modinfo in self.get_core_modules(): |
| 301 |
modules.append(modinfo) |
| 302 |
seen.add(modinfo.name) |
| 303 |
for setup_file in self.setup_files: |
| 304 |
for modinfo in self.parse_setup_file(setup_file): |
| 305 |
if modinfo.name not in seen: |
| 306 |
modules.append(modinfo) |
| 307 |
seen.add(modinfo.name) |
| 308 |
for modinfo in self.get_sysconfig_modules(): |
| 309 |
if modinfo.name not in seen: |
| 310 |
modules.append(modinfo) |
| 311 |
seen.add(modinfo.name) |
| 312 |
logger.debug("Found %i modules in total", len(modules)) |
| 313 |
modules.sort() |
| 314 |
return modules |
| 315 |
|
| 316 |
def get_core_modules(self) -> Iterable[ModuleInfo]: |
| 317 |
"""Get hard-coded core modules""" |
| 318 |
for name in CORE_MODULES: |
| 319 |
modinfo = ModuleInfo(name, ModuleState.BUILTIN) |
| 320 |
logger.debug("Found core module %s", modinfo) |
| 321 |
yield modinfo |
| 322 |
|
| 323 |
def get_sysconfig_modules(self) -> Iterable[ModuleInfo]: |
| 324 |
"""Get modules defined in Makefile through sysconfig |
| 325 |
|
| 326 |
MODBUILT_NAMES: modules in *static* block |
| 327 |
MODSHARED_NAMES: modules in *shared* block |
| 328 |
MODDISABLED_NAMES: modules in *disabled* block |
| 329 |
""" |
| 330 |
moddisabled = set(sysconfig.get_config_var("MODDISABLED_NAMES").split()) |
| 331 |
if self.cross_compiling: |
| 332 |
modbuiltin = set(sysconfig.get_config_var("MODBUILT_NAMES").split()) |
| 333 |
else: |
| 334 |
modbuiltin = set(sys.builtin_module_names) |
| 335 |
|
| 336 |
for key, value in sysconfig.get_config_vars().items(): |
| 337 |
if not key.startswith("MODULE_") or not key.endswith("_STATE"): |
| 338 |
continue |
| 339 |
if value not in {"yes", "disabled", "missing", "n/a"}: |
| 340 |
raise ValueError(f"Unsupported value '{value}' for {key}") |
| 341 |
|
| 342 |
modname = key[7:-6].lower() |
| 343 |
if modname in moddisabled: |
| 344 |
# Setup "*disabled*" rule |
| 345 |
state = ModuleState.DISABLED_SETUP |
| 346 |
elif value in {"disabled", "missing", "n/a"}: |
| 347 |
state = ModuleState(value) |
| 348 |
elif modname in modbuiltin: |
| 349 |
assert value == "yes" |
| 350 |
state = ModuleState.BUILTIN |
| 351 |
else: |
| 352 |
assert value == "yes" |
| 353 |
state = ModuleState.SHARED |
| 354 |
|
| 355 |
modinfo = ModuleInfo(modname, state) |
| 356 |
logger.debug("Found %s in Makefile", modinfo) |
| 357 |
yield modinfo |
| 358 |
|
| 359 |
def parse_setup_file(self, setup_file: pathlib.Path) -> Iterable[ModuleInfo]: |
| 360 |
"""Parse a Modules/Setup file""" |
| 361 |
assign_var = re.compile(r"^\w+=") # EGG_SPAM=foo |
| 362 |
# default to static module |
| 363 |
state = ModuleState.BUILTIN |
| 364 |
logger.debug("Parsing Setup file %s", setup_file) |
| 365 |
with open(setup_file, encoding="utf-8") as f: |
| 366 |
for line in f: |
| 367 |
line = line.strip() |
| 368 |
if not line or line.startswith("#") or assign_var.match(line): |
| 369 |
continue |
| 370 |
match line.split(): |
| 371 |
case ["*shared*"]: |
| 372 |
state = ModuleState.SHARED |
| 373 |
case ["*static*"]: |
| 374 |
state = ModuleState.BUILTIN |
| 375 |
case ["*disabled*"]: |
| 376 |
state = ModuleState.DISABLED |
| 377 |
case ["*noconfig*"]: |
| 378 |
continue |
| 379 |
case [*items]: |
| 380 |
if state == ModuleState.DISABLED: |
| 381 |
# *disabled* can disable multiple modules per line |
| 382 |
for item in items: |
| 383 |
modinfo = ModuleInfo(item, state) |
| 384 |
logger.debug("Found %s in %s", modinfo, setup_file) |
| 385 |
yield modinfo |
| 386 |
elif state in {ModuleState.SHARED, ModuleState.BUILTIN}: |
| 387 |
# *shared* and *static*, first item is the name of the module. |
| 388 |
modinfo = ModuleInfo(items[0], state) |
| 389 |
logger.debug("Found %s in %s", modinfo, setup_file) |
| 390 |
yield modinfo |
| 391 |
|
| 392 |
def get_spec(self, modinfo: ModuleInfo) -> ModuleSpec: |
| 393 |
"""Get ModuleSpec for builtin or extension module""" |
| 394 |
if modinfo.state == ModuleState.SHARED: |
| 395 |
mod_location = self.get_location(modinfo) |
| 396 |
assert mod_location is not None |
| 397 |
location = os.fspath(mod_location) |
| 398 |
loader = ExtensionFileLoader(modinfo.name, location) |
| 399 |
spec = spec_from_file_location(modinfo.name, location, loader=loader) |
| 400 |
assert spec is not None |
| 401 |
return spec |
| 402 |
elif modinfo.state == ModuleState.BUILTIN: |
| 403 |
spec = spec_from_loader(modinfo.name, loader=BuiltinImporter) |
| 404 |
assert spec is not None |
| 405 |
return spec |
| 406 |
else: |
| 407 |
raise ValueError(modinfo) |
| 408 |
|
| 409 |
def get_location(self, modinfo: ModuleInfo) -> pathlib.Path | None: |
| 410 |
"""Get shared library location in build directory""" |
| 411 |
if modinfo.state == ModuleState.SHARED: |
| 412 |
return self.builddir / f"{modinfo.name}{self.ext_suffix}" |
| 413 |
else: |
| 414 |
return None |
| 415 |
|
| 416 |
def _check_file(self, modinfo: ModuleInfo, spec: ModuleSpec) -> None: |
| 417 |
"""Check that the module file is present and not empty""" |
| 418 |
if spec.loader is BuiltinImporter: # type: ignore[comparison-overlap] |
| 419 |
return |
| 420 |
try: |
| 421 |
assert spec.origin is not None |
| 422 |
st = os.stat(spec.origin) |
| 423 |
except FileNotFoundError: |
| 424 |
logger.error("%s (%s) is missing", modinfo.name, spec.origin) |
| 425 |
raise |
| 426 |
if not st.st_size: |
| 427 |
raise ImportError(f"{spec.origin} is an empty file") |
| 428 |
|
| 429 |
def check_module_import(self, modinfo: ModuleInfo) -> None: |
| 430 |
"""Attempt to import module and report errors""" |
| 431 |
spec = self.get_spec(modinfo) |
| 432 |
self._check_file(modinfo, spec) |
| 433 |
try: |
| 434 |
with warnings.catch_warnings(): |
| 435 |
# ignore deprecation warning from deprecated modules |
| 436 |
warnings.simplefilter("ignore", DeprecationWarning) |
| 437 |
bootstrap_load(spec) |
| 438 |
except ImportError as e: |
| 439 |
logger.error("%s failed to import: %s", modinfo.name, e) |
| 440 |
raise |
| 441 |
except Exception: |
| 442 |
if not hasattr(_imp, 'create_dynamic'): |
| 443 |
logger.warning("Dynamic extension '%s' ignored", modinfo.name) |
| 444 |
return |
| 445 |
logger.exception("Importing extension '%s' failed!", modinfo.name) |
| 446 |
raise |
| 447 |
|
| 448 |
def check_module_cross(self, modinfo: ModuleInfo) -> None: |
| 449 |
"""Sanity check for cross compiling""" |
| 450 |
spec = self.get_spec(modinfo) |
| 451 |
self._check_file(modinfo, spec) |
| 452 |
|
| 453 |
def rename_module(self, modinfo: ModuleInfo) -> None: |
| 454 |
"""Rename module file""" |
| 455 |
if modinfo.state == ModuleState.BUILTIN: |
| 456 |
logger.error("Cannot mark builtin module '%s' as failed!", modinfo.name) |
| 457 |
return |
| 458 |
|
| 459 |
failed_name = f"{modinfo.name}_failed{self.ext_suffix}" |
| 460 |
builddir_path = self.get_location(modinfo) |
| 461 |
assert builddir_path is not None |
| 462 |
if builddir_path.is_symlink(): |
| 463 |
symlink = builddir_path |
| 464 |
module_path = builddir_path.resolve().relative_to(os.getcwd()) |
| 465 |
failed_path = module_path.parent / failed_name |
| 466 |
else: |
| 467 |
symlink = None |
| 468 |
module_path = builddir_path |
| 469 |
failed_path = self.builddir / failed_name |
| 470 |
|
| 471 |
# remove old failed file |
| 472 |
failed_path.unlink(missing_ok=True) |
| 473 |
# remove symlink |
| 474 |
if symlink is not None: |
| 475 |
symlink.unlink(missing_ok=True) |
| 476 |
# rename shared extension file |
| 477 |
try: |
| 478 |
module_path.rename(failed_path) |
| 479 |
except FileNotFoundError: |
| 480 |
logger.debug("Shared extension file '%s' does not exist.", module_path) |
| 481 |
else: |
| 482 |
logger.debug("Rename '%s' -> '%s'", module_path, failed_path) |
| 483 |
|
| 484 |
|
| 485 |
def main() -> None: |
| 486 |
args = parser.parse_args() |
| 487 |
if args.debug: |
| 488 |
args.verbose = True |
| 489 |
logging.basicConfig( |
| 490 |
level=logging.DEBUG if args.debug else logging.INFO, |
| 491 |
format="[%(levelname)s] %(message)s", |
| 492 |
) |
| 493 |
|
| 494 |
checker = ModuleChecker( |
| 495 |
cross_compiling=args.cross_compiling, |
| 496 |
strict=args.strict, |
| 497 |
) |
| 498 |
if args.list_module_names: |
| 499 |
names = checker.list_module_names(all=True) |
| 500 |
for name in sorted(names): |
| 501 |
print(name) |
| 502 |
else: |
| 503 |
checker.check() |
| 504 |
checker.summary(verbose=args.verbose) |
| 505 |
try: |
| 506 |
checker.check_strict_build() |
| 507 |
except RuntimeError as e: |
| 508 |
parser.exit(1, f"\nError: {e}\n") |
| 509 |
|
| 510 |
|
| 511 |
if __name__ == "__main__": |
| 512 |
main() |