| 1 |
# This script lists the names of standard library modules |
| 2 |
# to update Python/stdlib_module_names.h |
| 3 |
from __future__ import annotations |
| 4 |
|
| 5 |
import _imp |
| 6 |
import os.path |
| 7 |
import sys |
| 8 |
import sysconfig |
| 9 |
from typing import TextIO |
| 10 |
|
| 11 |
from check_extension_modules import ModuleChecker |
| 12 |
|
| 13 |
SCRIPT_NAME = 'Tools/build/generate_stdlib_module_names.py' |
| 14 |
|
| 15 |
SRC_DIR = os.path.dirname(os.path.dirname(os.path.dirname(__file__))) |
| 16 |
STDLIB_PATH = os.path.join(SRC_DIR, 'Lib') |
| 17 |
|
| 18 |
IGNORE = { |
| 19 |
'__init__', |
| 20 |
'__pycache__', |
| 21 |
'site-packages', |
| 22 |
|
| 23 |
# Test modules and packages |
| 24 |
'__hello__', |
| 25 |
'__phello__', |
| 26 |
'__hello_alias__', |
| 27 |
'__phello_alias__', |
| 28 |
'__hello_only__', |
| 29 |
'_ctypes_test', |
| 30 |
'_testbuffer', |
| 31 |
'_testcapi', |
| 32 |
'_testclinic', |
| 33 |
'_testclinic_limited', |
| 34 |
'_testconsole', |
| 35 |
'_testimportmultiple', |
| 36 |
'_testinternalcapi', |
| 37 |
'_testlimitedcapi', |
| 38 |
'_testmultiphase', |
| 39 |
'_testsinglephase', |
| 40 |
'_xxtestfuzz', |
| 41 |
'idlelib.idle_test', |
| 42 |
'test', |
| 43 |
'xxlimited', |
| 44 |
'xxlimited_35', |
| 45 |
'xxsubtype', |
| 46 |
} |
| 47 |
|
| 48 |
ALLOW_TEST_MODULES = { |
| 49 |
'doctest', |
| 50 |
'unittest', |
| 51 |
} |
| 52 |
|
| 53 |
# Built-in modules |
| 54 |
def list_builtin_modules(names: set[str]) -> None: |
| 55 |
names |= set(sys.builtin_module_names) |
| 56 |
|
| 57 |
|
| 58 |
# Pure Python modules (Lib/*.py) |
| 59 |
def list_python_modules(names: set[str]) -> None: |
| 60 |
for filename in os.listdir(STDLIB_PATH): |
| 61 |
if not filename.endswith(".py"): |
| 62 |
continue |
| 63 |
name = filename.removesuffix(".py") |
| 64 |
names.add(name) |
| 65 |
|
| 66 |
|
| 67 |
# Packages in Lib/ |
| 68 |
def list_packages(names: set[str]) -> None: |
| 69 |
for name in os.listdir(STDLIB_PATH): |
| 70 |
if name in IGNORE: |
| 71 |
continue |
| 72 |
package_path = os.path.join(STDLIB_PATH, name) |
| 73 |
if not os.path.isdir(package_path): |
| 74 |
continue |
| 75 |
if any(package_file.endswith(".py") |
| 76 |
for package_file in os.listdir(package_path)): |
| 77 |
names.add(name) |
| 78 |
|
| 79 |
|
| 80 |
# Built-in and extension modules built by Modules/Setup* |
| 81 |
# includes Windows and macOS extensions. |
| 82 |
def list_modules_setup_extensions(names: set[str]) -> None: |
| 83 |
checker = ModuleChecker() |
| 84 |
names.update(checker.list_module_names(all=True)) |
| 85 |
|
| 86 |
|
| 87 |
# List frozen modules of the PyImport_FrozenModules list (Python/frozen.c). |
| 88 |
# Use the "./Programs/_testembed list_frozen" command. |
| 89 |
def list_frozen(names: set[str]) -> None: |
| 90 |
submodules = set() |
| 91 |
for name in _imp._frozen_module_names(): # type: ignore[attr-defined] |
| 92 |
# To skip __hello__, __hello_alias__ and etc. |
| 93 |
if name.startswith('__'): |
| 94 |
continue |
| 95 |
if '.' in name: |
| 96 |
submodules.add(name) |
| 97 |
else: |
| 98 |
names.add(name) |
| 99 |
# Make sure all frozen submodules have a known parent. |
| 100 |
for name in list(submodules): |
| 101 |
if name.partition('.')[0] in names: |
| 102 |
submodules.remove(name) |
| 103 |
if submodules: |
| 104 |
raise Exception(f'unexpected frozen submodules: {sorted(submodules)}') |
| 105 |
|
| 106 |
|
| 107 |
def list_modules() -> set[str]: |
| 108 |
names: set[str] = set() |
| 109 |
|
| 110 |
list_builtin_modules(names) |
| 111 |
list_modules_setup_extensions(names) |
| 112 |
list_packages(names) |
| 113 |
list_python_modules(names) |
| 114 |
list_frozen(names) |
| 115 |
|
| 116 |
# Remove ignored packages and modules |
| 117 |
for name in list(names): |
| 118 |
package_name = name.split('.')[0] |
| 119 |
# package_name can be equal to name |
| 120 |
if package_name in IGNORE: |
| 121 |
names.discard(name) |
| 122 |
|
| 123 |
# Sanity checks |
| 124 |
for name in names: |
| 125 |
if "." in name: |
| 126 |
raise Exception(f"sub-modules must not be listed: {name}") |
| 127 |
if ("test" in name or "xx" in name) and name not in ALLOW_TEST_MODULES: |
| 128 |
raise Exception(f"test modules must not be listed: {name}") |
| 129 |
|
| 130 |
return names |
| 131 |
|
| 132 |
|
| 133 |
def write_modules(fp: TextIO, names: set[str]) -> None: |
| 134 |
print(f"// Auto-generated by {SCRIPT_NAME}.", |
| 135 |
file=fp) |
| 136 |
print("// List used to create sys.stdlib_module_names.", file=fp) |
| 137 |
print(file=fp) |
| 138 |
print("static const char* _Py_stdlib_module_names[] = {", file=fp) |
| 139 |
for name in sorted(names): |
| 140 |
print(f'"{name}",', file=fp) |
| 141 |
print("};", file=fp) |
| 142 |
|
| 143 |
|
| 144 |
def main() -> None: |
| 145 |
if not sysconfig.is_python_build(): |
| 146 |
print(f"ERROR: {sys.executable} is not a Python build", |
| 147 |
file=sys.stderr) |
| 148 |
sys.exit(1) |
| 149 |
|
| 150 |
fp = sys.stdout |
| 151 |
names = list_modules() |
| 152 |
write_modules(fp, names) |
| 153 |
|
| 154 |
|
| 155 |
if __name__ == "__main__": |
| 156 |
main() |