Debian Codegraph @

Tools/build/generate_stdlib_module_names.py

raw
1 # This script lists the names of standard library modules
2 # to update Python/stdlib_module_names.h
3 from __future__ import annotations
4
5 import _imp
6 import os.path
7 import sys
8 import sysconfig
9 from typing import TextIO
10
11 from check_extension_modules import ModuleChecker
12
13 SCRIPT_NAME = 'Tools/build/generate_stdlib_module_names.py'
14
15 SRC_DIR = os.path.dirname(os.path.dirname(os.path.dirname(__file__)))
16 STDLIB_PATH = os.path.join(SRC_DIR, 'Lib')
17
18 IGNORE = {
19 '__init__',
20 '__pycache__',
21 'site-packages',
22
23 # Test modules and packages
24 '__hello__',
25 '__phello__',
26 '__hello_alias__',
27 '__phello_alias__',
28 '__hello_only__',
29 '_ctypes_test',
30 '_testbuffer',
31 '_testcapi',
32 '_testclinic',
33 '_testclinic_limited',
34 '_testconsole',
35 '_testimportmultiple',
36 '_testinternalcapi',
37 '_testlimitedcapi',
38 '_testmultiphase',
39 '_testsinglephase',
40 '_xxtestfuzz',
41 'idlelib.idle_test',
42 'test',
43 'xxlimited',
44 'xxlimited_35',
45 'xxsubtype',
46 }
47
48 ALLOW_TEST_MODULES = {
49 'doctest',
50 'unittest',
51 }
52
53 # Built-in modules
54 def list_builtin_modules(names: set[str]) -> None:
55 names |= set(sys.builtin_module_names)
56
57
58 # Pure Python modules (Lib/*.py)
59 def list_python_modules(names: set[str]) -> None:
60 for filename in os.listdir(STDLIB_PATH):
61 if not filename.endswith(".py"):
62 continue
63 name = filename.removesuffix(".py")
64 names.add(name)
65
66
67 # Packages in Lib/
68 def list_packages(names: set[str]) -> None:
69 for name in os.listdir(STDLIB_PATH):
70 if name in IGNORE:
71 continue
72 package_path = os.path.join(STDLIB_PATH, name)
73 if not os.path.isdir(package_path):
74 continue
75 if any(package_file.endswith(".py")
76 for package_file in os.listdir(package_path)):
77 names.add(name)
78
79
80 # Built-in and extension modules built by Modules/Setup*
81 # includes Windows and macOS extensions.
82 def list_modules_setup_extensions(names: set[str]) -> None:
83 checker = ModuleChecker()
84 names.update(checker.list_module_names(all=True))
85
86
87 # List frozen modules of the PyImport_FrozenModules list (Python/frozen.c).
88 # Use the "./Programs/_testembed list_frozen" command.
89 def list_frozen(names: set[str]) -> None:
90 submodules = set()
91 for name in _imp._frozen_module_names(): # type: ignore[attr-defined]
92 # To skip __hello__, __hello_alias__ and etc.
93 if name.startswith('__'):
94 continue
95 if '.' in name:
96 submodules.add(name)
97 else:
98 names.add(name)
99 # Make sure all frozen submodules have a known parent.
100 for name in list(submodules):
101 if name.partition('.')[0] in names:
102 submodules.remove(name)
103 if submodules:
104 raise Exception(f'unexpected frozen submodules: {sorted(submodules)}')
105
106
107 def list_modules() -> set[str]:
108 names: set[str] = set()
109
110 list_builtin_modules(names)
111 list_modules_setup_extensions(names)
112 list_packages(names)
113 list_python_modules(names)
114 list_frozen(names)
115
116 # Remove ignored packages and modules
117 for name in list(names):
118 package_name = name.split('.')[0]
119 # package_name can be equal to name
120 if package_name in IGNORE:
121 names.discard(name)
122
123 # Sanity checks
124 for name in names:
125 if "." in name:
126 raise Exception(f"sub-modules must not be listed: {name}")
127 if ("test" in name or "xx" in name) and name not in ALLOW_TEST_MODULES:
128 raise Exception(f"test modules must not be listed: {name}")
129
130 return names
131
132
133 def write_modules(fp: TextIO, names: set[str]) -> None:
134 print(f"// Auto-generated by {SCRIPT_NAME}.",
135 file=fp)
136 print("// List used to create sys.stdlib_module_names.", file=fp)
137 print(file=fp)
138 print("static const char* _Py_stdlib_module_names[] = {", file=fp)
139 for name in sorted(names):
140 print(f'"{name}",', file=fp)
141 print("};", file=fp)
142
143
144 def main() -> None:
145 if not sysconfig.is_python_build():
146 print(f"ERROR: {sys.executable} is not a Python build",
147 file=sys.stderr)
148 sys.exit(1)
149
150 fp = sys.stdout
151 names = list_modules()
152 write_modules(fp, names)
153
154
155 if __name__ == "__main__":
156 main()