Debian Codegraph @

Tools/build/generate_stdlib_module_names.py

raw
1 # This script lists the names of standard library modules
2 # to update Python/stdlib_module_names.h
3 from __future__ import annotations
4
5 import _imp
6 import os.path
7 import sys
8 import sysconfig
9 from typing import TextIO
10
11 from check_extension_modules import ModuleChecker
12
13 SCRIPT_NAME = 'Tools/build/generate_stdlib_module_names.py'
14
15 SRC_DIR = os.path.dirname(os.path.dirname(os.path.dirname(__file__)))
16 STDLIB_PATH = os.path.join(SRC_DIR, 'Lib')
17
18 IGNORE = {
19 '__init__',
20 '__pycache__',
21 'site-packages',
22
23 # Test modules and packages
24 '__hello__',
25 '__phello__',
26 '__hello_alias__',
27 '__phello_alias__',
28 '__hello_only__',
29 '_ctypes_test',
30 '_testbuffer',
31 '_testcapi',
32 '_testclinic',
33 '_testclinic_limited',
34 '_testconsole',
35 '_testimportmultiple',
36 '_testinternalcapi',
37 '_testlimitedcapi',
38 '_testmultiphase',
39 '_testsinglephase',
40 '_testexternalinspection',
41 '_xxtestfuzz',
42 'idlelib.idle_test',
43 'test',
44 'xxlimited',
45 'xxlimited_35',
46 'xxsubtype',
47 }
48
49 ALLOW_TEST_MODULES = {
50 'doctest',
51 'unittest',
52 }
53
54 # Built-in modules
55 def list_builtin_modules(names: set[str]) -> None:
56 names |= set(sys.builtin_module_names)
57
58
59 # Pure Python modules (Lib/*.py)
60 def list_python_modules(names: set[str]) -> None:
61 for filename in os.listdir(STDLIB_PATH):
62 if not filename.endswith(".py"):
63 continue
64 name = filename.removesuffix(".py")
65 names.add(name)
66
67
68 # Packages in Lib/
69 def list_packages(names: set[str]) -> None:
70 for name in os.listdir(STDLIB_PATH):
71 if name in IGNORE:
72 continue
73 package_path = os.path.join(STDLIB_PATH, name)
74 if not os.path.isdir(package_path):
75 continue
76 if any(package_file.endswith(".py")
77 for package_file in os.listdir(package_path)):
78 names.add(name)
79
80
81 # Built-in and extension modules built by Modules/Setup*
82 # includes Windows and macOS extensions.
83 def list_modules_setup_extensions(names: set[str]) -> None:
84 checker = ModuleChecker()
85 names.update(checker.list_module_names(all=True))
86
87
88 # List frozen modules of the PyImport_FrozenModules list (Python/frozen.c).
89 # Use the "./Programs/_testembed list_frozen" command.
90 def list_frozen(names: set[str]) -> None:
91 submodules = set()
92 for name in _imp._frozen_module_names(): # type: ignore[attr-defined]
93 # To skip __hello__, __hello_alias__ and etc.
94 if name.startswith('__'):
95 continue
96 if '.' in name:
97 submodules.add(name)
98 else:
99 names.add(name)
100 # Make sure all frozen submodules have a known parent.
101 for name in list(submodules):
102 if name.partition('.')[0] in names:
103 submodules.remove(name)
104 if submodules:
105 raise Exception(f'unexpected frozen submodules: {sorted(submodules)}')
106
107
108 def list_modules() -> set[str]:
109 names: set[str] = set()
110
111 list_builtin_modules(names)
112 list_modules_setup_extensions(names)
113 list_packages(names)
114 list_python_modules(names)
115 list_frozen(names)
116
117 # Remove ignored packages and modules
118 for name in list(names):
119 package_name = name.split('.')[0]
120 # package_name can be equal to name
121 if package_name in IGNORE:
122 names.discard(name)
123
124 # Sanity checks
125 for name in names:
126 if "." in name:
127 raise Exception(f"sub-modules must not be listed: {name}")
128 if ("test" in name or "xx" in name) and name not in ALLOW_TEST_MODULES:
129 raise Exception(f"test modules must not be listed: {name}")
130
131 return names
132
133
134 def write_modules(fp: TextIO, names: set[str]) -> None:
135 print(f"// Auto-generated by {SCRIPT_NAME}.",
136 file=fp)
137 print("// List used to create sys.stdlib_module_names.", file=fp)
138 print(file=fp)
139 print("static const char* _Py_stdlib_module_names[] = {", file=fp)
140 for name in sorted(names):
141 print(f'"{name}",', file=fp)
142 print("};", file=fp)
143
144
145 def main() -> None:
146 if not sysconfig.is_python_build():
147 print(f"ERROR: {sys.executable} is not a Python build",
148 file=sys.stderr)
149 sys.exit(1)
150
151 fp = sys.stdout
152 names = list_modules()
153 write_modules(fp, names)
154
155
156 if __name__ == "__main__":
157 main()