Debian Codegraph @

Tools/build/generate_global_objects.py

raw
1 import contextlib
2 import io
3 import os.path
4 import re
5
6 SCRIPT_NAME = 'Tools/build/generate_global_objects.py'
7 __file__ = os.path.abspath(__file__)
8 ROOT = os.path.dirname(os.path.dirname(os.path.dirname(__file__)))
9 INTERNAL = os.path.join(ROOT, 'Include', 'internal')
10
11
12 IGNORED = {
13 'ACTION', # Python/_warnings.c
14 'ATTR', # Python/_warnings.c and Objects/funcobject.c
15 'DUNDER', # Objects/typeobject.c
16 'RDUNDER', # Objects/typeobject.c
17 'SPECIAL', # Objects/weakrefobject.c
18 'NAME', # Objects/typeobject.c
19 }
20 IDENTIFIERS = [
21 # from ADD() Python/_warnings.c
22 'default',
23 'ignore',
24
25 # from GET_WARNINGS_ATTR() in Python/_warnings.c
26 'WarningMessage',
27 '_showwarnmsg',
28 '_warn_unawaited_coroutine',
29 'defaultaction',
30 'filters',
31 'onceregistry',
32
33 # from WRAP_METHOD() in Objects/weakrefobject.c
34 '__bytes__',
35 '__reversed__',
36
37 # from COPY_ATTR() in Objects/funcobject.c
38 '__module__',
39 '__name__',
40 '__qualname__',
41 '__doc__',
42 '__annotations__',
43
44 # from SLOT* in Objects/typeobject.c
45 '__abs__',
46 '__add__',
47 '__aiter__',
48 '__and__',
49 '__anext__',
50 '__await__',
51 '__bool__',
52 '__call__',
53 '__contains__',
54 '__del__',
55 '__delattr__',
56 '__delete__',
57 '__delitem__',
58 '__eq__',
59 '__float__',
60 '__floordiv__',
61 '__ge__',
62 '__get__',
63 '__getattr__',
64 '__getattribute__',
65 '__getitem__',
66 '__gt__',
67 '__hash__',
68 '__iadd__',
69 '__iand__',
70 '__ifloordiv__',
71 '__ilshift__',
72 '__imatmul__',
73 '__imod__',
74 '__imul__',
75 '__index__',
76 '__init__',
77 '__int__',
78 '__invert__',
79 '__ior__',
80 '__ipow__',
81 '__irshift__',
82 '__isub__',
83 '__iter__',
84 '__itruediv__',
85 '__ixor__',
86 '__le__',
87 '__len__',
88 '__lshift__',
89 '__lt__',
90 '__matmul__',
91 '__mod__',
92 '__mul__',
93 '__ne__',
94 '__neg__',
95 '__new__',
96 '__next__',
97 '__or__',
98 '__pos__',
99 '__pow__',
100 '__radd__',
101 '__rand__',
102 '__repr__',
103 '__rfloordiv__',
104 '__rlshift__',
105 '__rmatmul__',
106 '__rmod__',
107 '__rmul__',
108 '__ror__',
109 '__rpow__',
110 '__rrshift__',
111 '__rshift__',
112 '__rsub__',
113 '__rtruediv__',
114 '__rxor__',
115 '__set__',
116 '__setattr__',
117 '__setitem__',
118 '__str__',
119 '__sub__',
120 '__truediv__',
121 '__xor__',
122 '__divmod__',
123 '__rdivmod__',
124 '__buffer__',
125 '__release_buffer__',
126
127 #Workarounds for GH-108918
128 'alias',
129 'args',
130 'exc_type',
131 'exc_value',
132 'self',
133 'traceback',
134 ]
135
136 NON_GENERATED_IMMORTAL_OBJECTS = [
137 # The generated ones come from generate_runtime_init().
138 '(PyObject *)&_Py_SINGLETON(bytes_empty)',
139 '(PyObject *)&_Py_SINGLETON(tuple_empty)',
140 '(PyObject *)&_Py_SINGLETON(hamt_bitmap_node_empty)',
141 '(PyObject *)&_Py_INTERP_SINGLETON(interp, hamt_empty)',
142 '(PyObject *)&_Py_SINGLETON(context_token_missing)',
143 ]
144
145
146 #######################################
147 # helpers
148
149 def iter_files():
150 for name in ('Modules', 'Objects', 'Parser', 'PC', 'Programs', 'Python'):
151 root = os.path.join(ROOT, name)
152 for dirname, _, files in os.walk(root):
153 for name in files:
154 if not name.endswith(('.c', '.h')):
155 continue
156 yield os.path.join(dirname, name)
157
158
159 def iter_global_strings():
160 id_regex = re.compile(r'\b_Py_ID\((\w+)\)')
161 str_regex = re.compile(r'\b_Py_DECLARE_STR\((\w+), "(.*?)"\)')
162 for filename in iter_files():
163 try:
164 infile = open(filename, encoding='utf-8')
165 except FileNotFoundError:
166 # The file must have been a temporary file.
167 continue
168 with infile:
169 for lno, line in enumerate(infile, 1):
170 for m in id_regex.finditer(line):
171 identifier, = m.groups()
172 yield identifier, None, filename, lno, line
173 for m in str_regex.finditer(line):
174 varname, string = m.groups()
175 yield varname, string, filename, lno, line
176
177
178 def iter_to_marker(lines, marker):
179 for line in lines:
180 if line.rstrip() == marker:
181 break
182 yield line
183
184
185 class Printer:
186
187 def __init__(self, file):
188 self.level = 0
189 self.file = file
190 self.continuation = [False]
191
192 @contextlib.contextmanager
193 def indent(self):
194 save_level = self.level
195 try:
196 self.level += 1
197 yield
198 finally:
199 self.level = save_level
200
201 def write(self, arg):
202 eol = '\n'
203 if self.continuation[-1]:
204 eol = f' \\{eol}' if arg else f'\\{eol}'
205 self.file.writelines((" "*self.level, arg, eol))
206
207 @contextlib.contextmanager
208 def block(self, prefix, suffix="", *, continuation=None):
209 if continuation is None:
210 continuation = self.continuation[-1]
211 self.continuation.append(continuation)
212
213 self.write(prefix + " {")
214 with self.indent():
215 yield
216 self.continuation.pop()
217 self.write("}" + suffix)
218
219
220 @contextlib.contextmanager
221 def open_for_changes(filename, orig):
222 """Like open() but only write to the file if it changed."""
223 outfile = io.StringIO()
224 yield outfile
225 text = outfile.getvalue()
226 if text != orig:
227 with open(filename, 'w', encoding='utf-8') as outfile:
228 outfile.write(text)
229 else:
230 print(f'# not changed: {filename}')
231
232
233 #######################################
234 # the global objects
235
236 START = f'/* The following is auto-generated by {SCRIPT_NAME}. */'
237 END = '/* End auto-generated code */'
238
239
240 def generate_global_strings(identifiers, strings):
241 filename = os.path.join(INTERNAL, 'pycore_global_strings.h')
242
243 # Read the non-generated part of the file.
244 with open(filename) as infile:
245 orig = infile.read()
246 lines = iter(orig.rstrip().splitlines())
247 before = '\n'.join(iter_to_marker(lines, START))
248 for _ in iter_to_marker(lines, END):
249 pass
250 after = '\n'.join(lines)
251
252 # Generate the file.
253 with open_for_changes(filename, orig) as outfile:
254 printer = Printer(outfile)
255 printer.write(before)
256 printer.write(START)
257 with printer.block('struct _Py_global_strings', ';'):
258 with printer.block('struct', ' literals;'):
259 for literal, name in sorted(strings.items(), key=lambda x: x[1]):
260 printer.write(f'STRUCT_FOR_STR({name}, "{literal}")')
261 outfile.write('\n')
262 with printer.block('struct', ' identifiers;'):
263 for name in sorted(identifiers):
264 assert name.isidentifier(), name
265 printer.write(f'STRUCT_FOR_ID({name})')
266 with printer.block('struct', ' ascii[128];'):
267 printer.write("PyASCIIObject _ascii;")
268 printer.write("uint8_t _data[2];")
269 with printer.block('struct', ' latin1[128];'):
270 printer.write("PyCompactUnicodeObject _latin1;")
271 printer.write("uint8_t _data[2];")
272 printer.write(END)
273 printer.write(after)
274
275
276 def generate_runtime_init(identifiers, strings):
277 # First get some info from the declarations.
278 nsmallposints = None
279 nsmallnegints = None
280 with open(os.path.join(INTERNAL, 'pycore_runtime_structs.h')) as infile:
281 for line in infile:
282 if line.startswith('#define _PY_NSMALLPOSINTS'):
283 nsmallposints = int(line.split()[-1])
284 elif line.startswith('#define _PY_NSMALLNEGINTS'):
285 nsmallnegints = int(line.split()[-1])
286 break
287 else:
288 raise NotImplementedError
289 assert nsmallposints
290 assert nsmallnegints
291
292 # Then target the runtime initializer.
293 filename = os.path.join(INTERNAL, 'pycore_runtime_init_generated.h')
294
295 # Read the non-generated part of the file.
296 with open(filename) as infile:
297 orig = infile.read()
298 lines = iter(orig.rstrip().splitlines())
299 before = '\n'.join(iter_to_marker(lines, START))
300 for _ in iter_to_marker(lines, END):
301 pass
302 after = '\n'.join(lines)
303
304 # Generate the file.
305 with open_for_changes(filename, orig) as outfile:
306 immortal_objects = []
307 printer = Printer(outfile)
308 printer.write(before)
309 printer.write(START)
310 with printer.block('#define _Py_small_ints_INIT', continuation=True):
311 for i in range(-nsmallnegints, nsmallposints):
312 printer.write(f'_PyLong_DIGIT_INIT({i}),')
313 immortal_objects.append(f'(PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + {i}]')
314 printer.write('')
315 with printer.block('#define _Py_bytes_characters_INIT', continuation=True):
316 for i in range(256):
317 printer.write(f'_PyBytes_CHAR_INIT({i}),')
318 immortal_objects.append(f'(PyObject *)&_Py_SINGLETON(bytes_characters)[{i}]')
319 printer.write('')
320 with printer.block('#define _Py_str_literals_INIT', continuation=True):
321 for literal, name in sorted(strings.items(), key=lambda x: x[1]):
322 printer.write(f'INIT_STR({name}, "{literal}"),')
323 immortal_objects.append(f'(PyObject *)&_Py_STR({name})')
324 printer.write('')
325 with printer.block('#define _Py_str_identifiers_INIT', continuation=True):
326 for name in sorted(identifiers):
327 assert name.isidentifier(), name
328 printer.write(f'INIT_ID({name}),')
329 immortal_objects.append(f'(PyObject *)&_Py_ID({name})')
330 printer.write('')
331 with printer.block('#define _Py_str_ascii_INIT', continuation=True):
332 for i in range(128):
333 printer.write(f'_PyASCIIObject_INIT("\\x{i:02x}"),')
334 immortal_objects.append(f'(PyObject *)&_Py_SINGLETON(strings).ascii[{i}]')
335 printer.write('')
336 with printer.block('#define _Py_str_latin1_INIT', continuation=True):
337 for i in range(128, 256):
338 utf8 = ['"']
339 for c in chr(i).encode('utf-8'):
340 utf8.append(f"\\x{c:02x}")
341 utf8.append('"')
342 printer.write(f'_PyUnicode_LATIN1_INIT("\\x{i:02x}", {"".join(utf8)}),')
343 immortal_objects.append(f'(PyObject *)&_Py_SINGLETON(strings).latin1[{i} - 128]')
344 printer.write(END)
345 printer.write(after)
346 return immortal_objects
347
348
349 def generate_static_strings_initializer(identifiers, strings):
350 # Target the runtime initializer.
351 filename = os.path.join(INTERNAL, 'pycore_unicodeobject_generated.h')
352
353 # Read the non-generated part of the file.
354 with open(filename) as infile:
355 orig = infile.read()
356 lines = iter(orig.rstrip().splitlines())
357 before = '\n'.join(iter_to_marker(lines, START))
358 for _ in iter_to_marker(lines, END):
359 pass
360 after = '\n'.join(lines)
361
362 # Generate the file.
363 with open_for_changes(filename, orig) as outfile:
364 printer = Printer(outfile)
365 printer.write(before)
366 printer.write(START)
367 printer.write("static inline void")
368 with printer.block("_PyUnicode_InitStaticStrings(PyInterpreterState *interp)"):
369 printer.write(f'PyObject *string;')
370 for i in sorted(identifiers):
371 # This use of _Py_ID() is ignored by iter_global_strings()
372 # since iter_files() ignores .h files.
373 printer.write(f'string = &_Py_ID({i});')
374 printer.write(f'_PyUnicode_InternStatic(interp, &string);')
375 printer.write(f'assert(_PyUnicode_CheckConsistency(string, 1));')
376 printer.write(f'assert(PyUnicode_GET_LENGTH(string) != 1);')
377 for value, name in sorted(strings.items()):
378 printer.write(f'string = &_Py_STR({name});')
379 printer.write(f'_PyUnicode_InternStatic(interp, &string);')
380 printer.write(f'assert(_PyUnicode_CheckConsistency(string, 1));')
381 printer.write(f'assert(PyUnicode_GET_LENGTH(string) != 1);')
382 printer.write(END)
383 printer.write(after)
384
385
386 def generate_global_object_finalizers(generated_immortal_objects):
387 # Target the runtime initializer.
388 filename = os.path.join(INTERNAL, 'pycore_global_objects_fini_generated.h')
389
390 # Read the non-generated part of the file.
391 with open(filename) as infile:
392 orig = infile.read()
393 lines = iter(orig.rstrip().splitlines())
394 before = '\n'.join(iter_to_marker(lines, START))
395 for _ in iter_to_marker(lines, END):
396 pass
397 after = '\n'.join(lines)
398
399 # Generate the file.
400 with open_for_changes(filename, orig) as outfile:
401 printer = Printer(outfile)
402 printer.write(before)
403 printer.write(START)
404 printer.write('#ifdef Py_DEBUG')
405 printer.write("static inline void")
406 with printer.block(
407 "_PyStaticObjects_CheckRefcnt(PyInterpreterState *interp)"):
408 printer.write('/* generated runtime-global */')
409 printer.write('// (see pycore_runtime_init_generated.h)')
410 for ref in generated_immortal_objects:
411 printer.write(f'_PyStaticObject_CheckRefcnt({ref});')
412 printer.write('/* non-generated */')
413 for ref in NON_GENERATED_IMMORTAL_OBJECTS:
414 printer.write(f'_PyStaticObject_CheckRefcnt({ref});')
415 printer.write('#endif // Py_DEBUG')
416 printer.write(END)
417 printer.write(after)
418
419
420 def get_identifiers_and_strings() -> 'tuple[set[str], dict[str, str]]':
421 identifiers = set(IDENTIFIERS)
422 strings = {}
423 # Note that we store strings as they appear in C source, so the checks here
424 # can be defeated, e.g.:
425 # - "a" and "\0x61" won't be reported as duplicate.
426 # - "\n" appears as 2 characters.
427 # Probably not worth adding a C string parser.
428 for name, string, *_ in iter_global_strings():
429 if string is None:
430 if name not in IGNORED:
431 identifiers.add(name)
432 else:
433 if len(string) == 1 and ord(string) < 256:
434 # Give a nice message for common mistakes.
435 # To cover tricky cases (like "\n") we also generate C asserts.
436 raise ValueError(
437 'do not use &_Py_ID or &_Py_STR for one-character latin-1 '
438 f'strings, use _Py_LATIN1_CHR instead: {string!r}')
439 if string not in strings:
440 strings[string] = name
441 elif name != strings[string]:
442 raise ValueError(f'name mismatch for string {string!r} ({name!r} != {strings[string]!r}')
443 overlap = identifiers & set(strings.keys())
444 if overlap:
445 raise ValueError(
446 'do not use both _Py_ID and _Py_DECLARE_STR for the same string: '
447 + repr(overlap))
448 return identifiers, strings
449
450
451 #######################################
452 # the script
453
454 def main() -> None:
455 identifiers, strings = get_identifiers_and_strings()
456
457 generate_global_strings(identifiers, strings)
458 generated_immortal_objects = generate_runtime_init(identifiers, strings)
459 generate_static_strings_initializer(identifiers, strings)
460 generate_global_object_finalizers(generated_immortal_objects)
461
462
463 if __name__ == '__main__':
464 main()