Debian Codegraph @

Tools/build/generate_token.py

raw
1 #! /usr/bin/env python3
2 # This script generates token related files from Grammar/Tokens:
3 #
4 # make_rst:
5 # Doc/library/token-list.inc
6 # Doc/library/token.rst (checked, not generated)
7 # make_h:
8 # Include/token.h
9 # make_c:
10 # Parser/token.c
11 # make_py:
12 # Lib/token.py
13
14 import re
15
16 SCRIPT_NAME = 'Tools/build/generate_token.py'
17 AUTO_GENERATED_BY_SCRIPT = f'Auto-generated by {SCRIPT_NAME}'
18 NT_OFFSET = 256
19
20 def load_tokens(path):
21 tok_names = []
22 string_to_tok = {}
23 ERRORTOKEN = None
24 with open(path) as fp:
25 for line in fp:
26 line = line.strip()
27 # strip comments
28 i = line.find('#')
29 if i >= 0:
30 line = line[:i].strip()
31 if not line:
32 continue
33 fields = line.split()
34 name = fields[0]
35 value = len(tok_names)
36 if name == 'ERRORTOKEN':
37 ERRORTOKEN = value
38 string = fields[1] if len(fields) > 1 else None
39 if string:
40 string = eval(string)
41 string_to_tok[string] = value
42 tok_names.append(name)
43 return tok_names, ERRORTOKEN, string_to_tok
44
45
46 def update_file(file, content):
47 try:
48 with open(file) as fobj:
49 if fobj.read() == content:
50 return False
51 except (OSError, ValueError):
52 pass
53 with open(file, 'w') as fobj:
54 fobj.write(content)
55 return True
56
57
58 token_h_template = f"""\
59 // {AUTO_GENERATED_BY_SCRIPT}
60 """
61 token_h_template += """\
62
63 /* Token types */
64 #ifndef Py_INTERNAL_TOKEN_H
65 #define Py_INTERNAL_TOKEN_H
66 #ifdef __cplusplus
67 extern "C" {
68 #endif
69
70 #ifndef Py_BUILD_CORE
71 # error "this header requires Py_BUILD_CORE define"
72 #endif
73
74 #undef TILDE /* Prevent clash of our definition with system macro. Ex AIX, ioctl.h */
75
76 %s\
77 #define N_TOKENS %d
78 #define NT_OFFSET %d
79
80 /* Special definitions for cooperation with parser */
81
82 #define ISTERMINAL(x) ((x) < NT_OFFSET)
83 #define ISNONTERMINAL(x) ((x) >= NT_OFFSET)
84 #define ISEOF(x) ((x) == ENDMARKER)
85 #define ISWHITESPACE(x) ((x) == ENDMARKER || \\
86 (x) == NEWLINE || \\
87 (x) == INDENT || \\
88 (x) == DEDENT)
89 #define ISSTRINGLIT(x) ((x) == STRING || \\
90 (x) == FSTRING_MIDDLE || \\
91 (x) == TSTRING_MIDDLE)
92
93
94 // Export these 4 symbols for 'test_peg_generator'
95 PyAPI_DATA(const char * const) _PyParser_TokenNames[]; /* Token names */
96 PyAPI_FUNC(int) _PyToken_OneChar(int);
97 PyAPI_FUNC(int) _PyToken_TwoChars(int, int);
98 PyAPI_FUNC(int) _PyToken_ThreeChars(int, int, int);
99
100 #ifdef __cplusplus
101 }
102 #endif
103 #endif // !Py_INTERNAL_TOKEN_H
104 """
105
106 def make_h(infile, outfile='Include/internal/pycore_token.h'):
107 tok_names, ERRORTOKEN, string_to_tok = load_tokens(infile)
108
109 defines = []
110 for value, name in enumerate(tok_names[:ERRORTOKEN + 1]):
111 defines.append("#define %-15s %d\n" % (name, value))
112
113 if update_file(outfile, token_h_template % (
114 ''.join(defines),
115 len(tok_names),
116 NT_OFFSET
117 )):
118 print("%s regenerated from %s" % (outfile, infile))
119
120
121 token_c_template = f"""\
122 /* {AUTO_GENERATED_BY_SCRIPT} */
123 """
124 token_c_template += """\
125
126 #include "Python.h"
127 #include "pycore_token.h"
128
129 /* Token names */
130
131 const char * const _PyParser_TokenNames[] = {
132 %s\
133 };
134
135 /* Return the token corresponding to a single character */
136
137 int
138 _PyToken_OneChar(int c1)
139 {
140 %s\
141 return OP;
142 }
143
144 int
145 _PyToken_TwoChars(int c1, int c2)
146 {
147 %s\
148 return OP;
149 }
150
151 int
152 _PyToken_ThreeChars(int c1, int c2, int c3)
153 {
154 %s\
155 return OP;
156 }
157 """
158
159 def generate_chars_to_token(mapping, n=1):
160 result = []
161 write = result.append
162 indent = ' ' * n
163 write(indent)
164 write('switch (c%d) {\n' % (n,))
165 for c in sorted(mapping):
166 write(indent)
167 value = mapping[c]
168 if isinstance(value, dict):
169 write("case '%s':\n" % (c,))
170 write(generate_chars_to_token(value, n + 1))
171 write(indent)
172 write(' break;\n')
173 else:
174 write("case '%s': return %s;\n" % (c, value))
175 write(indent)
176 write('}\n')
177 return ''.join(result)
178
179 def make_c(infile, outfile='Parser/token.c'):
180 tok_names, ERRORTOKEN, string_to_tok = load_tokens(infile)
181 string_to_tok['<>'] = string_to_tok['!=']
182 chars_to_token = {}
183 for string, value in string_to_tok.items():
184 assert 1 <= len(string) <= 3
185 name = tok_names[value]
186 m = chars_to_token.setdefault(len(string), {})
187 for c in string[:-1]:
188 m = m.setdefault(c, {})
189 m[string[-1]] = name
190
191 names = []
192 for value, name in enumerate(tok_names):
193 if value >= ERRORTOKEN:
194 name = '<%s>' % name
195 names.append(' "%s",\n' % name)
196 names.append(' "<N_TOKENS>",\n')
197
198 if update_file(outfile, token_c_template % (
199 ''.join(names),
200 generate_chars_to_token(chars_to_token[1]),
201 generate_chars_to_token(chars_to_token[2]),
202 generate_chars_to_token(chars_to_token[3])
203 )):
204 print("%s regenerated from %s" % (outfile, infile))
205
206
207 token_inc_template = f"""\
208 .. {AUTO_GENERATED_BY_SCRIPT}
209
210 .. list-table::
211 :align: left
212 :header-rows: 1
213
214 * - Token
215 - Value
216 %s
217 """
218
219 def make_rst(infile, outfile='Doc/library/token-list.inc',
220 rstfile='Doc/library/token.rst'):
221 tok_names, ERRORTOKEN, string_to_tok = load_tokens(infile)
222 tok_to_string = {value: s for s, value in string_to_tok.items()}
223
224 needs_handwritten_doc = set()
225
226 names = []
227 for value, name in enumerate(tok_names):
228 if value in tok_to_string:
229 assert name.isupper()
230 names.append(f' * - .. data:: {name}')
231 names.append(f' - ``"{tok_to_string[value]}"``')
232 else:
233 needs_handwritten_doc.add(name)
234
235 has_handwritten_doc = set()
236 with open(rstfile) as fileobj:
237 tokendef_re = re.compile(r'.. data:: ([0-9A-Z_]+)\s*')
238 for line in fileobj:
239 if match := tokendef_re.fullmatch(line):
240 has_handwritten_doc.add(match[1])
241
242 # Exclude non-token constants in token.py
243 has_handwritten_doc -= {'N_TOKENS', 'NT_OFFSET', 'EXACT_TOKEN_TYPES'}
244
245 if needs_handwritten_doc != has_handwritten_doc:
246 message_parts = [f'ERROR: {rstfile} does not document all tokens!']
247 undocumented = needs_handwritten_doc - has_handwritten_doc
248 extra = has_handwritten_doc - needs_handwritten_doc
249 if undocumented:
250 message_parts.append(f'Undocumented tokens: {undocumented}')
251 if extra:
252 message_parts.append(f'Documented nonexistent tokens: {extra}')
253 exit('\n'.join(message_parts))
254
255 if update_file(outfile, token_inc_template % '\n'.join(names)):
256 print("%s regenerated from %s" % (outfile, infile))
257
258
259 token_py_template = f'''\
260 """Token constants."""
261 # {AUTO_GENERATED_BY_SCRIPT}
262 '''
263 token_py_template += '''
264 __all__ = ['tok_name', 'ISTERMINAL', 'ISNONTERMINAL', 'ISEOF',
265 'EXACT_TOKEN_TYPES']
266
267 %s
268 N_TOKENS = %d
269 # Special definitions for cooperation with parser
270 NT_OFFSET = %d
271
272 tok_name = {value: name
273 for name, value in globals().items()
274 if isinstance(value, int) and not name.startswith('_')}
275 __all__.extend(tok_name.values())
276
277 EXACT_TOKEN_TYPES = {
278 %s
279 }
280
281 def ISTERMINAL(x: int) -> bool:
282 return x < NT_OFFSET
283
284 def ISNONTERMINAL(x: int) -> bool:
285 return x >= NT_OFFSET
286
287 def ISEOF(x: int) -> bool:
288 return x == ENDMARKER
289 '''
290
291 def make_py(infile, outfile='Lib/token.py'):
292 tok_names, ERRORTOKEN, string_to_tok = load_tokens(infile)
293
294 constants = []
295 for value, name in enumerate(tok_names):
296 constants.append('%s = %d' % (name, value))
297 constants.insert(ERRORTOKEN,
298 "# These aren't used by the C tokenizer but are needed for tokenize.py")
299
300 token_types = []
301 for s, value in sorted(string_to_tok.items()):
302 token_types.append(' %r: %s,' % (s, tok_names[value]))
303
304 if update_file(outfile, token_py_template % (
305 '\n'.join(constants),
306 len(tok_names),
307 NT_OFFSET,
308 '\n'.join(token_types),
309 )):
310 print("%s regenerated from %s" % (outfile, infile))
311
312
313 def main(op, infile='Grammar/Tokens', *args):
314 make = globals()['make_' + op]
315 make(infile, *args)
316
317
318 if __name__ == '__main__':
319 import sys
320 main(*sys.argv[1:])