Debian Codegraph @

Tools/build/generate_token.py

raw
1 #! /usr/bin/env python3
2 # This script generates token related files from Grammar/Tokens:
3 #
4 # make_rst:
5 # Doc/library/token-list.inc
6 # Doc/library/token.rst (checked, not generated)
7 # make_h:
8 # Include/token.h
9 # make_c:
10 # Parser/token.c
11 # make_py:
12 # Lib/token.py
13
14 import re
15
16 SCRIPT_NAME = 'Tools/build/generate_token.py'
17 AUTO_GENERATED_BY_SCRIPT = f'Auto-generated by {SCRIPT_NAME}'
18 NT_OFFSET = 256
19
20 def load_tokens(path):
21 tok_names = []
22 string_to_tok = {}
23 ERRORTOKEN = None
24 with open(path) as fp:
25 for line in fp:
26 line = line.strip()
27 # strip comments
28 i = line.find('#')
29 if i >= 0:
30 line = line[:i].strip()
31 if not line:
32 continue
33 fields = line.split()
34 name = fields[0]
35 value = len(tok_names)
36 if name == 'ERRORTOKEN':
37 ERRORTOKEN = value
38 string = fields[1] if len(fields) > 1 else None
39 if string:
40 string = eval(string)
41 string_to_tok[string] = value
42 tok_names.append(name)
43 return tok_names, ERRORTOKEN, string_to_tok
44
45
46 def update_file(file, content):
47 try:
48 with open(file) as fobj:
49 if fobj.read() == content:
50 return False
51 except (OSError, ValueError):
52 pass
53 with open(file, 'w') as fobj:
54 fobj.write(content)
55 return True
56
57
58 token_h_template = f"""\
59 // {AUTO_GENERATED_BY_SCRIPT}
60 """
61 token_h_template += """\
62
63 /* Token types */
64 #ifndef Py_INTERNAL_TOKEN_H
65 #define Py_INTERNAL_TOKEN_H
66 #ifdef __cplusplus
67 extern "C" {
68 #endif
69
70 #ifndef Py_BUILD_CORE
71 # error "this header requires Py_BUILD_CORE define"
72 #endif
73
74 #undef TILDE /* Prevent clash of our definition with system macro. Ex AIX, ioctl.h */
75
76 %s\
77 #define N_TOKENS %d
78 #define NT_OFFSET %d
79
80 /* Special definitions for cooperation with parser */
81
82 #define ISTERMINAL(x) ((x) < NT_OFFSET)
83 #define ISNONTERMINAL(x) ((x) >= NT_OFFSET)
84 #define ISEOF(x) ((x) == ENDMARKER)
85 #define ISWHITESPACE(x) ((x) == ENDMARKER || \\
86 (x) == NEWLINE || \\
87 (x) == INDENT || \\
88 (x) == DEDENT)
89 #define ISSTRINGLIT(x) ((x) == STRING || \\
90 (x) == FSTRING_MIDDLE)
91
92
93 // Export these 4 symbols for 'test_peg_generator'
94 PyAPI_DATA(const char * const) _PyParser_TokenNames[]; /* Token names */
95 PyAPI_FUNC(int) _PyToken_OneChar(int);
96 PyAPI_FUNC(int) _PyToken_TwoChars(int, int);
97 PyAPI_FUNC(int) _PyToken_ThreeChars(int, int, int);
98
99 #ifdef __cplusplus
100 }
101 #endif
102 #endif // !Py_INTERNAL_TOKEN_H
103 """
104
105 def make_h(infile, outfile='Include/internal/pycore_token.h'):
106 tok_names, ERRORTOKEN, string_to_tok = load_tokens(infile)
107
108 defines = []
109 for value, name in enumerate(tok_names[:ERRORTOKEN + 1]):
110 defines.append("#define %-15s %d\n" % (name, value))
111
112 if update_file(outfile, token_h_template % (
113 ''.join(defines),
114 len(tok_names),
115 NT_OFFSET
116 )):
117 print("%s regenerated from %s" % (outfile, infile))
118
119
120 token_c_template = f"""\
121 /* {AUTO_GENERATED_BY_SCRIPT} */
122 """
123 token_c_template += """\
124
125 #include "Python.h"
126 #include "pycore_token.h"
127
128 /* Token names */
129
130 const char * const _PyParser_TokenNames[] = {
131 %s\
132 };
133
134 /* Return the token corresponding to a single character */
135
136 int
137 _PyToken_OneChar(int c1)
138 {
139 %s\
140 return OP;
141 }
142
143 int
144 _PyToken_TwoChars(int c1, int c2)
145 {
146 %s\
147 return OP;
148 }
149
150 int
151 _PyToken_ThreeChars(int c1, int c2, int c3)
152 {
153 %s\
154 return OP;
155 }
156 """
157
158 def generate_chars_to_token(mapping, n=1):
159 result = []
160 write = result.append
161 indent = ' ' * n
162 write(indent)
163 write('switch (c%d) {\n' % (n,))
164 for c in sorted(mapping):
165 write(indent)
166 value = mapping[c]
167 if isinstance(value, dict):
168 write("case '%s':\n" % (c,))
169 write(generate_chars_to_token(value, n + 1))
170 write(indent)
171 write(' break;\n')
172 else:
173 write("case '%s': return %s;\n" % (c, value))
174 write(indent)
175 write('}\n')
176 return ''.join(result)
177
178 def make_c(infile, outfile='Parser/token.c'):
179 tok_names, ERRORTOKEN, string_to_tok = load_tokens(infile)
180 string_to_tok['<>'] = string_to_tok['!=']
181 chars_to_token = {}
182 for string, value in string_to_tok.items():
183 assert 1 <= len(string) <= 3
184 name = tok_names[value]
185 m = chars_to_token.setdefault(len(string), {})
186 for c in string[:-1]:
187 m = m.setdefault(c, {})
188 m[string[-1]] = name
189
190 names = []
191 for value, name in enumerate(tok_names):
192 if value >= ERRORTOKEN:
193 name = '<%s>' % name
194 names.append(' "%s",\n' % name)
195 names.append(' "<N_TOKENS>",\n')
196
197 if update_file(outfile, token_c_template % (
198 ''.join(names),
199 generate_chars_to_token(chars_to_token[1]),
200 generate_chars_to_token(chars_to_token[2]),
201 generate_chars_to_token(chars_to_token[3])
202 )):
203 print("%s regenerated from %s" % (outfile, infile))
204
205
206 token_inc_template = f"""\
207 .. {AUTO_GENERATED_BY_SCRIPT}
208
209 .. list-table::
210 :align: left
211 :header-rows: 1
212
213 * - Token
214 - Value
215 %s
216 """
217
218 def make_rst(infile, outfile='Doc/library/token-list.inc',
219 rstfile='Doc/library/token.rst'):
220 tok_names, ERRORTOKEN, string_to_tok = load_tokens(infile)
221 tok_to_string = {value: s for s, value in string_to_tok.items()}
222
223 needs_handwritten_doc = set()
224
225 names = []
226 for value, name in enumerate(tok_names):
227 if value in tok_to_string:
228 assert name.isupper()
229 names.append(f' * - .. data:: {name}')
230 names.append(f' - ``"{tok_to_string[value]}"``')
231 else:
232 needs_handwritten_doc.add(name)
233
234 has_handwritten_doc = set()
235 with open(rstfile) as fileobj:
236 tokendef_re = re.compile(r'.. data:: ([0-9A-Z_]+)\s*')
237 for line in fileobj:
238 if match := tokendef_re.fullmatch(line):
239 has_handwritten_doc.add(match[1])
240
241 # Exclude non-token constants in token.py
242 has_handwritten_doc -= {'N_TOKENS', 'NT_OFFSET', 'EXACT_TOKEN_TYPES'}
243
244 if needs_handwritten_doc != has_handwritten_doc:
245 message_parts = [f'ERROR: {rstfile} does not document all tokens!']
246 undocumented = needs_handwritten_doc - has_handwritten_doc
247 extra = has_handwritten_doc - needs_handwritten_doc
248 if undocumented:
249 message_parts.append(f'Undocumented tokens: {undocumented}')
250 if extra:
251 message_parts.append(f'Documented nonexistent tokens: {extra}')
252 exit('\n'.join(message_parts))
253
254 if update_file(outfile, token_inc_template % '\n'.join(names)):
255 print("%s regenerated from %s" % (outfile, infile))
256
257
258 token_py_template = f'''\
259 """Token constants."""
260 # {AUTO_GENERATED_BY_SCRIPT}
261 '''
262 token_py_template += '''
263 __all__ = ['tok_name', 'ISTERMINAL', 'ISNONTERMINAL', 'ISEOF',
264 'EXACT_TOKEN_TYPES']
265
266 %s
267 N_TOKENS = %d
268 # Special definitions for cooperation with parser
269 NT_OFFSET = %d
270
271 tok_name = {value: name
272 for name, value in globals().items()
273 if isinstance(value, int) and not name.startswith('_')}
274 __all__.extend(tok_name.values())
275
276 EXACT_TOKEN_TYPES = {
277 %s
278 }
279
280 def ISTERMINAL(x):
281 return x < NT_OFFSET
282
283 def ISNONTERMINAL(x):
284 return x >= NT_OFFSET
285
286 def ISEOF(x):
287 return x == ENDMARKER
288 '''
289
290 def make_py(infile, outfile='Lib/token.py'):
291 tok_names, ERRORTOKEN, string_to_tok = load_tokens(infile)
292
293 constants = []
294 for value, name in enumerate(tok_names):
295 constants.append('%s = %d' % (name, value))
296 constants.insert(ERRORTOKEN,
297 "# These aren't used by the C tokenizer but are needed for tokenize.py")
298
299 token_types = []
300 for s, value in sorted(string_to_tok.items()):
301 token_types.append(' %r: %s,' % (s, tok_names[value]))
302
303 if update_file(outfile, token_py_template % (
304 '\n'.join(constants),
305 len(tok_names),
306 NT_OFFSET,
307 '\n'.join(token_types),
308 )):
309 print("%s regenerated from %s" % (outfile, infile))
310
311
312 def main(op, infile='Grammar/Tokens', *args):
313 make = globals()['make_' + op]
314 make(infile, *args)
315
316
317 if __name__ == '__main__':
318 import sys
319 main(*sys.argv[1:])