Debian Codegraph @

Tools/peg_generator/pegen/python_generator.py

raw
1 import os.path
2 import token
3 from collections.abc import Sequence
4 from typing import IO, Any
5
6 from pegen import grammar
7 from pegen.grammar import (
8 Alt,
9 Cut,
10 Forced,
11 Gather,
12 GrammarVisitor,
13 Group,
14 Lookahead,
15 NamedItem,
16 NameLeaf,
17 NegativeLookahead,
18 Opt,
19 PositiveLookahead,
20 Repeat0,
21 Repeat1,
22 Rhs,
23 Rule,
24 StringLeaf,
25 )
26 from pegen.parser_generator import ParserGenerator
27
28 MODULE_PREFIX = """\
29 #!/usr/bin/env python3.8
30 # @generated by pegen from {filename}
31
32 import ast
33 import sys
34 import tokenize
35
36 from typing import Any, Optional
37
38 from pegen.parser import memoize, memoize_left_rec, logger, Parser
39
40 """
41 MODULE_SUFFIX = """
42
43 if __name__ == '__main__':
44 from pegen.parser import simple_parser_main
45 simple_parser_main({class_name})
46 """
47
48
49 class InvalidNodeVisitor(GrammarVisitor):
50 def visit_NameLeaf(self, node: NameLeaf) -> bool:
51 name = node.value
52 return name.startswith("invalid")
53
54 def visit_StringLeaf(self, node: StringLeaf) -> bool:
55 return False
56
57 def visit_NamedItem(self, node: NamedItem) -> bool:
58 return self.visit(node.item)
59
60 def visit_Rhs(self, node: Rhs) -> bool:
61 return any(self.visit(alt) for alt in node.alts)
62
63 def visit_Alt(self, node: Alt) -> bool:
64 return any(self.visit(item) for item in node.items)
65
66 def lookahead_call_helper(self, node: Lookahead) -> bool:
67 return self.visit(node.node)
68
69 def visit_PositiveLookahead(self, node: PositiveLookahead) -> bool:
70 return self.lookahead_call_helper(node)
71
72 def visit_NegativeLookahead(self, node: NegativeLookahead) -> bool:
73 return self.lookahead_call_helper(node)
74
75 def visit_Opt(self, node: Opt) -> bool:
76 return self.visit(node.node)
77
78 def visit_Repeat(self, node: Repeat0) -> tuple[str, str]:
79 return self.visit(node.node)
80
81 def visit_Gather(self, node: Gather) -> tuple[str, str]:
82 return self.visit(node.node)
83
84 def visit_Group(self, node: Group) -> bool:
85 return self.visit(node.rhs)
86
87 def visit_Cut(self, node: Cut) -> bool:
88 return False
89
90 def visit_Forced(self, node: Forced) -> bool:
91 return self.visit(node.node)
92
93
94 class PythonCallMakerVisitor(GrammarVisitor):
95 def __init__(self, parser_generator: ParserGenerator):
96 self.gen = parser_generator
97 self.cache: dict[Any, Any] = {}
98
99 def visit_NameLeaf(self, node: NameLeaf) -> tuple[str | None, str]:
100 name = node.value
101 if name == "SOFT_KEYWORD":
102 return "soft_keyword", "self.soft_keyword()"
103 if name in ("NAME", "NUMBER", "STRING", "OP", "TYPE_COMMENT"):
104 name = name.lower()
105 return name, f"self.{name}()"
106 if name in ("NEWLINE", "DEDENT", "INDENT", "ENDMARKER"):
107 # Avoid using names that can be Python keywords
108 return "_" + name.lower(), f"self.expect({name!r})"
109 return name, f"self.{name}()"
110
111 def visit_StringLeaf(self, node: StringLeaf) -> tuple[str, str]:
112 return "literal", f"self.expect({node.value})"
113
114 def visit_Rhs(self, node: Rhs) -> tuple[str | None, str]:
115 if node in self.cache:
116 return self.cache[node]
117 if len(node.alts) == 1 and len(node.alts[0].items) == 1:
118 self.cache[node] = self.visit(node.alts[0].items[0])
119 else:
120 name = self.gen.artifical_rule_from_rhs(node)
121 self.cache[node] = name, f"self.{name}()"
122 return self.cache[node]
123
124 def visit_NamedItem(self, node: NamedItem) -> tuple[str | None, str]:
125 name, call = self.visit(node.item)
126 if node.name:
127 name = node.name
128 return name, call
129
130 def lookahead_call_helper(self, node: Lookahead) -> tuple[str, str]:
131 name, call = self.visit(node.node)
132 head, tail = call.split("(", 1)
133 assert tail[-1] == ")"
134 tail = tail[:-1]
135 return head, tail
136
137 def visit_PositiveLookahead(self, node: PositiveLookahead) -> tuple[None, str]:
138 head, tail = self.lookahead_call_helper(node)
139 return None, f"self.positive_lookahead({head}, {tail})"
140
141 def visit_NegativeLookahead(self, node: NegativeLookahead) -> tuple[None, str]:
142 head, tail = self.lookahead_call_helper(node)
143 return None, f"self.negative_lookahead({head}, {tail})"
144
145 def visit_Opt(self, node: Opt) -> tuple[str, str]:
146 name, call = self.visit(node.node)
147 # Note trailing comma (the call may already have one comma
148 # at the end, for example when rules have both repeat0 and optional
149 # markers, e.g: [rule*])
150 if call.endswith(","):
151 return "opt", call
152 else:
153 return "opt", f"{call},"
154
155 def visit_Repeat0(self, node: Repeat0) -> tuple[str, str]:
156 if node in self.cache:
157 return self.cache[node]
158 name = self.gen.artificial_rule_from_repeat(node.node, False)
159 self.cache[node] = name, f"self.{name}()," # Also a trailing comma!
160 return self.cache[node]
161
162 def visit_Repeat1(self, node: Repeat1) -> tuple[str, str]:
163 if node in self.cache:
164 return self.cache[node]
165 name = self.gen.artificial_rule_from_repeat(node.node, True)
166 self.cache[node] = name, f"self.{name}()" # But no trailing comma here!
167 return self.cache[node]
168
169 def visit_Gather(self, node: Gather) -> tuple[str, str]:
170 if node in self.cache:
171 return self.cache[node]
172 name = self.gen.artifical_rule_from_gather(node)
173 self.cache[node] = name, f"self.{name}()" # No trailing comma here either!
174 return self.cache[node]
175
176 def visit_Group(self, node: Group) -> tuple[str | None, str]:
177 return self.visit(node.rhs)
178
179 def visit_Cut(self, node: Cut) -> tuple[str, str]:
180 return "cut", "True"
181
182 def visit_Forced(self, node: Forced) -> tuple[str, str]:
183 if isinstance(node.node, Group):
184 _, val = self.visit(node.node.rhs)
185 return "forced", f"self.expect_forced({val}, '''({node.node.rhs!s})''')"
186 else:
187 return (
188 "forced",
189 f"self.expect_forced(self.expect({node.node.value}), {node.node.value!r})",
190 )
191
192
193 class PythonParserGenerator(ParserGenerator, GrammarVisitor):
194 def __init__(
195 self,
196 grammar: grammar.Grammar,
197 file: IO[str] | None,
198 tokens: set[str] = set(token.tok_name.values()),
199 location_formatting: str | None = None,
200 unreachable_formatting: str | None = None,
201 ):
202 tokens.add("SOFT_KEYWORD")
203 super().__init__(grammar, tokens, file)
204 self.callmakervisitor: PythonCallMakerVisitor = PythonCallMakerVisitor(self)
205 self.invalidvisitor: InvalidNodeVisitor = InvalidNodeVisitor()
206 self.unreachable_formatting = unreachable_formatting or "None # pragma: no cover"
207 self.location_formatting = (
208 location_formatting
209 or "lineno=start_lineno, col_offset=start_col_offset, "
210 "end_lineno=end_lineno, end_col_offset=end_col_offset"
211 )
212
213 def generate(self, filename: str) -> None:
214 self.collect_rules()
215 header = self.grammar.metas.get("header", MODULE_PREFIX)
216 if header is not None:
217 basename = os.path.basename(filename)
218 self.print(header.rstrip("\n").format(filename=basename))
219 subheader = self.grammar.metas.get("subheader", "")
220 if subheader:
221 self.print(subheader)
222 cls_name = self.grammar.metas.get("class", "GeneratedParser")
223 self.print("# Keywords and soft keywords are listed at the end of the parser definition.")
224 self.print(f"class {cls_name}(Parser):")
225 for rule in self.all_rules.values():
226 self.print()
227 with self.indent():
228 self.visit(rule)
229
230 self.print()
231 with self.indent():
232 self.print(f"KEYWORDS = {tuple(self.keywords)}")
233 self.print(f"SOFT_KEYWORDS = {tuple(self.soft_keywords)}")
234
235 trailer = self.grammar.metas.get("trailer", MODULE_SUFFIX.format(class_name=cls_name))
236 if trailer is not None:
237 self.print(trailer.rstrip("\n"))
238
239 def alts_uses_locations(self, alts: Sequence[Alt]) -> bool:
240 for alt in alts:
241 if alt.action and "LOCATIONS" in alt.action:
242 return True
243 for n in alt.items:
244 if isinstance(n.item, Group) and self.alts_uses_locations(n.item.rhs.alts):
245 return True
246 return False
247
248 def visit_Rule(self, node: Rule) -> None:
249 is_loop = node.is_loop()
250 is_gather = node.is_gather()
251 rhs = node.flatten()
252 if node.left_recursive:
253 if node.leader:
254 self.print("@memoize_left_rec")
255 else:
256 # Non-leader rules in a cycle are not memoized,
257 # but they must still be logged.
258 self.print("@logger")
259 else:
260 self.print("@memoize")
261 node_type = node.type or "Any"
262 self.print(f"def {node.name}(self) -> Optional[{node_type}]:")
263 with self.indent():
264 self.print(f"# {node.name}: {rhs}")
265 self.print("mark = self._mark()")
266 if self.alts_uses_locations(node.rhs.alts):
267 self.print("tok = self._tokenizer.peek()")
268 self.print("start_lineno, start_col_offset = tok.start")
269 if is_loop:
270 self.print("children = []")
271 self.visit(rhs, is_loop=is_loop, is_gather=is_gather)
272 if is_loop:
273 self.print("return children")
274 else:
275 self.print("return None")
276
277 def visit_NamedItem(self, node: NamedItem) -> None:
278 name, call = self.callmakervisitor.visit(node.item)
279 if node.name:
280 name = node.name
281 if not name:
282 self.print(call)
283 else:
284 if name != "cut":
285 name = self.dedupe(name)
286 self.print(f"({name} := {call})")
287
288 def visit_Rhs(self, node: Rhs, is_loop: bool = False, is_gather: bool = False) -> None:
289 if is_loop:
290 assert len(node.alts) == 1
291 for alt in node.alts:
292 self.visit(alt, is_loop=is_loop, is_gather=is_gather)
293
294 def visit_Alt(self, node: Alt, is_loop: bool, is_gather: bool) -> None:
295 has_cut = any(isinstance(item.item, Cut) for item in node.items)
296 with self.local_variable_context():
297 if has_cut:
298 self.print("cut = False")
299 if is_loop:
300 self.print("while (")
301 else:
302 self.print("if (")
303 with self.indent():
304 first = True
305 for item in node.items:
306 if first:
307 first = False
308 else:
309 self.print("and")
310 self.visit(item)
311 if is_gather:
312 self.print("is not None")
313
314 self.print("):")
315 with self.indent():
316 action = node.action
317 if not action:
318 if is_gather:
319 assert len(self.local_variable_names) == 2
320 action = (
321 f"[{self.local_variable_names[0]}] + {self.local_variable_names[1]}"
322 )
323 else:
324 if self.invalidvisitor.visit(node):
325 action = "UNREACHABLE"
326 elif len(self.local_variable_names) == 1:
327 action = f"{self.local_variable_names[0]}"
328 else:
329 action = f"[{', '.join(self.local_variable_names)}]"
330 elif "LOCATIONS" in action:
331 self.print("tok = self._tokenizer.get_last_non_whitespace_token()")
332 self.print("end_lineno, end_col_offset = tok.end")
333 action = action.replace("LOCATIONS", self.location_formatting)
334
335 if is_loop:
336 self.print(f"children.append({action})")
337 self.print("mark = self._mark()")
338 else:
339 if "UNREACHABLE" in action:
340 action = action.replace("UNREACHABLE", self.unreachable_formatting)
341 self.print(f"return {action}")
342
343 self.print("self._reset(mark)")
344 # Skip remaining alternatives if a cut was reached.
345 if has_cut:
346 self.print("if cut: return None")