Debian Codegraph @

Tools/peg_generator/pegen/python_generator.py

raw
1 import os.path
2 import token
3 from collections.abc import Callable, Sequence
4 from typing import IO, Any
5
6 from pegen import grammar
7 from pegen.grammar import (
8 Alt,
9 Cut,
10 Forced,
11 Gather,
12 GrammarVisitor,
13 Group,
14 Lookahead,
15 NamedItem,
16 NameLeaf,
17 NegativeLookahead,
18 Opt,
19 PositiveLookahead,
20 Repeat0,
21 Repeat1,
22 Rhs,
23 Rule,
24 StringLeaf,
25 )
26 from pegen.parser_generator import ParserGenerator
27
28 MODULE_PREFIX = """\
29 #!/usr/bin/env python3.8
30 # @generated by pegen from {filename}
31
32 import ast
33 import sys
34 import tokenize
35
36 from typing import Any, Optional
37
38 from pegen.parser import memoize, memoize_left_rec, logger, Parser
39
40 """
41 MODULE_SUFFIX = """
42
43 if __name__ == '__main__':
44 from pegen.parser import simple_parser_main
45 simple_parser_main({class_name})
46 """
47
48
49 class InvalidNodeVisitor(GrammarVisitor):
50 def visit_NameLeaf(self, node: NameLeaf) -> bool:
51 name = node.value
52 return name.startswith("invalid")
53
54 def visit_StringLeaf(self, node: StringLeaf) -> bool:
55 return False
56
57 def visit_NamedItem(self, node: NamedItem) -> bool:
58 return self.visit(node.item)
59
60 def visit_Rhs(self, node: Rhs) -> bool:
61 return any(self.visit(alt) for alt in node.alts)
62
63 def visit_Alt(self, node: Alt) -> bool:
64 return any(self.visit(item) for item in node.items)
65
66 def lookahead_call_helper(self, node: Lookahead) -> bool:
67 return self.visit(node.node)
68
69 def visit_PositiveLookahead(self, node: PositiveLookahead) -> bool:
70 return self.lookahead_call_helper(node)
71
72 def visit_NegativeLookahead(self, node: NegativeLookahead) -> bool:
73 return self.lookahead_call_helper(node)
74
75 def visit_Opt(self, node: Opt) -> bool:
76 return self.visit(node.node)
77
78 def visit_Repeat(self, node: Repeat0) -> tuple[str, str]:
79 return self.visit(node.node)
80
81 def visit_Gather(self, node: Gather) -> tuple[str, str]:
82 return self.visit(node.node)
83
84 def visit_Group(self, node: Group) -> bool:
85 return self.visit(node.rhs)
86
87 def visit_Cut(self, node: Cut) -> bool:
88 return False
89
90 def visit_Forced(self, node: Forced) -> bool:
91 return self.visit(node.node)
92
93
94 class PythonCallMakerVisitor(GrammarVisitor):
95 def __init__(self, parser_generator: ParserGenerator):
96 self.gen = parser_generator
97 self.cache: dict[str, tuple[str, str]] = {}
98
99 def visit_NameLeaf(self, node: NameLeaf) -> tuple[str | None, str]:
100 name = node.value
101 if name == "SOFT_KEYWORD":
102 return "soft_keyword", "self.soft_keyword()"
103 if name in ("NAME", "NUMBER", "STRING", "OP", "TYPE_COMMENT",
104 "FSTRING_END", "FSTRING_MIDDLE", "FSTRING_START"):
105 name = name.lower()
106 return name, f"self.{name}()"
107 if name in ("NEWLINE", "DEDENT", "INDENT", "ENDMARKER"):
108 # Avoid using names that can be Python keywords
109 return "_" + name.lower(), f"self.expect({name!r})"
110 return name, f"self.{name}()"
111
112 def visit_StringLeaf(self, node: StringLeaf) -> tuple[str, str]:
113 return "literal", f"self.expect({node.value})"
114
115 def visit_NamedItem(self, node: NamedItem) -> tuple[str | None, str]:
116 name, call = self.visit(node.item)
117 if node.name:
118 name = node.name
119 return name, call
120
121 def lookahead_call_helper(self, node: Lookahead) -> tuple[str, str]:
122 name, call = self.visit(node.node)
123 head, tail = call.split("(", 1)
124 assert tail[-1] == ")"
125 tail = tail[:-1]
126 return head, tail
127
128 def visit_PositiveLookahead(self, node: PositiveLookahead) -> tuple[None, str]:
129 head, tail = self.lookahead_call_helper(node)
130 return None, f"self.positive_lookahead({head}, {tail})"
131
132 def visit_NegativeLookahead(self, node: NegativeLookahead) -> tuple[None, str]:
133 head, tail = self.lookahead_call_helper(node)
134 return None, f"self.negative_lookahead({head}, {tail})"
135
136 def visit_Opt(self, node: Opt) -> tuple[str, str]:
137 name, call = self.visit(node.node)
138 # Note trailing comma (the call may already have one comma
139 # at the end, for example when rules have both repeat0 and optional
140 # markers, e.g: [rule*])
141 if call.endswith(","):
142 return "opt", call
143 else:
144 return "opt", f"{call},"
145
146 def _generate_artificial_rule_call(
147 self,
148 node: Any,
149 prefix: str,
150 call_by_name_func: Callable[[str], str],
151 rule_generation_func: Callable[[], str],
152 ) -> tuple[str, str]:
153 node_str = f"{node}"
154 key = f"{prefix}_{node_str}"
155 if key in self.cache:
156 return self.cache[key]
157
158 name = rule_generation_func()
159 call = call_by_name_func(name)
160 self.cache[key] = name, call
161 return self.cache[key]
162
163 def visit_Rhs(self, node: Rhs) -> tuple[str, str]:
164 if len(node.alts) == 1 and len(node.alts[0].items) == 1:
165 return self.visit(node.alts[0].items[0])
166
167 return self._generate_artificial_rule_call(
168 node,
169 "rhs",
170 lambda name: f"self.{name}()",
171 lambda: self.gen.artificial_rule_from_rhs(node),
172 )
173
174 def visit_Repeat0(self, node: Repeat0) -> tuple[str, str]:
175 return self._generate_artificial_rule_call(
176 node,
177 "repeat0",
178 lambda name: f"self.{name}(),", # Also a trailing comma!
179 lambda: self.gen.artificial_rule_from_repeat(node.node, is_repeat1=False),
180 )
181
182 def visit_Repeat1(self, node: Repeat1) -> tuple[str, str]:
183 return self._generate_artificial_rule_call(
184 node,
185 "repeat1",
186 lambda name: f"self.{name}()", # But no trailing comma here!
187 lambda: self.gen.artificial_rule_from_repeat(node.node, is_repeat1=True),
188 )
189
190 def visit_Gather(self, node: Gather) -> tuple[str, str]:
191 return self._generate_artificial_rule_call(
192 node,
193 "gather",
194 lambda name: f"self.{name}()", # No trailing comma here either!
195 lambda: self.gen.artificial_rule_from_gather(node),
196 )
197
198 def visit_Group(self, node: Group) -> tuple[str | None, str]:
199 return self.visit(node.rhs)
200
201 def visit_Cut(self, node: Cut) -> tuple[str, str]:
202 return "cut", "True"
203
204 def visit_Forced(self, node: Forced) -> tuple[str, str]:
205 if isinstance(node.node, Group):
206 _, val = self.visit(node.node.rhs)
207 return "forced", f"self.expect_forced({val}, '''({node.node.rhs!s})''')"
208 else:
209 return (
210 "forced",
211 f"self.expect_forced(self.expect({node.node.value}), {node.node.value!r})",
212 )
213
214
215 class PythonParserGenerator(ParserGenerator, GrammarVisitor):
216 def __init__(
217 self,
218 grammar: grammar.Grammar,
219 file: IO[str] | None,
220 tokens: set[str] = set(token.tok_name.values()),
221 location_formatting: str | None = None,
222 unreachable_formatting: str | None = None,
223 ):
224 tokens.add("SOFT_KEYWORD")
225 super().__init__(grammar, tokens, file)
226 self.callmakervisitor: PythonCallMakerVisitor = PythonCallMakerVisitor(self)
227 self.invalidvisitor: InvalidNodeVisitor = InvalidNodeVisitor()
228 self.unreachable_formatting = unreachable_formatting or "None # pragma: no cover"
229 self.location_formatting = (
230 location_formatting
231 or "lineno=start_lineno, col_offset=start_col_offset, "
232 "end_lineno=end_lineno, end_col_offset=end_col_offset"
233 )
234
235 def generate(self, filename: str) -> None:
236 self.collect_rules()
237 header = self.grammar.metas.get("header", MODULE_PREFIX)
238 if header is not None:
239 basename = os.path.basename(filename)
240 self.print(header.rstrip("\n").format(filename=basename))
241 subheader = self.grammar.metas.get("subheader", "")
242 if subheader:
243 self.print(subheader)
244 cls_name = self.grammar.metas.get("class", "GeneratedParser")
245 self.print("# Keywords and soft keywords are listed at the end of the parser definition.")
246 self.print(f"class {cls_name}(Parser):")
247 for rule in self.all_rules.values():
248 self.print()
249 with self.indent():
250 self.visit(rule)
251
252 self.print()
253 with self.indent():
254 self.print(f"KEYWORDS = {tuple(self.keywords)}")
255 self.print(f"SOFT_KEYWORDS = {tuple(self.soft_keywords)}")
256
257 trailer = self.grammar.metas.get("trailer", MODULE_SUFFIX.format(class_name=cls_name))
258 if trailer is not None:
259 self.print(trailer.rstrip("\n"))
260
261 def alts_uses_locations(self, alts: Sequence[Alt]) -> bool:
262 for alt in alts:
263 if alt.action and "LOCATIONS" in alt.action:
264 return True
265 for n in alt.items:
266 if isinstance(n.item, Group) and self.alts_uses_locations(n.item.rhs.alts):
267 return True
268 return False
269
270 def visit_Rule(self, node: Rule) -> None:
271 is_loop = node.is_loop()
272 is_gather = node.is_gather()
273 rhs = node.flatten()
274 if node.left_recursive:
275 if node.leader:
276 self.print("@memoize_left_rec")
277 else:
278 # Non-leader rules in a cycle are not memoized,
279 # but they must still be logged.
280 self.print("@logger")
281 else:
282 self.print("@memoize")
283 node_type = node.type or "Any"
284 self.print(f"def {node.name}(self) -> Optional[{node_type}]:")
285 with self.indent():
286 self.print(f"# {node.name}: {rhs}")
287 self.print("mark = self._mark()")
288 if self.alts_uses_locations(node.rhs.alts):
289 self.print("tok = self._tokenizer.peek()")
290 self.print("start_lineno, start_col_offset = tok.start")
291 if is_loop:
292 self.print("children = []")
293 self.visit(rhs, is_loop=is_loop, is_gather=is_gather)
294 if is_loop:
295 self.print("return children")
296 else:
297 self.print("return None")
298
299 def visit_NamedItem(self, node: NamedItem) -> None:
300 name, call = self.callmakervisitor.visit(node.item)
301 if node.name:
302 name = node.name
303 if not name:
304 self.print(call)
305 else:
306 if name != "cut":
307 name = self.dedupe(name)
308 self.print(f"({name} := {call})")
309
310 def visit_Rhs(self, node: Rhs, is_loop: bool = False, is_gather: bool = False) -> None:
311 if is_loop:
312 assert len(node.alts) == 1
313 for alt in node.alts:
314 self.visit(alt, is_loop=is_loop, is_gather=is_gather)
315
316 def visit_Alt(self, node: Alt, is_loop: bool, is_gather: bool) -> None:
317 has_cut = any(isinstance(item.item, Cut) for item in node.items)
318 with self.local_variable_context():
319 if has_cut:
320 self.print("cut = False")
321 if is_loop:
322 self.print("while (")
323 else:
324 self.print("if (")
325 with self.indent():
326 first = True
327 for item in node.items:
328 if first:
329 first = False
330 else:
331 self.print("and")
332 self.visit(item)
333 if is_gather:
334 self.print("is not None")
335
336 self.print("):")
337 with self.indent():
338 action = node.action
339 if not action:
340 if is_gather:
341 assert len(self.local_variable_names) == 2
342 action = (
343 f"[{self.local_variable_names[0]}] + {self.local_variable_names[1]}"
344 )
345 else:
346 if self.invalidvisitor.visit(node):
347 action = "UNREACHABLE"
348 elif len(self.local_variable_names) == 1:
349 action = f"{self.local_variable_names[0]}"
350 else:
351 action = f"[{', '.join(self.local_variable_names)}]"
352 elif "LOCATIONS" in action:
353 self.print("tok = self._tokenizer.get_last_non_whitespace_token()")
354 self.print("end_lineno, end_col_offset = tok.end")
355 action = action.replace("LOCATIONS", self.location_formatting)
356
357 if is_loop:
358 self.print(f"children.append({action})")
359 self.print("mark = self._mark()")
360 else:
361 if "UNREACHABLE" in action:
362 action = action.replace("UNREACHABLE", self.unreachable_formatting)
363 self.print(f"return {action}")
364
365 self.print("self._reset(mark)")
366 # Skip remaining alternatives if a cut was reached.
367 if has_cut:
368 self.print("if cut: return None")