| 1 |
import os.path |
| 2 |
import token |
| 3 |
from collections.abc import Callable, Sequence |
| 4 |
from typing import IO, Any |
| 5 |
|
| 6 |
from pegen import grammar |
| 7 |
from pegen.grammar import ( |
| 8 |
Alt, |
| 9 |
Cut, |
| 10 |
Forced, |
| 11 |
Gather, |
| 12 |
GrammarVisitor, |
| 13 |
Group, |
| 14 |
Lookahead, |
| 15 |
NamedItem, |
| 16 |
NameLeaf, |
| 17 |
NegativeLookahead, |
| 18 |
Opt, |
| 19 |
PositiveLookahead, |
| 20 |
Repeat0, |
| 21 |
Repeat1, |
| 22 |
Rhs, |
| 23 |
Rule, |
| 24 |
StringLeaf, |
| 25 |
) |
| 26 |
from pegen.parser_generator import ParserGenerator |
| 27 |
|
| 28 |
MODULE_PREFIX = """\ |
| 29 |
#!/usr/bin/env python3.8 |
| 30 |
# @generated by pegen from {filename} |
| 31 |
|
| 32 |
import ast |
| 33 |
import sys |
| 34 |
import tokenize |
| 35 |
|
| 36 |
from typing import Any, Optional |
| 37 |
|
| 38 |
from pegen.parser import memoize, memoize_left_rec, logger, Parser |
| 39 |
|
| 40 |
""" |
| 41 |
MODULE_SUFFIX = """ |
| 42 |
|
| 43 |
if __name__ == '__main__': |
| 44 |
from pegen.parser import simple_parser_main |
| 45 |
simple_parser_main({class_name}) |
| 46 |
""" |
| 47 |
|
| 48 |
|
| 49 |
class InvalidNodeVisitor(GrammarVisitor): |
| 50 |
def visit_NameLeaf(self, node: NameLeaf) -> bool: |
| 51 |
name = node.value |
| 52 |
return name.startswith("invalid") |
| 53 |
|
| 54 |
def visit_StringLeaf(self, node: StringLeaf) -> bool: |
| 55 |
return False |
| 56 |
|
| 57 |
def visit_NamedItem(self, node: NamedItem) -> bool: |
| 58 |
return self.visit(node.item) |
| 59 |
|
| 60 |
def visit_Rhs(self, node: Rhs) -> bool: |
| 61 |
return any(self.visit(alt) for alt in node.alts) |
| 62 |
|
| 63 |
def visit_Alt(self, node: Alt) -> bool: |
| 64 |
return any(self.visit(item) for item in node.items) |
| 65 |
|
| 66 |
def lookahead_call_helper(self, node: Lookahead) -> bool: |
| 67 |
return self.visit(node.node) |
| 68 |
|
| 69 |
def visit_PositiveLookahead(self, node: PositiveLookahead) -> bool: |
| 70 |
return self.lookahead_call_helper(node) |
| 71 |
|
| 72 |
def visit_NegativeLookahead(self, node: NegativeLookahead) -> bool: |
| 73 |
return self.lookahead_call_helper(node) |
| 74 |
|
| 75 |
def visit_Opt(self, node: Opt) -> bool: |
| 76 |
return self.visit(node.node) |
| 77 |
|
| 78 |
def visit_Repeat(self, node: Repeat0) -> tuple[str, str]: |
| 79 |
return self.visit(node.node) |
| 80 |
|
| 81 |
def visit_Gather(self, node: Gather) -> tuple[str, str]: |
| 82 |
return self.visit(node.node) |
| 83 |
|
| 84 |
def visit_Group(self, node: Group) -> bool: |
| 85 |
return self.visit(node.rhs) |
| 86 |
|
| 87 |
def visit_Cut(self, node: Cut) -> bool: |
| 88 |
return False |
| 89 |
|
| 90 |
def visit_Forced(self, node: Forced) -> bool: |
| 91 |
return self.visit(node.node) |
| 92 |
|
| 93 |
|
| 94 |
class PythonCallMakerVisitor(GrammarVisitor): |
| 95 |
def __init__(self, parser_generator: ParserGenerator): |
| 96 |
self.gen = parser_generator |
| 97 |
self.cache: dict[str, tuple[str, str]] = {} |
| 98 |
|
| 99 |
def visit_NameLeaf(self, node: NameLeaf) -> tuple[str | None, str]: |
| 100 |
name = node.value |
| 101 |
if name == "SOFT_KEYWORD": |
| 102 |
return "soft_keyword", "self.soft_keyword()" |
| 103 |
if name in ("NAME", "NUMBER", "STRING", "OP", "TYPE_COMMENT", |
| 104 |
"FSTRING_END", "FSTRING_MIDDLE", "FSTRING_START"): |
| 105 |
name = name.lower() |
| 106 |
return name, f"self.{name}()" |
| 107 |
if name in ("NEWLINE", "DEDENT", "INDENT", "ENDMARKER"): |
| 108 |
# Avoid using names that can be Python keywords |
| 109 |
return "_" + name.lower(), f"self.expect({name!r})" |
| 110 |
return name, f"self.{name}()" |
| 111 |
|
| 112 |
def visit_StringLeaf(self, node: StringLeaf) -> tuple[str, str]: |
| 113 |
return "literal", f"self.expect({node.value})" |
| 114 |
|
| 115 |
def visit_NamedItem(self, node: NamedItem) -> tuple[str | None, str]: |
| 116 |
name, call = self.visit(node.item) |
| 117 |
if node.name: |
| 118 |
name = node.name |
| 119 |
return name, call |
| 120 |
|
| 121 |
def lookahead_call_helper(self, node: Lookahead) -> tuple[str, str]: |
| 122 |
name, call = self.visit(node.node) |
| 123 |
head, tail = call.split("(", 1) |
| 124 |
assert tail[-1] == ")" |
| 125 |
tail = tail[:-1] |
| 126 |
return head, tail |
| 127 |
|
| 128 |
def visit_PositiveLookahead(self, node: PositiveLookahead) -> tuple[None, str]: |
| 129 |
head, tail = self.lookahead_call_helper(node) |
| 130 |
return None, f"self.positive_lookahead({head}, {tail})" |
| 131 |
|
| 132 |
def visit_NegativeLookahead(self, node: NegativeLookahead) -> tuple[None, str]: |
| 133 |
head, tail = self.lookahead_call_helper(node) |
| 134 |
return None, f"self.negative_lookahead({head}, {tail})" |
| 135 |
|
| 136 |
def visit_Opt(self, node: Opt) -> tuple[str, str]: |
| 137 |
name, call = self.visit(node.node) |
| 138 |
# Note trailing comma (the call may already have one comma |
| 139 |
# at the end, for example when rules have both repeat0 and optional |
| 140 |
# markers, e.g: [rule*]) |
| 141 |
if call.endswith(","): |
| 142 |
return "opt", call |
| 143 |
else: |
| 144 |
return "opt", f"{call}," |
| 145 |
|
| 146 |
def _generate_artificial_rule_call( |
| 147 |
self, |
| 148 |
node: Any, |
| 149 |
prefix: str, |
| 150 |
call_by_name_func: Callable[[str], str], |
| 151 |
rule_generation_func: Callable[[], str], |
| 152 |
) -> tuple[str, str]: |
| 153 |
node_str = f"{node}" |
| 154 |
key = f"{prefix}_{node_str}" |
| 155 |
if key in self.cache: |
| 156 |
return self.cache[key] |
| 157 |
|
| 158 |
name = rule_generation_func() |
| 159 |
call = call_by_name_func(name) |
| 160 |
self.cache[key] = name, call |
| 161 |
return self.cache[key] |
| 162 |
|
| 163 |
def visit_Rhs(self, node: Rhs) -> tuple[str, str]: |
| 164 |
if len(node.alts) == 1 and len(node.alts[0].items) == 1: |
| 165 |
return self.visit(node.alts[0].items[0]) |
| 166 |
|
| 167 |
return self._generate_artificial_rule_call( |
| 168 |
node, |
| 169 |
"rhs", |
| 170 |
lambda name: f"self.{name}()", |
| 171 |
lambda: self.gen.artificial_rule_from_rhs(node), |
| 172 |
) |
| 173 |
|
| 174 |
def visit_Repeat0(self, node: Repeat0) -> tuple[str, str]: |
| 175 |
return self._generate_artificial_rule_call( |
| 176 |
node, |
| 177 |
"repeat0", |
| 178 |
lambda name: f"self.{name}(),", # Also a trailing comma! |
| 179 |
lambda: self.gen.artificial_rule_from_repeat(node.node, is_repeat1=False), |
| 180 |
) |
| 181 |
|
| 182 |
def visit_Repeat1(self, node: Repeat1) -> tuple[str, str]: |
| 183 |
return self._generate_artificial_rule_call( |
| 184 |
node, |
| 185 |
"repeat1", |
| 186 |
lambda name: f"self.{name}()", # But no trailing comma here! |
| 187 |
lambda: self.gen.artificial_rule_from_repeat(node.node, is_repeat1=True), |
| 188 |
) |
| 189 |
|
| 190 |
def visit_Gather(self, node: Gather) -> tuple[str, str]: |
| 191 |
return self._generate_artificial_rule_call( |
| 192 |
node, |
| 193 |
"gather", |
| 194 |
lambda name: f"self.{name}()", # No trailing comma here either! |
| 195 |
lambda: self.gen.artificial_rule_from_gather(node), |
| 196 |
) |
| 197 |
|
| 198 |
def visit_Group(self, node: Group) -> tuple[str | None, str]: |
| 199 |
return self.visit(node.rhs) |
| 200 |
|
| 201 |
def visit_Cut(self, node: Cut) -> tuple[str, str]: |
| 202 |
return "cut", "True" |
| 203 |
|
| 204 |
def visit_Forced(self, node: Forced) -> tuple[str, str]: |
| 205 |
if isinstance(node.node, Group): |
| 206 |
_, val = self.visit(node.node.rhs) |
| 207 |
return "forced", f"self.expect_forced({val}, '''({node.node.rhs!s})''')" |
| 208 |
else: |
| 209 |
return ( |
| 210 |
"forced", |
| 211 |
f"self.expect_forced(self.expect({node.node.value}), {node.node.value!r})", |
| 212 |
) |
| 213 |
|
| 214 |
|
| 215 |
class PythonParserGenerator(ParserGenerator, GrammarVisitor): |
| 216 |
def __init__( |
| 217 |
self, |
| 218 |
grammar: grammar.Grammar, |
| 219 |
file: IO[str] | None, |
| 220 |
tokens: set[str] = set(token.tok_name.values()), |
| 221 |
location_formatting: str | None = None, |
| 222 |
unreachable_formatting: str | None = None, |
| 223 |
): |
| 224 |
tokens.add("SOFT_KEYWORD") |
| 225 |
super().__init__(grammar, tokens, file) |
| 226 |
self.callmakervisitor: PythonCallMakerVisitor = PythonCallMakerVisitor(self) |
| 227 |
self.invalidvisitor: InvalidNodeVisitor = InvalidNodeVisitor() |
| 228 |
self.unreachable_formatting = unreachable_formatting or "None # pragma: no cover" |
| 229 |
self.location_formatting = ( |
| 230 |
location_formatting |
| 231 |
or "lineno=start_lineno, col_offset=start_col_offset, " |
| 232 |
"end_lineno=end_lineno, end_col_offset=end_col_offset" |
| 233 |
) |
| 234 |
|
| 235 |
def generate(self, filename: str) -> None: |
| 236 |
self.collect_rules() |
| 237 |
header = self.grammar.metas.get("header", MODULE_PREFIX) |
| 238 |
if header is not None: |
| 239 |
basename = os.path.basename(filename) |
| 240 |
self.print(header.rstrip("\n").format(filename=basename)) |
| 241 |
subheader = self.grammar.metas.get("subheader", "") |
| 242 |
if subheader: |
| 243 |
self.print(subheader) |
| 244 |
cls_name = self.grammar.metas.get("class", "GeneratedParser") |
| 245 |
self.print("# Keywords and soft keywords are listed at the end of the parser definition.") |
| 246 |
self.print(f"class {cls_name}(Parser):") |
| 247 |
for rule in self.all_rules.values(): |
| 248 |
self.print() |
| 249 |
with self.indent(): |
| 250 |
self.visit(rule) |
| 251 |
|
| 252 |
self.print() |
| 253 |
with self.indent(): |
| 254 |
self.print(f"KEYWORDS = {tuple(self.keywords)}") |
| 255 |
self.print(f"SOFT_KEYWORDS = {tuple(self.soft_keywords)}") |
| 256 |
|
| 257 |
trailer = self.grammar.metas.get("trailer", MODULE_SUFFIX.format(class_name=cls_name)) |
| 258 |
if trailer is not None: |
| 259 |
self.print(trailer.rstrip("\n")) |
| 260 |
|
| 261 |
def alts_uses_locations(self, alts: Sequence[Alt]) -> bool: |
| 262 |
for alt in alts: |
| 263 |
if alt.action and "LOCATIONS" in alt.action: |
| 264 |
return True |
| 265 |
for n in alt.items: |
| 266 |
if isinstance(n.item, Group) and self.alts_uses_locations(n.item.rhs.alts): |
| 267 |
return True |
| 268 |
return False |
| 269 |
|
| 270 |
def visit_Rule(self, node: Rule) -> None: |
| 271 |
is_loop = node.is_loop() |
| 272 |
is_gather = node.is_gather() |
| 273 |
rhs = node.flatten() |
| 274 |
if node.left_recursive: |
| 275 |
if node.leader: |
| 276 |
self.print("@memoize_left_rec") |
| 277 |
else: |
| 278 |
# Non-leader rules in a cycle are not memoized, |
| 279 |
# but they must still be logged. |
| 280 |
self.print("@logger") |
| 281 |
else: |
| 282 |
self.print("@memoize") |
| 283 |
node_type = node.type or "Any" |
| 284 |
self.print(f"def {node.name}(self) -> Optional[{node_type}]:") |
| 285 |
with self.indent(): |
| 286 |
self.print(f"# {node.name}: {rhs}") |
| 287 |
self.print("mark = self._mark()") |
| 288 |
if self.alts_uses_locations(node.rhs.alts): |
| 289 |
self.print("tok = self._tokenizer.peek()") |
| 290 |
self.print("start_lineno, start_col_offset = tok.start") |
| 291 |
if is_loop: |
| 292 |
self.print("children = []") |
| 293 |
self.visit(rhs, is_loop=is_loop, is_gather=is_gather) |
| 294 |
if is_loop: |
| 295 |
self.print("return children") |
| 296 |
else: |
| 297 |
self.print("return None") |
| 298 |
|
| 299 |
def visit_NamedItem(self, node: NamedItem) -> None: |
| 300 |
name, call = self.callmakervisitor.visit(node.item) |
| 301 |
if node.name: |
| 302 |
name = node.name |
| 303 |
if not name: |
| 304 |
self.print(call) |
| 305 |
else: |
| 306 |
if name != "cut": |
| 307 |
name = self.dedupe(name) |
| 308 |
self.print(f"({name} := {call})") |
| 309 |
|
| 310 |
def visit_Rhs(self, node: Rhs, is_loop: bool = False, is_gather: bool = False) -> None: |
| 311 |
if is_loop: |
| 312 |
assert len(node.alts) == 1 |
| 313 |
for alt in node.alts: |
| 314 |
self.visit(alt, is_loop=is_loop, is_gather=is_gather) |
| 315 |
|
| 316 |
def visit_Alt(self, node: Alt, is_loop: bool, is_gather: bool) -> None: |
| 317 |
has_cut = any(isinstance(item.item, Cut) for item in node.items) |
| 318 |
with self.local_variable_context(): |
| 319 |
if has_cut: |
| 320 |
self.print("cut = False") |
| 321 |
if is_loop: |
| 322 |
self.print("while (") |
| 323 |
else: |
| 324 |
self.print("if (") |
| 325 |
with self.indent(): |
| 326 |
first = True |
| 327 |
for item in node.items: |
| 328 |
if first: |
| 329 |
first = False |
| 330 |
else: |
| 331 |
self.print("and") |
| 332 |
self.visit(item) |
| 333 |
if is_gather: |
| 334 |
self.print("is not None") |
| 335 |
|
| 336 |
self.print("):") |
| 337 |
with self.indent(): |
| 338 |
action = node.action |
| 339 |
if not action: |
| 340 |
if is_gather: |
| 341 |
assert len(self.local_variable_names) == 2 |
| 342 |
action = ( |
| 343 |
f"[{self.local_variable_names[0]}] + {self.local_variable_names[1]}" |
| 344 |
) |
| 345 |
else: |
| 346 |
if self.invalidvisitor.visit(node): |
| 347 |
action = "UNREACHABLE" |
| 348 |
elif len(self.local_variable_names) == 1: |
| 349 |
action = f"{self.local_variable_names[0]}" |
| 350 |
else: |
| 351 |
action = f"[{', '.join(self.local_variable_names)}]" |
| 352 |
elif "LOCATIONS" in action: |
| 353 |
self.print("tok = self._tokenizer.get_last_non_whitespace_token()") |
| 354 |
self.print("end_lineno, end_col_offset = tok.end") |
| 355 |
action = action.replace("LOCATIONS", self.location_formatting) |
| 356 |
|
| 357 |
if is_loop: |
| 358 |
self.print(f"children.append({action})") |
| 359 |
self.print("mark = self._mark()") |
| 360 |
else: |
| 361 |
if "UNREACHABLE" in action: |
| 362 |
action = action.replace("UNREACHABLE", self.unreachable_formatting) |
| 363 |
self.print(f"return {action}") |
| 364 |
|
| 365 |
self.print("self._reset(mark)") |
| 366 |
# Skip remaining alternatives if a cut was reached. |
| 367 |
if has_cut: |
| 368 |
self.print("if cut: return None") |