| 1 |
import ast |
| 2 |
import os.path |
| 3 |
import re |
| 4 |
from collections.abc import Callable |
| 5 |
from dataclasses import dataclass, field |
| 6 |
from enum import Enum |
| 7 |
from typing import IO, Any |
| 8 |
|
| 9 |
from pegen import grammar |
| 10 |
from pegen.grammar import ( |
| 11 |
Alt, |
| 12 |
Cut, |
| 13 |
Forced, |
| 14 |
Gather, |
| 15 |
GrammarVisitor, |
| 16 |
Group, |
| 17 |
Leaf, |
| 18 |
Lookahead, |
| 19 |
NamedItem, |
| 20 |
NameLeaf, |
| 21 |
NegativeLookahead, |
| 22 |
Opt, |
| 23 |
PositiveLookahead, |
| 24 |
Repeat0, |
| 25 |
Repeat1, |
| 26 |
Rhs, |
| 27 |
Rule, |
| 28 |
StringLeaf, |
| 29 |
) |
| 30 |
from pegen.parser_generator import ParserGenerator |
| 31 |
|
| 32 |
EXTENSION_PREFIX = """\ |
| 33 |
#include "pegen.h" |
| 34 |
|
| 35 |
#if defined(Py_DEBUG) && defined(Py_BUILD_CORE) |
| 36 |
# define D(x) if (p->debug) { x; } |
| 37 |
#else |
| 38 |
# define D(x) |
| 39 |
#endif |
| 40 |
|
| 41 |
#ifdef __wasi__ |
| 42 |
# ifdef Py_DEBUG |
| 43 |
# define MAXSTACK 1000 |
| 44 |
# else |
| 45 |
# define MAXSTACK 4000 |
| 46 |
# endif |
| 47 |
#else |
| 48 |
# define MAXSTACK 6000 |
| 49 |
#endif |
| 50 |
|
| 51 |
""" |
| 52 |
|
| 53 |
|
| 54 |
EXTENSION_SUFFIX = """ |
| 55 |
void * |
| 56 |
_PyPegen_parse(Parser *p) |
| 57 |
{ |
| 58 |
// Initialize keywords |
| 59 |
p->keywords = reserved_keywords; |
| 60 |
p->n_keyword_lists = n_keyword_lists; |
| 61 |
p->soft_keywords = soft_keywords; |
| 62 |
|
| 63 |
return start_rule(p); |
| 64 |
} |
| 65 |
""" |
| 66 |
|
| 67 |
|
| 68 |
class NodeTypes(Enum): |
| 69 |
NAME_TOKEN = 0 |
| 70 |
NUMBER_TOKEN = 1 |
| 71 |
STRING_TOKEN = 2 |
| 72 |
GENERIC_TOKEN = 3 |
| 73 |
KEYWORD = 4 |
| 74 |
SOFT_KEYWORD = 5 |
| 75 |
CUT_OPERATOR = 6 |
| 76 |
F_STRING_CHUNK = 7 |
| 77 |
|
| 78 |
|
| 79 |
BASE_NODETYPES = { |
| 80 |
"NAME": NodeTypes.NAME_TOKEN, |
| 81 |
"NUMBER": NodeTypes.NUMBER_TOKEN, |
| 82 |
"STRING": NodeTypes.STRING_TOKEN, |
| 83 |
"SOFT_KEYWORD": NodeTypes.SOFT_KEYWORD, |
| 84 |
} |
| 85 |
|
| 86 |
|
| 87 |
@dataclass |
| 88 |
class FunctionCall: |
| 89 |
function: str |
| 90 |
arguments: list[Any] = field(default_factory=list) |
| 91 |
assigned_variable: str | None = None |
| 92 |
assigned_variable_type: str | None = None |
| 93 |
return_type: str | None = None |
| 94 |
nodetype: NodeTypes | None = None |
| 95 |
force_true: bool = False |
| 96 |
comment: str | None = None |
| 97 |
|
| 98 |
def __str__(self) -> str: |
| 99 |
parts = [] |
| 100 |
parts.append(self.function) |
| 101 |
if self.arguments: |
| 102 |
parts.append(f"({', '.join(map(str, self.arguments))})") |
| 103 |
if self.force_true: |
| 104 |
parts.append(", !p->error_indicator") |
| 105 |
if self.assigned_variable: |
| 106 |
if self.assigned_variable_type: |
| 107 |
parts = [ |
| 108 |
"(", |
| 109 |
self.assigned_variable, |
| 110 |
" = ", |
| 111 |
"(", |
| 112 |
self.assigned_variable_type, |
| 113 |
")", |
| 114 |
*parts, |
| 115 |
")", |
| 116 |
] |
| 117 |
else: |
| 118 |
parts = ["(", self.assigned_variable, " = ", *parts, ")"] |
| 119 |
if self.comment: |
| 120 |
parts.append(f" // {self.comment}") |
| 121 |
return "".join(parts) |
| 122 |
|
| 123 |
|
| 124 |
class CCallMakerVisitor(GrammarVisitor): |
| 125 |
def __init__( |
| 126 |
self, |
| 127 |
parser_generator: ParserGenerator, |
| 128 |
exact_tokens: dict[str, int], |
| 129 |
non_exact_tokens: set[str], |
| 130 |
): |
| 131 |
self.gen = parser_generator |
| 132 |
self.exact_tokens = exact_tokens |
| 133 |
self.non_exact_tokens = non_exact_tokens |
| 134 |
self.cache: dict[str, str] = {} |
| 135 |
self.cleanup_statements: list[str] = [] |
| 136 |
|
| 137 |
def keyword_helper(self, keyword: str) -> FunctionCall: |
| 138 |
return FunctionCall( |
| 139 |
assigned_variable="_keyword", |
| 140 |
function="_PyPegen_expect_token", |
| 141 |
arguments=["p", self.gen.keywords[keyword]], |
| 142 |
return_type="Token *", |
| 143 |
nodetype=NodeTypes.KEYWORD, |
| 144 |
comment=f"token='{keyword}'", |
| 145 |
) |
| 146 |
|
| 147 |
def soft_keyword_helper(self, value: str) -> FunctionCall: |
| 148 |
return FunctionCall( |
| 149 |
assigned_variable="_keyword", |
| 150 |
function="_PyPegen_expect_soft_keyword", |
| 151 |
arguments=["p", value], |
| 152 |
return_type="expr_ty", |
| 153 |
nodetype=NodeTypes.SOFT_KEYWORD, |
| 154 |
comment=f"soft_keyword='{value}'", |
| 155 |
) |
| 156 |
|
| 157 |
def visit_NameLeaf(self, node: NameLeaf) -> FunctionCall: |
| 158 |
name = node.value |
| 159 |
if name in self.non_exact_tokens: |
| 160 |
if name in BASE_NODETYPES: |
| 161 |
return FunctionCall( |
| 162 |
assigned_variable=f"{name.lower()}_var", |
| 163 |
function=f"_PyPegen_{name.lower()}_token", |
| 164 |
arguments=["p"], |
| 165 |
nodetype=BASE_NODETYPES[name], |
| 166 |
return_type="expr_ty", |
| 167 |
comment=name, |
| 168 |
) |
| 169 |
return FunctionCall( |
| 170 |
assigned_variable=f"{name.lower()}_var", |
| 171 |
function="_PyPegen_expect_token", |
| 172 |
arguments=["p", name], |
| 173 |
nodetype=NodeTypes.GENERIC_TOKEN, |
| 174 |
return_type="Token *", |
| 175 |
comment=f"token='{name}'", |
| 176 |
) |
| 177 |
|
| 178 |
type = None |
| 179 |
rule = self.gen.all_rules.get(name.lower()) |
| 180 |
if rule is not None: |
| 181 |
type = "asdl_seq *" if rule.is_loop() or rule.is_gather() else rule.type |
| 182 |
|
| 183 |
return FunctionCall( |
| 184 |
assigned_variable=f"{name}_var", |
| 185 |
function=f"{name}_rule", |
| 186 |
arguments=["p"], |
| 187 |
return_type=type, |
| 188 |
comment=f"{node}", |
| 189 |
) |
| 190 |
|
| 191 |
def visit_StringLeaf(self, node: StringLeaf) -> FunctionCall: |
| 192 |
val = ast.literal_eval(node.value) |
| 193 |
if re.match(r"[a-zA-Z_]\w*\Z", val): # This is a keyword |
| 194 |
if node.value.endswith("'"): |
| 195 |
return self.keyword_helper(val) |
| 196 |
else: |
| 197 |
return self.soft_keyword_helper(node.value) |
| 198 |
else: |
| 199 |
assert val in self.exact_tokens, f"{node.value} is not a known literal" |
| 200 |
type = self.exact_tokens[val] |
| 201 |
return FunctionCall( |
| 202 |
assigned_variable="_literal", |
| 203 |
function="_PyPegen_expect_token", |
| 204 |
arguments=["p", type], |
| 205 |
nodetype=NodeTypes.GENERIC_TOKEN, |
| 206 |
return_type="Token *", |
| 207 |
comment=f"token='{val}'", |
| 208 |
) |
| 209 |
|
| 210 |
def visit_NamedItem(self, node: NamedItem) -> FunctionCall: |
| 211 |
call = self.generate_call(node.item) |
| 212 |
if node.name: |
| 213 |
call.assigned_variable = node.name |
| 214 |
if node.type: |
| 215 |
call.assigned_variable_type = node.type |
| 216 |
return call |
| 217 |
|
| 218 |
def assert_no_undefined_behavior( |
| 219 |
self, call: FunctionCall, wrapper: str, expected_rtype: str | None, |
| 220 |
) -> None: |
| 221 |
if call.return_type != expected_rtype: |
| 222 |
raise RuntimeError( |
| 223 |
f"{call.function} return type is incompatible with {wrapper}: " |
| 224 |
f"expect: {expected_rtype}, actual: {call.return_type}" |
| 225 |
) |
| 226 |
|
| 227 |
def lookahead_call_helper(self, node: Lookahead, positive: int) -> FunctionCall: |
| 228 |
call = self.generate_call(node.node) |
| 229 |
comment = None |
| 230 |
if call.nodetype is NodeTypes.NAME_TOKEN: |
| 231 |
function = "_PyPegen_lookahead_for_expr" |
| 232 |
self.assert_no_undefined_behavior(call, function, "expr_ty") |
| 233 |
elif call.nodetype is NodeTypes.STRING_TOKEN: |
| 234 |
# _PyPegen_string_token() returns 'void *' instead of 'Token *'; |
| 235 |
# in addition, the overall function call would return 'expr_ty'. |
| 236 |
assert call.function == "_PyPegen_string_token" |
| 237 |
function = "_PyPegen_lookahead" |
| 238 |
self.assert_no_undefined_behavior(call, function, "expr_ty") |
| 239 |
elif call.nodetype == NodeTypes.SOFT_KEYWORD: |
| 240 |
function = "_PyPegen_lookahead_with_string" |
| 241 |
self.assert_no_undefined_behavior(call, function, "expr_ty") |
| 242 |
elif call.nodetype in {NodeTypes.GENERIC_TOKEN, NodeTypes.KEYWORD}: |
| 243 |
function = "_PyPegen_lookahead_with_int" |
| 244 |
self.assert_no_undefined_behavior(call, function, "Token *") |
| 245 |
comment = f"token={node.node}" |
| 246 |
elif call.return_type == "expr_ty": |
| 247 |
function = "_PyPegen_lookahead_for_expr" |
| 248 |
elif call.return_type == "stmt_ty": |
| 249 |
function = "_PyPegen_lookahead_for_stmt" |
| 250 |
else: |
| 251 |
function = "_PyPegen_lookahead" |
| 252 |
self.assert_no_undefined_behavior(call, function, None) |
| 253 |
return FunctionCall( |
| 254 |
function=function, |
| 255 |
arguments=[positive, call.function, *call.arguments], |
| 256 |
return_type="int", |
| 257 |
comment=comment, |
| 258 |
) |
| 259 |
|
| 260 |
def visit_PositiveLookahead(self, node: PositiveLookahead) -> FunctionCall: |
| 261 |
return self.lookahead_call_helper(node, 1) |
| 262 |
|
| 263 |
def visit_NegativeLookahead(self, node: NegativeLookahead) -> FunctionCall: |
| 264 |
return self.lookahead_call_helper(node, 0) |
| 265 |
|
| 266 |
def visit_Forced(self, node: Forced) -> FunctionCall: |
| 267 |
call = self.generate_call(node.node) |
| 268 |
if isinstance(node.node, Leaf): |
| 269 |
assert isinstance(node.node, Leaf) |
| 270 |
val = ast.literal_eval(node.node.value) |
| 271 |
assert val in self.exact_tokens, f"{node.node.value} is not a known literal" |
| 272 |
type = self.exact_tokens[val] |
| 273 |
return FunctionCall( |
| 274 |
assigned_variable="_literal", |
| 275 |
function="_PyPegen_expect_forced_token", |
| 276 |
arguments=["p", type, f'"{val}"'], |
| 277 |
nodetype=NodeTypes.GENERIC_TOKEN, |
| 278 |
return_type="Token *", |
| 279 |
comment=f"forced_token='{val}'", |
| 280 |
) |
| 281 |
if isinstance(node.node, Group): |
| 282 |
call = self.visit(node.node.rhs) |
| 283 |
call.assigned_variable = None |
| 284 |
call.comment = None |
| 285 |
return FunctionCall( |
| 286 |
assigned_variable="_literal", |
| 287 |
function="_PyPegen_expect_forced_result", |
| 288 |
arguments=["p", str(call), f'"{node.node.rhs!s}"'], |
| 289 |
return_type="void *", |
| 290 |
comment=f"forced_token=({node.node.rhs!s})", |
| 291 |
) |
| 292 |
else: |
| 293 |
raise NotImplementedError(f"Forced tokens don't work with {node.node} nodes") |
| 294 |
|
| 295 |
def visit_Opt(self, node: Opt) -> FunctionCall: |
| 296 |
call = self.generate_call(node.node) |
| 297 |
return FunctionCall( |
| 298 |
assigned_variable="_opt_var", |
| 299 |
function=call.function, |
| 300 |
arguments=call.arguments, |
| 301 |
force_true=True, |
| 302 |
comment=f"{node}", |
| 303 |
) |
| 304 |
|
| 305 |
def _generate_artificial_rule_call( |
| 306 |
self, |
| 307 |
node: Any, |
| 308 |
prefix: str, |
| 309 |
rule_generation_func: Callable[[], str], |
| 310 |
return_type: str | None = None, |
| 311 |
) -> FunctionCall: |
| 312 |
node_str = f"{node}" |
| 313 |
key = f"{prefix}_{node_str}" |
| 314 |
if key in self.cache: |
| 315 |
name = self.cache[key] |
| 316 |
else: |
| 317 |
name = rule_generation_func() |
| 318 |
self.cache[key] = name |
| 319 |
|
| 320 |
return FunctionCall( |
| 321 |
assigned_variable=f"{name}_var", |
| 322 |
function=f"{name}_rule", |
| 323 |
arguments=["p"], |
| 324 |
return_type=return_type, |
| 325 |
comment=node_str, |
| 326 |
) |
| 327 |
|
| 328 |
def visit_Rhs(self, node: Rhs) -> FunctionCall: |
| 329 |
if node.can_be_inlined: |
| 330 |
return self.generate_call(node.alts[0].items[0]) |
| 331 |
|
| 332 |
return self._generate_artificial_rule_call( |
| 333 |
node, |
| 334 |
"rhs", |
| 335 |
lambda: self.gen.artificial_rule_from_rhs(node), |
| 336 |
) |
| 337 |
|
| 338 |
def visit_Repeat0(self, node: Repeat0) -> FunctionCall: |
| 339 |
return self._generate_artificial_rule_call( |
| 340 |
node, |
| 341 |
"repeat0", |
| 342 |
lambda: self.gen.artificial_rule_from_repeat(node.node, is_repeat1=False), |
| 343 |
"asdl_seq *", |
| 344 |
) |
| 345 |
|
| 346 |
def visit_Repeat1(self, node: Repeat1) -> FunctionCall: |
| 347 |
return self._generate_artificial_rule_call( |
| 348 |
node, |
| 349 |
"repeat1", |
| 350 |
lambda: self.gen.artificial_rule_from_repeat(node.node, is_repeat1=True), |
| 351 |
"asdl_seq *", |
| 352 |
) |
| 353 |
|
| 354 |
def visit_Gather(self, node: Gather) -> FunctionCall: |
| 355 |
return self._generate_artificial_rule_call( |
| 356 |
node, |
| 357 |
"gather", |
| 358 |
lambda: self.gen.artificial_rule_from_gather(node), |
| 359 |
"asdl_seq *", |
| 360 |
) |
| 361 |
|
| 362 |
def visit_Group(self, node: Group) -> FunctionCall: |
| 363 |
return self.generate_call(node.rhs) |
| 364 |
|
| 365 |
def visit_Cut(self, node: Cut) -> FunctionCall: |
| 366 |
return FunctionCall( |
| 367 |
assigned_variable="_cut_var", |
| 368 |
return_type="int", |
| 369 |
function="1", |
| 370 |
nodetype=NodeTypes.CUT_OPERATOR, |
| 371 |
) |
| 372 |
|
| 373 |
def generate_call(self, node: Any) -> FunctionCall: |
| 374 |
return super().visit(node) |
| 375 |
|
| 376 |
|
| 377 |
class CParserGenerator(ParserGenerator, GrammarVisitor): |
| 378 |
def __init__( |
| 379 |
self, |
| 380 |
grammar: grammar.Grammar, |
| 381 |
tokens: dict[int, str], |
| 382 |
exact_tokens: dict[str, int], |
| 383 |
non_exact_tokens: set[str], |
| 384 |
file: IO[str] | None, |
| 385 |
debug: bool = False, |
| 386 |
skip_actions: bool = False, |
| 387 |
): |
| 388 |
super().__init__(grammar, set(tokens.values()), file) |
| 389 |
self.callmakervisitor: CCallMakerVisitor = CCallMakerVisitor( |
| 390 |
self, exact_tokens, non_exact_tokens |
| 391 |
) |
| 392 |
self._varname_counter = 0 |
| 393 |
self.debug = debug |
| 394 |
self.skip_actions = skip_actions |
| 395 |
self.cleanup_statements: list[str] = [] |
| 396 |
|
| 397 |
def add_level(self) -> None: |
| 398 |
self.print("if (p->level++ == MAXSTACK || _Py_ReachedRecursionLimitWithMargin(PyThreadState_Get(), 1)) {") |
| 399 |
with self.indent(): |
| 400 |
self.print("_Pypegen_stack_overflow(p);") |
| 401 |
self.print("}") |
| 402 |
|
| 403 |
def remove_level(self) -> None: |
| 404 |
self.print("p->level--;") |
| 405 |
|
| 406 |
def add_return(self, ret_val: str) -> None: |
| 407 |
for stmt in self.cleanup_statements: |
| 408 |
self.print(stmt) |
| 409 |
self.remove_level() |
| 410 |
self.print(f"return {ret_val};") |
| 411 |
|
| 412 |
def unique_varname(self, name: str = "tmpvar") -> str: |
| 413 |
new_var = name + "_" + str(self._varname_counter) |
| 414 |
self._varname_counter += 1 |
| 415 |
return new_var |
| 416 |
|
| 417 |
def call_with_errorcheck_return(self, call_text: str, returnval: str) -> None: |
| 418 |
error_var = self.unique_varname() |
| 419 |
self.print(f"int {error_var} = {call_text};") |
| 420 |
self.print(f"if ({error_var}) {{") |
| 421 |
with self.indent(): |
| 422 |
self.add_return(returnval) |
| 423 |
self.print("}") |
| 424 |
|
| 425 |
def call_with_errorcheck_goto(self, call_text: str, goto_target: str) -> None: |
| 426 |
error_var = self.unique_varname() |
| 427 |
self.print(f"int {error_var} = {call_text};") |
| 428 |
self.print(f"if ({error_var}) {{") |
| 429 |
with self.indent(): |
| 430 |
self.print(f"goto {goto_target};") |
| 431 |
self.print("}") |
| 432 |
|
| 433 |
def out_of_memory_return( |
| 434 |
self, |
| 435 |
expr: str, |
| 436 |
cleanup_code: str | None = None, |
| 437 |
) -> None: |
| 438 |
self.print(f"if ({expr}) {{") |
| 439 |
with self.indent(): |
| 440 |
if cleanup_code is not None: |
| 441 |
self.print(cleanup_code) |
| 442 |
self.print("p->error_indicator = 1;") |
| 443 |
self.print("PyErr_NoMemory();") |
| 444 |
self.add_return("NULL") |
| 445 |
self.print("}") |
| 446 |
|
| 447 |
def out_of_memory_goto(self, expr: str, goto_target: str) -> None: |
| 448 |
self.print(f"if ({expr}) {{") |
| 449 |
with self.indent(): |
| 450 |
self.print("PyErr_NoMemory();") |
| 451 |
self.print(f"goto {goto_target};") |
| 452 |
self.print("}") |
| 453 |
|
| 454 |
def generate(self, filename: str) -> None: |
| 455 |
self.collect_rules() |
| 456 |
basename = os.path.basename(filename) |
| 457 |
self.print(f"// @generated by pegen from {basename}") |
| 458 |
header = self.grammar.metas.get("header", EXTENSION_PREFIX) |
| 459 |
if header: |
| 460 |
self.print(header.rstrip("\n")) |
| 461 |
subheader = self.grammar.metas.get("subheader", "") |
| 462 |
if subheader: |
| 463 |
self.print(subheader) |
| 464 |
self._setup_keywords() |
| 465 |
self._setup_soft_keywords() |
| 466 |
for i, (rulename, rule) in enumerate(self.all_rules.items(), 1000): |
| 467 |
comment = " // Left-recursive" if rule.left_recursive else "" |
| 468 |
self.print(f"#define {rulename}_type {i}{comment}") |
| 469 |
self.print() |
| 470 |
for rulename, rule in self.all_rules.items(): |
| 471 |
if rule.is_loop() or rule.is_gather(): |
| 472 |
type = "asdl_seq *" |
| 473 |
elif rule.type: |
| 474 |
type = rule.type + " " |
| 475 |
else: |
| 476 |
type = "void *" |
| 477 |
self.print(f"static {type}{rulename}_rule(Parser *p);") |
| 478 |
self.print() |
| 479 |
for rulename, rule in list(self.all_rules.items()): |
| 480 |
self.print() |
| 481 |
if rule.left_recursive: |
| 482 |
self.print("// Left-recursive") |
| 483 |
self.visit(rule) |
| 484 |
if self.skip_actions: |
| 485 |
mode = 0 |
| 486 |
else: |
| 487 |
mode = int(self.rules["start"].type == "mod_ty") if "start" in self.rules else 1 |
| 488 |
if mode == 1 and self.grammar.metas.get("bytecode"): |
| 489 |
mode += 1 |
| 490 |
modulename = self.grammar.metas.get("modulename", "parse") |
| 491 |
trailer = self.grammar.metas.get("trailer", EXTENSION_SUFFIX) |
| 492 |
if trailer: |
| 493 |
self.print(trailer.rstrip("\n") % dict(mode=mode, modulename=modulename)) |
| 494 |
|
| 495 |
def _group_keywords_by_length(self) -> dict[int, list[tuple[str, int]]]: |
| 496 |
groups: dict[int, list[tuple[str, int]]] = {} |
| 497 |
for keyword_str, keyword_type in self.keywords.items(): |
| 498 |
length = len(keyword_str) |
| 499 |
if length in groups: |
| 500 |
groups[length].append((keyword_str, keyword_type)) |
| 501 |
else: |
| 502 |
groups[length] = [(keyword_str, keyword_type)] |
| 503 |
return groups |
| 504 |
|
| 505 |
def _setup_keywords(self) -> None: |
| 506 |
n_keyword_lists = ( |
| 507 |
len(max(self.keywords.keys(), key=len)) + 1 if len(self.keywords) > 0 else 0 |
| 508 |
) |
| 509 |
self.print(f"static const int n_keyword_lists = {n_keyword_lists};") |
| 510 |
groups = self._group_keywords_by_length() |
| 511 |
self.print("static KeywordToken *reserved_keywords[] = {") |
| 512 |
with self.indent(): |
| 513 |
num_groups = max(groups) + 1 if groups else 1 |
| 514 |
for keywords_length in range(num_groups): |
| 515 |
if keywords_length not in groups.keys(): |
| 516 |
self.print("(KeywordToken[]) {{NULL, -1}},") |
| 517 |
else: |
| 518 |
self.print("(KeywordToken[]) {") |
| 519 |
with self.indent(): |
| 520 |
for keyword_str, keyword_type in groups[keywords_length]: |
| 521 |
self.print(f'{{"{keyword_str}", {keyword_type}}},') |
| 522 |
self.print("{NULL, -1},") |
| 523 |
self.print("},") |
| 524 |
self.print("};") |
| 525 |
|
| 526 |
def _setup_soft_keywords(self) -> None: |
| 527 |
soft_keywords = sorted(self.soft_keywords) |
| 528 |
self.print("static char *soft_keywords[] = {") |
| 529 |
with self.indent(): |
| 530 |
for keyword in soft_keywords: |
| 531 |
self.print(f'"{keyword}",') |
| 532 |
self.print("NULL,") |
| 533 |
self.print("};") |
| 534 |
|
| 535 |
def _set_up_token_start_metadata_extraction(self) -> None: |
| 536 |
self.print("if (p->mark == p->fill && _PyPegen_fill_token(p) < 0) {") |
| 537 |
with self.indent(): |
| 538 |
self.print("p->error_indicator = 1;") |
| 539 |
self.add_return("NULL") |
| 540 |
self.print("}") |
| 541 |
self.print("int _start_lineno = p->tokens[_mark]->lineno;") |
| 542 |
self.print("UNUSED(_start_lineno); // Only used by EXTRA macro") |
| 543 |
self.print("int _start_col_offset = p->tokens[_mark]->col_offset;") |
| 544 |
self.print("UNUSED(_start_col_offset); // Only used by EXTRA macro") |
| 545 |
|
| 546 |
def _set_up_token_end_metadata_extraction(self) -> None: |
| 547 |
self.print("Token *_token = _PyPegen_get_last_nonnwhitespace_token(p);") |
| 548 |
self.print("if (_token == NULL) {") |
| 549 |
with self.indent(): |
| 550 |
self.add_return("NULL") |
| 551 |
self.print("}") |
| 552 |
self.print("int _end_lineno = _token->end_lineno;") |
| 553 |
self.print("UNUSED(_end_lineno); // Only used by EXTRA macro") |
| 554 |
self.print("int _end_col_offset = _token->end_col_offset;") |
| 555 |
self.print("UNUSED(_end_col_offset); // Only used by EXTRA macro") |
| 556 |
|
| 557 |
def _check_for_errors(self) -> None: |
| 558 |
self.print("if (p->error_indicator) {") |
| 559 |
with self.indent(): |
| 560 |
self.add_return("NULL") |
| 561 |
self.print("}") |
| 562 |
|
| 563 |
def _set_up_rule_memoization(self, node: Rule, result_type: str) -> None: |
| 564 |
self.print("{") |
| 565 |
with self.indent(): |
| 566 |
self.add_level() |
| 567 |
self.print(f"{result_type} _res = NULL;") |
| 568 |
self.print(f"if (_PyPegen_is_memoized(p, {node.name}_type, &_res)) {{") |
| 569 |
with self.indent(): |
| 570 |
self.add_return("_res") |
| 571 |
self.print("}") |
| 572 |
self.print("int _mark = p->mark;") |
| 573 |
self.print("int _resmark = p->mark;") |
| 574 |
self.print("while (1) {") |
| 575 |
with self.indent(): |
| 576 |
self.call_with_errorcheck_return( |
| 577 |
f"_PyPegen_update_memo(p, _mark, {node.name}_type, _res)", "_res" |
| 578 |
) |
| 579 |
self.print("p->mark = _mark;") |
| 580 |
self.print(f"void *_raw = {node.name}_raw(p);") |
| 581 |
self.print("if (p->error_indicator) {") |
| 582 |
with self.indent(): |
| 583 |
self.add_return("NULL") |
| 584 |
self.print("}") |
| 585 |
self.print("if (_raw == NULL || p->mark <= _resmark)") |
| 586 |
with self.indent(): |
| 587 |
self.print("break;") |
| 588 |
self.print("_resmark = p->mark;") |
| 589 |
self.print("_res = _raw;") |
| 590 |
self.print("}") |
| 591 |
self.print("p->mark = _resmark;") |
| 592 |
self.add_return("_res") |
| 593 |
self.print("}") |
| 594 |
self.print(f"static {result_type}") |
| 595 |
self.print(f"{node.name}_raw(Parser *p)") |
| 596 |
|
| 597 |
def _should_memoize(self, node: Rule) -> bool: |
| 598 |
return node.memo and not node.left_recursive |
| 599 |
|
| 600 |
def _handle_default_rule_body(self, node: Rule, rhs: Rhs, result_type: str) -> None: |
| 601 |
memoize = self._should_memoize(node) |
| 602 |
|
| 603 |
with self.indent(): |
| 604 |
self.add_level() |
| 605 |
self._check_for_errors() |
| 606 |
self.print(f"{result_type} _res = NULL;") |
| 607 |
if memoize: |
| 608 |
self.print(f"if (_PyPegen_is_memoized(p, {node.name}_type, &_res)) {{") |
| 609 |
with self.indent(): |
| 610 |
self.add_return("_res") |
| 611 |
self.print("}") |
| 612 |
self.print("int _mark = p->mark;") |
| 613 |
if any(alt.action and "EXTRA" in alt.action for alt in rhs.alts): |
| 614 |
self._set_up_token_start_metadata_extraction() |
| 615 |
self.visit( |
| 616 |
rhs, |
| 617 |
is_loop=False, |
| 618 |
is_gather=node.is_gather(), |
| 619 |
rulename=node.name, |
| 620 |
) |
| 621 |
if self.debug: |
| 622 |
self.print(f'D(fprintf(stderr, "Fail at %d: {node.name}\\n", p->mark));') |
| 623 |
self.print("_res = NULL;") |
| 624 |
self.print(" done:") |
| 625 |
with self.indent(): |
| 626 |
if memoize: |
| 627 |
self.print(f"_PyPegen_insert_memo(p, _mark, {node.name}_type, _res);") |
| 628 |
self.add_return("_res") |
| 629 |
|
| 630 |
def _handle_loop_rule_body(self, node: Rule, rhs: Rhs) -> None: |
| 631 |
memoize = self._should_memoize(node) |
| 632 |
is_repeat1 = node.name.startswith("_loop1") |
| 633 |
|
| 634 |
with self.indent(): |
| 635 |
self.add_level() |
| 636 |
self._check_for_errors() |
| 637 |
self.print("void *_res = NULL;") |
| 638 |
if memoize: |
| 639 |
self.print(f"if (_PyPegen_is_memoized(p, {node.name}_type, &_res)) {{") |
| 640 |
with self.indent(): |
| 641 |
self.add_return("_res") |
| 642 |
self.print("}") |
| 643 |
self.print("int _mark = p->mark;") |
| 644 |
if memoize: |
| 645 |
self.print("int _start_mark = p->mark;") |
| 646 |
self.print("void **_children = PyMem_Malloc(sizeof(void *));") |
| 647 |
self.out_of_memory_return("!_children") |
| 648 |
self.print("Py_ssize_t _children_capacity = 1;") |
| 649 |
self.print("Py_ssize_t _n = 0;") |
| 650 |
if any(alt.action and "EXTRA" in alt.action for alt in rhs.alts): |
| 651 |
self._set_up_token_start_metadata_extraction() |
| 652 |
self.visit( |
| 653 |
rhs, |
| 654 |
is_loop=True, |
| 655 |
is_gather=node.is_gather(), |
| 656 |
rulename=node.name, |
| 657 |
) |
| 658 |
if is_repeat1: |
| 659 |
self.print("if (_n == 0 || p->error_indicator) {") |
| 660 |
with self.indent(): |
| 661 |
self.print("PyMem_Free(_children);") |
| 662 |
self.add_return("NULL") |
| 663 |
self.print("}") |
| 664 |
self.print("asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena);") |
| 665 |
self.out_of_memory_return("!_seq", cleanup_code="PyMem_Free(_children);") |
| 666 |
self.print("for (Py_ssize_t i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]);") |
| 667 |
self.print("PyMem_Free(_children);") |
| 668 |
if memoize and node.name: |
| 669 |
self.print(f"_PyPegen_insert_memo(p, _start_mark, {node.name}_type, _seq);") |
| 670 |
self.add_return("_seq") |
| 671 |
|
| 672 |
def visit_Rule(self, node: Rule) -> None: |
| 673 |
is_loop = node.is_loop() |
| 674 |
is_gather = node.is_gather() |
| 675 |
rhs = node.flatten() |
| 676 |
if is_loop or is_gather: |
| 677 |
result_type = "asdl_seq *" |
| 678 |
elif node.type: |
| 679 |
result_type = node.type |
| 680 |
else: |
| 681 |
result_type = "void *" |
| 682 |
|
| 683 |
for line in str(node).splitlines(): |
| 684 |
self.print(f"// {line}") |
| 685 |
if node.left_recursive and node.leader: |
| 686 |
self.print(f"static {result_type} {node.name}_raw(Parser *);") |
| 687 |
|
| 688 |
self.print(f"static {result_type}") |
| 689 |
self.print(f"{node.name}_rule(Parser *p)") |
| 690 |
|
| 691 |
if node.left_recursive and node.leader: |
| 692 |
self._set_up_rule_memoization(node, result_type) |
| 693 |
|
| 694 |
self.print("{") |
| 695 |
|
| 696 |
if node.name.endswith("without_invalid"): |
| 697 |
with self.indent(): |
| 698 |
self.print("int _prev_call_invalid = p->call_invalid_rules;") |
| 699 |
self.print("p->call_invalid_rules = 0;") |
| 700 |
self.cleanup_statements.append("p->call_invalid_rules = _prev_call_invalid;") |
| 701 |
|
| 702 |
if is_loop: |
| 703 |
self._handle_loop_rule_body(node, rhs) |
| 704 |
else: |
| 705 |
self._handle_default_rule_body(node, rhs, result_type) |
| 706 |
|
| 707 |
if node.name.endswith("without_invalid"): |
| 708 |
self.cleanup_statements.pop() |
| 709 |
|
| 710 |
self.print("}") |
| 711 |
|
| 712 |
def visit_NamedItem(self, node: NamedItem) -> None: |
| 713 |
call = self.callmakervisitor.generate_call(node) |
| 714 |
if call.assigned_variable: |
| 715 |
call.assigned_variable = self.dedupe(call.assigned_variable) |
| 716 |
self.print(call) |
| 717 |
|
| 718 |
def visit_Rhs( |
| 719 |
self, node: Rhs, is_loop: bool, is_gather: bool, rulename: str | None |
| 720 |
) -> None: |
| 721 |
if is_loop: |
| 722 |
assert len(node.alts) == 1 |
| 723 |
for alt in node.alts: |
| 724 |
self.visit(alt, is_loop=is_loop, is_gather=is_gather, rulename=rulename) |
| 725 |
|
| 726 |
def join_conditions(self, keyword: str, node: Any) -> None: |
| 727 |
self.print(f"{keyword} (") |
| 728 |
with self.indent(): |
| 729 |
first = True |
| 730 |
for item in node.items: |
| 731 |
if first: |
| 732 |
first = False |
| 733 |
else: |
| 734 |
self.print("&&") |
| 735 |
self.visit(item) |
| 736 |
self.print(")") |
| 737 |
|
| 738 |
def emit_action(self, node: Alt, cleanup_code: str | None = None) -> None: |
| 739 |
self.print(f"_res = {node.action};") |
| 740 |
|
| 741 |
self.print("if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) {") |
| 742 |
with self.indent(): |
| 743 |
self.print("p->error_indicator = 1;") |
| 744 |
if cleanup_code: |
| 745 |
self.print(cleanup_code) |
| 746 |
self.add_return("NULL") |
| 747 |
self.print("}") |
| 748 |
|
| 749 |
if self.debug: |
| 750 |
self.print( |
| 751 |
f'D(fprintf(stderr, "Hit with action [%d-%d]: %s\\n", _mark, p->mark, "{node}"));' |
| 752 |
) |
| 753 |
|
| 754 |
def emit_default_action(self, is_gather: bool, node: Alt) -> None: |
| 755 |
if len(self.local_variable_names) > 1: |
| 756 |
if is_gather: |
| 757 |
assert len(self.local_variable_names) == 2 |
| 758 |
self.print( |
| 759 |
f"_res = _PyPegen_seq_insert_in_front(p, " |
| 760 |
f"{self.local_variable_names[0]}, {self.local_variable_names[1]});" |
| 761 |
) |
| 762 |
else: |
| 763 |
if self.debug: |
| 764 |
self.print( |
| 765 |
f'D(fprintf(stderr, "Hit without action [%d:%d]: %s\\n", _mark, p->mark, "{node}"));' |
| 766 |
) |
| 767 |
self.print( |
| 768 |
f"_res = _PyPegen_dummy_name(p, {', '.join(self.local_variable_names)});" |
| 769 |
) |
| 770 |
else: |
| 771 |
if self.debug: |
| 772 |
self.print( |
| 773 |
f'D(fprintf(stderr, "Hit with default action [%d:%d]: %s\\n", _mark, p->mark, "{node}"));' |
| 774 |
) |
| 775 |
self.print(f"_res = {self.local_variable_names[0]};") |
| 776 |
|
| 777 |
def emit_dummy_action(self) -> None: |
| 778 |
self.print("_res = _PyPegen_dummy_name(p);") |
| 779 |
|
| 780 |
def handle_alt_normal(self, node: Alt, is_gather: bool, rulename: str | None) -> None: |
| 781 |
self.join_conditions(keyword="if", node=node) |
| 782 |
self.print("{") |
| 783 |
# We have parsed successfully all the conditions for the option. |
| 784 |
with self.indent(): |
| 785 |
node_str = str(node).replace('"', '\\"') |
| 786 |
self.print( |
| 787 |
f'D(fprintf(stderr, "%*c+ {rulename}[%d-%d]: %s succeeded!\\n", p->level, \' \', _mark, p->mark, "{node_str}"));' |
| 788 |
) |
| 789 |
# Prepare to emit the rule action and do so |
| 790 |
if node.action and "EXTRA" in node.action: |
| 791 |
self._set_up_token_end_metadata_extraction() |
| 792 |
if self.skip_actions: |
| 793 |
self.emit_dummy_action() |
| 794 |
elif node.action: |
| 795 |
self.emit_action(node) |
| 796 |
else: |
| 797 |
self.emit_default_action(is_gather, node) |
| 798 |
|
| 799 |
# As the current option has parsed correctly, do not continue with the rest. |
| 800 |
self.print("goto done;") |
| 801 |
self.print("}") |
| 802 |
|
| 803 |
def handle_alt_loop(self, node: Alt, is_gather: bool, rulename: str | None) -> None: |
| 804 |
# Condition of the main body of the alternative |
| 805 |
self.join_conditions(keyword="while", node=node) |
| 806 |
self.print("{") |
| 807 |
# We have parsed successfully one item! |
| 808 |
with self.indent(): |
| 809 |
# Prepare to emit the rule action and do so |
| 810 |
if node.action and "EXTRA" in node.action: |
| 811 |
self._set_up_token_end_metadata_extraction() |
| 812 |
if self.skip_actions: |
| 813 |
self.emit_dummy_action() |
| 814 |
elif node.action: |
| 815 |
self.emit_action(node, cleanup_code="PyMem_Free(_children);") |
| 816 |
else: |
| 817 |
self.emit_default_action(is_gather, node) |
| 818 |
|
| 819 |
# Add the result of rule to the temporary buffer of children. This buffer |
| 820 |
# will populate later an asdl_seq with all elements to return. |
| 821 |
self.print("if (_n == _children_capacity) {") |
| 822 |
with self.indent(): |
| 823 |
self.print("_children_capacity *= 2;") |
| 824 |
self.print( |
| 825 |
"void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *));" |
| 826 |
) |
| 827 |
self.out_of_memory_return("!_new_children", cleanup_code="PyMem_Free(_children);") |
| 828 |
self.print("_children = _new_children;") |
| 829 |
self.print("}") |
| 830 |
self.print("_children[_n++] = _res;") |
| 831 |
self.print("_mark = p->mark;") |
| 832 |
self.print("}") |
| 833 |
|
| 834 |
def visit_Alt( |
| 835 |
self, node: Alt, is_loop: bool, is_gather: bool, rulename: str | None |
| 836 |
) -> None: |
| 837 |
if len(node.items) == 1 and str(node.items[0]).startswith("invalid_"): |
| 838 |
self.print(f"if (p->call_invalid_rules) {{ // {node}") |
| 839 |
else: |
| 840 |
self.print(f"{{ // {node}") |
| 841 |
with self.indent(): |
| 842 |
self._check_for_errors() |
| 843 |
node_str = str(node).replace('"', '\\"') |
| 844 |
self.print( |
| 845 |
f'D(fprintf(stderr, "%*c> {rulename}[%d-%d]: %s\\n", p->level, \' \', _mark, p->mark, "{node_str}"));' |
| 846 |
) |
| 847 |
# Prepare variable declarations for the alternative |
| 848 |
vars = self.collect_vars(node) |
| 849 |
for v, var_type in sorted(item for item in vars.items() if item[0] is not None): |
| 850 |
if not var_type: |
| 851 |
var_type = "void *" |
| 852 |
else: |
| 853 |
var_type += " " |
| 854 |
if v == "_cut_var": |
| 855 |
v += " = 0" # cut_var must be initialized |
| 856 |
self.print(f"{var_type}{v};") |
| 857 |
if v and v.startswith("_opt_var"): |
| 858 |
self.print(f"UNUSED({v}); // Silence compiler warnings") |
| 859 |
|
| 860 |
with self.local_variable_context(): |
| 861 |
if is_loop: |
| 862 |
self.handle_alt_loop(node, is_gather, rulename) |
| 863 |
else: |
| 864 |
self.handle_alt_normal(node, is_gather, rulename) |
| 865 |
|
| 866 |
self.print("p->mark = _mark;") |
| 867 |
node_str = str(node).replace('"', '\\"') |
| 868 |
self.print( |
| 869 |
f"D(fprintf(stderr, \"%*c%s {rulename}[%d-%d]: %s failed!\\n\", p->level, ' ',\n" |
| 870 |
f' p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "{node_str}"));' |
| 871 |
) |
| 872 |
if "_cut_var" in vars: |
| 873 |
self.print("if (_cut_var) {") |
| 874 |
with self.indent(): |
| 875 |
self.add_return("NULL") |
| 876 |
self.print("}") |
| 877 |
self.print("}") |
| 878 |
|
| 879 |
def collect_vars(self, node: Alt) -> dict[str | None, str | None]: |
| 880 |
types = {} |
| 881 |
with self.local_variable_context(): |
| 882 |
for item in node.items: |
| 883 |
name, type = self.add_var(item) |
| 884 |
types[name] = type |
| 885 |
return types |
| 886 |
|
| 887 |
def add_var(self, node: NamedItem) -> tuple[str | None, str | None]: |
| 888 |
call = self.callmakervisitor.generate_call(node.item) |
| 889 |
name = node.name if node.name else call.assigned_variable |
| 890 |
if name is not None: |
| 891 |
name = self.dedupe(name) |
| 892 |
return_type = call.return_type if node.type is None else node.type |
| 893 |
return name, return_type |