Debian Codegraph @

Tools/peg_generator/pegen/c_generator.py

raw
1 import ast
2 import os.path
3 import re
4 from collections.abc import Callable
5 from dataclasses import dataclass, field
6 from enum import Enum
7 from typing import IO, Any
8
9 from pegen import grammar
10 from pegen.grammar import (
11 Alt,
12 Cut,
13 Forced,
14 Gather,
15 GrammarVisitor,
16 Group,
17 Leaf,
18 Lookahead,
19 NamedItem,
20 NameLeaf,
21 NegativeLookahead,
22 Opt,
23 PositiveLookahead,
24 Repeat0,
25 Repeat1,
26 Rhs,
27 Rule,
28 StringLeaf,
29 )
30 from pegen.parser_generator import ParserGenerator
31
32 EXTENSION_PREFIX = """\
33 #include "pegen.h"
34
35 #if defined(Py_DEBUG) && defined(Py_BUILD_CORE)
36 # define D(x) if (p->debug) { x; }
37 #else
38 # define D(x)
39 #endif
40
41 #ifdef __wasi__
42 # ifdef Py_DEBUG
43 # define MAXSTACK 1000
44 # else
45 # define MAXSTACK 4000
46 # endif
47 #else
48 # define MAXSTACK 6000
49 #endif
50
51 """
52
53
54 EXTENSION_SUFFIX = """
55 void *
56 _PyPegen_parse(Parser *p)
57 {
58 // Initialize keywords
59 p->keywords = reserved_keywords;
60 p->n_keyword_lists = n_keyword_lists;
61 p->soft_keywords = soft_keywords;
62
63 return start_rule(p);
64 }
65 """
66
67
68 class NodeTypes(Enum):
69 NAME_TOKEN = 0
70 NUMBER_TOKEN = 1
71 STRING_TOKEN = 2
72 GENERIC_TOKEN = 3
73 KEYWORD = 4
74 SOFT_KEYWORD = 5
75 CUT_OPERATOR = 6
76 F_STRING_CHUNK = 7
77
78
79 BASE_NODETYPES = {
80 "NAME": NodeTypes.NAME_TOKEN,
81 "NUMBER": NodeTypes.NUMBER_TOKEN,
82 "STRING": NodeTypes.STRING_TOKEN,
83 "SOFT_KEYWORD": NodeTypes.SOFT_KEYWORD,
84 }
85
86
87 @dataclass
88 class FunctionCall:
89 function: str
90 arguments: list[Any] = field(default_factory=list)
91 assigned_variable: str | None = None
92 assigned_variable_type: str | None = None
93 return_type: str | None = None
94 nodetype: NodeTypes | None = None
95 force_true: bool = False
96 comment: str | None = None
97
98 def __str__(self) -> str:
99 parts = []
100 parts.append(self.function)
101 if self.arguments:
102 parts.append(f"({', '.join(map(str, self.arguments))})")
103 if self.force_true:
104 parts.append(", !p->error_indicator")
105 if self.assigned_variable:
106 if self.assigned_variable_type:
107 parts = [
108 "(",
109 self.assigned_variable,
110 " = ",
111 "(",
112 self.assigned_variable_type,
113 ")",
114 *parts,
115 ")",
116 ]
117 else:
118 parts = ["(", self.assigned_variable, " = ", *parts, ")"]
119 if self.comment:
120 parts.append(f" // {self.comment}")
121 return "".join(parts)
122
123
124 class CCallMakerVisitor(GrammarVisitor):
125 def __init__(
126 self,
127 parser_generator: ParserGenerator,
128 exact_tokens: dict[str, int],
129 non_exact_tokens: set[str],
130 ):
131 self.gen = parser_generator
132 self.exact_tokens = exact_tokens
133 self.non_exact_tokens = non_exact_tokens
134 self.cache: dict[str, str] = {}
135 self.cleanup_statements: list[str] = []
136
137 def keyword_helper(self, keyword: str) -> FunctionCall:
138 return FunctionCall(
139 assigned_variable="_keyword",
140 function="_PyPegen_expect_token",
141 arguments=["p", self.gen.keywords[keyword]],
142 return_type="Token *",
143 nodetype=NodeTypes.KEYWORD,
144 comment=f"token='{keyword}'",
145 )
146
147 def soft_keyword_helper(self, value: str) -> FunctionCall:
148 return FunctionCall(
149 assigned_variable="_keyword",
150 function="_PyPegen_expect_soft_keyword",
151 arguments=["p", value],
152 return_type="expr_ty",
153 nodetype=NodeTypes.SOFT_KEYWORD,
154 comment=f"soft_keyword='{value}'",
155 )
156
157 def visit_NameLeaf(self, node: NameLeaf) -> FunctionCall:
158 name = node.value
159 if name in self.non_exact_tokens:
160 if name in BASE_NODETYPES:
161 return FunctionCall(
162 assigned_variable=f"{name.lower()}_var",
163 function=f"_PyPegen_{name.lower()}_token",
164 arguments=["p"],
165 nodetype=BASE_NODETYPES[name],
166 return_type="expr_ty",
167 comment=name,
168 )
169 return FunctionCall(
170 assigned_variable=f"{name.lower()}_var",
171 function="_PyPegen_expect_token",
172 arguments=["p", name],
173 nodetype=NodeTypes.GENERIC_TOKEN,
174 return_type="Token *",
175 comment=f"token='{name}'",
176 )
177
178 type = None
179 rule = self.gen.all_rules.get(name.lower())
180 if rule is not None:
181 type = "asdl_seq *" if rule.is_loop() or rule.is_gather() else rule.type
182
183 return FunctionCall(
184 assigned_variable=f"{name}_var",
185 function=f"{name}_rule",
186 arguments=["p"],
187 return_type=type,
188 comment=f"{node}",
189 )
190
191 def visit_StringLeaf(self, node: StringLeaf) -> FunctionCall:
192 val = ast.literal_eval(node.value)
193 if re.match(r"[a-zA-Z_]\w*\Z", val): # This is a keyword
194 if node.value.endswith("'"):
195 return self.keyword_helper(val)
196 else:
197 return self.soft_keyword_helper(node.value)
198 else:
199 assert val in self.exact_tokens, f"{node.value} is not a known literal"
200 type = self.exact_tokens[val]
201 return FunctionCall(
202 assigned_variable="_literal",
203 function="_PyPegen_expect_token",
204 arguments=["p", type],
205 nodetype=NodeTypes.GENERIC_TOKEN,
206 return_type="Token *",
207 comment=f"token='{val}'",
208 )
209
210 def visit_NamedItem(self, node: NamedItem) -> FunctionCall:
211 call = self.generate_call(node.item)
212 if node.name:
213 call.assigned_variable = node.name
214 if node.type:
215 call.assigned_variable_type = node.type
216 return call
217
218 def assert_no_undefined_behavior(
219 self, call: FunctionCall, wrapper: str, expected_rtype: str | None,
220 ) -> None:
221 if call.return_type != expected_rtype:
222 raise RuntimeError(
223 f"{call.function} return type is incompatible with {wrapper}: "
224 f"expect: {expected_rtype}, actual: {call.return_type}"
225 )
226
227 def lookahead_call_helper(self, node: Lookahead, positive: int) -> FunctionCall:
228 call = self.generate_call(node.node)
229 comment = None
230 if call.nodetype is NodeTypes.NAME_TOKEN:
231 function = "_PyPegen_lookahead_for_expr"
232 self.assert_no_undefined_behavior(call, function, "expr_ty")
233 elif call.nodetype is NodeTypes.STRING_TOKEN:
234 # _PyPegen_string_token() returns 'void *' instead of 'Token *';
235 # in addition, the overall function call would return 'expr_ty'.
236 assert call.function == "_PyPegen_string_token"
237 function = "_PyPegen_lookahead"
238 self.assert_no_undefined_behavior(call, function, "expr_ty")
239 elif call.nodetype == NodeTypes.SOFT_KEYWORD:
240 function = "_PyPegen_lookahead_with_string"
241 self.assert_no_undefined_behavior(call, function, "expr_ty")
242 elif call.nodetype in {NodeTypes.GENERIC_TOKEN, NodeTypes.KEYWORD}:
243 function = "_PyPegen_lookahead_with_int"
244 self.assert_no_undefined_behavior(call, function, "Token *")
245 comment = f"token={node.node}"
246 elif call.return_type == "expr_ty":
247 function = "_PyPegen_lookahead_for_expr"
248 elif call.return_type == "stmt_ty":
249 function = "_PyPegen_lookahead_for_stmt"
250 else:
251 function = "_PyPegen_lookahead"
252 self.assert_no_undefined_behavior(call, function, None)
253 return FunctionCall(
254 function=function,
255 arguments=[positive, call.function, *call.arguments],
256 return_type="int",
257 comment=comment,
258 )
259
260 def visit_PositiveLookahead(self, node: PositiveLookahead) -> FunctionCall:
261 return self.lookahead_call_helper(node, 1)
262
263 def visit_NegativeLookahead(self, node: NegativeLookahead) -> FunctionCall:
264 return self.lookahead_call_helper(node, 0)
265
266 def visit_Forced(self, node: Forced) -> FunctionCall:
267 call = self.generate_call(node.node)
268 if isinstance(node.node, Leaf):
269 assert isinstance(node.node, Leaf)
270 val = ast.literal_eval(node.node.value)
271 assert val in self.exact_tokens, f"{node.node.value} is not a known literal"
272 type = self.exact_tokens[val]
273 return FunctionCall(
274 assigned_variable="_literal",
275 function="_PyPegen_expect_forced_token",
276 arguments=["p", type, f'"{val}"'],
277 nodetype=NodeTypes.GENERIC_TOKEN,
278 return_type="Token *",
279 comment=f"forced_token='{val}'",
280 )
281 if isinstance(node.node, Group):
282 call = self.visit(node.node.rhs)
283 call.assigned_variable = None
284 call.comment = None
285 return FunctionCall(
286 assigned_variable="_literal",
287 function="_PyPegen_expect_forced_result",
288 arguments=["p", str(call), f'"{node.node.rhs!s}"'],
289 return_type="void *",
290 comment=f"forced_token=({node.node.rhs!s})",
291 )
292 else:
293 raise NotImplementedError(f"Forced tokens don't work with {node.node} nodes")
294
295 def visit_Opt(self, node: Opt) -> FunctionCall:
296 call = self.generate_call(node.node)
297 return FunctionCall(
298 assigned_variable="_opt_var",
299 function=call.function,
300 arguments=call.arguments,
301 force_true=True,
302 comment=f"{node}",
303 )
304
305 def _generate_artificial_rule_call(
306 self,
307 node: Any,
308 prefix: str,
309 rule_generation_func: Callable[[], str],
310 return_type: str | None = None,
311 ) -> FunctionCall:
312 node_str = f"{node}"
313 key = f"{prefix}_{node_str}"
314 if key in self.cache:
315 name = self.cache[key]
316 else:
317 name = rule_generation_func()
318 self.cache[key] = name
319
320 return FunctionCall(
321 assigned_variable=f"{name}_var",
322 function=f"{name}_rule",
323 arguments=["p"],
324 return_type=return_type,
325 comment=node_str,
326 )
327
328 def visit_Rhs(self, node: Rhs) -> FunctionCall:
329 if node.can_be_inlined:
330 return self.generate_call(node.alts[0].items[0])
331
332 return self._generate_artificial_rule_call(
333 node,
334 "rhs",
335 lambda: self.gen.artificial_rule_from_rhs(node),
336 )
337
338 def visit_Repeat0(self, node: Repeat0) -> FunctionCall:
339 return self._generate_artificial_rule_call(
340 node,
341 "repeat0",
342 lambda: self.gen.artificial_rule_from_repeat(node.node, is_repeat1=False),
343 "asdl_seq *",
344 )
345
346 def visit_Repeat1(self, node: Repeat1) -> FunctionCall:
347 return self._generate_artificial_rule_call(
348 node,
349 "repeat1",
350 lambda: self.gen.artificial_rule_from_repeat(node.node, is_repeat1=True),
351 "asdl_seq *",
352 )
353
354 def visit_Gather(self, node: Gather) -> FunctionCall:
355 return self._generate_artificial_rule_call(
356 node,
357 "gather",
358 lambda: self.gen.artificial_rule_from_gather(node),
359 "asdl_seq *",
360 )
361
362 def visit_Group(self, node: Group) -> FunctionCall:
363 return self.generate_call(node.rhs)
364
365 def visit_Cut(self, node: Cut) -> FunctionCall:
366 return FunctionCall(
367 assigned_variable="_cut_var",
368 return_type="int",
369 function="1",
370 nodetype=NodeTypes.CUT_OPERATOR,
371 )
372
373 def generate_call(self, node: Any) -> FunctionCall:
374 return super().visit(node)
375
376
377 class CParserGenerator(ParserGenerator, GrammarVisitor):
378 def __init__(
379 self,
380 grammar: grammar.Grammar,
381 tokens: dict[int, str],
382 exact_tokens: dict[str, int],
383 non_exact_tokens: set[str],
384 file: IO[str] | None,
385 debug: bool = False,
386 skip_actions: bool = False,
387 ):
388 super().__init__(grammar, set(tokens.values()), file)
389 self.callmakervisitor: CCallMakerVisitor = CCallMakerVisitor(
390 self, exact_tokens, non_exact_tokens
391 )
392 self._varname_counter = 0
393 self.debug = debug
394 self.skip_actions = skip_actions
395 self.cleanup_statements: list[str] = []
396
397 def add_level(self) -> None:
398 self.print("if (p->level++ == MAXSTACK || _Py_ReachedRecursionLimitWithMargin(PyThreadState_Get(), 1)) {")
399 with self.indent():
400 self.print("_Pypegen_stack_overflow(p);")
401 self.print("}")
402
403 def remove_level(self) -> None:
404 self.print("p->level--;")
405
406 def add_return(self, ret_val: str) -> None:
407 for stmt in self.cleanup_statements:
408 self.print(stmt)
409 self.remove_level()
410 self.print(f"return {ret_val};")
411
412 def unique_varname(self, name: str = "tmpvar") -> str:
413 new_var = name + "_" + str(self._varname_counter)
414 self._varname_counter += 1
415 return new_var
416
417 def call_with_errorcheck_return(self, call_text: str, returnval: str) -> None:
418 error_var = self.unique_varname()
419 self.print(f"int {error_var} = {call_text};")
420 self.print(f"if ({error_var}) {{")
421 with self.indent():
422 self.add_return(returnval)
423 self.print("}")
424
425 def call_with_errorcheck_goto(self, call_text: str, goto_target: str) -> None:
426 error_var = self.unique_varname()
427 self.print(f"int {error_var} = {call_text};")
428 self.print(f"if ({error_var}) {{")
429 with self.indent():
430 self.print(f"goto {goto_target};")
431 self.print("}")
432
433 def out_of_memory_return(
434 self,
435 expr: str,
436 cleanup_code: str | None = None,
437 ) -> None:
438 self.print(f"if ({expr}) {{")
439 with self.indent():
440 if cleanup_code is not None:
441 self.print(cleanup_code)
442 self.print("p->error_indicator = 1;")
443 self.print("PyErr_NoMemory();")
444 self.add_return("NULL")
445 self.print("}")
446
447 def out_of_memory_goto(self, expr: str, goto_target: str) -> None:
448 self.print(f"if ({expr}) {{")
449 with self.indent():
450 self.print("PyErr_NoMemory();")
451 self.print(f"goto {goto_target};")
452 self.print("}")
453
454 def generate(self, filename: str) -> None:
455 self.collect_rules()
456 basename = os.path.basename(filename)
457 self.print(f"// @generated by pegen from {basename}")
458 header = self.grammar.metas.get("header", EXTENSION_PREFIX)
459 if header:
460 self.print(header.rstrip("\n"))
461 subheader = self.grammar.metas.get("subheader", "")
462 if subheader:
463 self.print(subheader)
464 self._setup_keywords()
465 self._setup_soft_keywords()
466 for i, (rulename, rule) in enumerate(self.all_rules.items(), 1000):
467 comment = " // Left-recursive" if rule.left_recursive else ""
468 self.print(f"#define {rulename}_type {i}{comment}")
469 self.print()
470 for rulename, rule in self.all_rules.items():
471 if rule.is_loop() or rule.is_gather():
472 type = "asdl_seq *"
473 elif rule.type:
474 type = rule.type + " "
475 else:
476 type = "void *"
477 self.print(f"static {type}{rulename}_rule(Parser *p);")
478 self.print()
479 for rulename, rule in list(self.all_rules.items()):
480 self.print()
481 if rule.left_recursive:
482 self.print("// Left-recursive")
483 self.visit(rule)
484 if self.skip_actions:
485 mode = 0
486 else:
487 mode = int(self.rules["start"].type == "mod_ty") if "start" in self.rules else 1
488 if mode == 1 and self.grammar.metas.get("bytecode"):
489 mode += 1
490 modulename = self.grammar.metas.get("modulename", "parse")
491 trailer = self.grammar.metas.get("trailer", EXTENSION_SUFFIX)
492 if trailer:
493 self.print(trailer.rstrip("\n") % dict(mode=mode, modulename=modulename))
494
495 def _group_keywords_by_length(self) -> dict[int, list[tuple[str, int]]]:
496 groups: dict[int, list[tuple[str, int]]] = {}
497 for keyword_str, keyword_type in self.keywords.items():
498 length = len(keyword_str)
499 if length in groups:
500 groups[length].append((keyword_str, keyword_type))
501 else:
502 groups[length] = [(keyword_str, keyword_type)]
503 return groups
504
505 def _setup_keywords(self) -> None:
506 n_keyword_lists = (
507 len(max(self.keywords.keys(), key=len)) + 1 if len(self.keywords) > 0 else 0
508 )
509 self.print(f"static const int n_keyword_lists = {n_keyword_lists};")
510 groups = self._group_keywords_by_length()
511 self.print("static KeywordToken *reserved_keywords[] = {")
512 with self.indent():
513 num_groups = max(groups) + 1 if groups else 1
514 for keywords_length in range(num_groups):
515 if keywords_length not in groups.keys():
516 self.print("(KeywordToken[]) {{NULL, -1}},")
517 else:
518 self.print("(KeywordToken[]) {")
519 with self.indent():
520 for keyword_str, keyword_type in groups[keywords_length]:
521 self.print(f'{{"{keyword_str}", {keyword_type}}},')
522 self.print("{NULL, -1},")
523 self.print("},")
524 self.print("};")
525
526 def _setup_soft_keywords(self) -> None:
527 soft_keywords = sorted(self.soft_keywords)
528 self.print("static char *soft_keywords[] = {")
529 with self.indent():
530 for keyword in soft_keywords:
531 self.print(f'"{keyword}",')
532 self.print("NULL,")
533 self.print("};")
534
535 def _set_up_token_start_metadata_extraction(self) -> None:
536 self.print("if (p->mark == p->fill && _PyPegen_fill_token(p) < 0) {")
537 with self.indent():
538 self.print("p->error_indicator = 1;")
539 self.add_return("NULL")
540 self.print("}")
541 self.print("int _start_lineno = p->tokens[_mark]->lineno;")
542 self.print("UNUSED(_start_lineno); // Only used by EXTRA macro")
543 self.print("int _start_col_offset = p->tokens[_mark]->col_offset;")
544 self.print("UNUSED(_start_col_offset); // Only used by EXTRA macro")
545
546 def _set_up_token_end_metadata_extraction(self) -> None:
547 self.print("Token *_token = _PyPegen_get_last_nonnwhitespace_token(p);")
548 self.print("if (_token == NULL) {")
549 with self.indent():
550 self.add_return("NULL")
551 self.print("}")
552 self.print("int _end_lineno = _token->end_lineno;")
553 self.print("UNUSED(_end_lineno); // Only used by EXTRA macro")
554 self.print("int _end_col_offset = _token->end_col_offset;")
555 self.print("UNUSED(_end_col_offset); // Only used by EXTRA macro")
556
557 def _check_for_errors(self) -> None:
558 self.print("if (p->error_indicator) {")
559 with self.indent():
560 self.add_return("NULL")
561 self.print("}")
562
563 def _set_up_rule_memoization(self, node: Rule, result_type: str) -> None:
564 self.print("{")
565 with self.indent():
566 self.add_level()
567 self.print(f"{result_type} _res = NULL;")
568 self.print(f"if (_PyPegen_is_memoized(p, {node.name}_type, &_res)) {{")
569 with self.indent():
570 self.add_return("_res")
571 self.print("}")
572 self.print("int _mark = p->mark;")
573 self.print("int _resmark = p->mark;")
574 self.print("while (1) {")
575 with self.indent():
576 self.call_with_errorcheck_return(
577 f"_PyPegen_update_memo(p, _mark, {node.name}_type, _res)", "_res"
578 )
579 self.print("p->mark = _mark;")
580 self.print(f"void *_raw = {node.name}_raw(p);")
581 self.print("if (p->error_indicator) {")
582 with self.indent():
583 self.add_return("NULL")
584 self.print("}")
585 self.print("if (_raw == NULL || p->mark <= _resmark)")
586 with self.indent():
587 self.print("break;")
588 self.print("_resmark = p->mark;")
589 self.print("_res = _raw;")
590 self.print("}")
591 self.print("p->mark = _resmark;")
592 self.add_return("_res")
593 self.print("}")
594 self.print(f"static {result_type}")
595 self.print(f"{node.name}_raw(Parser *p)")
596
597 def _should_memoize(self, node: Rule) -> bool:
598 return node.memo and not node.left_recursive
599
600 def _handle_default_rule_body(self, node: Rule, rhs: Rhs, result_type: str) -> None:
601 memoize = self._should_memoize(node)
602
603 with self.indent():
604 self.add_level()
605 self._check_for_errors()
606 self.print(f"{result_type} _res = NULL;")
607 if memoize:
608 self.print(f"if (_PyPegen_is_memoized(p, {node.name}_type, &_res)) {{")
609 with self.indent():
610 self.add_return("_res")
611 self.print("}")
612 self.print("int _mark = p->mark;")
613 if any(alt.action and "EXTRA" in alt.action for alt in rhs.alts):
614 self._set_up_token_start_metadata_extraction()
615 self.visit(
616 rhs,
617 is_loop=False,
618 is_gather=node.is_gather(),
619 rulename=node.name,
620 )
621 if self.debug:
622 self.print(f'D(fprintf(stderr, "Fail at %d: {node.name}\\n", p->mark));')
623 self.print("_res = NULL;")
624 self.print(" done:")
625 with self.indent():
626 if memoize:
627 self.print(f"_PyPegen_insert_memo(p, _mark, {node.name}_type, _res);")
628 self.add_return("_res")
629
630 def _handle_loop_rule_body(self, node: Rule, rhs: Rhs) -> None:
631 memoize = self._should_memoize(node)
632 is_repeat1 = node.name.startswith("_loop1")
633
634 with self.indent():
635 self.add_level()
636 self._check_for_errors()
637 self.print("void *_res = NULL;")
638 if memoize:
639 self.print(f"if (_PyPegen_is_memoized(p, {node.name}_type, &_res)) {{")
640 with self.indent():
641 self.add_return("_res")
642 self.print("}")
643 self.print("int _mark = p->mark;")
644 if memoize:
645 self.print("int _start_mark = p->mark;")
646 self.print("void **_children = PyMem_Malloc(sizeof(void *));")
647 self.out_of_memory_return("!_children")
648 self.print("Py_ssize_t _children_capacity = 1;")
649 self.print("Py_ssize_t _n = 0;")
650 if any(alt.action and "EXTRA" in alt.action for alt in rhs.alts):
651 self._set_up_token_start_metadata_extraction()
652 self.visit(
653 rhs,
654 is_loop=True,
655 is_gather=node.is_gather(),
656 rulename=node.name,
657 )
658 if is_repeat1:
659 self.print("if (_n == 0 || p->error_indicator) {")
660 with self.indent():
661 self.print("PyMem_Free(_children);")
662 self.add_return("NULL")
663 self.print("}")
664 self.print("asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena);")
665 self.out_of_memory_return("!_seq", cleanup_code="PyMem_Free(_children);")
666 self.print("for (Py_ssize_t i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]);")
667 self.print("PyMem_Free(_children);")
668 if memoize and node.name:
669 self.print(f"_PyPegen_insert_memo(p, _start_mark, {node.name}_type, _seq);")
670 self.add_return("_seq")
671
672 def visit_Rule(self, node: Rule) -> None:
673 is_loop = node.is_loop()
674 is_gather = node.is_gather()
675 rhs = node.flatten()
676 if is_loop or is_gather:
677 result_type = "asdl_seq *"
678 elif node.type:
679 result_type = node.type
680 else:
681 result_type = "void *"
682
683 for line in str(node).splitlines():
684 self.print(f"// {line}")
685 if node.left_recursive and node.leader:
686 self.print(f"static {result_type} {node.name}_raw(Parser *);")
687
688 self.print(f"static {result_type}")
689 self.print(f"{node.name}_rule(Parser *p)")
690
691 if node.left_recursive and node.leader:
692 self._set_up_rule_memoization(node, result_type)
693
694 self.print("{")
695
696 if node.name.endswith("without_invalid"):
697 with self.indent():
698 self.print("int _prev_call_invalid = p->call_invalid_rules;")
699 self.print("p->call_invalid_rules = 0;")
700 self.cleanup_statements.append("p->call_invalid_rules = _prev_call_invalid;")
701
702 if is_loop:
703 self._handle_loop_rule_body(node, rhs)
704 else:
705 self._handle_default_rule_body(node, rhs, result_type)
706
707 if node.name.endswith("without_invalid"):
708 self.cleanup_statements.pop()
709
710 self.print("}")
711
712 def visit_NamedItem(self, node: NamedItem) -> None:
713 call = self.callmakervisitor.generate_call(node)
714 if call.assigned_variable:
715 call.assigned_variable = self.dedupe(call.assigned_variable)
716 self.print(call)
717
718 def visit_Rhs(
719 self, node: Rhs, is_loop: bool, is_gather: bool, rulename: str | None
720 ) -> None:
721 if is_loop:
722 assert len(node.alts) == 1
723 for alt in node.alts:
724 self.visit(alt, is_loop=is_loop, is_gather=is_gather, rulename=rulename)
725
726 def join_conditions(self, keyword: str, node: Any) -> None:
727 self.print(f"{keyword} (")
728 with self.indent():
729 first = True
730 for item in node.items:
731 if first:
732 first = False
733 else:
734 self.print("&&")
735 self.visit(item)
736 self.print(")")
737
738 def emit_action(self, node: Alt, cleanup_code: str | None = None) -> None:
739 self.print(f"_res = {node.action};")
740
741 self.print("if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) {")
742 with self.indent():
743 self.print("p->error_indicator = 1;")
744 if cleanup_code:
745 self.print(cleanup_code)
746 self.add_return("NULL")
747 self.print("}")
748
749 if self.debug:
750 self.print(
751 f'D(fprintf(stderr, "Hit with action [%d-%d]: %s\\n", _mark, p->mark, "{node}"));'
752 )
753
754 def emit_default_action(self, is_gather: bool, node: Alt) -> None:
755 if len(self.local_variable_names) > 1:
756 if is_gather:
757 assert len(self.local_variable_names) == 2
758 self.print(
759 f"_res = _PyPegen_seq_insert_in_front(p, "
760 f"{self.local_variable_names[0]}, {self.local_variable_names[1]});"
761 )
762 else:
763 if self.debug:
764 self.print(
765 f'D(fprintf(stderr, "Hit without action [%d:%d]: %s\\n", _mark, p->mark, "{node}"));'
766 )
767 self.print(
768 f"_res = _PyPegen_dummy_name(p, {', '.join(self.local_variable_names)});"
769 )
770 else:
771 if self.debug:
772 self.print(
773 f'D(fprintf(stderr, "Hit with default action [%d:%d]: %s\\n", _mark, p->mark, "{node}"));'
774 )
775 self.print(f"_res = {self.local_variable_names[0]};")
776
777 def emit_dummy_action(self) -> None:
778 self.print("_res = _PyPegen_dummy_name(p);")
779
780 def handle_alt_normal(self, node: Alt, is_gather: bool, rulename: str | None) -> None:
781 self.join_conditions(keyword="if", node=node)
782 self.print("{")
783 # We have parsed successfully all the conditions for the option.
784 with self.indent():
785 node_str = str(node).replace('"', '\\"')
786 self.print(
787 f'D(fprintf(stderr, "%*c+ {rulename}[%d-%d]: %s succeeded!\\n", p->level, \' \', _mark, p->mark, "{node_str}"));'
788 )
789 # Prepare to emit the rule action and do so
790 if node.action and "EXTRA" in node.action:
791 self._set_up_token_end_metadata_extraction()
792 if self.skip_actions:
793 self.emit_dummy_action()
794 elif node.action:
795 self.emit_action(node)
796 else:
797 self.emit_default_action(is_gather, node)
798
799 # As the current option has parsed correctly, do not continue with the rest.
800 self.print("goto done;")
801 self.print("}")
802
803 def handle_alt_loop(self, node: Alt, is_gather: bool, rulename: str | None) -> None:
804 # Condition of the main body of the alternative
805 self.join_conditions(keyword="while", node=node)
806 self.print("{")
807 # We have parsed successfully one item!
808 with self.indent():
809 # Prepare to emit the rule action and do so
810 if node.action and "EXTRA" in node.action:
811 self._set_up_token_end_metadata_extraction()
812 if self.skip_actions:
813 self.emit_dummy_action()
814 elif node.action:
815 self.emit_action(node, cleanup_code="PyMem_Free(_children);")
816 else:
817 self.emit_default_action(is_gather, node)
818
819 # Add the result of rule to the temporary buffer of children. This buffer
820 # will populate later an asdl_seq with all elements to return.
821 self.print("if (_n == _children_capacity) {")
822 with self.indent():
823 self.print("_children_capacity *= 2;")
824 self.print(
825 "void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *));"
826 )
827 self.out_of_memory_return("!_new_children", cleanup_code="PyMem_Free(_children);")
828 self.print("_children = _new_children;")
829 self.print("}")
830 self.print("_children[_n++] = _res;")
831 self.print("_mark = p->mark;")
832 self.print("}")
833
834 def visit_Alt(
835 self, node: Alt, is_loop: bool, is_gather: bool, rulename: str | None
836 ) -> None:
837 if len(node.items) == 1 and str(node.items[0]).startswith("invalid_"):
838 self.print(f"if (p->call_invalid_rules) {{ // {node}")
839 else:
840 self.print(f"{{ // {node}")
841 with self.indent():
842 self._check_for_errors()
843 node_str = str(node).replace('"', '\\"')
844 self.print(
845 f'D(fprintf(stderr, "%*c> {rulename}[%d-%d]: %s\\n", p->level, \' \', _mark, p->mark, "{node_str}"));'
846 )
847 # Prepare variable declarations for the alternative
848 vars = self.collect_vars(node)
849 for v, var_type in sorted(item for item in vars.items() if item[0] is not None):
850 if not var_type:
851 var_type = "void *"
852 else:
853 var_type += " "
854 if v == "_cut_var":
855 v += " = 0" # cut_var must be initialized
856 self.print(f"{var_type}{v};")
857 if v and v.startswith("_opt_var"):
858 self.print(f"UNUSED({v}); // Silence compiler warnings")
859
860 with self.local_variable_context():
861 if is_loop:
862 self.handle_alt_loop(node, is_gather, rulename)
863 else:
864 self.handle_alt_normal(node, is_gather, rulename)
865
866 self.print("p->mark = _mark;")
867 node_str = str(node).replace('"', '\\"')
868 self.print(
869 f"D(fprintf(stderr, \"%*c%s {rulename}[%d-%d]: %s failed!\\n\", p->level, ' ',\n"
870 f' p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "{node_str}"));'
871 )
872 if "_cut_var" in vars:
873 self.print("if (_cut_var) {")
874 with self.indent():
875 self.add_return("NULL")
876 self.print("}")
877 self.print("}")
878
879 def collect_vars(self, node: Alt) -> dict[str | None, str | None]:
880 types = {}
881 with self.local_variable_context():
882 for item in node.items:
883 name, type = self.add_var(item)
884 types[name] = type
885 return types
886
887 def add_var(self, node: NamedItem) -> tuple[str | None, str | None]:
888 call = self.callmakervisitor.generate_call(node.item)
889 name = node.name if node.name else call.assigned_variable
890 if name is not None:
891 name = self.dedupe(name)
892 return_type = call.return_type if node.type is None else node.type
893 return name, return_type