Debian Codegraph @

Tools/peg_generator/pegen/c_generator.py

raw
1 import ast
2 import os.path
3 import re
4 from dataclasses import dataclass, field
5 from enum import Enum
6 from typing import IO, Any
7
8 from pegen import grammar
9 from pegen.grammar import (
10 Alt,
11 Cut,
12 Forced,
13 Gather,
14 GrammarVisitor,
15 Group,
16 Leaf,
17 Lookahead,
18 NamedItem,
19 NameLeaf,
20 NegativeLookahead,
21 Opt,
22 PositiveLookahead,
23 Repeat0,
24 Repeat1,
25 Rhs,
26 Rule,
27 StringLeaf,
28 )
29 from pegen.parser_generator import ParserGenerator
30
31 EXTENSION_PREFIX = """\
32 #include "pegen.h"
33
34 #if defined(Py_DEBUG) && defined(Py_BUILD_CORE)
35 # define D(x) if (p->debug) { x; }
36 #else
37 # define D(x)
38 #endif
39
40 #ifdef __wasi__
41 # ifdef Py_DEBUG
42 # define MAXSTACK 1000
43 # else
44 # define MAXSTACK 4000
45 # endif
46 #else
47 # define MAXSTACK 6000
48 #endif
49
50 """
51
52
53 EXTENSION_SUFFIX = """
54 void *
55 _PyPegen_parse(Parser *p)
56 {
57 // Initialize keywords
58 p->keywords = reserved_keywords;
59 p->n_keyword_lists = n_keyword_lists;
60 p->soft_keywords = soft_keywords;
61
62 return start_rule(p);
63 }
64 """
65
66
67 class NodeTypes(Enum):
68 NAME_TOKEN = 0
69 NUMBER_TOKEN = 1
70 STRING_TOKEN = 2
71 GENERIC_TOKEN = 3
72 KEYWORD = 4
73 SOFT_KEYWORD = 5
74 CUT_OPERATOR = 6
75 F_STRING_CHUNK = 7
76
77
78 BASE_NODETYPES = {
79 "NAME": NodeTypes.NAME_TOKEN,
80 "NUMBER": NodeTypes.NUMBER_TOKEN,
81 "STRING": NodeTypes.STRING_TOKEN,
82 "SOFT_KEYWORD": NodeTypes.SOFT_KEYWORD,
83 }
84
85
86 @dataclass
87 class FunctionCall:
88 function: str
89 arguments: list[Any] = field(default_factory=list)
90 assigned_variable: str | None = None
91 assigned_variable_type: str | None = None
92 return_type: str | None = None
93 nodetype: NodeTypes | None = None
94 force_true: bool = False
95 comment: str | None = None
96
97 def __str__(self) -> str:
98 parts = []
99 parts.append(self.function)
100 if self.arguments:
101 parts.append(f"({', '.join(map(str, self.arguments))})")
102 if self.force_true:
103 parts.append(", !p->error_indicator")
104 if self.assigned_variable:
105 if self.assigned_variable_type:
106 parts = [
107 "(",
108 self.assigned_variable,
109 " = ",
110 "(",
111 self.assigned_variable_type,
112 ")",
113 *parts,
114 ")",
115 ]
116 else:
117 parts = ["(", self.assigned_variable, " = ", *parts, ")"]
118 if self.comment:
119 parts.append(f" // {self.comment}")
120 return "".join(parts)
121
122
123 class CCallMakerVisitor(GrammarVisitor):
124 def __init__(
125 self,
126 parser_generator: ParserGenerator,
127 exact_tokens: dict[str, int],
128 non_exact_tokens: set[str],
129 ):
130 self.gen = parser_generator
131 self.exact_tokens = exact_tokens
132 self.non_exact_tokens = non_exact_tokens
133 self.cache: dict[Any, FunctionCall] = {}
134 self.cleanup_statements: list[str] = []
135
136 def keyword_helper(self, keyword: str) -> FunctionCall:
137 return FunctionCall(
138 assigned_variable="_keyword",
139 function="_PyPegen_expect_token",
140 arguments=["p", self.gen.keywords[keyword]],
141 return_type="Token *",
142 nodetype=NodeTypes.KEYWORD,
143 comment=f"token='{keyword}'",
144 )
145
146 def soft_keyword_helper(self, value: str) -> FunctionCall:
147 return FunctionCall(
148 assigned_variable="_keyword",
149 function="_PyPegen_expect_soft_keyword",
150 arguments=["p", value],
151 return_type="expr_ty",
152 nodetype=NodeTypes.SOFT_KEYWORD,
153 comment=f"soft_keyword='{value}'",
154 )
155
156 def visit_NameLeaf(self, node: NameLeaf) -> FunctionCall:
157 name = node.value
158 if name in self.non_exact_tokens:
159 if name in BASE_NODETYPES:
160 return FunctionCall(
161 assigned_variable=f"{name.lower()}_var",
162 function=f"_PyPegen_{name.lower()}_token",
163 arguments=["p"],
164 nodetype=BASE_NODETYPES[name],
165 return_type="expr_ty",
166 comment=name,
167 )
168 return FunctionCall(
169 assigned_variable=f"{name.lower()}_var",
170 function="_PyPegen_expect_token",
171 arguments=["p", name],
172 nodetype=NodeTypes.GENERIC_TOKEN,
173 return_type="Token *",
174 comment=f"token='{name}'",
175 )
176
177 type = None
178 rule = self.gen.all_rules.get(name.lower())
179 if rule is not None:
180 type = "asdl_seq *" if rule.is_loop() or rule.is_gather() else rule.type
181
182 return FunctionCall(
183 assigned_variable=f"{name}_var",
184 function=f"{name}_rule",
185 arguments=["p"],
186 return_type=type,
187 comment=f"{node}",
188 )
189
190 def visit_StringLeaf(self, node: StringLeaf) -> FunctionCall:
191 val = ast.literal_eval(node.value)
192 if re.match(r"[a-zA-Z_]\w*\Z", val): # This is a keyword
193 if node.value.endswith("'"):
194 return self.keyword_helper(val)
195 else:
196 return self.soft_keyword_helper(node.value)
197 else:
198 assert val in self.exact_tokens, f"{node.value} is not a known literal"
199 type = self.exact_tokens[val]
200 return FunctionCall(
201 assigned_variable="_literal",
202 function="_PyPegen_expect_token",
203 arguments=["p", type],
204 nodetype=NodeTypes.GENERIC_TOKEN,
205 return_type="Token *",
206 comment=f"token='{val}'",
207 )
208
209 def visit_Rhs(self, node: Rhs) -> FunctionCall:
210 if node in self.cache:
211 return self.cache[node]
212 if node.can_be_inlined:
213 self.cache[node] = self.generate_call(node.alts[0].items[0])
214 else:
215 name = self.gen.artifical_rule_from_rhs(node)
216 self.cache[node] = FunctionCall(
217 assigned_variable=f"{name}_var",
218 function=f"{name}_rule",
219 arguments=["p"],
220 comment=f"{node}",
221 )
222 return self.cache[node]
223
224 def visit_NamedItem(self, node: NamedItem) -> FunctionCall:
225 call = self.generate_call(node.item)
226 if node.name:
227 call.assigned_variable = node.name
228 if node.type:
229 call.assigned_variable_type = node.type
230 return call
231
232 def assert_no_undefined_behavior(
233 self, call: FunctionCall, wrapper: str, expected_rtype: str | None,
234 ) -> None:
235 if call.return_type != expected_rtype:
236 raise RuntimeError(
237 f"{call.function} return type is incompatible with {wrapper}: "
238 f"expect: {expected_rtype}, actual: {call.return_type}"
239 )
240
241 def lookahead_call_helper(self, node: Lookahead, positive: int) -> FunctionCall:
242 call = self.generate_call(node.node)
243 comment = None
244 if call.nodetype is NodeTypes.NAME_TOKEN:
245 function = "_PyPegen_lookahead_for_expr"
246 self.assert_no_undefined_behavior(call, function, "expr_ty")
247 elif call.nodetype is NodeTypes.STRING_TOKEN:
248 # _PyPegen_string_token() returns 'void *' instead of 'Token *';
249 # in addition, the overall function call would return 'expr_ty'.
250 assert call.function == "_PyPegen_string_token"
251 function = "_PyPegen_lookahead"
252 self.assert_no_undefined_behavior(call, function, "expr_ty")
253 elif call.nodetype == NodeTypes.SOFT_KEYWORD:
254 function = "_PyPegen_lookahead_with_string"
255 self.assert_no_undefined_behavior(call, function, "expr_ty")
256 elif call.nodetype in {NodeTypes.GENERIC_TOKEN, NodeTypes.KEYWORD}:
257 function = "_PyPegen_lookahead_with_int"
258 self.assert_no_undefined_behavior(call, function, "Token *")
259 comment = f"token={node.node}"
260 elif call.return_type == "expr_ty":
261 function = "_PyPegen_lookahead_for_expr"
262 elif call.return_type == "stmt_ty":
263 function = "_PyPegen_lookahead_for_stmt"
264 else:
265 function = "_PyPegen_lookahead"
266 self.assert_no_undefined_behavior(call, function, None)
267 return FunctionCall(
268 function=function,
269 arguments=[positive, call.function, *call.arguments],
270 return_type="int",
271 comment=comment,
272 )
273
274 def visit_PositiveLookahead(self, node: PositiveLookahead) -> FunctionCall:
275 return self.lookahead_call_helper(node, 1)
276
277 def visit_NegativeLookahead(self, node: NegativeLookahead) -> FunctionCall:
278 return self.lookahead_call_helper(node, 0)
279
280 def visit_Forced(self, node: Forced) -> FunctionCall:
281 call = self.generate_call(node.node)
282 if isinstance(node.node, Leaf):
283 assert isinstance(node.node, Leaf)
284 val = ast.literal_eval(node.node.value)
285 assert val in self.exact_tokens, f"{node.node.value} is not a known literal"
286 type = self.exact_tokens[val]
287 return FunctionCall(
288 assigned_variable="_literal",
289 function="_PyPegen_expect_forced_token",
290 arguments=["p", type, f'"{val}"'],
291 nodetype=NodeTypes.GENERIC_TOKEN,
292 return_type="Token *",
293 comment=f"forced_token='{val}'",
294 )
295 if isinstance(node.node, Group):
296 call = self.visit(node.node.rhs)
297 call.assigned_variable = None
298 call.comment = None
299 return FunctionCall(
300 assigned_variable="_literal",
301 function="_PyPegen_expect_forced_result",
302 arguments=["p", str(call), f'"{node.node.rhs!s}"'],
303 return_type="void *",
304 comment=f"forced_token=({node.node.rhs!s})",
305 )
306 else:
307 raise NotImplementedError(f"Forced tokens don't work with {node.node} nodes")
308
309 def visit_Opt(self, node: Opt) -> FunctionCall:
310 call = self.generate_call(node.node)
311 return FunctionCall(
312 assigned_variable="_opt_var",
313 function=call.function,
314 arguments=call.arguments,
315 force_true=True,
316 comment=f"{node}",
317 )
318
319 def visit_Repeat0(self, node: Repeat0) -> FunctionCall:
320 if node in self.cache:
321 return self.cache[node]
322 name = self.gen.artificial_rule_from_repeat(node.node, False)
323 self.cache[node] = FunctionCall(
324 assigned_variable=f"{name}_var",
325 function=f"{name}_rule",
326 arguments=["p"],
327 return_type="asdl_seq *",
328 comment=f"{node}",
329 )
330 return self.cache[node]
331
332 def visit_Repeat1(self, node: Repeat1) -> FunctionCall:
333 if node in self.cache:
334 return self.cache[node]
335 name = self.gen.artificial_rule_from_repeat(node.node, True)
336 self.cache[node] = FunctionCall(
337 assigned_variable=f"{name}_var",
338 function=f"{name}_rule",
339 arguments=["p"],
340 return_type="asdl_seq *",
341 comment=f"{node}",
342 )
343 return self.cache[node]
344
345 def visit_Gather(self, node: Gather) -> FunctionCall:
346 if node in self.cache:
347 return self.cache[node]
348 name = self.gen.artifical_rule_from_gather(node)
349 self.cache[node] = FunctionCall(
350 assigned_variable=f"{name}_var",
351 function=f"{name}_rule",
352 arguments=["p"],
353 return_type="asdl_seq *",
354 comment=f"{node}",
355 )
356 return self.cache[node]
357
358 def visit_Group(self, node: Group) -> FunctionCall:
359 return self.generate_call(node.rhs)
360
361 def visit_Cut(self, node: Cut) -> FunctionCall:
362 return FunctionCall(
363 assigned_variable="_cut_var",
364 return_type="int",
365 function="1",
366 nodetype=NodeTypes.CUT_OPERATOR,
367 )
368
369 def generate_call(self, node: Any) -> FunctionCall:
370 return super().visit(node)
371
372
373 class CParserGenerator(ParserGenerator, GrammarVisitor):
374 def __init__(
375 self,
376 grammar: grammar.Grammar,
377 tokens: dict[int, str],
378 exact_tokens: dict[str, int],
379 non_exact_tokens: set[str],
380 file: IO[str] | None,
381 debug: bool = False,
382 skip_actions: bool = False,
383 ):
384 super().__init__(grammar, set(tokens.values()), file)
385 self.callmakervisitor: CCallMakerVisitor = CCallMakerVisitor(
386 self, exact_tokens, non_exact_tokens
387 )
388 self._varname_counter = 0
389 self.debug = debug
390 self.skip_actions = skip_actions
391 self.cleanup_statements: list[str] = []
392
393 def add_level(self) -> None:
394 self.print("if (p->level++ == MAXSTACK) {")
395 with self.indent():
396 self.print("_Pypegen_stack_overflow(p);")
397 self.print("}")
398
399 def remove_level(self) -> None:
400 self.print("p->level--;")
401
402 def add_return(self, ret_val: str) -> None:
403 for stmt in self.cleanup_statements:
404 self.print(stmt)
405 self.remove_level()
406 self.print(f"return {ret_val};")
407
408 def unique_varname(self, name: str = "tmpvar") -> str:
409 new_var = name + "_" + str(self._varname_counter)
410 self._varname_counter += 1
411 return new_var
412
413 def call_with_errorcheck_return(self, call_text: str, returnval: str) -> None:
414 error_var = self.unique_varname()
415 self.print(f"int {error_var} = {call_text};")
416 self.print(f"if ({error_var}) {{")
417 with self.indent():
418 self.add_return(returnval)
419 self.print("}")
420
421 def call_with_errorcheck_goto(self, call_text: str, goto_target: str) -> None:
422 error_var = self.unique_varname()
423 self.print(f"int {error_var} = {call_text};")
424 self.print(f"if ({error_var}) {{")
425 with self.indent():
426 self.print(f"goto {goto_target};")
427 self.print("}")
428
429 def out_of_memory_return(
430 self,
431 expr: str,
432 cleanup_code: str | None = None,
433 ) -> None:
434 self.print(f"if ({expr}) {{")
435 with self.indent():
436 if cleanup_code is not None:
437 self.print(cleanup_code)
438 self.print("p->error_indicator = 1;")
439 self.print("PyErr_NoMemory();")
440 self.add_return("NULL")
441 self.print("}")
442
443 def out_of_memory_goto(self, expr: str, goto_target: str) -> None:
444 self.print(f"if ({expr}) {{")
445 with self.indent():
446 self.print("PyErr_NoMemory();")
447 self.print(f"goto {goto_target};")
448 self.print("}")
449
450 def generate(self, filename: str) -> None:
451 self.collect_rules()
452 basename = os.path.basename(filename)
453 self.print(f"// @generated by pegen from {basename}")
454 header = self.grammar.metas.get("header", EXTENSION_PREFIX)
455 if header:
456 self.print(header.rstrip("\n"))
457 subheader = self.grammar.metas.get("subheader", "")
458 if subheader:
459 self.print(subheader)
460 self._setup_keywords()
461 self._setup_soft_keywords()
462 for i, (rulename, rule) in enumerate(self.all_rules.items(), 1000):
463 comment = " // Left-recursive" if rule.left_recursive else ""
464 self.print(f"#define {rulename}_type {i}{comment}")
465 self.print()
466 for rulename, rule in self.all_rules.items():
467 if rule.is_loop() or rule.is_gather():
468 type = "asdl_seq *"
469 elif rule.type:
470 type = rule.type + " "
471 else:
472 type = "void *"
473 self.print(f"static {type}{rulename}_rule(Parser *p);")
474 self.print()
475 for rulename, rule in list(self.all_rules.items()):
476 self.print()
477 if rule.left_recursive:
478 self.print("// Left-recursive")
479 self.visit(rule)
480 if self.skip_actions:
481 mode = 0
482 else:
483 mode = int(self.rules["start"].type == "mod_ty") if "start" in self.rules else 1
484 if mode == 1 and self.grammar.metas.get("bytecode"):
485 mode += 1
486 modulename = self.grammar.metas.get("modulename", "parse")
487 trailer = self.grammar.metas.get("trailer", EXTENSION_SUFFIX)
488 if trailer:
489 self.print(trailer.rstrip("\n") % dict(mode=mode, modulename=modulename))
490
491 def _group_keywords_by_length(self) -> dict[int, list[tuple[str, int]]]:
492 groups: dict[int, list[tuple[str, int]]] = {}
493 for keyword_str, keyword_type in self.keywords.items():
494 length = len(keyword_str)
495 if length in groups:
496 groups[length].append((keyword_str, keyword_type))
497 else:
498 groups[length] = [(keyword_str, keyword_type)]
499 return groups
500
501 def _setup_keywords(self) -> None:
502 n_keyword_lists = (
503 len(max(self.keywords.keys(), key=len)) + 1 if len(self.keywords) > 0 else 0
504 )
505 self.print(f"static const int n_keyword_lists = {n_keyword_lists};")
506 groups = self._group_keywords_by_length()
507 self.print("static KeywordToken *reserved_keywords[] = {")
508 with self.indent():
509 num_groups = max(groups) + 1 if groups else 1
510 for keywords_length in range(num_groups):
511 if keywords_length not in groups.keys():
512 self.print("(KeywordToken[]) {{NULL, -1}},")
513 else:
514 self.print("(KeywordToken[]) {")
515 with self.indent():
516 for keyword_str, keyword_type in groups[keywords_length]:
517 self.print(f'{{"{keyword_str}", {keyword_type}}},')
518 self.print("{NULL, -1},")
519 self.print("},")
520 self.print("};")
521
522 def _setup_soft_keywords(self) -> None:
523 soft_keywords = sorted(self.soft_keywords)
524 self.print("static char *soft_keywords[] = {")
525 with self.indent():
526 for keyword in soft_keywords:
527 self.print(f'"{keyword}",')
528 self.print("NULL,")
529 self.print("};")
530
531 def _set_up_token_start_metadata_extraction(self) -> None:
532 self.print("if (p->mark == p->fill && _PyPegen_fill_token(p) < 0) {")
533 with self.indent():
534 self.print("p->error_indicator = 1;")
535 self.add_return("NULL")
536 self.print("}")
537 self.print("int _start_lineno = p->tokens[_mark]->lineno;")
538 self.print("UNUSED(_start_lineno); // Only used by EXTRA macro")
539 self.print("int _start_col_offset = p->tokens[_mark]->col_offset;")
540 self.print("UNUSED(_start_col_offset); // Only used by EXTRA macro")
541
542 def _set_up_token_end_metadata_extraction(self) -> None:
543 self.print("Token *_token = _PyPegen_get_last_nonnwhitespace_token(p);")
544 self.print("if (_token == NULL) {")
545 with self.indent():
546 self.add_return("NULL")
547 self.print("}")
548 self.print("int _end_lineno = _token->end_lineno;")
549 self.print("UNUSED(_end_lineno); // Only used by EXTRA macro")
550 self.print("int _end_col_offset = _token->end_col_offset;")
551 self.print("UNUSED(_end_col_offset); // Only used by EXTRA macro")
552
553 def _check_for_errors(self) -> None:
554 self.print("if (p->error_indicator) {")
555 with self.indent():
556 self.add_return("NULL")
557 self.print("}")
558
559 def _set_up_rule_memoization(self, node: Rule, result_type: str) -> None:
560 self.print("{")
561 with self.indent():
562 self.add_level()
563 self.print(f"{result_type} _res = NULL;")
564 self.print(f"if (_PyPegen_is_memoized(p, {node.name}_type, &_res)) {{")
565 with self.indent():
566 self.add_return("_res")
567 self.print("}")
568 self.print("int _mark = p->mark;")
569 self.print("int _resmark = p->mark;")
570 self.print("while (1) {")
571 with self.indent():
572 self.call_with_errorcheck_return(
573 f"_PyPegen_update_memo(p, _mark, {node.name}_type, _res)", "_res"
574 )
575 self.print("p->mark = _mark;")
576 self.print(f"void *_raw = {node.name}_raw(p);")
577 self.print("if (p->error_indicator) {")
578 with self.indent():
579 self.add_return("NULL")
580 self.print("}")
581 self.print("if (_raw == NULL || p->mark <= _resmark)")
582 with self.indent():
583 self.print("break;")
584 self.print("_resmark = p->mark;")
585 self.print("_res = _raw;")
586 self.print("}")
587 self.print("p->mark = _resmark;")
588 self.add_return("_res")
589 self.print("}")
590 self.print(f"static {result_type}")
591 self.print(f"{node.name}_raw(Parser *p)")
592
593 def _should_memoize(self, node: Rule) -> bool:
594 return node.memo and not node.left_recursive
595
596 def _handle_default_rule_body(self, node: Rule, rhs: Rhs, result_type: str) -> None:
597 memoize = self._should_memoize(node)
598
599 with self.indent():
600 self.add_level()
601 self._check_for_errors()
602 self.print(f"{result_type} _res = NULL;")
603 if memoize:
604 self.print(f"if (_PyPegen_is_memoized(p, {node.name}_type, &_res)) {{")
605 with self.indent():
606 self.add_return("_res")
607 self.print("}")
608 self.print("int _mark = p->mark;")
609 if any(alt.action and "EXTRA" in alt.action for alt in rhs.alts):
610 self._set_up_token_start_metadata_extraction()
611 self.visit(
612 rhs,
613 is_loop=False,
614 is_gather=node.is_gather(),
615 rulename=node.name,
616 )
617 if self.debug:
618 self.print(f'D(fprintf(stderr, "Fail at %d: {node.name}\\n", p->mark));')
619 self.print("_res = NULL;")
620 self.print(" done:")
621 with self.indent():
622 if memoize:
623 self.print(f"_PyPegen_insert_memo(p, _mark, {node.name}_type, _res);")
624 self.add_return("_res")
625
626 def _handle_loop_rule_body(self, node: Rule, rhs: Rhs) -> None:
627 memoize = self._should_memoize(node)
628 is_repeat1 = node.name.startswith("_loop1")
629
630 with self.indent():
631 self.add_level()
632 self._check_for_errors()
633 self.print("void *_res = NULL;")
634 if memoize:
635 self.print(f"if (_PyPegen_is_memoized(p, {node.name}_type, &_res)) {{")
636 with self.indent():
637 self.add_return("_res")
638 self.print("}")
639 self.print("int _mark = p->mark;")
640 if memoize:
641 self.print("int _start_mark = p->mark;")
642 self.print("void **_children = PyMem_Malloc(sizeof(void *));")
643 self.out_of_memory_return("!_children")
644 self.print("Py_ssize_t _children_capacity = 1;")
645 self.print("Py_ssize_t _n = 0;")
646 if any(alt.action and "EXTRA" in alt.action for alt in rhs.alts):
647 self._set_up_token_start_metadata_extraction()
648 self.visit(
649 rhs,
650 is_loop=True,
651 is_gather=node.is_gather(),
652 rulename=node.name,
653 )
654 if is_repeat1:
655 self.print("if (_n == 0 || p->error_indicator) {")
656 with self.indent():
657 self.print("PyMem_Free(_children);")
658 self.add_return("NULL")
659 self.print("}")
660 self.print("asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena);")
661 self.out_of_memory_return("!_seq", cleanup_code="PyMem_Free(_children);")
662 self.print("for (int i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]);")
663 self.print("PyMem_Free(_children);")
664 if memoize and node.name:
665 self.print(f"_PyPegen_insert_memo(p, _start_mark, {node.name}_type, _seq);")
666 self.add_return("_seq")
667
668 def visit_Rule(self, node: Rule) -> None:
669 is_loop = node.is_loop()
670 is_gather = node.is_gather()
671 rhs = node.flatten()
672 if is_loop or is_gather:
673 result_type = "asdl_seq *"
674 elif node.type:
675 result_type = node.type
676 else:
677 result_type = "void *"
678
679 for line in str(node).splitlines():
680 self.print(f"// {line}")
681 if node.left_recursive and node.leader:
682 self.print(f"static {result_type} {node.name}_raw(Parser *);")
683
684 self.print(f"static {result_type}")
685 self.print(f"{node.name}_rule(Parser *p)")
686
687 if node.left_recursive and node.leader:
688 self._set_up_rule_memoization(node, result_type)
689
690 self.print("{")
691
692 if node.name.endswith("without_invalid"):
693 with self.indent():
694 self.print("int _prev_call_invalid = p->call_invalid_rules;")
695 self.print("p->call_invalid_rules = 0;")
696 self.cleanup_statements.append("p->call_invalid_rules = _prev_call_invalid;")
697
698 if is_loop:
699 self._handle_loop_rule_body(node, rhs)
700 else:
701 self._handle_default_rule_body(node, rhs, result_type)
702
703 if node.name.endswith("without_invalid"):
704 self.cleanup_statements.pop()
705
706 self.print("}")
707
708 def visit_NamedItem(self, node: NamedItem) -> None:
709 call = self.callmakervisitor.generate_call(node)
710 if call.assigned_variable:
711 call.assigned_variable = self.dedupe(call.assigned_variable)
712 self.print(call)
713
714 def visit_Rhs(
715 self, node: Rhs, is_loop: bool, is_gather: bool, rulename: str | None
716 ) -> None:
717 if is_loop:
718 assert len(node.alts) == 1
719 for alt in node.alts:
720 self.visit(alt, is_loop=is_loop, is_gather=is_gather, rulename=rulename)
721
722 def join_conditions(self, keyword: str, node: Any) -> None:
723 self.print(f"{keyword} (")
724 with self.indent():
725 first = True
726 for item in node.items:
727 if first:
728 first = False
729 else:
730 self.print("&&")
731 self.visit(item)
732 self.print(")")
733
734 def emit_action(self, node: Alt, cleanup_code: str | None = None) -> None:
735 self.print(f"_res = {node.action};")
736
737 self.print("if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) {")
738 with self.indent():
739 self.print("p->error_indicator = 1;")
740 if cleanup_code:
741 self.print(cleanup_code)
742 self.add_return("NULL")
743 self.print("}")
744
745 if self.debug:
746 self.print(
747 f'D(fprintf(stderr, "Hit with action [%d-%d]: %s\\n", _mark, p->mark, "{node}"));'
748 )
749
750 def emit_default_action(self, is_gather: bool, node: Alt) -> None:
751 if len(self.local_variable_names) > 1:
752 if is_gather:
753 assert len(self.local_variable_names) == 2
754 self.print(
755 f"_res = _PyPegen_seq_insert_in_front(p, "
756 f"{self.local_variable_names[0]}, {self.local_variable_names[1]});"
757 )
758 else:
759 if self.debug:
760 self.print(
761 f'D(fprintf(stderr, "Hit without action [%d:%d]: %s\\n", _mark, p->mark, "{node}"));'
762 )
763 self.print(
764 f"_res = _PyPegen_dummy_name(p, {', '.join(self.local_variable_names)});"
765 )
766 else:
767 if self.debug:
768 self.print(
769 f'D(fprintf(stderr, "Hit with default action [%d:%d]: %s\\n", _mark, p->mark, "{node}"));'
770 )
771 self.print(f"_res = {self.local_variable_names[0]};")
772
773 def emit_dummy_action(self) -> None:
774 self.print("_res = _PyPegen_dummy_name(p);")
775
776 def handle_alt_normal(self, node: Alt, is_gather: bool, rulename: str | None) -> None:
777 self.join_conditions(keyword="if", node=node)
778 self.print("{")
779 # We have parsed successfully all the conditions for the option.
780 with self.indent():
781 node_str = str(node).replace('"', '\\"')
782 self.print(
783 f'D(fprintf(stderr, "%*c+ {rulename}[%d-%d]: %s succeeded!\\n", p->level, \' \', _mark, p->mark, "{node_str}"));'
784 )
785 # Prepare to emit the rule action and do so
786 if node.action and "EXTRA" in node.action:
787 self._set_up_token_end_metadata_extraction()
788 if self.skip_actions:
789 self.emit_dummy_action()
790 elif node.action:
791 self.emit_action(node)
792 else:
793 self.emit_default_action(is_gather, node)
794
795 # As the current option has parsed correctly, do not continue with the rest.
796 self.print("goto done;")
797 self.print("}")
798
799 def handle_alt_loop(self, node: Alt, is_gather: bool, rulename: str | None) -> None:
800 # Condition of the main body of the alternative
801 self.join_conditions(keyword="while", node=node)
802 self.print("{")
803 # We have parsed successfully one item!
804 with self.indent():
805 # Prepare to emit the rule action and do so
806 if node.action and "EXTRA" in node.action:
807 self._set_up_token_end_metadata_extraction()
808 if self.skip_actions:
809 self.emit_dummy_action()
810 elif node.action:
811 self.emit_action(node, cleanup_code="PyMem_Free(_children);")
812 else:
813 self.emit_default_action(is_gather, node)
814
815 # Add the result of rule to the temporary buffer of children. This buffer
816 # will populate later an asdl_seq with all elements to return.
817 self.print("if (_n == _children_capacity) {")
818 with self.indent():
819 self.print("_children_capacity *= 2;")
820 self.print(
821 "void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *));"
822 )
823 self.out_of_memory_return("!_new_children", cleanup_code="PyMem_Free(_children);")
824 self.print("_children = _new_children;")
825 self.print("}")
826 self.print("_children[_n++] = _res;")
827 self.print("_mark = p->mark;")
828 self.print("}")
829
830 def visit_Alt(
831 self, node: Alt, is_loop: bool, is_gather: bool, rulename: str | None
832 ) -> None:
833 if len(node.items) == 1 and str(node.items[0]).startswith("invalid_"):
834 self.print(f"if (p->call_invalid_rules) {{ // {node}")
835 else:
836 self.print(f"{{ // {node}")
837 with self.indent():
838 self._check_for_errors()
839 node_str = str(node).replace('"', '\\"')
840 self.print(
841 f'D(fprintf(stderr, "%*c> {rulename}[%d-%d]: %s\\n", p->level, \' \', _mark, p->mark, "{node_str}"));'
842 )
843 # Prepare variable declarations for the alternative
844 vars = self.collect_vars(node)
845 for v, var_type in sorted(item for item in vars.items() if item[0] is not None):
846 if not var_type:
847 var_type = "void *"
848 else:
849 var_type += " "
850 if v == "_cut_var":
851 v += " = 0" # cut_var must be initialized
852 self.print(f"{var_type}{v};")
853 if v and v.startswith("_opt_var"):
854 self.print(f"UNUSED({v}); // Silence compiler warnings")
855
856 with self.local_variable_context():
857 if is_loop:
858 self.handle_alt_loop(node, is_gather, rulename)
859 else:
860 self.handle_alt_normal(node, is_gather, rulename)
861
862 self.print("p->mark = _mark;")
863 node_str = str(node).replace('"', '\\"')
864 self.print(
865 f"D(fprintf(stderr, \"%*c%s {rulename}[%d-%d]: %s failed!\\n\", p->level, ' ',\n"
866 f' p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "{node_str}"));'
867 )
868 if "_cut_var" in vars:
869 self.print("if (_cut_var) {")
870 with self.indent():
871 self.add_return("NULL")
872 self.print("}")
873 self.print("}")
874
875 def collect_vars(self, node: Alt) -> dict[str | None, str | None]:
876 types = {}
877 with self.local_variable_context():
878 for item in node.items:
879 name, type = self.add_var(item)
880 types[name] = type
881 return types
882
883 def add_var(self, node: NamedItem) -> tuple[str | None, str | None]:
884 call = self.callmakervisitor.generate_call(node.item)
885 name = node.name if node.name else call.assigned_variable
886 if name is not None:
887 name = self.dedupe(name)
888 return_type = call.return_type if node.type is None else node.type
889 return name, return_type