Debian Codegraph @

Tools/peg_generator/pegen/__main__.py

raw
1 #!/usr/bin/env python3.8
2
3 """pegen -- PEG Generator.
4
5 Search the web for PEG Parsers for reference.
6 """
7
8 import argparse
9 import sys
10 import time
11 import token
12 import traceback
13
14 from pegen.grammar import Grammar
15 from pegen.parser import Parser
16 from pegen.parser_generator import ParserGenerator
17 from pegen.tokenizer import Tokenizer
18 from pegen.validator import validate_grammar
19
20
21 def generate_c_code(
22 args: argparse.Namespace,
23 ) -> tuple[Grammar, Parser, Tokenizer, ParserGenerator]:
24 from pegen.build import build_c_parser_and_generator
25
26 verbose = args.verbose
27 verbose_tokenizer = verbose >= 3
28 verbose_parser = verbose == 2 or verbose >= 4
29 try:
30 grammar, parser, tokenizer, gen = build_c_parser_and_generator(
31 args.grammar_filename,
32 args.tokens_filename,
33 args.output,
34 args.compile_extension,
35 verbose_tokenizer,
36 verbose_parser,
37 args.verbose,
38 keep_asserts_in_extension=False if args.optimized else True,
39 skip_actions=args.skip_actions,
40 )
41 return grammar, parser, tokenizer, gen
42 except Exception as err:
43 if args.verbose:
44 raise # Show traceback
45 traceback.print_exception(err.__class__, err, None)
46 sys.stderr.write("For full traceback, use -v\n")
47 sys.exit(1)
48
49
50 def generate_python_code(
51 args: argparse.Namespace,
52 ) -> tuple[Grammar, Parser, Tokenizer, ParserGenerator]:
53 from pegen.build import build_python_parser_and_generator
54
55 verbose = args.verbose
56 verbose_tokenizer = verbose >= 3
57 verbose_parser = verbose == 2 or verbose >= 4
58 try:
59 grammar, parser, tokenizer, gen = build_python_parser_and_generator(
60 args.grammar_filename,
61 args.output,
62 verbose_tokenizer,
63 verbose_parser,
64 skip_actions=args.skip_actions,
65 )
66 return grammar, parser, tokenizer, gen
67 except Exception as err:
68 if args.verbose:
69 raise # Show traceback
70 traceback.print_exception(err.__class__, err, None)
71 sys.stderr.write("For full traceback, use -v\n")
72 sys.exit(1)
73
74
75 argparser = argparse.ArgumentParser(
76 prog="pegen", description="Experimental PEG-like parser generator"
77 )
78 argparser.add_argument("-q", "--quiet", action="store_true", help="Don't print the parsed grammar")
79 argparser.add_argument(
80 "-v",
81 "--verbose",
82 action="count",
83 default=0,
84 help="Print timing stats; repeat for more debug output",
85 )
86 subparsers = argparser.add_subparsers(help="target language for the generated code")
87
88 c_parser = subparsers.add_parser("c", help="Generate C code for inclusion into CPython")
89 c_parser.set_defaults(func=generate_c_code)
90 c_parser.add_argument("grammar_filename", help="Grammar description")
91 c_parser.add_argument("tokens_filename", help="Tokens description")
92 c_parser.add_argument(
93 "-o", "--output", metavar="OUT", default="parse.c", help="Where to write the generated parser"
94 )
95 c_parser.add_argument(
96 "--compile-extension",
97 action="store_true",
98 help="Compile generated C code into an extension module",
99 )
100 c_parser.add_argument(
101 "--optimized", action="store_true", help="Compile the extension in optimized mode"
102 )
103 c_parser.add_argument(
104 "--skip-actions",
105 action="store_true",
106 help="Suppress code emission for rule actions",
107 )
108
109 python_parser = subparsers.add_parser(
110 "python",
111 help="Generate Python code, needs grammar definition with Python actions",
112 )
113 python_parser.set_defaults(func=generate_python_code)
114 python_parser.add_argument("grammar_filename", help="Grammar description")
115 python_parser.add_argument(
116 "-o",
117 "--output",
118 metavar="OUT",
119 default="parse.py",
120 help="Where to write the generated parser",
121 )
122 python_parser.add_argument(
123 "--skip-actions",
124 action="store_true",
125 help="Suppress code emission for rule actions",
126 )
127
128
129 def main() -> None:
130 from pegen.testutil import print_memstats
131
132 args = argparser.parse_args()
133 if "func" not in args:
134 argparser.error("Must specify the target language mode ('c' or 'python')")
135
136 t0 = time.time()
137 grammar, parser, tokenizer, gen = args.func(args)
138 t1 = time.time()
139
140 validate_grammar(grammar)
141
142 if not args.quiet:
143 if args.verbose:
144 print("Raw Grammar:")
145 for line in repr(grammar).splitlines():
146 print(" ", line)
147
148 print("Clean Grammar:")
149 for line in str(grammar).splitlines():
150 print(" ", line)
151
152 if args.verbose:
153 print("First Graph:")
154 for src, dsts in gen.first_graph.items():
155 print(f" {src} -> {', '.join(dsts)}")
156 print("First SCCS:")
157 for scc in gen.first_sccs:
158 print(" ", scc, end="")
159 if len(scc) > 1:
160 print(
161 " # Indirectly left-recursive; leaders:",
162 {name for name in scc if grammar.rules[name].leader},
163 )
164 else:
165 name = next(iter(scc))
166 if name in gen.first_graph[name]:
167 print(" # Left-recursive")
168 else:
169 print()
170
171 if args.verbose:
172 dt = t1 - t0
173 diag = tokenizer.diagnose()
174 nlines = diag.end[0]
175 if diag.type == token.ENDMARKER:
176 nlines -= 1
177 print(f"Total time: {dt:.3f} sec; {nlines} lines", end="")
178 if dt:
179 print(f"; {nlines / dt:.0f} lines/sec")
180 else:
181 print()
182 print("Caches sizes:")
183 print(f" token array : {len(tokenizer._tokens):10}")
184 print(f" cache : {len(parser._cache):10}")
185 if not print_memstats():
186 print("(Can't find psutil; install it for memory stats.)")
187
188
189 if __name__ == "__main__":
190 if sys.version_info < (3, 8): # noqa: UP036
191 print("ERROR: using pegen requires at least Python 3.8!", file=sys.stderr)
192 sys.exit(1)
193 main()