Debian Codegraph @

Tools/peg_generator/pegen/keywordgen.py

raw
1 """Generate Lib/keyword.py from the Grammar and Tokens files using pgen"""
2
3 import argparse
4
5 from .build import build_parser, generate_token_definitions
6 from .c_generator import CParserGenerator
7
8 TEMPLATE = r'''
9 """Keywords (from "Grammar/python.gram")
10
11 This file is automatically generated; please don't muck it up!
12
13 To update the symbols in this file, 'cd' to the top directory of
14 the python source tree and run:
15
16 PYTHONPATH=Tools/peg_generator python3 -m pegen.keywordgen \
17 Grammar/python.gram \
18 Grammar/Tokens \
19 Lib/keyword.py
20
21 Alternatively, you can run 'make regen-keyword'.
22 """
23
24 __all__ = ["iskeyword", "issoftkeyword", "kwlist", "softkwlist"]
25
26 kwlist = [
27 {keywords}
28 ]
29
30 softkwlist = [
31 {soft_keywords}
32 ]
33
34 iskeyword = frozenset(kwlist).__contains__
35 issoftkeyword = frozenset(softkwlist).__contains__
36 '''.lstrip()
37
38
39 def main() -> None:
40 parser = argparse.ArgumentParser(
41 description="Generate the Lib/keywords.py file from the grammar."
42 )
43 parser.add_argument(
44 "grammar", help="The file with the grammar definition in PEG format"
45 )
46 parser.add_argument(
47 "tokens_file", help="The file with the token definitions"
48 )
49 parser.add_argument(
50 "keyword_file",
51 help="The path to write the keyword definitions",
52 )
53 args = parser.parse_args()
54
55 grammar, _, _ = build_parser(args.grammar)
56 with open(args.tokens_file) as tok_file:
57 all_tokens, exact_tok, non_exact_tok = generate_token_definitions(tok_file)
58 gen = CParserGenerator(grammar, all_tokens, exact_tok, non_exact_tok, file=None)
59 gen.collect_rules()
60
61 with open(args.keyword_file, 'w') as thefile:
62 all_keywords = sorted(list(gen.keywords.keys()))
63 all_soft_keywords = sorted(gen.soft_keywords)
64
65 keywords = "" if not all_keywords else " " + ",\n ".join(map(repr, all_keywords))
66 soft_keywords = (
67 "" if not all_soft_keywords else " " + ",\n ".join(map(repr, all_soft_keywords))
68 )
69 thefile.write(TEMPLATE.format(keywords=keywords, soft_keywords=soft_keywords))
70
71
72 if __name__ == "__main__":
73 main()