Debian Codegraph @

Grammar/Tokens

raw
1 # When adding new tokens, remember to update the PEG generator in
2 # Tools/peg_generator/pegen/parser_generator.py
3 # This will ensure that older versions of Python can generate a Python parser
4 # using "python -m pegen python <GRAMMAR FILE>".
5
6 ENDMARKER
7 NAME
8 NUMBER
9 STRING
10 NEWLINE
11 INDENT
12 DEDENT
13
14 LPAR '('
15 RPAR ')'
16 LSQB '['
17 RSQB ']'
18 COLON ':'
19 COMMA ','
20 SEMI ';'
21 PLUS '+'
22 MINUS '-'
23 STAR '*'
24 SLASH '/'
25 VBAR '|'
26 AMPER '&'
27 LESS '<'
28 GREATER '>'
29 EQUAL '='
30 DOT '.'
31 PERCENT '%'
32 LBRACE '{'
33 RBRACE '}'
34 EQEQUAL '=='
35 NOTEQUAL '!='
36 LESSEQUAL '<='
37 GREATEREQUAL '>='
38 TILDE '~'
39 CIRCUMFLEX '^'
40 LEFTSHIFT '<<'
41 RIGHTSHIFT '>>'
42 DOUBLESTAR '**'
43 PLUSEQUAL '+='
44 MINEQUAL '-='
45 STAREQUAL '*='
46 SLASHEQUAL '/='
47 PERCENTEQUAL '%='
48 AMPEREQUAL '&='
49 VBAREQUAL '|='
50 CIRCUMFLEXEQUAL '^='
51 LEFTSHIFTEQUAL '<<='
52 RIGHTSHIFTEQUAL '>>='
53 DOUBLESTAREQUAL '**='
54 DOUBLESLASH '//'
55 DOUBLESLASHEQUAL '//='
56 AT '@'
57 ATEQUAL '@='
58 RARROW '->'
59 ELLIPSIS '...'
60 COLONEQUAL ':='
61 EXCLAMATION '!'
62
63 OP
64 TYPE_IGNORE
65 TYPE_COMMENT
66 SOFT_KEYWORD
67 FSTRING_START
68 FSTRING_MIDDLE
69 FSTRING_END
70 TSTRING_START
71 TSTRING_MIDDLE
72 TSTRING_END
73 COMMENT
74 NL
75 ERRORTOKEN
76
77 # These aren't used by the C tokenizer but are needed for tokenize.py
78 ENCODING