| 1 |
# When adding new tokens, remember to update the PEG generator in |
| 2 |
# Tools/peg_generator/pegen/parser_generator.py |
| 3 |
# This will ensure that older versions of Python can generate a Python parser |
| 4 |
# using "python -m pegen python <GRAMMAR FILE>". |
| 5 |
|
| 6 |
ENDMARKER |
| 7 |
NAME |
| 8 |
NUMBER |
| 9 |
STRING |
| 10 |
NEWLINE |
| 11 |
INDENT |
| 12 |
DEDENT |
| 13 |
|
| 14 |
LPAR '(' |
| 15 |
RPAR ')' |
| 16 |
LSQB '[' |
| 17 |
RSQB ']' |
| 18 |
COLON ':' |
| 19 |
COMMA ',' |
| 20 |
SEMI ';' |
| 21 |
PLUS '+' |
| 22 |
MINUS '-' |
| 23 |
STAR '*' |
| 24 |
SLASH '/' |
| 25 |
VBAR '|' |
| 26 |
AMPER '&' |
| 27 |
LESS '<' |
| 28 |
GREATER '>' |
| 29 |
EQUAL '=' |
| 30 |
DOT '.' |
| 31 |
PERCENT '%' |
| 32 |
LBRACE '{' |
| 33 |
RBRACE '}' |
| 34 |
EQEQUAL '==' |
| 35 |
NOTEQUAL '!=' |
| 36 |
LESSEQUAL '<=' |
| 37 |
GREATEREQUAL '>=' |
| 38 |
TILDE '~' |
| 39 |
CIRCUMFLEX '^' |
| 40 |
LEFTSHIFT '<<' |
| 41 |
RIGHTSHIFT '>>' |
| 42 |
DOUBLESTAR '**' |
| 43 |
PLUSEQUAL '+=' |
| 44 |
MINEQUAL '-=' |
| 45 |
STAREQUAL '*=' |
| 46 |
SLASHEQUAL '/=' |
| 47 |
PERCENTEQUAL '%=' |
| 48 |
AMPEREQUAL '&=' |
| 49 |
VBAREQUAL '|=' |
| 50 |
CIRCUMFLEXEQUAL '^=' |
| 51 |
LEFTSHIFTEQUAL '<<=' |
| 52 |
RIGHTSHIFTEQUAL '>>=' |
| 53 |
DOUBLESTAREQUAL '**=' |
| 54 |
DOUBLESLASH '//' |
| 55 |
DOUBLESLASHEQUAL '//=' |
| 56 |
AT '@' |
| 57 |
ATEQUAL '@=' |
| 58 |
RARROW '->' |
| 59 |
ELLIPSIS '...' |
| 60 |
COLONEQUAL ':=' |
| 61 |
EXCLAMATION '!' |
| 62 |
|
| 63 |
OP |
| 64 |
TYPE_IGNORE |
| 65 |
TYPE_COMMENT |
| 66 |
SOFT_KEYWORD |
| 67 |
FSTRING_START |
| 68 |
FSTRING_MIDDLE |
| 69 |
FSTRING_END |
| 70 |
TSTRING_START |
| 71 |
TSTRING_MIDDLE |
| 72 |
TSTRING_END |
| 73 |
COMMENT |
| 74 |
NL |
| 75 |
ERRORTOKEN |
| 76 |
|
| 77 |
# These aren't used by the C tokenizer but are needed for tokenize.py |
| 78 |
ENCODING |