Debian Codegraph @

Parser/lexer/state.h

raw
1 #ifndef _PY_LEXER_H_
2 #define _PY_LEXER_H_
3
4 #include "object.h"
5
6 #define MAXINDENT 100 /* Max indentation level */
7 #define MAXLEVEL 200 /* Max parentheses level */
8 #define MAXFSTRINGLEVEL 150 /* Max f-string nesting level */
9
10 #define INSIDE_FSTRING(tok) (tok->tok_mode_stack_index > 0)
11 #define INSIDE_FSTRING_EXPR(tok) (tok->curly_bracket_expr_start_depth >= 0)
12 #define INSIDE_FSTRING_EXPR_AT_TOP(tok) \
13 (tok->curly_bracket_depth - tok->curly_bracket_expr_start_depth == 1)
14
15 enum decoding_state {
16 STATE_INIT,
17 STATE_SEEK_CODING,
18 STATE_NORMAL
19 };
20
21 enum interactive_underflow_t {
22 /* Normal mode of operation: return a new token when asked in interactive mode */
23 IUNDERFLOW_NORMAL,
24 /* Forcefully return ENDMARKER when asked for a new token in interactive mode. This
25 * can be used to prevent the tokenizer to prompt the user for new tokens */
26 IUNDERFLOW_STOP,
27 };
28
29 struct token {
30 int level;
31 int lineno, col_offset, end_lineno, end_col_offset;
32 const char *start, *end;
33 PyObject *metadata;
34 };
35
36 enum tokenizer_mode_kind_t {
37 TOK_REGULAR_MODE,
38 TOK_FSTRING_MODE,
39 };
40
41 enum string_kind_t {
42 FSTRING,
43 TSTRING,
44 };
45
46 #define MAX_EXPR_NESTING 3
47
48 typedef struct _tokenizer_mode {
49 enum tokenizer_mode_kind_t kind;
50
51 int curly_bracket_depth;
52 int curly_bracket_expr_start_depth;
53
54 char quote;
55 int quote_size;
56 int raw;
57 const char* start;
58 const char* multi_line_start;
59 int first_line;
60
61 Py_ssize_t start_offset;
62 Py_ssize_t multi_line_start_offset;
63
64 Py_ssize_t last_expr_size;
65 Py_ssize_t last_expr_end;
66 char* last_expr_buffer;
67 int in_debug;
68 int in_format_spec;
69
70 enum string_kind_t string_kind;
71 } tokenizer_mode;
72
73 /* Tokenizer state */
74 struct tok_state {
75 /* Input state; buf <= cur <= inp <= end */
76 /* NB an entire line is held in the buffer */
77 char *buf; /* Input buffer, or NULL; malloc'ed if fp != NULL or readline != NULL */
78 char *cur; /* Next character in buffer */
79 char *inp; /* End of data in buffer */
80 int fp_interactive; /* If the file descriptor is interactive */
81 char *interactive_src_start; /* The start of the source parsed so far in interactive mode */
82 char *interactive_src_end; /* The end of the source parsed so far in interactive mode */
83 const char *end; /* End of input buffer if buf != NULL */
84 const char *start; /* Start of current token if not NULL */
85 int done; /* E_OK normally, E_EOF at EOF, otherwise error code */
86 /* NB If done != E_OK, cur must be == inp!!! */
87 FILE *fp; /* Rest of input; NULL if tokenizing a string */
88 int tabsize; /* Tab spacing */
89 int indent; /* Current indentation index */
90 int indstack[MAXINDENT]; /* Stack of indents */
91 int atbol; /* Nonzero if at begin of new line */
92 int pendin; /* Pending indents (if > 0) or dedents (if < 0) */
93 const char *prompt, *nextprompt; /* For interactive prompting */
94 int lineno; /* Current line number */
95 int first_lineno; /* First line of a single line or multi line string
96 expression (cf. issue 16806) */
97 int starting_col_offset; /* The column offset at the beginning of a token */
98 int col_offset; /* Current col offset */
99 int level; /* () [] {} Parentheses nesting level */
100 /* Used to allow free continuations inside them */
101 char parenstack[MAXLEVEL];
102 int parenlinenostack[MAXLEVEL];
103 int parencolstack[MAXLEVEL];
104 PyObject *filename;
105 /* Stuff for checking on different tab sizes */
106 int altindstack[MAXINDENT]; /* Stack of alternate indents */
107 /* Stuff for PEP 0263 */
108 enum decoding_state decoding_state;
109 int decoding_erred; /* whether erred in decoding */
110 char *encoding; /* Source encoding. */
111 int cont_line; /* whether we are in a continuation line. */
112 const char* line_start; /* pointer to start of current line */
113 const char* multi_line_start; /* pointer to start of first line of
114 a single line or multi line string
115 expression (cf. issue 16806) */
116 PyObject *decoding_readline; /* open(...).readline */
117 PyObject *decoding_buffer;
118 PyObject *readline; /* readline() function */
119 const char* enc; /* Encoding for the current str. */
120 char* str; /* Source string being tokenized (if tokenizing from a string)*/
121 char* input; /* Tokenizer's newline translated copy of the string. */
122
123 int type_comments; /* Whether to look for type comments */
124
125 /* How to proceed when asked for a new token in interactive mode */
126 enum interactive_underflow_t interactive_underflow;
127 int (*underflow)(struct tok_state *); /* Function to call when buffer is empty and we need to refill it*/
128
129 int report_warnings;
130 // TODO: Factor this into its own thing
131 tokenizer_mode tok_mode_stack[MAXFSTRINGLEVEL];
132 int tok_mode_stack_index;
133 int tok_extra_tokens;
134 int comment_newline;
135 int implicit_newline;
136 #ifdef Py_DEBUG
137 int debug;
138 #endif
139 };
140
141 int _PyLexer_type_comment_token_setup(struct tok_state *tok, struct token *token, int type, int col_offset,
142 int end_col_offset, const char *start, const char *end);
143 int _PyLexer_token_setup(struct tok_state *tok, struct token *token, int type, const char *start, const char *end);
144
145 struct tok_state *_PyTokenizer_tok_new(void);
146 void _PyTokenizer_Free(struct tok_state *);
147 void _PyToken_Free(struct token *);
148 void _PyToken_Init(struct token *);
149
150
151 #endif