Debian Codegraph @

Parser/lexer/state.h

raw
1 #ifndef _PY_LEXER_H_
2 #define _PY_LEXER_H_
3
4 #include "Python.h"
5 #include "object.h"
6
7 #define MAXINDENT 100 /* Max indentation level */
8 #define MAXLEVEL 200 /* Max parentheses level */
9 #define MAXFSTRINGLEVEL 150 /* Max f-string nesting level */
10
11 #define INSIDE_FSTRING(tok) (tok->tok_mode_stack_index > 0)
12 #define INSIDE_FSTRING_EXPR(tok) (tok->curly_bracket_expr_start_depth >= 0)
13 #define INSIDE_FSTRING_EXPR_AT_TOP(tok) \
14 (tok->curly_bracket_depth - tok->curly_bracket_expr_start_depth == 1)
15
16 enum decoding_state {
17 STATE_INIT,
18 STATE_SEEK_CODING,
19 STATE_NORMAL
20 };
21
22 enum interactive_underflow_t {
23 /* Normal mode of operation: return a new token when asked in interactive mode */
24 IUNDERFLOW_NORMAL,
25 /* Forcefully return ENDMARKER when asked for a new token in interactive mode. This
26 * can be used to prevent the tokenizer to prompt the user for new tokens */
27 IUNDERFLOW_STOP,
28 };
29
30 struct token {
31 int level;
32 int lineno, col_offset, end_lineno, end_col_offset;
33 const char *start, *end;
34 PyObject *metadata;
35 };
36
37 enum tokenizer_mode_kind_t {
38 TOK_REGULAR_MODE,
39 TOK_FSTRING_MODE,
40 };
41
42 #define MAX_EXPR_NESTING 3
43
44 typedef struct _tokenizer_mode {
45 enum tokenizer_mode_kind_t kind;
46
47 int curly_bracket_depth;
48 int curly_bracket_expr_start_depth;
49
50 char f_string_quote;
51 int f_string_quote_size;
52 int f_string_raw;
53 const char* f_string_start;
54 const char* f_string_multi_line_start;
55 int f_string_line_start;
56
57 Py_ssize_t f_string_start_offset;
58 Py_ssize_t f_string_multi_line_start_offset;
59
60 Py_ssize_t last_expr_size;
61 Py_ssize_t last_expr_end;
62 char* last_expr_buffer;
63 int f_string_debug;
64 int in_format_spec;
65 } tokenizer_mode;
66
67 /* Tokenizer state */
68 struct tok_state {
69 /* Input state; buf <= cur <= inp <= end */
70 /* NB an entire line is held in the buffer */
71 char *buf; /* Input buffer, or NULL; malloc'ed if fp != NULL or readline != NULL */
72 char *cur; /* Next character in buffer */
73 char *inp; /* End of data in buffer */
74 int fp_interactive; /* If the file descriptor is interactive */
75 char *interactive_src_start; /* The start of the source parsed so far in interactive mode */
76 char *interactive_src_end; /* The end of the source parsed so far in interactive mode */
77 const char *end; /* End of input buffer if buf != NULL */
78 const char *start; /* Start of current token if not NULL */
79 int done; /* E_OK normally, E_EOF at EOF, otherwise error code */
80 /* NB If done != E_OK, cur must be == inp!!! */
81 FILE *fp; /* Rest of input; NULL if tokenizing a string */
82 int tabsize; /* Tab spacing */
83 int indent; /* Current indentation index */
84 int indstack[MAXINDENT]; /* Stack of indents */
85 int atbol; /* Nonzero if at begin of new line */
86 int pendin; /* Pending indents (if > 0) or dedents (if < 0) */
87 const char *prompt, *nextprompt; /* For interactive prompting */
88 int lineno; /* Current line number */
89 int first_lineno; /* First line of a single line or multi line string
90 expression (cf. issue 16806) */
91 int starting_col_offset; /* The column offset at the beginning of a token */
92 int col_offset; /* Current col offset */
93 int level; /* () [] {} Parentheses nesting level */
94 /* Used to allow free continuations inside them */
95 char parenstack[MAXLEVEL];
96 int parenlinenostack[MAXLEVEL];
97 int parencolstack[MAXLEVEL];
98 PyObject *filename;
99 /* Stuff for checking on different tab sizes */
100 int altindstack[MAXINDENT]; /* Stack of alternate indents */
101 /* Stuff for PEP 0263 */
102 enum decoding_state decoding_state;
103 int decoding_erred; /* whether erred in decoding */
104 char *encoding; /* Source encoding. */
105 int cont_line; /* whether we are in a continuation line. */
106 const char* line_start; /* pointer to start of current line */
107 const char* multi_line_start; /* pointer to start of first line of
108 a single line or multi line string
109 expression (cf. issue 16806) */
110 PyObject *decoding_readline; /* open(...).readline */
111 PyObject *decoding_buffer;
112 PyObject *readline; /* readline() function */
113 const char* enc; /* Encoding for the current str. */
114 char* str; /* Source string being tokenized (if tokenizing from a string)*/
115 char* input; /* Tokenizer's newline translated copy of the string. */
116
117 int type_comments; /* Whether to look for type comments */
118
119 /* How to proceed when asked for a new token in interactive mode */
120 enum interactive_underflow_t interactive_underflow;
121 int (*underflow)(struct tok_state *); /* Function to call when buffer is empty and we need to refill it*/
122
123 int report_warnings;
124 // TODO: Factor this into its own thing
125 tokenizer_mode tok_mode_stack[MAXFSTRINGLEVEL];
126 int tok_mode_stack_index;
127 int tok_extra_tokens;
128 int comment_newline;
129 int implicit_newline;
130 #ifdef Py_DEBUG
131 int debug;
132 #endif
133 };
134
135 int _PyLexer_type_comment_token_setup(struct tok_state *tok, struct token *token, int type, int col_offset,
136 int end_col_offset, const char *start, const char *end);
137 int _PyLexer_token_setup(struct tok_state *tok, struct token *token, int type, const char *start, const char *end);
138
139 struct tok_state *_PyTokenizer_tok_new(void);
140 void _PyTokenizer_Free(struct tok_state *);
141 void _PyToken_Free(struct token *);
142 void _PyToken_Init(struct token *);
143
144 #ifdef Py_DEBUG
145 static inline tokenizer_mode* TOK_GET_MODE(struct tok_state* tok) {
146 assert(tok->tok_mode_stack_index >= 0);
147 assert(tok->tok_mode_stack_index < MAXFSTRINGLEVEL);
148 return &(tok->tok_mode_stack[tok->tok_mode_stack_index]);
149 }
150 static inline tokenizer_mode* TOK_NEXT_MODE(struct tok_state* tok) {
151 assert(tok->tok_mode_stack_index >= 0);
152 assert(tok->tok_mode_stack_index + 1 < MAXFSTRINGLEVEL);
153 return &(tok->tok_mode_stack[++tok->tok_mode_stack_index]);
154 }
155 #else
156 #define TOK_GET_MODE(tok) (&(tok->tok_mode_stack[tok->tok_mode_stack_index]))
157 #define TOK_NEXT_MODE(tok) (&(tok->tok_mode_stack[++tok->tok_mode_stack_index]))
158 #endif
159
160 #endif