Debian Codegraph @

Parser/lexer/state.c

raw
1 #include "Python.h"
2 #include "pycore_pystate.h"
3 #include "pycore_token.h"
4 #include "errcode.h"
5
6 #include "state.h"
7
8 /* Never change this */
9 #define TABSIZE 8
10
11 /* Create and initialize a new tok_state structure */
12 struct tok_state *
13 _PyTokenizer_tok_new(void)
14 {
15 struct tok_state *tok = (struct tok_state *)PyMem_Calloc(
16 1,
17 sizeof(struct tok_state));
18 if (tok == NULL) {
19 PyErr_NoMemory();
20 return NULL;
21 }
22
23 tok->buf = tok->cur = tok->inp = NULL;
24 tok->fp_interactive = 0;
25 tok->interactive_src_start = NULL;
26 tok->interactive_src_end = NULL;
27 tok->start = NULL;
28 tok->end = NULL;
29 tok->done = E_OK;
30 tok->fp = NULL;
31 tok->input = NULL;
32 tok->tabsize = TABSIZE;
33 tok->indent = 0;
34 tok->indstack[0] = 0;
35 tok->atbol = 1;
36 tok->pendin = 0;
37 tok->prompt = tok->nextprompt = NULL;
38 tok->lineno = 0;
39 tok->starting_col_offset = -1;
40 tok->col_offset = -1;
41 tok->level = 0;
42 tok->altindstack[0] = 0;
43 tok->decoding_state = STATE_INIT;
44 tok->decoding_erred = 0;
45 tok->enc = NULL;
46 tok->encoding = NULL;
47 tok->cont_line = 0;
48 tok->filename = NULL;
49 tok->decoding_readline = NULL;
50 tok->decoding_buffer = NULL;
51 tok->readline = NULL;
52 tok->type_comments = 0;
53 tok->interactive_underflow = IUNDERFLOW_NORMAL;
54 tok->underflow = NULL;
55 tok->str = NULL;
56 tok->report_warnings = 1;
57 tok->tok_extra_tokens = 0;
58 tok->comment_newline = 0;
59 tok->implicit_newline = 0;
60 tok->tok_mode_stack[0] = (tokenizer_mode){.kind =TOK_REGULAR_MODE, .quote='\0', .quote_size = 0, .in_debug=0};
61 tok->tok_mode_stack_index = 0;
62 #ifdef Py_DEBUG
63 tok->debug = _Py_GetConfig()->parser_debug;
64 #endif
65 return tok;
66 }
67
68 static void
69 free_fstring_expressions(struct tok_state *tok)
70 {
71 int index;
72 tokenizer_mode *mode;
73
74 for (index = tok->tok_mode_stack_index; index >= 0; --index) {
75 mode = &(tok->tok_mode_stack[index]);
76 if (mode->last_expr_buffer != NULL) {
77 PyMem_Free(mode->last_expr_buffer);
78 mode->last_expr_buffer = NULL;
79 mode->last_expr_size = 0;
80 mode->last_expr_end = -1;
81 mode->in_format_spec = 0;
82 }
83 }
84 }
85
86 /* Free a tok_state structure */
87 void
88 _PyTokenizer_Free(struct tok_state *tok)
89 {
90 if (tok->encoding != NULL) {
91 PyMem_Free(tok->encoding);
92 }
93 Py_XDECREF(tok->decoding_readline);
94 Py_XDECREF(tok->decoding_buffer);
95 Py_XDECREF(tok->readline);
96 Py_XDECREF(tok->filename);
97 if ((tok->readline != NULL || tok->fp != NULL ) && tok->buf != NULL) {
98 PyMem_Free(tok->buf);
99 }
100 if (tok->input) {
101 PyMem_Free(tok->input);
102 }
103 if (tok->interactive_src_start != NULL) {
104 PyMem_Free(tok->interactive_src_start);
105 }
106 free_fstring_expressions(tok);
107 PyMem_Free(tok);
108 }
109
110 void
111 _PyToken_Free(struct token *token) {
112 Py_XDECREF(token->metadata);
113 }
114
115 void
116 _PyToken_Init(struct token *token) {
117 token->metadata = NULL;
118 }
119
120 int
121 _PyLexer_type_comment_token_setup(struct tok_state *tok, struct token *token, int type, int col_offset,
122 int end_col_offset, const char *start, const char *end)
123 {
124 token->level = tok->level;
125 token->lineno = token->end_lineno = tok->lineno;
126 token->col_offset = col_offset;
127 token->end_col_offset = end_col_offset;
128 token->start = start;
129 token->end = end;
130 return type;
131 }
132
133 int
134 _PyLexer_token_setup(struct tok_state *tok, struct token *token, int type, const char *start, const char *end)
135 {
136 assert((start == NULL && end == NULL) || (start != NULL && end != NULL));
137 token->level = tok->level;
138 if (ISSTRINGLIT(type)) {
139 token->lineno = tok->first_lineno;
140 }
141 else {
142 token->lineno = tok->lineno;
143 }
144 token->end_lineno = tok->lineno;
145 token->col_offset = token->end_col_offset = -1;
146 token->start = start;
147 token->end = end;
148
149 if (start != NULL && end != NULL) {
150 token->col_offset = tok->starting_col_offset;
151 token->end_col_offset = tok->col_offset;
152 }
153 return type;
154 }