Debian Codegraph @

Parser/pegen.h

raw
1 #ifndef PEGEN_H
2 #define PEGEN_H
3
4 #include <Python.h>
5 #include <pycore_ast.h>
6 #include <pycore_token.h>
7
8 #if 0
9 #define PyPARSE_YIELD_IS_KEYWORD 0x0001
10 #endif
11
12 #define PyPARSE_DONT_IMPLY_DEDENT 0x0002
13
14 #if 0
15 #define PyPARSE_WITH_IS_KEYWORD 0x0003
16 #define PyPARSE_PRINT_IS_FUNCTION 0x0004
17 #define PyPARSE_UNICODE_LITERALS 0x0008
18 #endif
19
20 #define PyPARSE_IGNORE_COOKIE 0x0010
21 #define PyPARSE_BARRY_AS_BDFL 0x0020
22 #define PyPARSE_TYPE_COMMENTS 0x0040
23 #define PyPARSE_ALLOW_INCOMPLETE_INPUT 0x0100
24
25 #define CURRENT_POS (-5)
26
27 typedef struct _memo {
28 int type;
29 void *node;
30 int mark;
31 struct _memo *next;
32 } Memo;
33
34 typedef struct {
35 int type;
36 PyObject *bytes;
37 int level;
38 int lineno, col_offset, end_lineno, end_col_offset;
39 Memo *memo;
40 PyObject *metadata;
41 } Token;
42
43 typedef struct {
44 const char *str;
45 int type;
46 } KeywordToken;
47
48
49 typedef struct {
50 struct {
51 int lineno;
52 char *comment; // The " <tag>" in "# type: ignore <tag>"
53 } *items;
54 size_t size;
55 size_t num_items;
56 } growable_comment_array;
57
58 typedef struct {
59 struct tok_state *tok;
60 Token **tokens;
61 int mark;
62 int fill, size;
63 PyArena *arena;
64 KeywordToken **keywords;
65 char **soft_keywords;
66 int n_keyword_lists;
67 int start_rule;
68 int *errcode;
69 int parsing_started;
70 PyObject* normalize;
71 int starting_lineno;
72 int starting_col_offset;
73 int error_indicator;
74 int flags;
75 int feature_version;
76 growable_comment_array type_ignore_comments;
77 Token *known_err_token;
78 int level;
79 int call_invalid_rules;
80 int debug;
81 } Parser;
82
83 typedef struct {
84 cmpop_ty cmpop;
85 expr_ty expr;
86 } CmpopExprPair;
87
88 typedef struct {
89 expr_ty key;
90 expr_ty value;
91 } KeyValuePair;
92
93 typedef struct {
94 expr_ty key;
95 pattern_ty pattern;
96 } KeyPatternPair;
97
98 typedef struct {
99 arg_ty arg;
100 expr_ty value;
101 } NameDefaultPair;
102
103 typedef struct {
104 asdl_arg_seq *plain_names;
105 asdl_seq *names_with_defaults; // asdl_seq* of NameDefaultsPair's
106 } SlashWithDefault;
107
108 typedef struct {
109 arg_ty vararg;
110 asdl_seq *kwonlyargs; // asdl_seq* of NameDefaultsPair's
111 arg_ty kwarg;
112 } StarEtc;
113
114 typedef struct { operator_ty kind; } AugOperator;
115 typedef struct {
116 void *element;
117 int is_keyword;
118 } KeywordOrStarred;
119
120 typedef struct {
121 void *result;
122 PyObject *metadata;
123 } ResultTokenWithMetadata;
124
125 // Internal parser functions
126 #if defined(Py_DEBUG)
127 void _PyPegen_clear_memo_statistics(void);
128 PyObject *_PyPegen_get_memo_statistics(void);
129 #endif
130
131 int _PyPegen_insert_memo(Parser *p, int mark, int type, void *node);
132 int _PyPegen_update_memo(Parser *p, int mark, int type, void *node);
133 int _PyPegen_is_memoized(Parser *p, int type, void *pres);
134
135 int _PyPegen_lookahead(int, void *(func)(Parser *), Parser *);
136 int _PyPegen_lookahead_for_expr(int, expr_ty (func)(Parser *), Parser *);
137 int _PyPegen_lookahead_for_stmt(int, stmt_ty (func)(Parser *), Parser *);
138 int _PyPegen_lookahead_with_int(int, Token *(func)(Parser *, int), Parser *, int);
139 int _PyPegen_lookahead_with_string(int, expr_ty (func)(Parser *, const char*), Parser *, const char*);
140
141 Token *_PyPegen_expect_token(Parser *p, int type);
142 void* _PyPegen_expect_forced_result(Parser *p, void* result, const char* expected);
143 Token *_PyPegen_expect_forced_token(Parser *p, int type, const char* expected);
144 expr_ty _PyPegen_expect_soft_keyword(Parser *p, const char *keyword);
145 expr_ty _PyPegen_soft_keyword_token(Parser *p);
146 expr_ty _PyPegen_fstring_middle_token(Parser* p);
147 Token *_PyPegen_get_last_nonnwhitespace_token(Parser *);
148 int _PyPegen_fill_token(Parser *p);
149 expr_ty _PyPegen_name_token(Parser *p);
150 expr_ty _PyPegen_number_token(Parser *p);
151 void *_PyPegen_string_token(Parser *p);
152 Py_ssize_t _PyPegen_byte_offset_to_character_offset_line(PyObject *line, Py_ssize_t col_offset, Py_ssize_t end_col_offset);
153 Py_ssize_t _PyPegen_byte_offset_to_character_offset(PyObject *line, Py_ssize_t col_offset);
154 Py_ssize_t _PyPegen_byte_offset_to_character_offset_raw(const char*, Py_ssize_t col_offset);
155
156 // Error handling functions and APIs
157 typedef enum {
158 STAR_TARGETS,
159 DEL_TARGETS,
160 FOR_TARGETS
161 } TARGETS_TYPE;
162
163 int _Pypegen_raise_decode_error(Parser *p);
164 void _PyPegen_raise_tokenizer_init_error(PyObject *filename);
165 int _Pypegen_tokenizer_error(Parser *p);
166 void *_PyPegen_raise_error(Parser *p, PyObject *errtype, int use_mark, const char *errmsg, ...);
167 void *_PyPegen_raise_error_known_location(Parser *p, PyObject *errtype,
168 Py_ssize_t lineno, Py_ssize_t col_offset,
169 Py_ssize_t end_lineno, Py_ssize_t end_col_offset,
170 const char *errmsg, va_list va);
171 void _Pypegen_set_syntax_error(Parser* p, Token* last_token);
172 void _Pypegen_stack_overflow(Parser *p);
173
174 Py_LOCAL_INLINE(void *)
175 RAISE_ERROR_KNOWN_LOCATION(Parser *p, PyObject *errtype,
176 Py_ssize_t lineno, Py_ssize_t col_offset,
177 Py_ssize_t end_lineno, Py_ssize_t end_col_offset,
178 const char *errmsg, ...)
179 {
180 va_list va;
181 va_start(va, errmsg);
182 Py_ssize_t _col_offset = (col_offset == CURRENT_POS ? CURRENT_POS : col_offset + 1);
183 Py_ssize_t _end_col_offset = (end_col_offset == CURRENT_POS ? CURRENT_POS : end_col_offset + 1);
184 _PyPegen_raise_error_known_location(p, errtype, lineno, _col_offset, end_lineno, _end_col_offset, errmsg, va);
185 va_end(va);
186 return NULL;
187 }
188 #define RAISE_SYNTAX_ERROR(msg, ...) _PyPegen_raise_error(p, PyExc_SyntaxError, 0, msg, ##__VA_ARGS__)
189 #define RAISE_INDENTATION_ERROR(msg, ...) _PyPegen_raise_error(p, PyExc_IndentationError, 0, msg, ##__VA_ARGS__)
190 #define RAISE_SYNTAX_ERROR_ON_NEXT_TOKEN(msg, ...) _PyPegen_raise_error(p, PyExc_SyntaxError, 1, msg, ##__VA_ARGS__)
191 #define RAISE_SYNTAX_ERROR_KNOWN_RANGE(a, b, msg, ...) \
192 RAISE_ERROR_KNOWN_LOCATION(p, PyExc_SyntaxError, (a)->lineno, (a)->col_offset, (b)->end_lineno, (b)->end_col_offset, msg, ##__VA_ARGS__)
193 #define RAISE_SYNTAX_ERROR_KNOWN_LOCATION(a, msg, ...) \
194 RAISE_ERROR_KNOWN_LOCATION(p, PyExc_SyntaxError, (a)->lineno, (a)->col_offset, (a)->end_lineno, (a)->end_col_offset, msg, ##__VA_ARGS__)
195 #define RAISE_SYNTAX_ERROR_STARTING_FROM(a, msg, ...) \
196 RAISE_ERROR_KNOWN_LOCATION(p, PyExc_SyntaxError, (a)->lineno, (a)->col_offset, CURRENT_POS, CURRENT_POS, msg, ##__VA_ARGS__)
197 #define RAISE_SYNTAX_ERROR_INVALID_TARGET(type, e) _RAISE_SYNTAX_ERROR_INVALID_TARGET(p, type, e)
198
199 Py_LOCAL_INLINE(void *)
200 CHECK_CALL(Parser *p, void *result)
201 {
202 if (result == NULL) {
203 assert(PyErr_Occurred());
204 p->error_indicator = 1;
205 }
206 return result;
207 }
208
209 /* This is needed for helper functions that are allowed to
210 return NULL without an error. Example: _PyPegen_seq_extract_starred_exprs */
211 Py_LOCAL_INLINE(void *)
212 CHECK_CALL_NULL_ALLOWED(Parser *p, void *result)
213 {
214 if (result == NULL && PyErr_Occurred()) {
215 p->error_indicator = 1;
216 }
217 return result;
218 }
219
220 #define CHECK(type, result) ((type) CHECK_CALL(p, result))
221 #define CHECK_NULL_ALLOWED(type, result) ((type) CHECK_CALL_NULL_ALLOWED(p, result))
222
223 expr_ty _PyPegen_get_invalid_target(expr_ty e, TARGETS_TYPE targets_type);
224 const char *_PyPegen_get_expr_name(expr_ty);
225 Py_LOCAL_INLINE(void *)
226 _RAISE_SYNTAX_ERROR_INVALID_TARGET(Parser *p, TARGETS_TYPE type, void *e)
227 {
228 expr_ty invalid_target = CHECK_NULL_ALLOWED(expr_ty, _PyPegen_get_invalid_target(e, type));
229 if (invalid_target != NULL) {
230 const char *msg;
231 if (type == STAR_TARGETS || type == FOR_TARGETS) {
232 msg = "cannot assign to %s";
233 }
234 else {
235 msg = "cannot delete %s";
236 }
237 return RAISE_SYNTAX_ERROR_KNOWN_LOCATION(
238 invalid_target,
239 msg,
240 _PyPegen_get_expr_name(invalid_target)
241 );
242 return RAISE_SYNTAX_ERROR_KNOWN_LOCATION(invalid_target, "invalid syntax");
243 }
244 return NULL;
245 }
246
247 // Action utility functions
248
249 void *_PyPegen_dummy_name(Parser *p, ...);
250 void * _PyPegen_seq_last_item(asdl_seq *seq);
251 #define PyPegen_last_item(seq, type) ((type)_PyPegen_seq_last_item((asdl_seq*)seq))
252 void * _PyPegen_seq_first_item(asdl_seq *seq);
253 #define PyPegen_first_item(seq, type) ((type)_PyPegen_seq_first_item((asdl_seq*)seq))
254 #define UNUSED(expr) do { (void)(expr); } while (0)
255 #define EXTRA_EXPR(head, tail) head->lineno, (head)->col_offset, (tail)->end_lineno, (tail)->end_col_offset, p->arena
256 #define EXTRA _start_lineno, _start_col_offset, _end_lineno, _end_col_offset, p->arena
257 PyObject *_PyPegen_new_type_comment(Parser *, const char *);
258
259 Py_LOCAL_INLINE(PyObject *)
260 NEW_TYPE_COMMENT(Parser *p, Token *tc)
261 {
262 if (tc == NULL) {
263 return NULL;
264 }
265 const char *bytes = PyBytes_AsString(tc->bytes);
266 if (bytes == NULL) {
267 goto error;
268 }
269 PyObject *tco = _PyPegen_new_type_comment(p, bytes);
270 if (tco == NULL) {
271 goto error;
272 }
273 return tco;
274 error:
275 p->error_indicator = 1; // Inline CHECK_CALL
276 return NULL;
277 }
278
279 Py_LOCAL_INLINE(void *)
280 INVALID_VERSION_CHECK(Parser *p, int version, char *msg, void *node)
281 {
282 if (node == NULL) {
283 p->error_indicator = 1; // Inline CHECK_CALL
284 return NULL;
285 }
286 if (p->feature_version < version) {
287 p->error_indicator = 1;
288 return RAISE_SYNTAX_ERROR("%s only supported in Python 3.%i and greater",
289 msg, version);
290 }
291 return node;
292 }
293
294 #define CHECK_VERSION(type, version, msg, node) ((type) INVALID_VERSION_CHECK(p, version, msg, node))
295
296 arg_ty _PyPegen_add_type_comment_to_arg(Parser *, arg_ty, Token *);
297 PyObject *_PyPegen_new_identifier(Parser *, const char *);
298 asdl_seq *_PyPegen_singleton_seq(Parser *, void *);
299 asdl_seq *_PyPegen_seq_insert_in_front(Parser *, void *, asdl_seq *);
300 asdl_seq *_PyPegen_seq_append_to_end(Parser *, asdl_seq *, void *);
301 asdl_seq *_PyPegen_seq_flatten(Parser *, asdl_seq *);
302 expr_ty _PyPegen_join_names_with_dot(Parser *, expr_ty, expr_ty);
303 int _PyPegen_seq_count_dots(asdl_seq *);
304 alias_ty _PyPegen_alias_for_star(Parser *, int, int, int, int, PyArena *);
305 asdl_identifier_seq *_PyPegen_map_names_to_ids(Parser *, asdl_expr_seq *);
306 CmpopExprPair *_PyPegen_cmpop_expr_pair(Parser *, cmpop_ty, expr_ty);
307 asdl_int_seq *_PyPegen_get_cmpops(Parser *p, asdl_seq *);
308 asdl_expr_seq *_PyPegen_get_exprs(Parser *, asdl_seq *);
309 expr_ty _PyPegen_set_expr_context(Parser *, expr_ty, expr_context_ty);
310 KeyValuePair *_PyPegen_key_value_pair(Parser *, expr_ty, expr_ty);
311 asdl_expr_seq *_PyPegen_get_keys(Parser *, asdl_seq *);
312 asdl_expr_seq *_PyPegen_get_values(Parser *, asdl_seq *);
313 KeyPatternPair *_PyPegen_key_pattern_pair(Parser *, expr_ty, pattern_ty);
314 asdl_expr_seq *_PyPegen_get_pattern_keys(Parser *, asdl_seq *);
315 asdl_pattern_seq *_PyPegen_get_patterns(Parser *, asdl_seq *);
316 NameDefaultPair *_PyPegen_name_default_pair(Parser *, arg_ty, expr_ty, Token *);
317 SlashWithDefault *_PyPegen_slash_with_default(Parser *, asdl_arg_seq *, asdl_seq *);
318 StarEtc *_PyPegen_star_etc(Parser *, arg_ty, asdl_seq *, arg_ty);
319 arguments_ty _PyPegen_make_arguments(Parser *, asdl_arg_seq *, SlashWithDefault *,
320 asdl_arg_seq *, asdl_seq *, StarEtc *);
321 arguments_ty _PyPegen_empty_arguments(Parser *);
322 expr_ty _PyPegen_formatted_value(Parser *, expr_ty, Token *, ResultTokenWithMetadata *, ResultTokenWithMetadata *, Token *,
323 int, int, int, int, PyArena *);
324 AugOperator *_PyPegen_augoperator(Parser*, operator_ty type);
325 stmt_ty _PyPegen_function_def_decorators(Parser *, asdl_expr_seq *, stmt_ty);
326 stmt_ty _PyPegen_class_def_decorators(Parser *, asdl_expr_seq *, stmt_ty);
327 KeywordOrStarred *_PyPegen_keyword_or_starred(Parser *, void *, int);
328 asdl_expr_seq *_PyPegen_seq_extract_starred_exprs(Parser *, asdl_seq *);
329 asdl_keyword_seq *_PyPegen_seq_delete_starred_exprs(Parser *, asdl_seq *);
330 expr_ty _PyPegen_collect_call_seqs(Parser *, asdl_expr_seq *, asdl_seq *,
331 int lineno, int col_offset, int end_lineno,
332 int end_col_offset, PyArena *arena);
333 expr_ty _PyPegen_constant_from_token(Parser* p, Token* tok);
334 expr_ty _PyPegen_decoded_constant_from_token(Parser* p, Token* tok);
335 expr_ty _PyPegen_constant_from_string(Parser* p, Token* tok);
336 expr_ty _PyPegen_concatenate_strings(Parser *p, asdl_expr_seq *, int, int, int, int, PyArena *);
337 expr_ty _PyPegen_FetchRawForm(Parser *p, int, int, int, int);
338 expr_ty _PyPegen_ensure_imaginary(Parser *p, expr_ty);
339 expr_ty _PyPegen_ensure_real(Parser *p, expr_ty);
340 asdl_seq *_PyPegen_join_sequences(Parser *, asdl_seq *, asdl_seq *);
341 int _PyPegen_check_barry_as_flufl(Parser *, Token *);
342 int _PyPegen_check_legacy_stmt(Parser *p, expr_ty t);
343 ResultTokenWithMetadata *_PyPegen_check_fstring_conversion(Parser *p, Token *, expr_ty t);
344 ResultTokenWithMetadata *_PyPegen_setup_full_format_spec(Parser *, Token *, asdl_expr_seq *, int, int,
345 int, int, PyArena *);
346 mod_ty _PyPegen_make_module(Parser *, asdl_stmt_seq *);
347 void *_PyPegen_arguments_parsing_error(Parser *, expr_ty);
348 expr_ty _PyPegen_get_last_comprehension_item(comprehension_ty comprehension);
349 void *_PyPegen_nonparen_genexp_in_call(Parser *p, expr_ty args, asdl_comprehension_seq *comprehensions);
350 stmt_ty _PyPegen_checked_future_import(Parser *p, identifier module, asdl_alias_seq *,
351 int , int, int , int , int , PyArena *);
352
353 // Parser API
354
355 Parser *_PyPegen_Parser_New(struct tok_state *, int, int, int, int *, PyArena *);
356 void _PyPegen_Parser_Free(Parser *);
357 mod_ty _PyPegen_run_parser_from_file_pointer(FILE *, int, PyObject *, const char *,
358 const char *, const char *, PyCompilerFlags *, int *, PyObject **,
359 PyArena *);
360 void *_PyPegen_run_parser(Parser *);
361 mod_ty _PyPegen_run_parser_from_string(const char *, int, PyObject *, PyCompilerFlags *, PyArena *);
362 asdl_stmt_seq *_PyPegen_interactive_exit(Parser *);
363
364 // TODO: move to the correct place in this file
365 expr_ty _PyPegen_joined_str(Parser *p, Token* a, asdl_expr_seq* expr, Token*b);
366
367 // Generated function in parse.c - function definition in python.gram
368 void *_PyPegen_parse(Parser *);
369
370 #endif