Debian Codegraph @

Parser/pegen.c

raw
1 #include <Python.h>
2 #include "pycore_ast.h" // _PyAST_Validate(),
3 #include "pycore_pystate.h" // _PyThreadState_GET()
4 #include "pycore_pyerrors.h" // PyExc_IncompleteInputError
5 #include <errcode.h>
6
7 #include "lexer/lexer.h"
8 #include "tokenizer/tokenizer.h"
9 #include "pegen.h"
10
11 // Internal parser functions
12
13 asdl_stmt_seq*
14 _PyPegen_interactive_exit(Parser *p)
15 {
16 if (p->errcode) {
17 *(p->errcode) = E_EOF;
18 }
19 return NULL;
20 }
21
22 Py_ssize_t
23 _PyPegen_byte_offset_to_character_offset_line(PyObject *line, Py_ssize_t col_offset, Py_ssize_t end_col_offset)
24 {
25 const char *data = PyUnicode_AsUTF8(line);
26
27 Py_ssize_t len = 0;
28 while (col_offset < end_col_offset) {
29 Py_UCS4 ch = data[col_offset];
30 if (ch < 0x80) {
31 col_offset += 1;
32 } else if ((ch & 0xe0) == 0xc0) {
33 col_offset += 2;
34 } else if ((ch & 0xf0) == 0xe0) {
35 col_offset += 3;
36 } else if ((ch & 0xf8) == 0xf0) {
37 col_offset += 4;
38 } else {
39 PyErr_SetString(PyExc_ValueError, "Invalid UTF-8 sequence");
40 return -1;
41 }
42 len++;
43 }
44 return len;
45 }
46
47 Py_ssize_t
48 _PyPegen_byte_offset_to_character_offset_raw(const char* str, Py_ssize_t col_offset)
49 {
50 Py_ssize_t len = strlen(str);
51 if (col_offset > len + 1) {
52 col_offset = len + 1;
53 }
54 assert(col_offset >= 0);
55 PyObject *text = PyUnicode_DecodeUTF8(str, col_offset, "replace");
56 if (!text) {
57 return -1;
58 }
59 Py_ssize_t size = PyUnicode_GET_LENGTH(text);
60 Py_DECREF(text);
61 return size;
62 }
63
64 Py_ssize_t
65 _PyPegen_byte_offset_to_character_offset(PyObject *line, Py_ssize_t col_offset)
66 {
67 const char *str = PyUnicode_AsUTF8(line);
68 if (!str) {
69 return -1;
70 }
71 return _PyPegen_byte_offset_to_character_offset_raw(str, col_offset);
72 }
73
74 // Here, mark is the start of the node, while p->mark is the end.
75 // If node==NULL, they should be the same.
76 int
77 _PyPegen_insert_memo(Parser *p, int mark, int type, void *node)
78 {
79 // Insert in front
80 Memo *m = _PyArena_Malloc(p->arena, sizeof(Memo));
81 if (m == NULL) {
82 return -1;
83 }
84 m->type = type;
85 m->node = node;
86 m->mark = p->mark;
87 m->next = p->tokens[mark]->memo;
88 p->tokens[mark]->memo = m;
89 return 0;
90 }
91
92 // Like _PyPegen_insert_memo(), but updates an existing node if found.
93 int
94 _PyPegen_update_memo(Parser *p, int mark, int type, void *node)
95 {
96 for (Memo *m = p->tokens[mark]->memo; m != NULL; m = m->next) {
97 if (m->type == type) {
98 // Update existing node.
99 m->node = node;
100 m->mark = p->mark;
101 return 0;
102 }
103 }
104 // Insert new node.
105 return _PyPegen_insert_memo(p, mark, type, node);
106 }
107
108 static int
109 init_normalization(Parser *p)
110 {
111 if (p->normalize) {
112 return 1;
113 }
114 p->normalize = _PyImport_GetModuleAttrString("unicodedata", "normalize");
115 if (!p->normalize)
116 {
117 return 0;
118 }
119 return 1;
120 }
121
122 static int
123 growable_comment_array_init(growable_comment_array *arr, size_t initial_size) {
124 assert(initial_size > 0);
125 arr->items = PyMem_Malloc(initial_size * sizeof(*arr->items));
126 arr->size = initial_size;
127 arr->num_items = 0;
128
129 return arr->items != NULL;
130 }
131
132 static int
133 growable_comment_array_add(growable_comment_array *arr, int lineno, char *comment) {
134 if (arr->num_items >= arr->size) {
135 size_t new_size = arr->size * 2;
136 void *new_items_array = PyMem_Realloc(arr->items, new_size * sizeof(*arr->items));
137 if (!new_items_array) {
138 return 0;
139 }
140 arr->items = new_items_array;
141 arr->size = new_size;
142 }
143
144 arr->items[arr->num_items].lineno = lineno;
145 arr->items[arr->num_items].comment = comment; // Take ownership
146 arr->num_items++;
147 return 1;
148 }
149
150 static void
151 growable_comment_array_deallocate(growable_comment_array *arr) {
152 for (unsigned i = 0; i < arr->num_items; i++) {
153 PyMem_Free(arr->items[i].comment);
154 }
155 PyMem_Free(arr->items);
156 }
157
158 static int
159 _get_keyword_or_name_type(Parser *p, struct token *new_token)
160 {
161 int name_len = new_token->end_col_offset - new_token->col_offset;
162 assert(name_len > 0);
163
164 if (name_len >= p->n_keyword_lists ||
165 p->keywords[name_len] == NULL ||
166 p->keywords[name_len]->type == -1) {
167 return NAME;
168 }
169 for (KeywordToken *k = p->keywords[name_len]; k != NULL && k->type != -1; k++) {
170 if (strncmp(k->str, new_token->start, name_len) == 0) {
171 return k->type;
172 }
173 }
174 return NAME;
175 }
176
177 static int
178 initialize_token(Parser *p, Token *parser_token, struct token *new_token, int token_type) {
179 assert(parser_token != NULL);
180
181 parser_token->type = (token_type == NAME) ? _get_keyword_or_name_type(p, new_token) : token_type;
182 parser_token->bytes = PyBytes_FromStringAndSize(new_token->start, new_token->end - new_token->start);
183 if (parser_token->bytes == NULL) {
184 return -1;
185 }
186 if (_PyArena_AddPyObject(p->arena, parser_token->bytes) < 0) {
187 Py_DECREF(parser_token->bytes);
188 return -1;
189 }
190
191 parser_token->metadata = NULL;
192 if (new_token->metadata != NULL) {
193 if (_PyArena_AddPyObject(p->arena, new_token->metadata) < 0) {
194 Py_DECREF(new_token->metadata);
195 return -1;
196 }
197 parser_token->metadata = new_token->metadata;
198 new_token->metadata = NULL;
199 }
200
201 parser_token->level = new_token->level;
202 parser_token->lineno = new_token->lineno;
203 parser_token->col_offset = p->tok->lineno == p->starting_lineno ? p->starting_col_offset + new_token->col_offset
204 : new_token->col_offset;
205 parser_token->end_lineno = new_token->end_lineno;
206 parser_token->end_col_offset = p->tok->lineno == p->starting_lineno ? p->starting_col_offset + new_token->end_col_offset
207 : new_token->end_col_offset;
208
209 p->fill += 1;
210
211 if (token_type == ERRORTOKEN && p->tok->done == E_DECODE) {
212 return _Pypegen_raise_decode_error(p);
213 }
214
215 return (token_type == ERRORTOKEN ? _Pypegen_tokenizer_error(p) : 0);
216 }
217
218 static int
219 _resize_tokens_array(Parser *p) {
220 int newsize = p->size * 2;
221 Token **new_tokens = PyMem_Realloc(p->tokens, newsize * sizeof(Token *));
222 if (new_tokens == NULL) {
223 PyErr_NoMemory();
224 return -1;
225 }
226 p->tokens = new_tokens;
227
228 for (int i = p->size; i < newsize; i++) {
229 p->tokens[i] = PyMem_Calloc(1, sizeof(Token));
230 if (p->tokens[i] == NULL) {
231 p->size = i; // Needed, in order to cleanup correctly after parser fails
232 PyErr_NoMemory();
233 return -1;
234 }
235 }
236 p->size = newsize;
237 return 0;
238 }
239
240 int
241 _PyPegen_fill_token(Parser *p)
242 {
243 struct token new_token;
244 _PyToken_Init(&new_token);
245 int type = _PyTokenizer_Get(p->tok, &new_token);
246
247 // Record and skip '# type: ignore' comments
248 while (type == TYPE_IGNORE) {
249 Py_ssize_t len = new_token.end_col_offset - new_token.col_offset;
250 char *tag = PyMem_Malloc(len + 1);
251 if (tag == NULL) {
252 PyErr_NoMemory();
253 goto error;
254 }
255 strncpy(tag, new_token.start, len);
256 tag[len] = '\0';
257 // Ownership of tag passes to the growable array
258 if (!growable_comment_array_add(&p->type_ignore_comments, p->tok->lineno, tag)) {
259 PyErr_NoMemory();
260 goto error;
261 }
262 type = _PyTokenizer_Get(p->tok, &new_token);
263 }
264
265 // If we have reached the end and we are in single input mode we need to insert a newline and reset the parsing
266 if (p->start_rule == Py_single_input && type == ENDMARKER && p->parsing_started) {
267 type = NEWLINE; /* Add an extra newline */
268 p->parsing_started = 0;
269
270 if (p->tok->indent && !(p->flags & PyPARSE_DONT_IMPLY_DEDENT)) {
271 p->tok->pendin = -p->tok->indent;
272 p->tok->indent = 0;
273 }
274 }
275 else {
276 p->parsing_started = 1;
277 }
278
279 // Check if we are at the limit of the token array capacity and resize if needed
280 if ((p->fill == p->size) && (_resize_tokens_array(p) != 0)) {
281 goto error;
282 }
283
284 Token *t = p->tokens[p->fill];
285 return initialize_token(p, t, &new_token, type);
286 error:
287 _PyToken_Free(&new_token);
288 return -1;
289 }
290
291 #if defined(Py_DEBUG)
292 // Instrumentation to count the effectiveness of memoization.
293 // The array counts the number of tokens skipped by memoization,
294 // indexed by type.
295
296 #define NSTATISTICS _PYPEGEN_NSTATISTICS
297 #define memo_statistics _PyRuntime.parser.memo_statistics
298
299 #ifdef Py_GIL_DISABLED
300 #define MUTEX_LOCK() PyMutex_Lock(&_PyRuntime.parser.mutex)
301 #define MUTEX_UNLOCK() PyMutex_Unlock(&_PyRuntime.parser.mutex)
302 #else
303 #define MUTEX_LOCK()
304 #define MUTEX_UNLOCK()
305 #endif
306
307 void
308 _PyPegen_clear_memo_statistics(void)
309 {
310 MUTEX_LOCK();
311 for (int i = 0; i < NSTATISTICS; i++) {
312 memo_statistics[i] = 0;
313 }
314 MUTEX_UNLOCK();
315 }
316
317 PyObject *
318 _PyPegen_get_memo_statistics(void)
319 {
320 PyObject *ret = PyList_New(NSTATISTICS);
321 if (ret == NULL) {
322 return NULL;
323 }
324
325 MUTEX_LOCK();
326 for (int i = 0; i < NSTATISTICS; i++) {
327 PyObject *value = PyLong_FromLong(memo_statistics[i]);
328 if (value == NULL) {
329 MUTEX_UNLOCK();
330 Py_DECREF(ret);
331 return NULL;
332 }
333 // PyList_SetItem borrows a reference to value.
334 if (PyList_SetItem(ret, i, value) < 0) {
335 MUTEX_UNLOCK();
336 Py_DECREF(ret);
337 return NULL;
338 }
339 }
340 MUTEX_UNLOCK();
341 return ret;
342 }
343 #endif
344
345 int // bool
346 _PyPegen_is_memoized(Parser *p, int type, void *pres)
347 {
348 if (p->mark == p->fill) {
349 if (_PyPegen_fill_token(p) < 0) {
350 p->error_indicator = 1;
351 return -1;
352 }
353 }
354
355 Token *t = p->tokens[p->mark];
356
357 for (Memo *m = t->memo; m != NULL; m = m->next) {
358 if (m->type == type) {
359 #if defined(Py_DEBUG)
360 if (0 <= type && type < NSTATISTICS) {
361 long count = m->mark - p->mark;
362 // A memoized negative result counts for one.
363 if (count <= 0) {
364 count = 1;
365 }
366 MUTEX_LOCK();
367 memo_statistics[type] += count;
368 MUTEX_UNLOCK();
369 }
370 #endif
371 p->mark = m->mark;
372 *(void **)(pres) = m->node;
373 return 1;
374 }
375 }
376 return 0;
377 }
378
379 #define LOOKAHEAD1(NAME, RES_TYPE) \
380 int \
381 NAME (int positive, RES_TYPE (func)(Parser *), Parser *p) \
382 { \
383 int mark = p->mark; \
384 void *res = func(p); \
385 p->mark = mark; \
386 return (res != NULL) == positive; \
387 }
388
389 LOOKAHEAD1(_PyPegen_lookahead, void *)
390 LOOKAHEAD1(_PyPegen_lookahead_for_expr, expr_ty)
391 LOOKAHEAD1(_PyPegen_lookahead_for_stmt, stmt_ty)
392 #undef LOOKAHEAD1
393
394 #define LOOKAHEAD2(NAME, RES_TYPE, T) \
395 int \
396 NAME (int positive, RES_TYPE (func)(Parser *, T), Parser *p, T arg) \
397 { \
398 int mark = p->mark; \
399 void *res = func(p, arg); \
400 p->mark = mark; \
401 return (res != NULL) == positive; \
402 }
403
404 LOOKAHEAD2(_PyPegen_lookahead_with_int, Token *, int)
405 LOOKAHEAD2(_PyPegen_lookahead_with_string, expr_ty, const char *)
406 #undef LOOKAHEAD2
407
408 Token *
409 _PyPegen_expect_token(Parser *p, int type)
410 {
411 if (p->mark == p->fill) {
412 if (_PyPegen_fill_token(p) < 0) {
413 p->error_indicator = 1;
414 return NULL;
415 }
416 }
417 Token *t = p->tokens[p->mark];
418 if (t->type != type) {
419 return NULL;
420 }
421 p->mark += 1;
422 return t;
423 }
424
425 void*
426 _PyPegen_expect_forced_result(Parser *p, void* result, const char* expected) {
427
428 if (p->error_indicator == 1) {
429 return NULL;
430 }
431 if (result == NULL) {
432 RAISE_SYNTAX_ERROR("expected (%s)", expected);
433 return NULL;
434 }
435 return result;
436 }
437
438 Token *
439 _PyPegen_expect_forced_token(Parser *p, int type, const char* expected) {
440
441 if (p->error_indicator == 1) {
442 return NULL;
443 }
444
445 if (p->mark == p->fill) {
446 if (_PyPegen_fill_token(p) < 0) {
447 p->error_indicator = 1;
448 return NULL;
449 }
450 }
451 Token *t = p->tokens[p->mark];
452 if (t->type != type) {
453 RAISE_SYNTAX_ERROR_KNOWN_LOCATION(t, "expected '%s'", expected);
454 return NULL;
455 }
456 p->mark += 1;
457 return t;
458 }
459
460 expr_ty
461 _PyPegen_expect_soft_keyword(Parser *p, const char *keyword)
462 {
463 if (p->mark == p->fill) {
464 if (_PyPegen_fill_token(p) < 0) {
465 p->error_indicator = 1;
466 return NULL;
467 }
468 }
469 Token *t = p->tokens[p->mark];
470 if (t->type != NAME) {
471 return NULL;
472 }
473 const char *s = PyBytes_AsString(t->bytes);
474 if (!s) {
475 p->error_indicator = 1;
476 return NULL;
477 }
478 if (strcmp(s, keyword) != 0) {
479 return NULL;
480 }
481 return _PyPegen_name_token(p);
482 }
483
484 Token *
485 _PyPegen_get_last_nonnwhitespace_token(Parser *p)
486 {
487 assert(p->mark >= 0);
488 Token *token = NULL;
489 for (int m = p->mark - 1; m >= 0; m--) {
490 token = p->tokens[m];
491 if (token->type != ENDMARKER && (token->type < NEWLINE || token->type > DEDENT)) {
492 break;
493 }
494 }
495 return token;
496 }
497
498 PyObject *
499 _PyPegen_new_identifier(Parser *p, const char *n)
500 {
501 PyObject *id = PyUnicode_DecodeUTF8(n, strlen(n), NULL);
502 if (!id) {
503 goto error;
504 }
505 /* PyUnicode_DecodeUTF8 should always return a ready string. */
506 assert(PyUnicode_IS_READY(id));
507 /* Check whether there are non-ASCII characters in the
508 identifier; if so, normalize to NFKC. */
509 if (!PyUnicode_IS_ASCII(id))
510 {
511 if (!init_normalization(p))
512 {
513 Py_DECREF(id);
514 goto error;
515 }
516 PyObject *form = PyUnicode_InternFromString("NFKC");
517 if (form == NULL)
518 {
519 Py_DECREF(id);
520 goto error;
521 }
522 PyObject *args[2] = {form, id};
523 PyObject *id2 = PyObject_Vectorcall(p->normalize, args, 2, NULL);
524 Py_DECREF(id);
525 Py_DECREF(form);
526 if (!id2) {
527 goto error;
528 }
529
530 if (!PyUnicode_Check(id2))
531 {
532 PyErr_Format(PyExc_TypeError,
533 "unicodedata.normalize() must return a string, not "
534 "%.200s",
535 _PyType_Name(Py_TYPE(id2)));
536 Py_DECREF(id2);
537 goto error;
538 }
539 id = id2;
540 }
541 static const char * const forbidden[] = {
542 "None",
543 "True",
544 "False",
545 NULL
546 };
547 for (int i = 0; forbidden[i] != NULL; i++) {
548 if (_PyUnicode_EqualToASCIIString(id, forbidden[i])) {
549 PyErr_Format(PyExc_ValueError,
550 "identifier field can't represent '%s' constant",
551 forbidden[i]);
552 Py_DECREF(id);
553 goto error;
554 }
555 }
556 PyInterpreterState *interp = _PyInterpreterState_GET();
557 _PyUnicode_InternImmortal(interp, &id);
558 if (_PyArena_AddPyObject(p->arena, id) < 0)
559 {
560 Py_DECREF(id);
561 goto error;
562 }
563 return id;
564
565 error:
566 p->error_indicator = 1;
567 return NULL;
568 }
569
570 static expr_ty
571 _PyPegen_name_from_token(Parser *p, Token* t)
572 {
573 if (t == NULL) {
574 return NULL;
575 }
576 const char *s = PyBytes_AsString(t->bytes);
577 if (!s) {
578 p->error_indicator = 1;
579 return NULL;
580 }
581 PyObject *id = _PyPegen_new_identifier(p, s);
582 if (id == NULL) {
583 p->error_indicator = 1;
584 return NULL;
585 }
586 return _PyAST_Name(id, Load, t->lineno, t->col_offset, t->end_lineno,
587 t->end_col_offset, p->arena);
588 }
589
590 expr_ty
591 _PyPegen_name_token(Parser *p)
592 {
593 Token *t = _PyPegen_expect_token(p, NAME);
594 return _PyPegen_name_from_token(p, t);
595 }
596
597 void *
598 _PyPegen_string_token(Parser *p)
599 {
600 return _PyPegen_expect_token(p, STRING);
601 }
602
603 expr_ty _PyPegen_soft_keyword_token(Parser *p) {
604 Token *t = _PyPegen_expect_token(p, NAME);
605 if (t == NULL) {
606 return NULL;
607 }
608 char *the_token;
609 Py_ssize_t size;
610 PyBytes_AsStringAndSize(t->bytes, &the_token, &size);
611 for (char **keyword = p->soft_keywords; *keyword != NULL; keyword++) {
612 if (strlen(*keyword) == (size_t)size &&
613 strncmp(*keyword, the_token, (size_t)size) == 0) {
614 return _PyPegen_name_from_token(p, t);
615 }
616 }
617 return NULL;
618 }
619
620 static PyObject *
621 parsenumber_raw(const char *s)
622 {
623 const char *end;
624 long x;
625 double dx;
626 Py_complex compl;
627 int imflag;
628
629 assert(s != NULL);
630 errno = 0;
631 end = s + strlen(s) - 1;
632 imflag = *end == 'j' || *end == 'J';
633 if (s[0] == '0') {
634 x = (long)PyOS_strtoul(s, (char **)&end, 0);
635 if (x < 0 && errno == 0) {
636 return PyLong_FromString(s, (char **)0, 0);
637 }
638 }
639 else {
640 x = PyOS_strtol(s, (char **)&end, 0);
641 }
642 if (*end == '\0') {
643 if (errno != 0) {
644 return PyLong_FromString(s, (char **)0, 0);
645 }
646 return PyLong_FromLong(x);
647 }
648 /* XXX Huge floats may silently fail */
649 if (imflag) {
650 compl.real = 0.;
651 compl.imag = PyOS_string_to_double(s, (char **)&end, NULL);
652 if (compl.imag == -1.0 && PyErr_Occurred()) {
653 return NULL;
654 }
655 return PyComplex_FromCComplex(compl);
656 }
657 dx = PyOS_string_to_double(s, NULL, NULL);
658 if (dx == -1.0 && PyErr_Occurred()) {
659 return NULL;
660 }
661 return PyFloat_FromDouble(dx);
662 }
663
664 static PyObject *
665 parsenumber(const char *s)
666 {
667 char *dup;
668 char *end;
669 PyObject *res = NULL;
670
671 assert(s != NULL);
672
673 if (strchr(s, '_') == NULL) {
674 return parsenumber_raw(s);
675 }
676 /* Create a duplicate without underscores. */
677 dup = PyMem_Malloc(strlen(s) + 1);
678 if (dup == NULL) {
679 return PyErr_NoMemory();
680 }
681 end = dup;
682 for (; *s; s++) {
683 if (*s != '_') {
684 *end++ = *s;
685 }
686 }
687 *end = '\0';
688 res = parsenumber_raw(dup);
689 PyMem_Free(dup);
690 return res;
691 }
692
693 expr_ty
694 _PyPegen_number_token(Parser *p)
695 {
696 Token *t = _PyPegen_expect_token(p, NUMBER);
697 if (t == NULL) {
698 return NULL;
699 }
700
701 const char *num_raw = PyBytes_AsString(t->bytes);
702 if (num_raw == NULL) {
703 p->error_indicator = 1;
704 return NULL;
705 }
706
707 if (p->feature_version < 6 && strchr(num_raw, '_') != NULL) {
708 p->error_indicator = 1;
709 return RAISE_SYNTAX_ERROR("Underscores in numeric literals are only supported "
710 "in Python 3.6 and greater");
711 }
712
713 PyObject *c = parsenumber(num_raw);
714
715 if (c == NULL) {
716 p->error_indicator = 1;
717 PyThreadState *tstate = _PyThreadState_GET();
718 // The only way a ValueError should happen in _this_ code is via
719 // PyLong_FromString hitting a length limit.
720 if (tstate->current_exception != NULL &&
721 Py_TYPE(tstate->current_exception) == (PyTypeObject *)PyExc_ValueError
722 ) {
723 PyObject *exc = PyErr_GetRaisedException();
724 /* Intentionally omitting columns to avoid a wall of 1000s of '^'s
725 * on the error message. Nobody is going to overlook their huge
726 * numeric literal once given the line. */
727 RAISE_ERROR_KNOWN_LOCATION(
728 p, PyExc_SyntaxError,
729 t->lineno, -1 /* col_offset */,
730 t->end_lineno, -1 /* end_col_offset */,
731 "%S - Consider hexadecimal for huge integer literals "
732 "to avoid decimal conversion limits.",
733 exc);
734 Py_DECREF(exc);
735 }
736 return NULL;
737 }
738
739 if (_PyArena_AddPyObject(p->arena, c) < 0) {
740 Py_DECREF(c);
741 p->error_indicator = 1;
742 return NULL;
743 }
744
745 return _PyAST_Constant(c, NULL, t->lineno, t->col_offset, t->end_lineno,
746 t->end_col_offset, p->arena);
747 }
748
749 /* Check that the source for a single input statement really is a single
750 statement by looking at what is left in the buffer after parsing.
751 Trailing whitespace and comments are OK. */
752 static int // bool
753 bad_single_statement(Parser *p)
754 {
755 char *cur = p->tok->cur;
756 char c = *cur;
757
758 for (;;) {
759 while (c == ' ' || c == '\t' || c == '\n' || c == '\014') {
760 c = *++cur;
761 }
762
763 if (!c) {
764 return 0;
765 }
766
767 if (c != '#') {
768 return 1;
769 }
770
771 /* Suck up comment. */
772 while (c && c != '\n') {
773 c = *++cur;
774 }
775 }
776 }
777
778 static int
779 compute_parser_flags(PyCompilerFlags *flags)
780 {
781 int parser_flags = 0;
782 if (!flags) {
783 return 0;
784 }
785 if (flags->cf_flags & PyCF_DONT_IMPLY_DEDENT) {
786 parser_flags |= PyPARSE_DONT_IMPLY_DEDENT;
787 }
788 if (flags->cf_flags & PyCF_IGNORE_COOKIE) {
789 parser_flags |= PyPARSE_IGNORE_COOKIE;
790 }
791 if (flags->cf_flags & CO_FUTURE_BARRY_AS_BDFL) {
792 parser_flags |= PyPARSE_BARRY_AS_BDFL;
793 }
794 if (flags->cf_flags & PyCF_TYPE_COMMENTS) {
795 parser_flags |= PyPARSE_TYPE_COMMENTS;
796 }
797 if (flags->cf_flags & PyCF_ALLOW_INCOMPLETE_INPUT) {
798 parser_flags |= PyPARSE_ALLOW_INCOMPLETE_INPUT;
799 }
800 return parser_flags;
801 }
802
803 // Parser API
804
805 Parser *
806 _PyPegen_Parser_New(struct tok_state *tok, int start_rule, int flags,
807 int feature_version, int *errcode, PyArena *arena)
808 {
809 Parser *p = PyMem_Malloc(sizeof(Parser));
810 if (p == NULL) {
811 return (Parser *) PyErr_NoMemory();
812 }
813 assert(tok != NULL);
814 tok->type_comments = (flags & PyPARSE_TYPE_COMMENTS) > 0;
815 p->tok = tok;
816 p->keywords = NULL;
817 p->n_keyword_lists = -1;
818 p->soft_keywords = NULL;
819 p->tokens = PyMem_Malloc(sizeof(Token *));
820 if (!p->tokens) {
821 PyMem_Free(p);
822 return (Parser *) PyErr_NoMemory();
823 }
824 p->tokens[0] = PyMem_Calloc(1, sizeof(Token));
825 if (!p->tokens[0]) {
826 PyMem_Free(p->tokens);
827 PyMem_Free(p);
828 return (Parser *) PyErr_NoMemory();
829 }
830 if (!growable_comment_array_init(&p->type_ignore_comments, 10)) {
831 PyMem_Free(p->tokens[0]);
832 PyMem_Free(p->tokens);
833 PyMem_Free(p);
834 return (Parser *) PyErr_NoMemory();
835 }
836
837 p->mark = 0;
838 p->fill = 0;
839 p->size = 1;
840
841 p->errcode = errcode;
842 p->arena = arena;
843 p->start_rule = start_rule;
844 p->parsing_started = 0;
845 p->normalize = NULL;
846 p->error_indicator = 0;
847
848 p->starting_lineno = 0;
849 p->starting_col_offset = 0;
850 p->flags = flags;
851 p->feature_version = feature_version;
852 p->known_err_token = NULL;
853 p->level = 0;
854 p->call_invalid_rules = 0;
855 #ifdef Py_DEBUG
856 p->debug = _Py_GetConfig()->parser_debug;
857 #endif
858 return p;
859 }
860
861 void
862 _PyPegen_Parser_Free(Parser *p)
863 {
864 Py_XDECREF(p->normalize);
865 for (int i = 0; i < p->size; i++) {
866 PyMem_Free(p->tokens[i]);
867 }
868 PyMem_Free(p->tokens);
869 growable_comment_array_deallocate(&p->type_ignore_comments);
870 PyMem_Free(p);
871 }
872
873 static void
874 reset_parser_state_for_error_pass(Parser *p)
875 {
876 for (int i = 0; i < p->fill; i++) {
877 p->tokens[i]->memo = NULL;
878 }
879 p->mark = 0;
880 p->call_invalid_rules = 1;
881 // Don't try to get extra tokens in interactive mode when trying to
882 // raise specialized errors in the second pass.
883 p->tok->interactive_underflow = IUNDERFLOW_STOP;
884 }
885
886 static inline int
887 _is_end_of_source(Parser *p) {
888 int err = p->tok->done;
889 return err == E_EOF || err == E_EOFS || err == E_EOLS;
890 }
891
892 void *
893 _PyPegen_run_parser(Parser *p)
894 {
895 void *res = _PyPegen_parse(p);
896 assert(p->level == 0);
897 if (res != NULL && PyErr_Occurred()) {
898 // Discard a result returned with an exception still pending
899 // (e.g. a MemoryError from a recovered-from allocation failure).
900 return NULL;
901 }
902 if (res == NULL) {
903 if ((p->flags & PyPARSE_ALLOW_INCOMPLETE_INPUT) && _is_end_of_source(p)) {
904 PyErr_Clear();
905 return _PyPegen_raise_error(p, PyExc_IncompleteInputError, 0, "incomplete input");
906 }
907 if (PyErr_Occurred() && !PyErr_ExceptionMatches(PyExc_SyntaxError)) {
908 return NULL;
909 }
910 // Make a second parser pass. In this pass we activate heavier and slower checks
911 // to produce better error messages and more complete diagnostics. Extra "invalid_*"
912 // rules will be active during parsing.
913 Token *last_token = p->tokens[p->fill - 1];
914 reset_parser_state_for_error_pass(p);
915 _PyPegen_parse(p);
916
917 // Set SyntaxErrors accordingly depending on the parser/tokenizer status at the failure
918 // point.
919 _Pypegen_set_syntax_error(p, last_token);
920 return NULL;
921 }
922
923 if (p->start_rule == Py_single_input && bad_single_statement(p)) {
924 p->tok->done = E_BADSINGLE; // This is not necessary for now, but might be in the future
925 return RAISE_SYNTAX_ERROR("multiple statements found while compiling a single statement");
926 }
927
928 // test_peg_generator defines _Py_TEST_PEGEN to not call PyAST_Validate()
929 #if defined(Py_DEBUG) && !defined(_Py_TEST_PEGEN)
930 if (p->start_rule == Py_single_input ||
931 p->start_rule == Py_file_input ||
932 p->start_rule == Py_eval_input)
933 {
934 if (!_PyAST_Validate(res)) {
935 return NULL;
936 }
937 }
938 #endif
939 return res;
940 }
941
942 mod_ty
943 _PyPegen_run_parser_from_file_pointer(FILE *fp, int start_rule, PyObject *filename_ob,
944 const char *enc, const char *ps1, const char *ps2,
945 PyCompilerFlags *flags, int *errcode,
946 PyObject **interactive_src, PyArena *arena)
947 {
948 struct tok_state *tok = _PyTokenizer_FromFile(fp, enc, ps1, ps2);
949 if (tok == NULL) {
950 if (PyErr_Occurred()) {
951 _PyPegen_raise_tokenizer_init_error(filename_ob);
952 }
953 else {
954 // The only silent tokenizer init failure is a failed allocation.
955 PyErr_NoMemory();
956 }
957 return NULL;
958 }
959 if (!tok->fp || ps1 != NULL || ps2 != NULL ||
960 PyUnicode_CompareWithASCIIString(filename_ob, "<stdin>") == 0) {
961 tok->fp_interactive = 1;
962 }
963 // This transfers the ownership to the tokenizer
964 tok->filename = Py_NewRef(filename_ob);
965
966 // From here on we need to clean up even if there's an error
967 mod_ty result = NULL;
968
969 int parser_flags = compute_parser_flags(flags);
970 Parser *p = _PyPegen_Parser_New(tok, start_rule, parser_flags, PY_MINOR_VERSION,
971 errcode, arena);
972 if (p == NULL) {
973 goto error;
974 }
975
976 result = _PyPegen_run_parser(p);
977 _PyPegen_Parser_Free(p);
978
979 if (tok->fp_interactive && tok->interactive_src_start && result && interactive_src != NULL) {
980 *interactive_src = PyUnicode_FromString(tok->interactive_src_start);
981 if (!interactive_src || _PyArena_AddPyObject(arena, *interactive_src) < 0) {
982 Py_XDECREF(interactive_src);
983 result = NULL;
984 goto error;
985 }
986 }
987
988 error:
989 _PyTokenizer_Free(tok);
990 return result;
991 }
992
993 mod_ty
994 _PyPegen_run_parser_from_string(const char *str, int start_rule, PyObject *filename_ob,
995 PyCompilerFlags *flags, PyArena *arena)
996 {
997 int exec_input = start_rule == Py_file_input;
998
999 struct tok_state *tok;
1000 if (flags != NULL && flags->cf_flags & PyCF_IGNORE_COOKIE) {
1001 tok = _PyTokenizer_FromUTF8(str, exec_input, 0);
1002 } else {
1003 tok = _PyTokenizer_FromString(str, exec_input, 0);
1004 }
1005 if (tok == NULL) {
1006 if (PyErr_Occurred()) {
1007 _PyPegen_raise_tokenizer_init_error(filename_ob);
1008 }
1009 else {
1010 // The only silent tokenizer init failure is a failed allocation.
1011 PyErr_NoMemory();
1012 }
1013 return NULL;
1014 }
1015 // This transfers the ownership to the tokenizer
1016 tok->filename = Py_NewRef(filename_ob);
1017
1018 // We need to clear up from here on
1019 mod_ty result = NULL;
1020
1021 int parser_flags = compute_parser_flags(flags);
1022 int feature_version = flags && (flags->cf_flags & PyCF_ONLY_AST) ?
1023 flags->cf_feature_version : PY_MINOR_VERSION;
1024 Parser *p = _PyPegen_Parser_New(tok, start_rule, parser_flags, feature_version,
1025 NULL, arena);
1026 if (p == NULL) {
1027 goto error;
1028 }
1029
1030 result = _PyPegen_run_parser(p);
1031 _PyPegen_Parser_Free(p);
1032
1033 error:
1034 _PyTokenizer_Free(tok);
1035 return result;
1036 }