Debian Codegraph @

Parser/lexer/lexer.c

raw
1 #include "Python.h"
2 #include "pycore_token.h"
3 #include "pycore_unicodeobject.h"
4 #include "errcode.h"
5
6 #include "state.h"
7 #include "../tokenizer/helpers.h"
8
9 /* Alternate tab spacing */
10 #define ALTTABSIZE 1
11
12 #define is_potential_identifier_start(c) (\
13 (c >= 'a' && c <= 'z')\
14 || (c >= 'A' && c <= 'Z')\
15 || c == '_'\
16 || (c >= 128))
17
18 #define is_potential_identifier_char(c) (\
19 (c >= 'a' && c <= 'z')\
20 || (c >= 'A' && c <= 'Z')\
21 || (c >= '0' && c <= '9')\
22 || c == '_'\
23 || (c >= 128))
24
25 #define MAKE_TOKEN(token_type) _PyLexer_token_setup(tok, token, token_type, p_start, p_end)
26 #define MAKE_TYPE_COMMENT_TOKEN(token_type, col_offset, end_col_offset) (\
27 _PyLexer_type_comment_token_setup(tok, token, token_type, col_offset, end_col_offset, p_start, p_end))
28
29 /* Spaces in this constant are treated as "zero or more spaces or tabs" when
30 tokenizing. */
31 static const char* type_comment_prefix = "# type: ";
32
33 static inline int
34 contains_null_bytes(const char* str, size_t size)
35 {
36 return memchr(str, 0, size) != NULL;
37 }
38
39 /* Get next char, updating state; error code goes into tok->done */
40 static int
41 tok_nextc(struct tok_state *tok)
42 {
43 int rc;
44 for (;;) {
45 if (tok->cur != tok->inp) {
46 if ((unsigned int) tok->col_offset >= (unsigned int) INT_MAX) {
47 tok->done = E_COLUMNOVERFLOW;
48 return EOF;
49 }
50 tok->col_offset++;
51 return Py_CHARMASK(*tok->cur++); /* Fast path */
52 }
53 if (tok->done != E_OK) {
54 return EOF;
55 }
56 rc = tok->underflow(tok);
57 #if defined(Py_DEBUG)
58 if (tok->debug) {
59 fprintf(stderr, "line[%d] = ", tok->lineno);
60 _PyTokenizer_print_escape(stderr, tok->cur, tok->inp - tok->cur);
61 fprintf(stderr, " tok->done = %d\n", tok->done);
62 }
63 #endif
64 if (!rc) {
65 tok->cur = tok->inp;
66 return EOF;
67 }
68 tok->line_start = tok->cur;
69
70 if (contains_null_bytes(tok->line_start, tok->inp - tok->line_start)) {
71 _PyTokenizer_syntaxerror(tok, "source code cannot contain null bytes");
72 tok->cur = tok->inp;
73 return EOF;
74 }
75 }
76 Py_UNREACHABLE();
77 }
78
79 /* Back-up one character */
80 static void
81 tok_backup(struct tok_state *tok, int c)
82 {
83 if (c != EOF) {
84 if (--tok->cur < tok->buf) {
85 Py_FatalError("tokenizer beginning of buffer");
86 }
87 if ((int)(unsigned char)*tok->cur != Py_CHARMASK(c)) {
88 Py_FatalError("tok_backup: wrong character");
89 }
90 tok->col_offset--;
91 }
92 }
93
94 static int
95 set_fstring_expr(struct tok_state* tok, struct token *token, char c) {
96 assert(token != NULL);
97 assert(c == '}' || c == ':' || c == '!');
98 tokenizer_mode *tok_mode = TOK_GET_MODE(tok);
99
100 if (!tok_mode->f_string_debug || token->metadata) {
101 return 0;
102 }
103 PyObject *res = NULL;
104
105 // Look for a # character outside of string literals
106 int hash_detected = 0;
107 int in_string = 0;
108 char quote_char = 0;
109
110 for (Py_ssize_t i = 0; i < tok_mode->last_expr_size - tok_mode->last_expr_end; i++) {
111 char ch = tok_mode->last_expr_buffer[i];
112
113 // Skip escaped characters
114 if (ch == '\\') {
115 i++;
116 continue;
117 }
118
119 // Handle quotes
120 if (ch == '"' || ch == '\'') {
121 // The following if/else block works becase there is an off number
122 // of quotes in STRING tokens and the lexer only ever reaches this
123 // function with valid STRING tokens.
124 // For example: """hello"""
125 // First quote: in_string = 1
126 // Second quote: in_string = 0
127 // Third quote: in_string = 1
128 if (!in_string) {
129 in_string = 1;
130 quote_char = ch;
131 }
132 else if (ch == quote_char) {
133 in_string = 0;
134 }
135 continue;
136 }
137
138 // Check for # outside strings
139 if (ch == '#' && !in_string) {
140 hash_detected = 1;
141 break;
142 }
143 }
144 // If we found a # character in the expression, we need to handle comments
145 if (hash_detected) {
146 // Allocate buffer for processed result
147 char *result = (char *)PyMem_Malloc((tok_mode->last_expr_size - tok_mode->last_expr_end + 1) * sizeof(char));
148 if (!result) {
149 return -1;
150 }
151
152 Py_ssize_t i = 0; // Input position
153 Py_ssize_t j = 0; // Output position
154 in_string = 0; // Whether we're in a string
155 quote_char = 0; // Current string quote char
156
157 // Process each character
158 while (i < tok_mode->last_expr_size - tok_mode->last_expr_end) {
159 char ch = tok_mode->last_expr_buffer[i];
160
161 // Handle string quotes
162 if (ch == '"' || ch == '\'') {
163 // See comment above to understand this part
164 if (!in_string) {
165 in_string = 1;
166 quote_char = ch;
167 } else if (ch == quote_char) {
168 in_string = 0;
169 }
170 result[j++] = ch;
171 }
172 // Skip comments
173 else if (ch == '#' && !in_string) {
174 while (i < tok_mode->last_expr_size - tok_mode->last_expr_end &&
175 tok_mode->last_expr_buffer[i] != '\n') {
176 i++;
177 }
178 if (i < tok_mode->last_expr_size - tok_mode->last_expr_end) {
179 result[j++] = '\n';
180 }
181 }
182 // Copy other chars
183 else {
184 result[j++] = ch;
185 }
186 i++;
187 }
188
189 result[j] = '\0'; // Null-terminate the result string
190 res = PyUnicode_DecodeUTF8(result, j, NULL);
191 PyMem_Free(result);
192 } else {
193 res = PyUnicode_DecodeUTF8(
194 tok_mode->last_expr_buffer,
195 tok_mode->last_expr_size - tok_mode->last_expr_end,
196 NULL
197 );
198 }
199
200 if (!res) {
201 return -1;
202 }
203 token->metadata = res;
204 return 0;
205 }
206
207 int
208 _PyLexer_update_fstring_expr(struct tok_state *tok, char cur)
209 {
210 assert(tok->cur != NULL);
211
212 Py_ssize_t size = strlen(tok->cur);
213 tokenizer_mode *tok_mode = TOK_GET_MODE(tok);
214
215 switch (cur) {
216 case 0:
217 if (!tok_mode->last_expr_buffer || tok_mode->last_expr_end >= 0) {
218 return 1;
219 }
220 char *new_buffer = PyMem_Realloc(
221 tok_mode->last_expr_buffer,
222 tok_mode->last_expr_size + size
223 );
224 if (new_buffer == NULL) {
225 PyMem_Free(tok_mode->last_expr_buffer);
226 goto error;
227 }
228 tok_mode->last_expr_buffer = new_buffer;
229 strncpy(tok_mode->last_expr_buffer + tok_mode->last_expr_size, tok->cur, size);
230 tok_mode->last_expr_size += size;
231 break;
232 case '{':
233 if (tok_mode->last_expr_buffer != NULL) {
234 PyMem_Free(tok_mode->last_expr_buffer);
235 }
236 tok_mode->last_expr_buffer = PyMem_Malloc(size);
237 if (tok_mode->last_expr_buffer == NULL) {
238 goto error;
239 }
240 tok_mode->last_expr_size = size;
241 tok_mode->last_expr_end = -1;
242 strncpy(tok_mode->last_expr_buffer, tok->cur, size);
243 break;
244 case '}':
245 case '!':
246 tok_mode->last_expr_end = strlen(tok->start);
247 break;
248 case ':':
249 if (tok_mode->last_expr_end == -1) {
250 tok_mode->last_expr_end = strlen(tok->start);
251 }
252 break;
253 default:
254 Py_UNREACHABLE();
255 }
256 return 1;
257 error:
258 tok->done = E_NOMEM;
259 return 0;
260 }
261
262 static int
263 lookahead(struct tok_state *tok, const char *test)
264 {
265 const char *s = test;
266 int res = 0;
267 while (1) {
268 int c = tok_nextc(tok);
269 if (*s == 0) {
270 res = !is_potential_identifier_char(c);
271 }
272 else if (c == *s) {
273 s++;
274 continue;
275 }
276
277 tok_backup(tok, c);
278 while (s != test) {
279 tok_backup(tok, *--s);
280 }
281 return res;
282 }
283 }
284
285 static int
286 verify_end_of_number(struct tok_state *tok, int c, const char *kind) {
287 if (tok->tok_extra_tokens) {
288 // When we are parsing extra tokens, we don't want to emit warnings
289 // about invalid literals, because we want to be a bit more liberal.
290 return 1;
291 }
292 /* Emit a deprecation warning only if the numeric literal is immediately
293 * followed by one of keywords which can occur after a numeric literal
294 * in valid code: "and", "else", "for", "if", "in", "is" and "or".
295 * It allows to gradually deprecate existing valid code without adding
296 * warning before error in most cases of invalid numeric literal (which
297 * would be confusing and break existing tests).
298 * Raise a syntax error with slightly better message than plain
299 * "invalid syntax" if the numeric literal is immediately followed by
300 * other keyword or identifier.
301 */
302 int r = 0;
303 if (c == 'a') {
304 r = lookahead(tok, "nd");
305 }
306 else if (c == 'e') {
307 r = lookahead(tok, "lse");
308 }
309 else if (c == 'f') {
310 r = lookahead(tok, "or");
311 }
312 else if (c == 'i') {
313 int c2 = tok_nextc(tok);
314 if (c2 == 'f' || c2 == 'n' || c2 == 's') {
315 r = 1;
316 }
317 tok_backup(tok, c2);
318 }
319 else if (c == 'o') {
320 r = lookahead(tok, "r");
321 }
322 else if (c == 'n') {
323 r = lookahead(tok, "ot");
324 }
325 if (r) {
326 tok_backup(tok, c);
327 if (_PyTokenizer_parser_warn(tok, PyExc_SyntaxWarning,
328 "invalid %s literal", kind))
329 {
330 return 0;
331 }
332 tok_nextc(tok);
333 }
334 else /* In future releases, only error will remain. */
335 if (c < 128 && is_potential_identifier_char(c)) {
336 tok_backup(tok, c);
337 _PyTokenizer_syntaxerror(tok, "invalid %s literal", kind);
338 return 0;
339 }
340 return 1;
341 }
342
343 /* Verify that the identifier follows PEP 3131.
344 All identifier strings are guaranteed to be "ready" unicode objects.
345 */
346 static int
347 verify_identifier(struct tok_state *tok)
348 {
349 if (tok->tok_extra_tokens) {
350 return 1;
351 }
352 PyObject *s;
353 if (tok->decoding_erred)
354 return 0;
355 s = PyUnicode_DecodeUTF8(tok->start, tok->cur - tok->start, NULL);
356 if (s == NULL) {
357 if (PyErr_ExceptionMatches(PyExc_UnicodeDecodeError)) {
358 tok->done = E_DECODE;
359 }
360 else {
361 tok->done = E_ERROR;
362 }
363 return 0;
364 }
365 Py_ssize_t invalid = _PyUnicode_ScanIdentifier(s);
366 if (invalid < 0) {
367 Py_DECREF(s);
368 tok->done = E_ERROR;
369 return 0;
370 }
371 assert(PyUnicode_GET_LENGTH(s) > 0);
372 if (invalid < PyUnicode_GET_LENGTH(s)) {
373 Py_UCS4 ch = PyUnicode_READ_CHAR(s, invalid);
374 if (invalid + 1 < PyUnicode_GET_LENGTH(s)) {
375 /* Determine the offset in UTF-8 encoded input */
376 Py_SETREF(s, PyUnicode_Substring(s, 0, invalid + 1));
377 if (s != NULL) {
378 Py_SETREF(s, PyUnicode_AsUTF8String(s));
379 }
380 if (s == NULL) {
381 tok->done = E_ERROR;
382 return 0;
383 }
384 tok->cur = (char *)tok->start + PyBytes_GET_SIZE(s);
385 }
386 Py_DECREF(s);
387 if (Py_UNICODE_ISPRINTABLE(ch)) {
388 _PyTokenizer_syntaxerror(tok, "invalid character '%c' (U+%04X)", ch, ch);
389 }
390 else {
391 _PyTokenizer_syntaxerror(tok, "invalid non-printable character U+%04X", ch);
392 }
393 return 0;
394 }
395 Py_DECREF(s);
396 return 1;
397 }
398
399 static int
400 tok_decimal_tail(struct tok_state *tok)
401 {
402 int c;
403
404 while (1) {
405 do {
406 c = tok_nextc(tok);
407 } while (Py_ISDIGIT(c));
408 if (c != '_') {
409 break;
410 }
411 c = tok_nextc(tok);
412 if (!Py_ISDIGIT(c)) {
413 tok_backup(tok, c);
414 _PyTokenizer_syntaxerror(tok, "invalid decimal literal");
415 return 0;
416 }
417 }
418 return c;
419 }
420
421 static inline int
422 tok_continuation_line(struct tok_state *tok) {
423 int c = tok_nextc(tok);
424 if (c == '\r') {
425 c = tok_nextc(tok);
426 }
427 if (c != '\n') {
428 tok->done = E_LINECONT;
429 return -1;
430 }
431 c = tok_nextc(tok);
432 if (c == EOF) {
433 tok->done = E_EOF;
434 tok->cur = tok->inp;
435 return -1;
436 } else {
437 tok_backup(tok, c);
438 }
439 return c;
440 }
441
442 static int
443 tok_get_normal_mode(struct tok_state *tok, tokenizer_mode* current_tok, struct token *token)
444 {
445 int c;
446 int blankline, nonascii;
447
448 const char *p_start = NULL;
449 const char *p_end = NULL;
450 nextline:
451 tok->start = NULL;
452 tok->starting_col_offset = -1;
453 blankline = 0;
454
455
456 /* Get indentation level */
457 if (tok->atbol) {
458 int col = 0;
459 int altcol = 0;
460 tok->atbol = 0;
461 int cont_line_col = 0;
462 for (;;) {
463 c = tok_nextc(tok);
464 if (c == ' ') {
465 col++, altcol++;
466 }
467 else if (c == '\t') {
468 col = (col / tok->tabsize + 1) * tok->tabsize;
469 altcol = (altcol / ALTTABSIZE + 1) * ALTTABSIZE;
470 }
471 else if (c == '\014') {/* Control-L (formfeed) */
472 col = altcol = 0; /* For Emacs users */
473 }
474 else if (c == '\\') {
475 // Indentation cannot be split over multiple physical lines
476 // using backslashes. This means that if we found a backslash
477 // preceded by whitespace, **the first one we find** determines
478 // the level of indentation of whatever comes next.
479 cont_line_col = cont_line_col ? cont_line_col : col;
480 if ((c = tok_continuation_line(tok)) == -1) {
481 return MAKE_TOKEN(ERRORTOKEN);
482 }
483 }
484 else if (c == EOF && PyErr_Occurred()) {
485 return MAKE_TOKEN(ERRORTOKEN);
486 }
487 else {
488 break;
489 }
490 }
491 tok_backup(tok, c);
492 if (c == '#' || c == '\n' || c == '\r') {
493 /* Lines with only whitespace and/or comments
494 shouldn't affect the indentation and are
495 not passed to the parser as NEWLINE tokens,
496 except *totally* empty lines in interactive
497 mode, which signal the end of a command group. */
498 if (col == 0 && c == '\n' && tok->prompt != NULL) {
499 blankline = 0; /* Let it through */
500 }
501 else if (tok->prompt != NULL && tok->lineno == 1) {
502 /* In interactive mode, if the first line contains
503 only spaces and/or a comment, let it through. */
504 blankline = 0;
505 col = altcol = 0;
506 }
507 else {
508 blankline = 1; /* Ignore completely */
509 }
510 /* We can't jump back right here since we still
511 may need to skip to the end of a comment */
512 }
513 if (!blankline && tok->level == 0) {
514 col = cont_line_col ? cont_line_col : col;
515 altcol = cont_line_col ? cont_line_col : altcol;
516 if (col == tok->indstack[tok->indent]) {
517 /* No change */
518 if (altcol != tok->altindstack[tok->indent]) {
519 return MAKE_TOKEN(_PyTokenizer_indenterror(tok));
520 }
521 }
522 else if (col > tok->indstack[tok->indent]) {
523 /* Indent -- always one */
524 if (tok->indent+1 >= MAXINDENT) {
525 tok->done = E_TOODEEP;
526 tok->cur = tok->inp;
527 return MAKE_TOKEN(ERRORTOKEN);
528 }
529 if (altcol <= tok->altindstack[tok->indent]) {
530 return MAKE_TOKEN(_PyTokenizer_indenterror(tok));
531 }
532 tok->pendin++;
533 tok->indstack[++tok->indent] = col;
534 tok->altindstack[tok->indent] = altcol;
535 }
536 else /* col < tok->indstack[tok->indent] */ {
537 /* Dedent -- any number, must be consistent */
538 while (tok->indent > 0 &&
539 col < tok->indstack[tok->indent]) {
540 tok->pendin--;
541 tok->indent--;
542 }
543 if (col != tok->indstack[tok->indent]) {
544 tok->done = E_DEDENT;
545 tok->cur = tok->inp;
546 return MAKE_TOKEN(ERRORTOKEN);
547 }
548 if (altcol != tok->altindstack[tok->indent]) {
549 return MAKE_TOKEN(_PyTokenizer_indenterror(tok));
550 }
551 }
552 }
553 }
554
555 tok->start = tok->cur;
556 tok->starting_col_offset = tok->col_offset;
557
558 /* Return pending indents/dedents */
559 if (tok->pendin != 0) {
560 if (tok->pendin < 0) {
561 if (tok->tok_extra_tokens) {
562 p_start = tok->cur;
563 p_end = tok->cur;
564 }
565 tok->pendin++;
566 return MAKE_TOKEN(DEDENT);
567 }
568 else {
569 if (tok->tok_extra_tokens) {
570 p_start = tok->buf;
571 p_end = tok->cur;
572 }
573 tok->pendin--;
574 return MAKE_TOKEN(INDENT);
575 }
576 }
577
578 /* Peek ahead at the next character */
579 c = tok_nextc(tok);
580 tok_backup(tok, c);
581
582 again:
583 tok->start = NULL;
584 /* Skip spaces */
585 do {
586 c = tok_nextc(tok);
587 } while (c == ' ' || c == '\t' || c == '\014');
588
589 /* Set start of current token */
590 tok->start = tok->cur == NULL ? NULL : tok->cur - 1;
591 tok->starting_col_offset = tok->col_offset - 1;
592
593 /* Skip comment, unless it's a type comment */
594 if (c == '#') {
595
596 const char* p = NULL;
597 const char *prefix, *type_start;
598 int current_starting_col_offset;
599
600 while (c != EOF && c != '\n' && c != '\r') {
601 c = tok_nextc(tok);
602 }
603
604 if (tok->tok_extra_tokens) {
605 p = tok->start;
606 }
607
608 if (tok->type_comments) {
609 p = tok->start;
610 current_starting_col_offset = tok->starting_col_offset;
611 prefix = type_comment_prefix;
612 while (*prefix && p < tok->cur) {
613 if (*prefix == ' ') {
614 while (*p == ' ' || *p == '\t') {
615 p++;
616 current_starting_col_offset++;
617 }
618 } else if (*prefix == *p) {
619 p++;
620 current_starting_col_offset++;
621 } else {
622 break;
623 }
624
625 prefix++;
626 }
627
628 /* This is a type comment if we matched all of type_comment_prefix. */
629 if (!*prefix) {
630 int is_type_ignore = 1;
631 // +6 in order to skip the word 'ignore'
632 const char *ignore_end = p + 6;
633 const int ignore_end_col_offset = current_starting_col_offset + 6;
634 tok_backup(tok, c); /* don't eat the newline or EOF */
635
636 type_start = p;
637
638 /* A TYPE_IGNORE is "type: ignore" followed by the end of the token
639 * or anything ASCII and non-alphanumeric. */
640 is_type_ignore = (
641 tok->cur >= ignore_end && memcmp(p, "ignore", 6) == 0
642 && !(tok->cur > ignore_end
643 && ((unsigned char)ignore_end[0] >= 128 || Py_ISALNUM(ignore_end[0]))));
644
645 if (is_type_ignore) {
646 p_start = ignore_end;
647 p_end = tok->cur;
648
649 /* If this type ignore is the only thing on the line, consume the newline also. */
650 if (blankline) {
651 tok_nextc(tok);
652 tok->atbol = 1;
653 }
654 return MAKE_TYPE_COMMENT_TOKEN(TYPE_IGNORE, ignore_end_col_offset, tok->col_offset);
655 } else {
656 p_start = type_start;
657 p_end = tok->cur;
658 return MAKE_TYPE_COMMENT_TOKEN(TYPE_COMMENT, current_starting_col_offset, tok->col_offset);
659 }
660 }
661 }
662 if (tok->tok_extra_tokens) {
663 tok_backup(tok, c); /* don't eat the newline or EOF */
664 p_start = p;
665 p_end = tok->cur;
666 tok->comment_newline = blankline;
667 return MAKE_TOKEN(COMMENT);
668 }
669 }
670
671 if (tok->done == E_INTERACT_STOP) {
672 return MAKE_TOKEN(ENDMARKER);
673 }
674
675 /* Check for EOF and errors now */
676 if (c == EOF) {
677 if (tok->level) {
678 return MAKE_TOKEN(ERRORTOKEN);
679 }
680 return MAKE_TOKEN(tok->done == E_EOF ? ENDMARKER : ERRORTOKEN);
681 }
682
683 /* Identifier (most frequent token!) */
684 nonascii = 0;
685 if (is_potential_identifier_start(c)) {
686 /* Process the various legal combinations of b"", r"", u"", and f"". */
687 int saw_b = 0, saw_r = 0, saw_u = 0, saw_f = 0;
688 while (1) {
689 if (!(saw_b || saw_u || saw_f) && (c == 'b' || c == 'B'))
690 saw_b = 1;
691 /* Since this is a backwards compatibility support literal we don't
692 want to support it in arbitrary order like byte literals. */
693 else if (!(saw_b || saw_u || saw_r || saw_f)
694 && (c == 'u'|| c == 'U')) {
695 saw_u = 1;
696 }
697 /* ur"" and ru"" are not supported */
698 else if (!(saw_r || saw_u) && (c == 'r' || c == 'R')) {
699 saw_r = 1;
700 }
701 else if (!(saw_f || saw_b || saw_u) && (c == 'f' || c == 'F')) {
702 saw_f = 1;
703 }
704 else {
705 break;
706 }
707 c = tok_nextc(tok);
708 if (c == '"' || c == '\'') {
709 if (saw_f) {
710 goto f_string_quote;
711 }
712 goto letter_quote;
713 }
714 }
715 while (is_potential_identifier_char(c)) {
716 if (c >= 128) {
717 nonascii = 1;
718 }
719 c = tok_nextc(tok);
720 }
721 tok_backup(tok, c);
722 if (nonascii && !verify_identifier(tok)) {
723 return MAKE_TOKEN(ERRORTOKEN);
724 }
725
726 p_start = tok->start;
727 p_end = tok->cur;
728
729 return MAKE_TOKEN(NAME);
730 }
731
732 if (c == '\r') {
733 c = tok_nextc(tok);
734 }
735
736 /* Newline */
737 if (c == '\n') {
738 tok->atbol = 1;
739 if (blankline || tok->level > 0) {
740 if (tok->tok_extra_tokens) {
741 if (tok->comment_newline) {
742 tok->comment_newline = 0;
743 }
744 p_start = tok->start;
745 p_end = tok->cur;
746 return MAKE_TOKEN(NL);
747 }
748 goto nextline;
749 }
750 if (tok->comment_newline && tok->tok_extra_tokens) {
751 tok->comment_newline = 0;
752 p_start = tok->start;
753 p_end = tok->cur;
754 return MAKE_TOKEN(NL);
755 }
756 p_start = tok->start;
757 p_end = tok->cur - 1; /* Leave '\n' out of the string */
758 tok->cont_line = 0;
759 return MAKE_TOKEN(NEWLINE);
760 }
761
762 /* Period or number starting with period? */
763 if (c == '.') {
764 c = tok_nextc(tok);
765 if (Py_ISDIGIT(c)) {
766 goto fraction;
767 } else if (c == '.') {
768 c = tok_nextc(tok);
769 if (c == '.') {
770 p_start = tok->start;
771 p_end = tok->cur;
772 return MAKE_TOKEN(ELLIPSIS);
773 }
774 else {
775 tok_backup(tok, c);
776 }
777 tok_backup(tok, '.');
778 }
779 else {
780 tok_backup(tok, c);
781 }
782 p_start = tok->start;
783 p_end = tok->cur;
784 return MAKE_TOKEN(DOT);
785 }
786
787 /* Number */
788 if (Py_ISDIGIT(c)) {
789 if (c == '0') {
790 /* Hex, octal or binary -- maybe. */
791 c = tok_nextc(tok);
792 if (c == 'x' || c == 'X') {
793 /* Hex */
794 c = tok_nextc(tok);
795 do {
796 if (c == '_') {
797 c = tok_nextc(tok);
798 }
799 if (!Py_ISXDIGIT(c)) {
800 tok_backup(tok, c);
801 return MAKE_TOKEN(_PyTokenizer_syntaxerror(tok, "invalid hexadecimal literal"));
802 }
803 do {
804 c = tok_nextc(tok);
805 } while (Py_ISXDIGIT(c));
806 } while (c == '_');
807 if (!verify_end_of_number(tok, c, "hexadecimal")) {
808 return MAKE_TOKEN(ERRORTOKEN);
809 }
810 }
811 else if (c == 'o' || c == 'O') {
812 /* Octal */
813 c = tok_nextc(tok);
814 do {
815 if (c == '_') {
816 c = tok_nextc(tok);
817 }
818 if (c < '0' || c >= '8') {
819 if (Py_ISDIGIT(c)) {
820 return MAKE_TOKEN(_PyTokenizer_syntaxerror(tok,
821 "invalid digit '%c' in octal literal", c));
822 }
823 else {
824 tok_backup(tok, c);
825 return MAKE_TOKEN(_PyTokenizer_syntaxerror(tok, "invalid octal literal"));
826 }
827 }
828 do {
829 c = tok_nextc(tok);
830 } while ('0' <= c && c < '8');
831 } while (c == '_');
832 if (Py_ISDIGIT(c)) {
833 return MAKE_TOKEN(_PyTokenizer_syntaxerror(tok,
834 "invalid digit '%c' in octal literal", c));
835 }
836 if (!verify_end_of_number(tok, c, "octal")) {
837 return MAKE_TOKEN(ERRORTOKEN);
838 }
839 }
840 else if (c == 'b' || c == 'B') {
841 /* Binary */
842 c = tok_nextc(tok);
843 do {
844 if (c == '_') {
845 c = tok_nextc(tok);
846 }
847 if (c != '0' && c != '1') {
848 if (Py_ISDIGIT(c)) {
849 return MAKE_TOKEN(_PyTokenizer_syntaxerror(tok, "invalid digit '%c' in binary literal", c));
850 }
851 else {
852 tok_backup(tok, c);
853 return MAKE_TOKEN(_PyTokenizer_syntaxerror(tok, "invalid binary literal"));
854 }
855 }
856 do {
857 c = tok_nextc(tok);
858 } while (c == '0' || c == '1');
859 } while (c == '_');
860 if (Py_ISDIGIT(c)) {
861 return MAKE_TOKEN(_PyTokenizer_syntaxerror(tok, "invalid digit '%c' in binary literal", c));
862 }
863 if (!verify_end_of_number(tok, c, "binary")) {
864 return MAKE_TOKEN(ERRORTOKEN);
865 }
866 }
867 else {
868 int nonzero = 0;
869 /* maybe old-style octal; c is first char of it */
870 /* in any case, allow '0' as a literal */
871 while (1) {
872 if (c == '_') {
873 c = tok_nextc(tok);
874 if (!Py_ISDIGIT(c)) {
875 tok_backup(tok, c);
876 return MAKE_TOKEN(_PyTokenizer_syntaxerror(tok, "invalid decimal literal"));
877 }
878 }
879 if (c != '0') {
880 break;
881 }
882 c = tok_nextc(tok);
883 }
884 char* zeros_end = tok->cur;
885 if (Py_ISDIGIT(c)) {
886 nonzero = 1;
887 c = tok_decimal_tail(tok);
888 if (c == 0) {
889 return MAKE_TOKEN(ERRORTOKEN);
890 }
891 }
892 if (c == '.') {
893 c = tok_nextc(tok);
894 goto fraction;
895 }
896 else if (c == 'e' || c == 'E') {
897 goto exponent;
898 }
899 else if (c == 'j' || c == 'J') {
900 goto imaginary;
901 }
902 else if (nonzero && !tok->tok_extra_tokens) {
903 /* Old-style octal: now disallowed. */
904 tok_backup(tok, c);
905 return MAKE_TOKEN(_PyTokenizer_syntaxerror_known_range(
906 tok, (int)(tok->start + 1 - tok->line_start),
907 (int)(zeros_end - tok->line_start),
908 "leading zeros in decimal integer "
909 "literals are not permitted; "
910 "use an 0o prefix for octal integers"));
911 }
912 if (!verify_end_of_number(tok, c, "decimal")) {
913 return MAKE_TOKEN(ERRORTOKEN);
914 }
915 }
916 }
917 else {
918 /* Decimal */
919 c = tok_decimal_tail(tok);
920 if (c == 0) {
921 return MAKE_TOKEN(ERRORTOKEN);
922 }
923 {
924 /* Accept floating-point numbers. */
925 if (c == '.') {
926 c = tok_nextc(tok);
927 fraction:
928 /* Fraction */
929 if (Py_ISDIGIT(c)) {
930 c = tok_decimal_tail(tok);
931 if (c == 0) {
932 return MAKE_TOKEN(ERRORTOKEN);
933 }
934 }
935 }
936 if (c == 'e' || c == 'E') {
937 int e;
938 exponent:
939 e = c;
940 /* Exponent part */
941 c = tok_nextc(tok);
942 if (c == '+' || c == '-') {
943 c = tok_nextc(tok);
944 if (!Py_ISDIGIT(c)) {
945 tok_backup(tok, c);
946 return MAKE_TOKEN(_PyTokenizer_syntaxerror(tok, "invalid decimal literal"));
947 }
948 } else if (!Py_ISDIGIT(c)) {
949 tok_backup(tok, c);
950 if (!verify_end_of_number(tok, e, "decimal")) {
951 return MAKE_TOKEN(ERRORTOKEN);
952 }
953 tok_backup(tok, e);
954 p_start = tok->start;
955 p_end = tok->cur;
956 return MAKE_TOKEN(NUMBER);
957 }
958 c = tok_decimal_tail(tok);
959 if (c == 0) {
960 return MAKE_TOKEN(ERRORTOKEN);
961 }
962 }
963 if (c == 'j' || c == 'J') {
964 /* Imaginary part */
965 imaginary:
966 c = tok_nextc(tok);
967 if (!verify_end_of_number(tok, c, "imaginary")) {
968 return MAKE_TOKEN(ERRORTOKEN);
969 }
970 }
971 else if (!verify_end_of_number(tok, c, "decimal")) {
972 return MAKE_TOKEN(ERRORTOKEN);
973 }
974 }
975 }
976 tok_backup(tok, c);
977 p_start = tok->start;
978 p_end = tok->cur;
979 return MAKE_TOKEN(NUMBER);
980 }
981
982 f_string_quote:
983 if (((Py_TOLOWER(*tok->start) == 'f' || Py_TOLOWER(*tok->start) == 'r') && (c == '\'' || c == '"'))) {
984 int quote = c;
985 int quote_size = 1; /* 1 or 3 */
986
987 /* Nodes of type STRING, especially multi line strings
988 must be handled differently in order to get both
989 the starting line number and the column offset right.
990 (cf. issue 16806) */
991 tok->first_lineno = tok->lineno;
992 tok->multi_line_start = tok->line_start;
993
994 /* Find the quote size and start of string */
995 int after_quote = tok_nextc(tok);
996 if (after_quote == quote) {
997 int after_after_quote = tok_nextc(tok);
998 if (after_after_quote == quote) {
999 quote_size = 3;
1000 }
1001 else {
1002 // TODO: Check this
1003 tok_backup(tok, after_after_quote);
1004 tok_backup(tok, after_quote);
1005 }
1006 }
1007 if (after_quote != quote) {
1008 tok_backup(tok, after_quote);
1009 }
1010
1011
1012 p_start = tok->start;
1013 p_end = tok->cur;
1014 if (tok->tok_mode_stack_index + 1 >= MAXFSTRINGLEVEL) {
1015 return MAKE_TOKEN(_PyTokenizer_syntaxerror(tok, "too many nested f-strings"));
1016 }
1017 tokenizer_mode *the_current_tok = TOK_NEXT_MODE(tok);
1018 the_current_tok->kind = TOK_FSTRING_MODE;
1019 the_current_tok->f_string_quote = quote;
1020 the_current_tok->f_string_quote_size = quote_size;
1021 the_current_tok->f_string_start = tok->start;
1022 the_current_tok->f_string_multi_line_start = tok->line_start;
1023 the_current_tok->f_string_line_start = tok->lineno;
1024 the_current_tok->f_string_start_offset = -1;
1025 the_current_tok->f_string_multi_line_start_offset = -1;
1026 the_current_tok->last_expr_buffer = NULL;
1027 the_current_tok->last_expr_size = 0;
1028 the_current_tok->last_expr_end = -1;
1029 the_current_tok->in_format_spec = 0;
1030 the_current_tok->f_string_debug = 0;
1031
1032 switch (*tok->start) {
1033 case 'F':
1034 case 'f':
1035 the_current_tok->f_string_raw = Py_TOLOWER(*(tok->start + 1)) == 'r';
1036 break;
1037 case 'R':
1038 case 'r':
1039 the_current_tok->f_string_raw = 1;
1040 break;
1041 default:
1042 Py_UNREACHABLE();
1043 }
1044
1045 the_current_tok->curly_bracket_depth = 0;
1046 the_current_tok->curly_bracket_expr_start_depth = -1;
1047 return MAKE_TOKEN(FSTRING_START);
1048 }
1049
1050 letter_quote:
1051 /* String */
1052 if (c == '\'' || c == '"') {
1053 int quote = c;
1054 int quote_size = 1; /* 1 or 3 */
1055 int end_quote_size = 0;
1056 int has_escaped_quote = 0;
1057
1058 /* Nodes of type STRING, especially multi line strings
1059 must be handled differently in order to get both
1060 the starting line number and the column offset right.
1061 (cf. issue 16806) */
1062 tok->first_lineno = tok->lineno;
1063 tok->multi_line_start = tok->line_start;
1064
1065 /* Find the quote size and start of string */
1066 c = tok_nextc(tok);
1067 if (c == quote) {
1068 c = tok_nextc(tok);
1069 if (c == quote) {
1070 quote_size = 3;
1071 }
1072 else {
1073 end_quote_size = 1; /* empty string found */
1074 }
1075 }
1076 if (c != quote) {
1077 tok_backup(tok, c);
1078 }
1079
1080 /* Get rest of string */
1081 while (end_quote_size != quote_size) {
1082 c = tok_nextc(tok);
1083 if (tok->done == E_ERROR) {
1084 return MAKE_TOKEN(ERRORTOKEN);
1085 }
1086 if (tok->done == E_DECODE) {
1087 break;
1088 }
1089 if (c == EOF || (quote_size == 1 && c == '\n')) {
1090 assert(tok->multi_line_start != NULL);
1091 // shift the tok_state's location into
1092 // the start of string, and report the error
1093 // from the initial quote character
1094 tok->cur = (char *)tok->start;
1095 tok->cur++;
1096 tok->line_start = tok->multi_line_start;
1097 int start = tok->lineno;
1098 tok->lineno = tok->first_lineno;
1099
1100 if (INSIDE_FSTRING(tok)) {
1101 /* When we are in an f-string, before raising the
1102 * unterminated string literal error, check whether
1103 * does the initial quote matches with f-strings quotes
1104 * and if it is, then this must be a missing '}' token
1105 * so raise the proper error */
1106 tokenizer_mode *the_current_tok = TOK_GET_MODE(tok);
1107 if (the_current_tok->f_string_quote == quote &&
1108 the_current_tok->f_string_quote_size == quote_size) {
1109 return MAKE_TOKEN(_PyTokenizer_syntaxerror(tok, "f-string: expecting '}'"));
1110 }
1111 }
1112
1113 if (quote_size == 3) {
1114 _PyTokenizer_syntaxerror(tok, "unterminated triple-quoted string literal"
1115 " (detected at line %d)", start);
1116 if (c != '\n') {
1117 tok->done = E_EOFS;
1118 }
1119 return MAKE_TOKEN(ERRORTOKEN);
1120 }
1121 else {
1122 if (has_escaped_quote) {
1123 _PyTokenizer_syntaxerror(
1124 tok,
1125 "unterminated string literal (detected at line %d); "
1126 "perhaps you escaped the end quote?",
1127 start
1128 );
1129 } else {
1130 _PyTokenizer_syntaxerror(
1131 tok, "unterminated string literal (detected at line %d)", start
1132 );
1133 }
1134 if (c != '\n') {
1135 tok->done = E_EOLS;
1136 }
1137 return MAKE_TOKEN(ERRORTOKEN);
1138 }
1139 }
1140 if (c == quote) {
1141 end_quote_size += 1;
1142 }
1143 else {
1144 end_quote_size = 0;
1145 if (c == '\\') {
1146 c = tok_nextc(tok); /* skip escaped char */
1147 if (c == quote) { /* but record whether the escaped char was a quote */
1148 has_escaped_quote = 1;
1149 }
1150 if (c == '\r') {
1151 c = tok_nextc(tok);
1152 }
1153 }
1154 }
1155 }
1156
1157 p_start = tok->start;
1158 p_end = tok->cur;
1159 return MAKE_TOKEN(STRING);
1160 }
1161
1162 /* Line continuation */
1163 if (c == '\\') {
1164 if ((c = tok_continuation_line(tok)) == -1) {
1165 return MAKE_TOKEN(ERRORTOKEN);
1166 }
1167 tok->cont_line = 1;
1168 goto again; /* Read next line */
1169 }
1170
1171 /* Punctuation character */
1172 int is_punctuation = (c == ':' || c == '}' || c == '!' || c == '{');
1173 if (is_punctuation && INSIDE_FSTRING(tok) && INSIDE_FSTRING_EXPR(current_tok)) {
1174 /* This code block gets executed before the curly_bracket_depth is incremented
1175 * by the `{` case, so for ensuring that we are on the 0th level, we need
1176 * to adjust it manually */
1177 int cursor = current_tok->curly_bracket_depth - (c != '{');
1178 int in_format_spec = current_tok->in_format_spec;
1179 int cursor_in_format_with_debug =
1180 cursor == 1 && (current_tok->f_string_debug || in_format_spec);
1181 int cursor_valid = cursor == 0 || cursor_in_format_with_debug;
1182 if ((cursor_valid) && !_PyLexer_update_fstring_expr(tok, c)) {
1183 return MAKE_TOKEN(ENDMARKER);
1184 }
1185 if ((cursor_valid) && c != '{' && set_fstring_expr(tok, token, c)) {
1186 return MAKE_TOKEN(ERRORTOKEN);
1187 }
1188
1189 if (c == ':' && cursor == current_tok->curly_bracket_expr_start_depth) {
1190 current_tok->kind = TOK_FSTRING_MODE;
1191 current_tok->in_format_spec = 1;
1192 p_start = tok->start;
1193 p_end = tok->cur;
1194 return MAKE_TOKEN(_PyToken_OneChar(c));
1195 }
1196 }
1197
1198 /* Check for two-character token */
1199 {
1200 int c2 = tok_nextc(tok);
1201 int current_token = _PyToken_TwoChars(c, c2);
1202 if (current_token != OP) {
1203 int c3 = tok_nextc(tok);
1204 int current_token3 = _PyToken_ThreeChars(c, c2, c3);
1205 if (current_token3 != OP) {
1206 current_token = current_token3;
1207 }
1208 else {
1209 tok_backup(tok, c3);
1210 }
1211 p_start = tok->start;
1212 p_end = tok->cur;
1213 return MAKE_TOKEN(current_token);
1214 }
1215 tok_backup(tok, c2);
1216 }
1217
1218 /* Keep track of parentheses nesting level */
1219 switch (c) {
1220 case '(':
1221 case '[':
1222 case '{':
1223 if (tok->level >= MAXLEVEL) {
1224 return MAKE_TOKEN(_PyTokenizer_syntaxerror(tok, "too many nested parentheses"));
1225 }
1226 tok->parenstack[tok->level] = c;
1227 tok->parenlinenostack[tok->level] = tok->lineno;
1228 tok->parencolstack[tok->level] = (int)(tok->start - tok->line_start);
1229 tok->level++;
1230 if (INSIDE_FSTRING(tok)) {
1231 current_tok->curly_bracket_depth++;
1232 }
1233 break;
1234 case ')':
1235 case ']':
1236 case '}':
1237 if (INSIDE_FSTRING(tok) && !current_tok->curly_bracket_depth && c == '}') {
1238 return MAKE_TOKEN(_PyTokenizer_syntaxerror(tok, "f-string: single '}' is not allowed"));
1239 }
1240 if (!tok->tok_extra_tokens && !tok->level) {
1241 return MAKE_TOKEN(_PyTokenizer_syntaxerror(tok, "unmatched '%c'", c));
1242 }
1243 if (tok->level > 0) {
1244 tok->level--;
1245 int opening = tok->parenstack[tok->level];
1246 if (!tok->tok_extra_tokens && !((opening == '(' && c == ')') ||
1247 (opening == '[' && c == ']') ||
1248 (opening == '{' && c == '}'))) {
1249 /* If the opening bracket belongs to an f-string's expression
1250 part (e.g. f"{)}") and the closing bracket is an arbitrary
1251 nested expression, then instead of matching a different
1252 syntactical construct with it; we'll throw an unmatched
1253 parentheses error. */
1254 if (INSIDE_FSTRING(tok) && opening == '{') {
1255 assert(current_tok->curly_bracket_depth >= 0);
1256 int previous_bracket = current_tok->curly_bracket_depth - 1;
1257 if (previous_bracket == current_tok->curly_bracket_expr_start_depth) {
1258 return MAKE_TOKEN(_PyTokenizer_syntaxerror(tok, "f-string: unmatched '%c'", c));
1259 }
1260 }
1261 if (tok->parenlinenostack[tok->level] != tok->lineno) {
1262 return MAKE_TOKEN(_PyTokenizer_syntaxerror(tok,
1263 "closing parenthesis '%c' does not match "
1264 "opening parenthesis '%c' on line %d",
1265 c, opening, tok->parenlinenostack[tok->level]));
1266 }
1267 else {
1268 return MAKE_TOKEN(_PyTokenizer_syntaxerror(tok,
1269 "closing parenthesis '%c' does not match "
1270 "opening parenthesis '%c'",
1271 c, opening));
1272 }
1273 }
1274 }
1275
1276 if (INSIDE_FSTRING(tok)) {
1277 current_tok->curly_bracket_depth--;
1278 if (current_tok->curly_bracket_depth < 0) {
1279 return MAKE_TOKEN(_PyTokenizer_syntaxerror(tok, "f-string: unmatched '%c'", c));
1280 }
1281 if (c == '}' && current_tok->curly_bracket_depth == current_tok->curly_bracket_expr_start_depth) {
1282 current_tok->curly_bracket_expr_start_depth--;
1283 current_tok->kind = TOK_FSTRING_MODE;
1284 current_tok->in_format_spec = 0;
1285 current_tok->f_string_debug = 0;
1286 }
1287 }
1288 break;
1289 default:
1290 break;
1291 }
1292
1293 if (!Py_UNICODE_ISPRINTABLE(c)) {
1294 return MAKE_TOKEN(_PyTokenizer_syntaxerror(tok, "invalid non-printable character U+%04X", c));
1295 }
1296
1297 if( c == '=' && INSIDE_FSTRING_EXPR_AT_TOP(current_tok)) {
1298 current_tok->f_string_debug = 1;
1299 }
1300
1301 /* Punctuation character */
1302 p_start = tok->start;
1303 p_end = tok->cur;
1304 return MAKE_TOKEN(_PyToken_OneChar(c));
1305 }
1306
1307 static int
1308 tok_get_fstring_mode(struct tok_state *tok, tokenizer_mode* current_tok, struct token *token)
1309 {
1310 const char *p_start = NULL;
1311 const char *p_end = NULL;
1312 int end_quote_size = 0;
1313 int unicode_escape = 0;
1314
1315 tok->start = tok->cur;
1316 tok->first_lineno = tok->lineno;
1317 tok->starting_col_offset = tok->col_offset;
1318
1319 // If we start with a bracket, we defer to the normal mode as there is nothing for us to tokenize
1320 // before it.
1321 int start_char = tok_nextc(tok);
1322 if (start_char == '{') {
1323 int peek1 = tok_nextc(tok);
1324 tok_backup(tok, peek1);
1325 tok_backup(tok, start_char);
1326 if (peek1 != '{') {
1327 current_tok->curly_bracket_expr_start_depth++;
1328 if (current_tok->curly_bracket_expr_start_depth >= MAX_EXPR_NESTING) {
1329 return MAKE_TOKEN(_PyTokenizer_syntaxerror(tok, "f-string: expressions nested too deeply"));
1330 }
1331 TOK_GET_MODE(tok)->kind = TOK_REGULAR_MODE;
1332 return tok_get_normal_mode(tok, current_tok, token);
1333 }
1334 }
1335 else {
1336 tok_backup(tok, start_char);
1337 }
1338
1339 // Check if we are at the end of the string
1340 for (int i = 0; i < current_tok->f_string_quote_size; i++) {
1341 int quote = tok_nextc(tok);
1342 if (quote != current_tok->f_string_quote) {
1343 tok_backup(tok, quote);
1344 goto f_string_middle;
1345 }
1346 }
1347
1348 if (current_tok->last_expr_buffer != NULL) {
1349 PyMem_Free(current_tok->last_expr_buffer);
1350 current_tok->last_expr_buffer = NULL;
1351 current_tok->last_expr_size = 0;
1352 current_tok->last_expr_end = -1;
1353 }
1354
1355 p_start = tok->start;
1356 p_end = tok->cur;
1357 tok->tok_mode_stack_index--;
1358 return MAKE_TOKEN(FSTRING_END);
1359
1360 f_string_middle:
1361
1362 // TODO: This is a bit of a hack, but it works for now. We need to find a better way to handle
1363 // this.
1364 tok->multi_line_start = tok->line_start;
1365 while (end_quote_size != current_tok->f_string_quote_size) {
1366 int c = tok_nextc(tok);
1367 if (tok->done == E_ERROR || tok->done == E_DECODE) {
1368 return MAKE_TOKEN(ERRORTOKEN);
1369 }
1370 int in_format_spec = (
1371 current_tok->in_format_spec
1372 &&
1373 INSIDE_FSTRING_EXPR(current_tok)
1374 );
1375
1376 if (c == EOF || (current_tok->f_string_quote_size == 1 && c == '\n')) {
1377 if (tok->decoding_erred) {
1378 return MAKE_TOKEN(ERRORTOKEN);
1379 }
1380
1381 // If we are in a format spec and we found a newline,
1382 // it means that the format spec ends here and we should
1383 // return to the regular mode.
1384 if (in_format_spec && c == '\n') {
1385 if (current_tok->f_string_quote_size == 1) {
1386 return MAKE_TOKEN(
1387 _PyTokenizer_syntaxerror(
1388 tok,
1389 "f-string: newlines are not allowed in format specifiers for single quoted f-strings"
1390 )
1391 );
1392 }
1393 tok_backup(tok, c);
1394 TOK_GET_MODE(tok)->kind = TOK_REGULAR_MODE;
1395 current_tok->in_format_spec = 0;
1396 p_start = tok->start;
1397 p_end = tok->cur;
1398 return MAKE_TOKEN(FSTRING_MIDDLE);
1399 }
1400
1401 assert(tok->multi_line_start != NULL);
1402 // shift the tok_state's location into
1403 // the start of string, and report the error
1404 // from the initial quote character
1405 tok->cur = (char *)current_tok->f_string_start;
1406 tok->cur++;
1407 tok->line_start = current_tok->f_string_multi_line_start;
1408 int start = tok->lineno;
1409
1410 tokenizer_mode *the_current_tok = TOK_GET_MODE(tok);
1411 tok->lineno = the_current_tok->f_string_line_start;
1412
1413 if (current_tok->f_string_quote_size == 3) {
1414 _PyTokenizer_syntaxerror(tok,
1415 "unterminated triple-quoted f-string literal"
1416 " (detected at line %d)", start);
1417 if (c != '\n') {
1418 tok->done = E_EOFS;
1419 }
1420 return MAKE_TOKEN(ERRORTOKEN);
1421 }
1422 else {
1423 return MAKE_TOKEN(_PyTokenizer_syntaxerror(tok,
1424 "unterminated f-string literal (detected at"
1425 " line %d)", start));
1426 }
1427 }
1428
1429 if (c == current_tok->f_string_quote) {
1430 end_quote_size += 1;
1431 continue;
1432 } else {
1433 end_quote_size = 0;
1434 }
1435
1436 if (c == '{') {
1437 if (!_PyLexer_update_fstring_expr(tok, c)) {
1438 return MAKE_TOKEN(ENDMARKER);
1439 }
1440 int peek = tok_nextc(tok);
1441 if (peek != '{' || in_format_spec) {
1442 tok_backup(tok, peek);
1443 tok_backup(tok, c);
1444 current_tok->curly_bracket_expr_start_depth++;
1445 if (current_tok->curly_bracket_expr_start_depth >= MAX_EXPR_NESTING) {
1446 return MAKE_TOKEN(_PyTokenizer_syntaxerror(tok, "f-string: expressions nested too deeply"));
1447 }
1448 TOK_GET_MODE(tok)->kind = TOK_REGULAR_MODE;
1449 current_tok->in_format_spec = 0;
1450 p_start = tok->start;
1451 p_end = tok->cur;
1452 } else {
1453 p_start = tok->start;
1454 p_end = tok->cur - 1;
1455 }
1456 return MAKE_TOKEN(FSTRING_MIDDLE);
1457 } else if (c == '}') {
1458 if (unicode_escape) {
1459 p_start = tok->start;
1460 p_end = tok->cur;
1461 return MAKE_TOKEN(FSTRING_MIDDLE);
1462 }
1463 int peek = tok_nextc(tok);
1464
1465 // The tokenizer can only be in the format spec if we have already completed the expression
1466 // scanning (indicated by the end of the expression being set) and we are not at the top level
1467 // of the bracket stack (-1 is the top level). Since format specifiers can't legally use double
1468 // brackets, we can bypass it here.
1469 int cursor = current_tok->curly_bracket_depth;
1470 if (peek == '}' && !in_format_spec && cursor == 0) {
1471 p_start = tok->start;
1472 p_end = tok->cur - 1;
1473 } else {
1474 tok_backup(tok, peek);
1475 tok_backup(tok, c);
1476 TOK_GET_MODE(tok)->kind = TOK_REGULAR_MODE;
1477 current_tok->in_format_spec = 0;
1478 p_start = tok->start;
1479 p_end = tok->cur;
1480 }
1481 return MAKE_TOKEN(FSTRING_MIDDLE);
1482 } else if (c == '\\') {
1483 int peek = tok_nextc(tok);
1484 if (peek == '\r') {
1485 peek = tok_nextc(tok);
1486 }
1487 // Special case when the backslash is right before a curly
1488 // brace. We have to restore and return the control back
1489 // to the loop for the next iteration.
1490 if (peek == '{' || peek == '}') {
1491 if (!current_tok->f_string_raw) {
1492 if (_PyTokenizer_warn_invalid_escape_sequence(tok, peek)) {
1493 return MAKE_TOKEN(ERRORTOKEN);
1494 }
1495 }
1496 tok_backup(tok, peek);
1497 continue;
1498 }
1499
1500 if (!current_tok->f_string_raw) {
1501 if (peek == 'N') {
1502 /* Handle named unicode escapes (\N{BULLET}) */
1503 peek = tok_nextc(tok);
1504 if (peek == '{') {
1505 unicode_escape = 1;
1506 } else {
1507 tok_backup(tok, peek);
1508 }
1509 }
1510 } /* else {
1511 skip the escaped character
1512 }*/
1513 }
1514 }
1515
1516 // Backup the f-string quotes to emit a final FSTRING_MIDDLE and
1517 // add the quotes to the FSTRING_END in the next tokenizer iteration.
1518 for (int i = 0; i < current_tok->f_string_quote_size; i++) {
1519 tok_backup(tok, current_tok->f_string_quote);
1520 }
1521 p_start = tok->start;
1522 p_end = tok->cur;
1523 return MAKE_TOKEN(FSTRING_MIDDLE);
1524 }
1525
1526 static int
1527 tok_get(struct tok_state *tok, struct token *token)
1528 {
1529 tokenizer_mode *current_tok = TOK_GET_MODE(tok);
1530 if (current_tok->kind == TOK_REGULAR_MODE) {
1531 return tok_get_normal_mode(tok, current_tok, token);
1532 } else {
1533 return tok_get_fstring_mode(tok, current_tok, token);
1534 }
1535 }
1536
1537 int
1538 _PyTokenizer_Get(struct tok_state *tok, struct token *token)
1539 {
1540 int result = tok_get(tok, token);
1541 if (tok->decoding_erred) {
1542 result = ERRORTOKEN;
1543 tok->done = E_DECODE;
1544 }
1545 return result;
1546 }