Debian Codegraph @

Parser/lexer/lexer.c

raw
1 #include "Python.h"
2 #include "pycore_token.h"
3 #include "pycore_unicodeobject.h"
4 #include "errcode.h"
5
6 #include "state.h"
7 #include "../tokenizer/helpers.h"
8
9 /* Alternate tab spacing */
10 #define ALTTABSIZE 1
11
12 #define is_potential_identifier_start(c) (\
13 (c >= 'a' && c <= 'z')\
14 || (c >= 'A' && c <= 'Z')\
15 || c == '_'\
16 || (c >= 128))
17
18 #define is_potential_identifier_char(c) (\
19 (c >= 'a' && c <= 'z')\
20 || (c >= 'A' && c <= 'Z')\
21 || (c >= '0' && c <= '9')\
22 || c == '_'\
23 || (c >= 128))
24
25 #ifdef Py_DEBUG
26 static inline tokenizer_mode* TOK_GET_MODE(struct tok_state* tok) {
27 assert(tok->tok_mode_stack_index >= 0);
28 assert(tok->tok_mode_stack_index < MAXFSTRINGLEVEL);
29 return &(tok->tok_mode_stack[tok->tok_mode_stack_index]);
30 }
31 static inline tokenizer_mode* TOK_NEXT_MODE(struct tok_state* tok) {
32 assert(tok->tok_mode_stack_index >= 0);
33 assert(tok->tok_mode_stack_index + 1 < MAXFSTRINGLEVEL);
34 return &(tok->tok_mode_stack[++tok->tok_mode_stack_index]);
35 }
36 #else
37 #define TOK_GET_MODE(tok) (&(tok->tok_mode_stack[tok->tok_mode_stack_index]))
38 #define TOK_NEXT_MODE(tok) (&(tok->tok_mode_stack[++tok->tok_mode_stack_index]))
39 #endif
40
41 #define FTSTRING_MIDDLE(tok_mode) (tok_mode->string_kind == TSTRING ? TSTRING_MIDDLE : FSTRING_MIDDLE)
42 #define FTSTRING_END(tok_mode) (tok_mode->string_kind == TSTRING ? TSTRING_END : FSTRING_END)
43 #define TOK_GET_STRING_PREFIX(tok) (TOK_GET_MODE(tok)->string_kind == TSTRING ? 't' : 'f')
44 #define MAKE_TOKEN(token_type) _PyLexer_token_setup(tok, token, token_type, p_start, p_end)
45 #define MAKE_TYPE_COMMENT_TOKEN(token_type, col_offset, end_col_offset) (\
46 _PyLexer_type_comment_token_setup(tok, token, token_type, col_offset, end_col_offset, p_start, p_end))
47
48 /* Spaces in this constant are treated as "zero or more spaces or tabs" when
49 tokenizing. */
50 static const char* type_comment_prefix = "# type: ";
51
52 static inline int
53 contains_null_bytes(const char* str, size_t size)
54 {
55 return memchr(str, 0, size) != NULL;
56 }
57
58 /* Get next char, updating state; error code goes into tok->done */
59 static int
60 tok_nextc(struct tok_state *tok)
61 {
62 int rc;
63 for (;;) {
64 if (tok->cur != tok->inp) {
65 if ((unsigned int) tok->col_offset >= (unsigned int) INT_MAX) {
66 tok->done = E_COLUMNOVERFLOW;
67 return EOF;
68 }
69 tok->col_offset++;
70 return Py_CHARMASK(*tok->cur++); /* Fast path */
71 }
72 if (tok->done != E_OK) {
73 return EOF;
74 }
75 rc = tok->underflow(tok);
76 #if defined(Py_DEBUG)
77 if (tok->debug) {
78 fprintf(stderr, "line[%d] = ", tok->lineno);
79 _PyTokenizer_print_escape(stderr, tok->cur, tok->inp - tok->cur);
80 fprintf(stderr, " tok->done = %d\n", tok->done);
81 }
82 #endif
83 if (!rc) {
84 tok->cur = tok->inp;
85 return EOF;
86 }
87 tok->line_start = tok->cur;
88
89 if (contains_null_bytes(tok->line_start, tok->inp - tok->line_start)) {
90 _PyTokenizer_syntaxerror(tok, "source code cannot contain null bytes");
91 tok->cur = tok->inp;
92 return EOF;
93 }
94 }
95 Py_UNREACHABLE();
96 }
97
98 /* Back-up one character */
99 static void
100 tok_backup(struct tok_state *tok, int c)
101 {
102 if (c != EOF) {
103 if (--tok->cur < tok->buf) {
104 Py_FatalError("tokenizer beginning of buffer");
105 }
106 if ((int)(unsigned char)*tok->cur != Py_CHARMASK(c)) {
107 Py_FatalError("tok_backup: wrong character");
108 }
109 tok->col_offset--;
110 }
111 }
112
113 static int
114 set_ftstring_expr(struct tok_state* tok, struct token *token, char c) {
115 assert(token != NULL);
116 assert(c == '}' || c == ':' || c == '!');
117 tokenizer_mode *tok_mode = TOK_GET_MODE(tok);
118
119 if (!(tok_mode->in_debug || tok_mode->string_kind == TSTRING) || token->metadata) {
120 return 0;
121 }
122 PyObject *res = NULL;
123
124 // Look for a # character outside of string literals
125 int hash_detected = 0;
126 int in_string = 0;
127 char quote_char = 0;
128
129 for (Py_ssize_t i = 0; i < tok_mode->last_expr_size - tok_mode->last_expr_end; i++) {
130 char ch = tok_mode->last_expr_buffer[i];
131
132 // Skip escaped characters
133 if (ch == '\\') {
134 i++;
135 continue;
136 }
137
138 // Handle quotes
139 if (ch == '"' || ch == '\'') {
140 // The following if/else block works becase there is an off number
141 // of quotes in STRING tokens and the lexer only ever reaches this
142 // function with valid STRING tokens.
143 // For example: """hello"""
144 // First quote: in_string = 1
145 // Second quote: in_string = 0
146 // Third quote: in_string = 1
147 if (!in_string) {
148 in_string = 1;
149 quote_char = ch;
150 }
151 else if (ch == quote_char) {
152 in_string = 0;
153 }
154 continue;
155 }
156
157 // Check for # outside strings
158 if (ch == '#' && !in_string) {
159 hash_detected = 1;
160 break;
161 }
162 }
163 // If we found a # character in the expression, we need to handle comments
164 if (hash_detected) {
165 // Allocate buffer for processed result
166 char *result = (char *)PyMem_Malloc((tok_mode->last_expr_size - tok_mode->last_expr_end + 1) * sizeof(char));
167 if (!result) {
168 return -1;
169 }
170
171 Py_ssize_t i = 0; // Input position
172 Py_ssize_t j = 0; // Output position
173 in_string = 0; // Whether we're in a string
174 quote_char = 0; // Current string quote char
175
176 // Process each character
177 while (i < tok_mode->last_expr_size - tok_mode->last_expr_end) {
178 char ch = tok_mode->last_expr_buffer[i];
179
180 // Handle string quotes
181 if (ch == '"' || ch == '\'') {
182 // See comment above to understand this part
183 if (!in_string) {
184 in_string = 1;
185 quote_char = ch;
186 } else if (ch == quote_char) {
187 in_string = 0;
188 }
189 result[j++] = ch;
190 }
191 // Skip comments
192 else if (ch == '#' && !in_string) {
193 while (i < tok_mode->last_expr_size - tok_mode->last_expr_end &&
194 tok_mode->last_expr_buffer[i] != '\n') {
195 i++;
196 }
197 if (i < tok_mode->last_expr_size - tok_mode->last_expr_end) {
198 result[j++] = '\n';
199 }
200 }
201 // Copy other chars
202 else {
203 result[j++] = ch;
204 }
205 i++;
206 }
207
208 result[j] = '\0'; // Null-terminate the result string
209 res = PyUnicode_DecodeUTF8(result, j, NULL);
210 PyMem_Free(result);
211 } else {
212 res = PyUnicode_DecodeUTF8(
213 tok_mode->last_expr_buffer,
214 tok_mode->last_expr_size - tok_mode->last_expr_end,
215 NULL
216 );
217 }
218
219 if (!res) {
220 return -1;
221 }
222 token->metadata = res;
223 return 0;
224 }
225
226 int
227 _PyLexer_update_ftstring_expr(struct tok_state *tok, char cur)
228 {
229 assert(tok->cur != NULL);
230
231 Py_ssize_t size = strlen(tok->cur);
232 tokenizer_mode *tok_mode = TOK_GET_MODE(tok);
233
234 switch (cur) {
235 case 0:
236 if (!tok_mode->last_expr_buffer || tok_mode->last_expr_end >= 0) {
237 return 1;
238 }
239 char *new_buffer = PyMem_Realloc(
240 tok_mode->last_expr_buffer,
241 tok_mode->last_expr_size + size
242 );
243 if (new_buffer == NULL) {
244 PyMem_Free(tok_mode->last_expr_buffer);
245 goto error;
246 }
247 tok_mode->last_expr_buffer = new_buffer;
248 strncpy(tok_mode->last_expr_buffer + tok_mode->last_expr_size, tok->cur, size);
249 tok_mode->last_expr_size += size;
250 break;
251 case '{':
252 if (tok_mode->last_expr_buffer != NULL) {
253 PyMem_Free(tok_mode->last_expr_buffer);
254 }
255 tok_mode->last_expr_buffer = PyMem_Malloc(size);
256 if (tok_mode->last_expr_buffer == NULL) {
257 goto error;
258 }
259 tok_mode->last_expr_size = size;
260 tok_mode->last_expr_end = -1;
261 strncpy(tok_mode->last_expr_buffer, tok->cur, size);
262 break;
263 case '}':
264 case '!':
265 tok_mode->last_expr_end = strlen(tok->start);
266 break;
267 case ':':
268 if (tok_mode->last_expr_end == -1) {
269 tok_mode->last_expr_end = strlen(tok->start);
270 }
271 break;
272 default:
273 Py_UNREACHABLE();
274 }
275 return 1;
276 error:
277 tok->done = E_NOMEM;
278 return 0;
279 }
280
281 static int
282 lookahead(struct tok_state *tok, const char *test)
283 {
284 const char *s = test;
285 int res = 0;
286 while (1) {
287 int c = tok_nextc(tok);
288 if (*s == 0) {
289 res = !is_potential_identifier_char(c);
290 }
291 else if (c == *s) {
292 s++;
293 continue;
294 }
295
296 tok_backup(tok, c);
297 while (s != test) {
298 tok_backup(tok, *--s);
299 }
300 return res;
301 }
302 }
303
304 static int
305 verify_end_of_number(struct tok_state *tok, int c, const char *kind) {
306 if (tok->tok_extra_tokens) {
307 // When we are parsing extra tokens, we don't want to emit warnings
308 // about invalid literals, because we want to be a bit more liberal.
309 return 1;
310 }
311 /* Emit a deprecation warning only if the numeric literal is immediately
312 * followed by one of keywords which can occur after a numeric literal
313 * in valid code: "and", "else", "for", "if", "in", "is" and "or".
314 * It allows to gradually deprecate existing valid code without adding
315 * warning before error in most cases of invalid numeric literal (which
316 * would be confusing and break existing tests).
317 * Raise a syntax error with slightly better message than plain
318 * "invalid syntax" if the numeric literal is immediately followed by
319 * other keyword or identifier.
320 */
321 int r = 0;
322 if (c == 'a') {
323 r = lookahead(tok, "nd");
324 }
325 else if (c == 'e') {
326 r = lookahead(tok, "lse");
327 }
328 else if (c == 'f') {
329 r = lookahead(tok, "or");
330 }
331 else if (c == 'i') {
332 int c2 = tok_nextc(tok);
333 if (c2 == 'f' || c2 == 'n' || c2 == 's') {
334 r = 1;
335 }
336 tok_backup(tok, c2);
337 }
338 else if (c == 'o') {
339 r = lookahead(tok, "r");
340 }
341 else if (c == 'n') {
342 r = lookahead(tok, "ot");
343 }
344 if (r) {
345 tok_backup(tok, c);
346 if (_PyTokenizer_parser_warn(tok, PyExc_SyntaxWarning,
347 "invalid %s literal", kind))
348 {
349 return 0;
350 }
351 tok_nextc(tok);
352 }
353 else /* In future releases, only error will remain. */
354 if (c < 128 && is_potential_identifier_char(c)) {
355 tok_backup(tok, c);
356 _PyTokenizer_syntaxerror(tok, "invalid %s literal", kind);
357 return 0;
358 }
359 return 1;
360 }
361
362 /* Verify that the identifier follows PEP 3131. */
363 static int
364 verify_identifier(struct tok_state *tok)
365 {
366 if (tok->tok_extra_tokens) {
367 return 1;
368 }
369 PyObject *s;
370 if (tok->decoding_erred)
371 return 0;
372 s = PyUnicode_DecodeUTF8(tok->start, tok->cur - tok->start, NULL);
373 if (s == NULL) {
374 if (PyErr_ExceptionMatches(PyExc_UnicodeDecodeError)) {
375 tok->done = E_DECODE;
376 }
377 else {
378 tok->done = E_ERROR;
379 }
380 return 0;
381 }
382 Py_ssize_t invalid = _PyUnicode_ScanIdentifier(s);
383 assert(invalid >= 0);
384 assert(PyUnicode_GET_LENGTH(s) > 0);
385 if (invalid < PyUnicode_GET_LENGTH(s)) {
386 Py_UCS4 ch = PyUnicode_READ_CHAR(s, invalid);
387 if (invalid + 1 < PyUnicode_GET_LENGTH(s)) {
388 /* Determine the offset in UTF-8 encoded input */
389 Py_SETREF(s, PyUnicode_Substring(s, 0, invalid + 1));
390 if (s != NULL) {
391 Py_SETREF(s, PyUnicode_AsUTF8String(s));
392 }
393 if (s == NULL) {
394 tok->done = E_ERROR;
395 return 0;
396 }
397 tok->cur = (char *)tok->start + PyBytes_GET_SIZE(s);
398 }
399 Py_DECREF(s);
400 if (Py_UNICODE_ISPRINTABLE(ch)) {
401 _PyTokenizer_syntaxerror(tok, "invalid character '%c' (U+%04X)", ch, ch);
402 }
403 else {
404 _PyTokenizer_syntaxerror(tok, "invalid non-printable character U+%04X", ch);
405 }
406 return 0;
407 }
408 Py_DECREF(s);
409 return 1;
410 }
411
412 static int
413 tok_decimal_tail(struct tok_state *tok)
414 {
415 int c;
416
417 while (1) {
418 do {
419 c = tok_nextc(tok);
420 } while (Py_ISDIGIT(c));
421 if (c != '_') {
422 break;
423 }
424 c = tok_nextc(tok);
425 if (!Py_ISDIGIT(c)) {
426 tok_backup(tok, c);
427 _PyTokenizer_syntaxerror(tok, "invalid decimal literal");
428 return 0;
429 }
430 }
431 return c;
432 }
433
434 static inline int
435 tok_continuation_line(struct tok_state *tok) {
436 int c = tok_nextc(tok);
437 if (c == '\r') {
438 c = tok_nextc(tok);
439 }
440 if (c != '\n') {
441 tok->done = E_LINECONT;
442 return -1;
443 }
444 c = tok_nextc(tok);
445 if (c == EOF) {
446 tok->done = E_EOF;
447 tok->cur = tok->inp;
448 return -1;
449 } else {
450 tok_backup(tok, c);
451 }
452 return c;
453 }
454
455 static int
456 maybe_raise_syntax_error_for_string_prefixes(struct tok_state *tok,
457 int saw_b, int saw_r, int saw_u,
458 int saw_f, int saw_t) {
459 // Supported: rb, rf, rt (in any order)
460 // Unsupported: ub, ur, uf, ut, bf, bt, ft (in any order)
461
462 #define RETURN_SYNTAX_ERROR(PREFIX1, PREFIX2) \
463 do { \
464 (void)_PyTokenizer_syntaxerror_known_range( \
465 tok, (int)(tok->start + 1 - tok->line_start), \
466 (int)(tok->cur - tok->line_start), \
467 "'" PREFIX1 "' and '" PREFIX2 "' prefixes are incompatible"); \
468 return -1; \
469 } while (0)
470
471 if (saw_u && saw_b) {
472 RETURN_SYNTAX_ERROR("u", "b");
473 }
474 if (saw_u && saw_r) {
475 RETURN_SYNTAX_ERROR("u", "r");
476 }
477 if (saw_u && saw_f) {
478 RETURN_SYNTAX_ERROR("u", "f");
479 }
480 if (saw_u && saw_t) {
481 RETURN_SYNTAX_ERROR("u", "t");
482 }
483
484 if (saw_b && saw_f) {
485 RETURN_SYNTAX_ERROR("b", "f");
486 }
487 if (saw_b && saw_t) {
488 RETURN_SYNTAX_ERROR("b", "t");
489 }
490
491 if (saw_f && saw_t) {
492 RETURN_SYNTAX_ERROR("f", "t");
493 }
494
495 #undef RETURN_SYNTAX_ERROR
496
497 return 0;
498 }
499
500 static int
501 tok_get_normal_mode(struct tok_state *tok, tokenizer_mode* current_tok, struct token *token)
502 {
503 int c;
504 int blankline, nonascii;
505
506 const char *p_start = NULL;
507 const char *p_end = NULL;
508 nextline:
509 tok->start = NULL;
510 tok->starting_col_offset = -1;
511 blankline = 0;
512
513
514 /* Get indentation level */
515 if (tok->atbol) {
516 int col = 0;
517 int altcol = 0;
518 tok->atbol = 0;
519 int cont_line_col = 0;
520 for (;;) {
521 c = tok_nextc(tok);
522 if (c == ' ') {
523 col++, altcol++;
524 }
525 else if (c == '\t') {
526 col = (col / tok->tabsize + 1) * tok->tabsize;
527 altcol = (altcol / ALTTABSIZE + 1) * ALTTABSIZE;
528 }
529 else if (c == '\014') {/* Control-L (formfeed) */
530 col = altcol = 0; /* For Emacs users */
531 }
532 else if (c == '\\') {
533 // Indentation cannot be split over multiple physical lines
534 // using backslashes. This means that if we found a backslash
535 // preceded by whitespace, **the first one we find** determines
536 // the level of indentation of whatever comes next.
537 cont_line_col = cont_line_col ? cont_line_col : col;
538 if ((c = tok_continuation_line(tok)) == -1) {
539 return MAKE_TOKEN(ERRORTOKEN);
540 }
541 }
542 else if (c == EOF && PyErr_Occurred()) {
543 return MAKE_TOKEN(ERRORTOKEN);
544 }
545 else {
546 break;
547 }
548 }
549 tok_backup(tok, c);
550 if (c == '#' || c == '\n' || c == '\r') {
551 /* Lines with only whitespace and/or comments
552 shouldn't affect the indentation and are
553 not passed to the parser as NEWLINE tokens,
554 except *totally* empty lines in interactive
555 mode, which signal the end of a command group. */
556 if (col == 0 && c == '\n' && tok->prompt != NULL) {
557 blankline = 0; /* Let it through */
558 }
559 else if (tok->prompt != NULL && tok->lineno == 1) {
560 /* In interactive mode, if the first line contains
561 only spaces and/or a comment, let it through. */
562 blankline = 0;
563 col = altcol = 0;
564 }
565 else {
566 blankline = 1; /* Ignore completely */
567 }
568 /* We can't jump back right here since we still
569 may need to skip to the end of a comment */
570 }
571 if (!blankline && tok->level == 0) {
572 col = cont_line_col ? cont_line_col : col;
573 altcol = cont_line_col ? cont_line_col : altcol;
574 if (col == tok->indstack[tok->indent]) {
575 /* No change */
576 if (altcol != tok->altindstack[tok->indent]) {
577 return MAKE_TOKEN(_PyTokenizer_indenterror(tok));
578 }
579 }
580 else if (col > tok->indstack[tok->indent]) {
581 /* Indent -- always one */
582 if (tok->indent+1 >= MAXINDENT) {
583 tok->done = E_TOODEEP;
584 tok->cur = tok->inp;
585 return MAKE_TOKEN(ERRORTOKEN);
586 }
587 if (altcol <= tok->altindstack[tok->indent]) {
588 return MAKE_TOKEN(_PyTokenizer_indenterror(tok));
589 }
590 tok->pendin++;
591 tok->indstack[++tok->indent] = col;
592 tok->altindstack[tok->indent] = altcol;
593 }
594 else /* col < tok->indstack[tok->indent] */ {
595 /* Dedent -- any number, must be consistent */
596 while (tok->indent > 0 &&
597 col < tok->indstack[tok->indent]) {
598 tok->pendin--;
599 tok->indent--;
600 }
601 if (col != tok->indstack[tok->indent]) {
602 tok->done = E_DEDENT;
603 tok->cur = tok->inp;
604 return MAKE_TOKEN(ERRORTOKEN);
605 }
606 if (altcol != tok->altindstack[tok->indent]) {
607 return MAKE_TOKEN(_PyTokenizer_indenterror(tok));
608 }
609 }
610 }
611 }
612
613 tok->start = tok->cur;
614 tok->starting_col_offset = tok->col_offset;
615
616 /* Return pending indents/dedents */
617 if (tok->pendin != 0) {
618 if (tok->pendin < 0) {
619 if (tok->tok_extra_tokens) {
620 p_start = tok->cur;
621 p_end = tok->cur;
622 }
623 tok->pendin++;
624 return MAKE_TOKEN(DEDENT);
625 }
626 else {
627 if (tok->tok_extra_tokens) {
628 p_start = tok->buf;
629 p_end = tok->cur;
630 }
631 tok->pendin--;
632 return MAKE_TOKEN(INDENT);
633 }
634 }
635
636 /* Peek ahead at the next character */
637 c = tok_nextc(tok);
638 tok_backup(tok, c);
639
640 again:
641 tok->start = NULL;
642 /* Skip spaces */
643 do {
644 c = tok_nextc(tok);
645 } while (c == ' ' || c == '\t' || c == '\014');
646
647 /* Set start of current token */
648 tok->start = tok->cur == NULL ? NULL : tok->cur - 1;
649 tok->starting_col_offset = tok->col_offset - 1;
650
651 /* Skip comment, unless it's a type comment */
652 if (c == '#') {
653
654 const char* p = NULL;
655 const char *prefix, *type_start;
656 int current_starting_col_offset;
657
658 while (c != EOF && c != '\n' && c != '\r') {
659 c = tok_nextc(tok);
660 }
661
662 if (tok->tok_extra_tokens) {
663 p = tok->start;
664 }
665
666 if (tok->type_comments) {
667 p = tok->start;
668 current_starting_col_offset = tok->starting_col_offset;
669 prefix = type_comment_prefix;
670 while (*prefix && p < tok->cur) {
671 if (*prefix == ' ') {
672 while (*p == ' ' || *p == '\t') {
673 p++;
674 current_starting_col_offset++;
675 }
676 } else if (*prefix == *p) {
677 p++;
678 current_starting_col_offset++;
679 } else {
680 break;
681 }
682
683 prefix++;
684 }
685
686 /* This is a type comment if we matched all of type_comment_prefix. */
687 if (!*prefix) {
688 int is_type_ignore = 1;
689 // +6 in order to skip the word 'ignore'
690 const char *ignore_end = p + 6;
691 const int ignore_end_col_offset = current_starting_col_offset + 6;
692 tok_backup(tok, c); /* don't eat the newline or EOF */
693
694 type_start = p;
695
696 /* A TYPE_IGNORE is "type: ignore" followed by the end of the token
697 * or anything ASCII and non-alphanumeric. */
698 is_type_ignore = (
699 tok->cur >= ignore_end && memcmp(p, "ignore", 6) == 0
700 && !(tok->cur > ignore_end
701 && ((unsigned char)ignore_end[0] >= 128 || Py_ISALNUM(ignore_end[0]))));
702
703 if (is_type_ignore) {
704 p_start = ignore_end;
705 p_end = tok->cur;
706
707 /* If this type ignore is the only thing on the line, consume the newline also. */
708 if (blankline) {
709 tok_nextc(tok);
710 tok->atbol = 1;
711 }
712 return MAKE_TYPE_COMMENT_TOKEN(TYPE_IGNORE, ignore_end_col_offset, tok->col_offset);
713 } else {
714 p_start = type_start;
715 p_end = tok->cur;
716 return MAKE_TYPE_COMMENT_TOKEN(TYPE_COMMENT, current_starting_col_offset, tok->col_offset);
717 }
718 }
719 }
720 if (tok->tok_extra_tokens) {
721 tok_backup(tok, c); /* don't eat the newline or EOF */
722 p_start = p;
723 p_end = tok->cur;
724 tok->comment_newline = blankline;
725 return MAKE_TOKEN(COMMENT);
726 }
727 }
728
729 if (tok->done == E_INTERACT_STOP) {
730 return MAKE_TOKEN(ENDMARKER);
731 }
732
733 /* Check for EOF and errors now */
734 if (c == EOF) {
735 if (tok->level) {
736 return MAKE_TOKEN(ERRORTOKEN);
737 }
738 return MAKE_TOKEN(tok->done == E_EOF ? ENDMARKER : ERRORTOKEN);
739 }
740
741 /* Identifier (most frequent token!) */
742 nonascii = 0;
743 if (is_potential_identifier_start(c)) {
744 /* Process the various legal combinations of b"", r"", u"", and f"". */
745 int saw_b = 0, saw_r = 0, saw_u = 0, saw_f = 0, saw_t = 0;
746 while (1) {
747 if (!saw_b && (c == 'b' || c == 'B')) {
748 saw_b = 1;
749 }
750 /* Since this is a backwards compatibility support literal we don't
751 want to support it in arbitrary order like byte literals. */
752 else if (!saw_u && (c == 'u'|| c == 'U')) {
753 saw_u = 1;
754 }
755 /* ur"" and ru"" are not supported */
756 else if (!saw_r && (c == 'r' || c == 'R')) {
757 saw_r = 1;
758 }
759 else if (!saw_f && (c == 'f' || c == 'F')) {
760 saw_f = 1;
761 }
762 else if (!saw_t && (c == 't' || c == 'T')) {
763 saw_t = 1;
764 }
765 else {
766 break;
767 }
768 c = tok_nextc(tok);
769 if (c == '"' || c == '\'') {
770 // Raise error on incompatible string prefixes:
771 int status = maybe_raise_syntax_error_for_string_prefixes(
772 tok, saw_b, saw_r, saw_u, saw_f, saw_t);
773 if (status < 0) {
774 return MAKE_TOKEN(ERRORTOKEN);
775 }
776
777 // Handle valid f or t string creation:
778 if (saw_f || saw_t) {
779 goto f_string_quote;
780 }
781 goto letter_quote;
782 }
783 }
784 while (is_potential_identifier_char(c)) {
785 if (c >= 128) {
786 nonascii = 1;
787 }
788 c = tok_nextc(tok);
789 }
790 tok_backup(tok, c);
791 if (nonascii && !verify_identifier(tok)) {
792 return MAKE_TOKEN(ERRORTOKEN);
793 }
794
795 p_start = tok->start;
796 p_end = tok->cur;
797
798 return MAKE_TOKEN(NAME);
799 }
800
801 if (c == '\r') {
802 c = tok_nextc(tok);
803 }
804
805 /* Newline */
806 if (c == '\n') {
807 tok->atbol = 1;
808 if (blankline || tok->level > 0) {
809 if (tok->tok_extra_tokens) {
810 if (tok->comment_newline) {
811 tok->comment_newline = 0;
812 }
813 p_start = tok->start;
814 p_end = tok->cur;
815 return MAKE_TOKEN(NL);
816 }
817 goto nextline;
818 }
819 if (tok->comment_newline && tok->tok_extra_tokens) {
820 tok->comment_newline = 0;
821 p_start = tok->start;
822 p_end = tok->cur;
823 return MAKE_TOKEN(NL);
824 }
825 p_start = tok->start;
826 p_end = tok->cur - 1; /* Leave '\n' out of the string */
827 tok->cont_line = 0;
828 return MAKE_TOKEN(NEWLINE);
829 }
830
831 /* Period or number starting with period? */
832 if (c == '.') {
833 c = tok_nextc(tok);
834 if (Py_ISDIGIT(c)) {
835 goto fraction;
836 } else if (c == '.') {
837 c = tok_nextc(tok);
838 if (c == '.') {
839 p_start = tok->start;
840 p_end = tok->cur;
841 return MAKE_TOKEN(ELLIPSIS);
842 }
843 else {
844 tok_backup(tok, c);
845 }
846 tok_backup(tok, '.');
847 }
848 else {
849 tok_backup(tok, c);
850 }
851 p_start = tok->start;
852 p_end = tok->cur;
853 return MAKE_TOKEN(DOT);
854 }
855
856 /* Number */
857 if (Py_ISDIGIT(c)) {
858 if (c == '0') {
859 /* Hex, octal or binary -- maybe. */
860 c = tok_nextc(tok);
861 if (c == 'x' || c == 'X') {
862 /* Hex */
863 c = tok_nextc(tok);
864 do {
865 if (c == '_') {
866 c = tok_nextc(tok);
867 }
868 if (!Py_ISXDIGIT(c)) {
869 tok_backup(tok, c);
870 return MAKE_TOKEN(_PyTokenizer_syntaxerror(tok, "invalid hexadecimal literal"));
871 }
872 do {
873 c = tok_nextc(tok);
874 } while (Py_ISXDIGIT(c));
875 } while (c == '_');
876 if (!verify_end_of_number(tok, c, "hexadecimal")) {
877 return MAKE_TOKEN(ERRORTOKEN);
878 }
879 }
880 else if (c == 'o' || c == 'O') {
881 /* Octal */
882 c = tok_nextc(tok);
883 do {
884 if (c == '_') {
885 c = tok_nextc(tok);
886 }
887 if (c < '0' || c >= '8') {
888 if (Py_ISDIGIT(c)) {
889 return MAKE_TOKEN(_PyTokenizer_syntaxerror(tok,
890 "invalid digit '%c' in octal literal", c));
891 }
892 else {
893 tok_backup(tok, c);
894 return MAKE_TOKEN(_PyTokenizer_syntaxerror(tok, "invalid octal literal"));
895 }
896 }
897 do {
898 c = tok_nextc(tok);
899 } while ('0' <= c && c < '8');
900 } while (c == '_');
901 if (Py_ISDIGIT(c)) {
902 return MAKE_TOKEN(_PyTokenizer_syntaxerror(tok,
903 "invalid digit '%c' in octal literal", c));
904 }
905 if (!verify_end_of_number(tok, c, "octal")) {
906 return MAKE_TOKEN(ERRORTOKEN);
907 }
908 }
909 else if (c == 'b' || c == 'B') {
910 /* Binary */
911 c = tok_nextc(tok);
912 do {
913 if (c == '_') {
914 c = tok_nextc(tok);
915 }
916 if (c != '0' && c != '1') {
917 if (Py_ISDIGIT(c)) {
918 return MAKE_TOKEN(_PyTokenizer_syntaxerror(tok, "invalid digit '%c' in binary literal", c));
919 }
920 else {
921 tok_backup(tok, c);
922 return MAKE_TOKEN(_PyTokenizer_syntaxerror(tok, "invalid binary literal"));
923 }
924 }
925 do {
926 c = tok_nextc(tok);
927 } while (c == '0' || c == '1');
928 } while (c == '_');
929 if (Py_ISDIGIT(c)) {
930 return MAKE_TOKEN(_PyTokenizer_syntaxerror(tok, "invalid digit '%c' in binary literal", c));
931 }
932 if (!verify_end_of_number(tok, c, "binary")) {
933 return MAKE_TOKEN(ERRORTOKEN);
934 }
935 }
936 else {
937 int nonzero = 0;
938 /* maybe old-style octal; c is first char of it */
939 /* in any case, allow '0' as a literal */
940 while (1) {
941 if (c == '_') {
942 c = tok_nextc(tok);
943 if (!Py_ISDIGIT(c)) {
944 tok_backup(tok, c);
945 return MAKE_TOKEN(_PyTokenizer_syntaxerror(tok, "invalid decimal literal"));
946 }
947 }
948 if (c != '0') {
949 break;
950 }
951 c = tok_nextc(tok);
952 }
953 char* zeros_end = tok->cur;
954 if (Py_ISDIGIT(c)) {
955 nonzero = 1;
956 c = tok_decimal_tail(tok);
957 if (c == 0) {
958 return MAKE_TOKEN(ERRORTOKEN);
959 }
960 }
961 if (c == '.') {
962 c = tok_nextc(tok);
963 goto fraction;
964 }
965 else if (c == 'e' || c == 'E') {
966 goto exponent;
967 }
968 else if (c == 'j' || c == 'J') {
969 goto imaginary;
970 }
971 else if (nonzero && !tok->tok_extra_tokens) {
972 /* Old-style octal: now disallowed. */
973 tok_backup(tok, c);
974 return MAKE_TOKEN(_PyTokenizer_syntaxerror_known_range(
975 tok, (int)(tok->start + 1 - tok->line_start),
976 (int)(zeros_end - tok->line_start),
977 "leading zeros in decimal integer "
978 "literals are not permitted; "
979 "use an 0o prefix for octal integers"));
980 }
981 if (!verify_end_of_number(tok, c, "decimal")) {
982 return MAKE_TOKEN(ERRORTOKEN);
983 }
984 }
985 }
986 else {
987 /* Decimal */
988 c = tok_decimal_tail(tok);
989 if (c == 0) {
990 return MAKE_TOKEN(ERRORTOKEN);
991 }
992 {
993 /* Accept floating-point numbers. */
994 if (c == '.') {
995 c = tok_nextc(tok);
996 fraction:
997 /* Fraction */
998 if (Py_ISDIGIT(c)) {
999 c = tok_decimal_tail(tok);
1000 if (c == 0) {
1001 return MAKE_TOKEN(ERRORTOKEN);
1002 }
1003 }
1004 }
1005 if (c == 'e' || c == 'E') {
1006 int e;
1007 exponent:
1008 e = c;
1009 /* Exponent part */
1010 c = tok_nextc(tok);
1011 if (c == '+' || c == '-') {
1012 c = tok_nextc(tok);
1013 if (!Py_ISDIGIT(c)) {
1014 tok_backup(tok, c);
1015 return MAKE_TOKEN(_PyTokenizer_syntaxerror(tok, "invalid decimal literal"));
1016 }
1017 } else if (!Py_ISDIGIT(c)) {
1018 tok_backup(tok, c);
1019 if (!verify_end_of_number(tok, e, "decimal")) {
1020 return MAKE_TOKEN(ERRORTOKEN);
1021 }
1022 tok_backup(tok, e);
1023 p_start = tok->start;
1024 p_end = tok->cur;
1025 return MAKE_TOKEN(NUMBER);
1026 }
1027 c = tok_decimal_tail(tok);
1028 if (c == 0) {
1029 return MAKE_TOKEN(ERRORTOKEN);
1030 }
1031 }
1032 if (c == 'j' || c == 'J') {
1033 /* Imaginary part */
1034 imaginary:
1035 c = tok_nextc(tok);
1036 if (!verify_end_of_number(tok, c, "imaginary")) {
1037 return MAKE_TOKEN(ERRORTOKEN);
1038 }
1039 }
1040 else if (!verify_end_of_number(tok, c, "decimal")) {
1041 return MAKE_TOKEN(ERRORTOKEN);
1042 }
1043 }
1044 }
1045 tok_backup(tok, c);
1046 p_start = tok->start;
1047 p_end = tok->cur;
1048 return MAKE_TOKEN(NUMBER);
1049 }
1050
1051 f_string_quote:
1052 if (((Py_TOLOWER(*tok->start) == 'f' || Py_TOLOWER(*tok->start) == 'r' || Py_TOLOWER(*tok->start) == 't')
1053 && (c == '\'' || c == '"'))) {
1054
1055 int quote = c;
1056 int quote_size = 1; /* 1 or 3 */
1057
1058 /* Nodes of type STRING, especially multi line strings
1059 must be handled differently in order to get both
1060 the starting line number and the column offset right.
1061 (cf. issue 16806) */
1062 tok->first_lineno = tok->lineno;
1063 tok->multi_line_start = tok->line_start;
1064
1065 /* Find the quote size and start of string */
1066 int after_quote = tok_nextc(tok);
1067 if (after_quote == quote) {
1068 int after_after_quote = tok_nextc(tok);
1069 if (after_after_quote == quote) {
1070 quote_size = 3;
1071 }
1072 else {
1073 // TODO: Check this
1074 tok_backup(tok, after_after_quote);
1075 tok_backup(tok, after_quote);
1076 }
1077 }
1078 if (after_quote != quote) {
1079 tok_backup(tok, after_quote);
1080 }
1081
1082
1083 p_start = tok->start;
1084 p_end = tok->cur;
1085 if (tok->tok_mode_stack_index + 1 >= MAXFSTRINGLEVEL) {
1086 return MAKE_TOKEN(_PyTokenizer_syntaxerror(tok, "too many nested f-strings or t-strings"));
1087 }
1088 tokenizer_mode *the_current_tok = TOK_NEXT_MODE(tok);
1089 the_current_tok->kind = TOK_FSTRING_MODE;
1090 the_current_tok->quote = quote;
1091 the_current_tok->quote_size = quote_size;
1092 the_current_tok->start = tok->start;
1093 the_current_tok->multi_line_start = tok->line_start;
1094 the_current_tok->first_line = tok->lineno;
1095 the_current_tok->start_offset = -1;
1096 the_current_tok->multi_line_start_offset = -1;
1097 the_current_tok->last_expr_buffer = NULL;
1098 the_current_tok->last_expr_size = 0;
1099 the_current_tok->last_expr_end = -1;
1100 the_current_tok->in_format_spec = 0;
1101 the_current_tok->in_debug = 0;
1102
1103 enum string_kind_t string_kind = FSTRING;
1104 switch (*tok->start) {
1105 case 'T':
1106 case 't':
1107 the_current_tok->raw = Py_TOLOWER(*(tok->start + 1)) == 'r';
1108 string_kind = TSTRING;
1109 break;
1110 case 'F':
1111 case 'f':
1112 the_current_tok->raw = Py_TOLOWER(*(tok->start + 1)) == 'r';
1113 break;
1114 case 'R':
1115 case 'r':
1116 the_current_tok->raw = 1;
1117 if (Py_TOLOWER(*(tok->start + 1)) == 't') {
1118 string_kind = TSTRING;
1119 }
1120 break;
1121 default:
1122 Py_UNREACHABLE();
1123 }
1124
1125 the_current_tok->string_kind = string_kind;
1126 the_current_tok->curly_bracket_depth = 0;
1127 the_current_tok->curly_bracket_expr_start_depth = -1;
1128 return string_kind == TSTRING ? MAKE_TOKEN(TSTRING_START) : MAKE_TOKEN(FSTRING_START);
1129 }
1130
1131 letter_quote:
1132 /* String */
1133 if (c == '\'' || c == '"') {
1134 int quote = c;
1135 int quote_size = 1; /* 1 or 3 */
1136 int end_quote_size = 0;
1137 int has_escaped_quote = 0;
1138
1139 /* Nodes of type STRING, especially multi line strings
1140 must be handled differently in order to get both
1141 the starting line number and the column offset right.
1142 (cf. issue 16806) */
1143 tok->first_lineno = tok->lineno;
1144 tok->multi_line_start = tok->line_start;
1145
1146 /* Find the quote size and start of string */
1147 c = tok_nextc(tok);
1148 if (c == quote) {
1149 c = tok_nextc(tok);
1150 if (c == quote) {
1151 quote_size = 3;
1152 }
1153 else {
1154 end_quote_size = 1; /* empty string found */
1155 }
1156 }
1157 if (c != quote) {
1158 tok_backup(tok, c);
1159 }
1160
1161 /* Get rest of string */
1162 while (end_quote_size != quote_size) {
1163 c = tok_nextc(tok);
1164 if (tok->done == E_ERROR) {
1165 return MAKE_TOKEN(ERRORTOKEN);
1166 }
1167 if (tok->done == E_DECODE) {
1168 break;
1169 }
1170 if (c == EOF || (quote_size == 1 && c == '\n')) {
1171 assert(tok->multi_line_start != NULL);
1172 // shift the tok_state's location into
1173 // the start of string, and report the error
1174 // from the initial quote character
1175 tok->cur = (char *)tok->start;
1176 tok->cur++;
1177 tok->line_start = tok->multi_line_start;
1178 int start = tok->lineno;
1179 tok->lineno = tok->first_lineno;
1180
1181 if (INSIDE_FSTRING(tok)) {
1182 /* When we are in an f-string, before raising the
1183 * unterminated string literal error, check whether
1184 * does the initial quote matches with f-strings quotes
1185 * and if it is, then this must be a missing '}' token
1186 * so raise the proper error */
1187 tokenizer_mode *the_current_tok = TOK_GET_MODE(tok);
1188 if (the_current_tok->quote == quote &&
1189 the_current_tok->quote_size == quote_size) {
1190 return MAKE_TOKEN(_PyTokenizer_syntaxerror(tok,
1191 "%c-string: expecting '}'", TOK_GET_STRING_PREFIX(tok)));
1192 }
1193 }
1194
1195 if (quote_size == 3) {
1196 _PyTokenizer_syntaxerror(tok, "unterminated triple-quoted string literal"
1197 " (detected at line %d)", start);
1198 if (c != '\n') {
1199 tok->done = E_EOFS;
1200 }
1201 return MAKE_TOKEN(ERRORTOKEN);
1202 }
1203 else {
1204 if (has_escaped_quote) {
1205 _PyTokenizer_syntaxerror(
1206 tok,
1207 "unterminated string literal (detected at line %d); "
1208 "perhaps you escaped the end quote?",
1209 start
1210 );
1211 } else {
1212 _PyTokenizer_syntaxerror(
1213 tok, "unterminated string literal (detected at line %d)", start
1214 );
1215 }
1216 if (c != '\n') {
1217 tok->done = E_EOLS;
1218 }
1219 return MAKE_TOKEN(ERRORTOKEN);
1220 }
1221 }
1222 if (c == quote) {
1223 end_quote_size += 1;
1224 }
1225 else {
1226 end_quote_size = 0;
1227 if (c == '\\') {
1228 c = tok_nextc(tok); /* skip escaped char */
1229 if (c == quote) { /* but record whether the escaped char was a quote */
1230 has_escaped_quote = 1;
1231 }
1232 if (c == '\r') {
1233 c = tok_nextc(tok);
1234 }
1235 }
1236 }
1237 }
1238
1239 p_start = tok->start;
1240 p_end = tok->cur;
1241 return MAKE_TOKEN(STRING);
1242 }
1243
1244 /* Line continuation */
1245 if (c == '\\') {
1246 if ((c = tok_continuation_line(tok)) == -1) {
1247 return MAKE_TOKEN(ERRORTOKEN);
1248 }
1249 tok->cont_line = 1;
1250 goto again; /* Read next line */
1251 }
1252
1253 /* Punctuation character */
1254 int is_punctuation = (c == ':' || c == '}' || c == '!' || c == '{');
1255 if (is_punctuation && INSIDE_FSTRING(tok) && INSIDE_FSTRING_EXPR(current_tok)) {
1256 /* This code block gets executed before the curly_bracket_depth is incremented
1257 * by the `{` case, so for ensuring that we are on the 0th level, we need
1258 * to adjust it manually */
1259 int cursor = current_tok->curly_bracket_depth - (c != '{');
1260 int in_format_spec = current_tok->in_format_spec;
1261 int cursor_in_format_with_debug =
1262 cursor == 1 && (current_tok->in_debug || in_format_spec);
1263 int cursor_valid = cursor == 0 || cursor_in_format_with_debug;
1264 if ((cursor_valid) && !_PyLexer_update_ftstring_expr(tok, c)) {
1265 return MAKE_TOKEN(ENDMARKER);
1266 }
1267 if ((cursor_valid) && c != '{' && set_ftstring_expr(tok, token, c)) {
1268 return MAKE_TOKEN(ERRORTOKEN);
1269 }
1270
1271 if (c == ':' && cursor == current_tok->curly_bracket_expr_start_depth) {
1272 current_tok->kind = TOK_FSTRING_MODE;
1273 current_tok->in_format_spec = 1;
1274 p_start = tok->start;
1275 p_end = tok->cur;
1276 return MAKE_TOKEN(_PyToken_OneChar(c));
1277 }
1278 }
1279
1280 /* Check for two-character token */
1281 {
1282 int c2 = tok_nextc(tok);
1283 int current_token = _PyToken_TwoChars(c, c2);
1284 if (current_token != OP) {
1285 int c3 = tok_nextc(tok);
1286 int current_token3 = _PyToken_ThreeChars(c, c2, c3);
1287 if (current_token3 != OP) {
1288 current_token = current_token3;
1289 }
1290 else {
1291 tok_backup(tok, c3);
1292 }
1293 p_start = tok->start;
1294 p_end = tok->cur;
1295 return MAKE_TOKEN(current_token);
1296 }
1297 tok_backup(tok, c2);
1298 }
1299
1300 /* Keep track of parentheses nesting level */
1301 switch (c) {
1302 case '(':
1303 case '[':
1304 case '{':
1305 if (tok->level >= MAXLEVEL) {
1306 return MAKE_TOKEN(_PyTokenizer_syntaxerror(tok, "too many nested parentheses"));
1307 }
1308 tok->parenstack[tok->level] = c;
1309 tok->parenlinenostack[tok->level] = tok->lineno;
1310 tok->parencolstack[tok->level] = (int)(tok->start - tok->line_start);
1311 tok->level++;
1312 if (INSIDE_FSTRING(tok)) {
1313 current_tok->curly_bracket_depth++;
1314 }
1315 break;
1316 case ')':
1317 case ']':
1318 case '}':
1319 if (INSIDE_FSTRING(tok) && !current_tok->curly_bracket_depth && c == '}') {
1320 return MAKE_TOKEN(_PyTokenizer_syntaxerror(tok,
1321 "%c-string: single '}' is not allowed", TOK_GET_STRING_PREFIX(tok)));
1322 }
1323 if (!tok->tok_extra_tokens && !tok->level) {
1324 return MAKE_TOKEN(_PyTokenizer_syntaxerror(tok, "unmatched '%c'", c));
1325 }
1326 if (tok->level > 0) {
1327 tok->level--;
1328 int opening = tok->parenstack[tok->level];
1329 if (!tok->tok_extra_tokens && !((opening == '(' && c == ')') ||
1330 (opening == '[' && c == ']') ||
1331 (opening == '{' && c == '}'))) {
1332 /* If the opening bracket belongs to an f-string's expression
1333 part (e.g. f"{)}") and the closing bracket is an arbitrary
1334 nested expression, then instead of matching a different
1335 syntactical construct with it; we'll throw an unmatched
1336 parentheses error. */
1337 if (INSIDE_FSTRING(tok) && opening == '{') {
1338 assert(current_tok->curly_bracket_depth >= 0);
1339 int previous_bracket = current_tok->curly_bracket_depth - 1;
1340 if (previous_bracket == current_tok->curly_bracket_expr_start_depth) {
1341 return MAKE_TOKEN(_PyTokenizer_syntaxerror(tok,
1342 "%c-string: unmatched '%c'", TOK_GET_STRING_PREFIX(tok), c));
1343 }
1344 }
1345 if (tok->parenlinenostack[tok->level] != tok->lineno) {
1346 return MAKE_TOKEN(_PyTokenizer_syntaxerror(tok,
1347 "closing parenthesis '%c' does not match "
1348 "opening parenthesis '%c' on line %d",
1349 c, opening, tok->parenlinenostack[tok->level]));
1350 }
1351 else {
1352 return MAKE_TOKEN(_PyTokenizer_syntaxerror(tok,
1353 "closing parenthesis '%c' does not match "
1354 "opening parenthesis '%c'",
1355 c, opening));
1356 }
1357 }
1358 }
1359
1360 if (INSIDE_FSTRING(tok)) {
1361 current_tok->curly_bracket_depth--;
1362 if (current_tok->curly_bracket_depth < 0) {
1363 return MAKE_TOKEN(_PyTokenizer_syntaxerror(tok, "%c-string: unmatched '%c'",
1364 TOK_GET_STRING_PREFIX(tok), c));
1365 }
1366 if (c == '}' && current_tok->curly_bracket_depth == current_tok->curly_bracket_expr_start_depth) {
1367 current_tok->curly_bracket_expr_start_depth--;
1368 current_tok->kind = TOK_FSTRING_MODE;
1369 current_tok->in_format_spec = 0;
1370 current_tok->in_debug = 0;
1371 }
1372 }
1373 break;
1374 default:
1375 break;
1376 }
1377
1378 if (!Py_UNICODE_ISPRINTABLE(c)) {
1379 return MAKE_TOKEN(_PyTokenizer_syntaxerror(tok, "invalid non-printable character U+%04X", c));
1380 }
1381
1382 if( c == '=' && INSIDE_FSTRING_EXPR_AT_TOP(current_tok)) {
1383 current_tok->in_debug = 1;
1384 }
1385
1386 /* Punctuation character */
1387 p_start = tok->start;
1388 p_end = tok->cur;
1389 return MAKE_TOKEN(_PyToken_OneChar(c));
1390 }
1391
1392 static int
1393 tok_get_fstring_mode(struct tok_state *tok, tokenizer_mode* current_tok, struct token *token)
1394 {
1395 const char *p_start = NULL;
1396 const char *p_end = NULL;
1397 int end_quote_size = 0;
1398 int unicode_escape = 0;
1399
1400 tok->start = tok->cur;
1401 tok->first_lineno = tok->lineno;
1402 tok->starting_col_offset = tok->col_offset;
1403
1404 // If we start with a bracket, we defer to the normal mode as there is nothing for us to tokenize
1405 // before it.
1406 int start_char = tok_nextc(tok);
1407 if (start_char == '{') {
1408 int peek1 = tok_nextc(tok);
1409 tok_backup(tok, peek1);
1410 tok_backup(tok, start_char);
1411 if (peek1 != '{') {
1412 current_tok->curly_bracket_expr_start_depth++;
1413 if (current_tok->curly_bracket_expr_start_depth >= MAX_EXPR_NESTING) {
1414 return MAKE_TOKEN(_PyTokenizer_syntaxerror(tok,
1415 "%c-string: expressions nested too deeply", TOK_GET_STRING_PREFIX(tok)));
1416 }
1417 TOK_GET_MODE(tok)->kind = TOK_REGULAR_MODE;
1418 return tok_get_normal_mode(tok, current_tok, token);
1419 }
1420 }
1421 else {
1422 tok_backup(tok, start_char);
1423 }
1424
1425 // Check if we are at the end of the string
1426 for (int i = 0; i < current_tok->quote_size; i++) {
1427 int quote = tok_nextc(tok);
1428 if (quote != current_tok->quote) {
1429 tok_backup(tok, quote);
1430 goto f_string_middle;
1431 }
1432 }
1433
1434 if (current_tok->last_expr_buffer != NULL) {
1435 PyMem_Free(current_tok->last_expr_buffer);
1436 current_tok->last_expr_buffer = NULL;
1437 current_tok->last_expr_size = 0;
1438 current_tok->last_expr_end = -1;
1439 }
1440
1441 p_start = tok->start;
1442 p_end = tok->cur;
1443 tok->tok_mode_stack_index--;
1444 return MAKE_TOKEN(FTSTRING_END(current_tok));
1445
1446 f_string_middle:
1447
1448 // TODO: This is a bit of a hack, but it works for now. We need to find a better way to handle
1449 // this.
1450 tok->multi_line_start = tok->line_start;
1451 while (end_quote_size != current_tok->quote_size) {
1452 int c = tok_nextc(tok);
1453 if (tok->done == E_ERROR || tok->done == E_DECODE) {
1454 return MAKE_TOKEN(ERRORTOKEN);
1455 }
1456 int in_format_spec = (
1457 current_tok->in_format_spec
1458 &&
1459 INSIDE_FSTRING_EXPR(current_tok)
1460 );
1461
1462 if (c == EOF || (current_tok->quote_size == 1 && c == '\n')) {
1463 if (tok->decoding_erred) {
1464 return MAKE_TOKEN(ERRORTOKEN);
1465 }
1466
1467 // If we are in a format spec and we found a newline,
1468 // it means that the format spec ends here and we should
1469 // return to the regular mode.
1470 if (in_format_spec && c == '\n') {
1471 if (current_tok->quote_size == 1) {
1472 return MAKE_TOKEN(
1473 _PyTokenizer_syntaxerror(
1474 tok,
1475 "%c-string: newlines are not allowed in format specifiers for single quoted %c-strings",
1476 TOK_GET_STRING_PREFIX(tok), TOK_GET_STRING_PREFIX(tok)
1477 )
1478 );
1479 }
1480 tok_backup(tok, c);
1481 TOK_GET_MODE(tok)->kind = TOK_REGULAR_MODE;
1482 current_tok->in_format_spec = 0;
1483 p_start = tok->start;
1484 p_end = tok->cur;
1485 return MAKE_TOKEN(FTSTRING_MIDDLE(current_tok));
1486 }
1487
1488 assert(tok->multi_line_start != NULL);
1489 // shift the tok_state's location into
1490 // the start of string, and report the error
1491 // from the initial quote character
1492 tok->cur = (char *)current_tok->start;
1493 tok->cur++;
1494 tok->line_start = current_tok->multi_line_start;
1495 int start = tok->lineno;
1496
1497 tokenizer_mode *the_current_tok = TOK_GET_MODE(tok);
1498 tok->lineno = the_current_tok->first_line;
1499
1500 if (current_tok->quote_size == 3) {
1501 _PyTokenizer_syntaxerror(tok,
1502 "unterminated triple-quoted %c-string literal"
1503 " (detected at line %d)",
1504 TOK_GET_STRING_PREFIX(tok), start);
1505 if (c != '\n') {
1506 tok->done = E_EOFS;
1507 }
1508 return MAKE_TOKEN(ERRORTOKEN);
1509 }
1510 else {
1511 return MAKE_TOKEN(_PyTokenizer_syntaxerror(tok,
1512 "unterminated %c-string literal (detected at"
1513 " line %d)", TOK_GET_STRING_PREFIX(tok), start));
1514 }
1515 }
1516
1517 if (c == current_tok->quote) {
1518 end_quote_size += 1;
1519 continue;
1520 } else {
1521 end_quote_size = 0;
1522 }
1523
1524 if (c == '{') {
1525 if (!_PyLexer_update_ftstring_expr(tok, c)) {
1526 return MAKE_TOKEN(ENDMARKER);
1527 }
1528 int peek = tok_nextc(tok);
1529 if (peek != '{' || in_format_spec) {
1530 tok_backup(tok, peek);
1531 tok_backup(tok, c);
1532 current_tok->curly_bracket_expr_start_depth++;
1533 if (current_tok->curly_bracket_expr_start_depth >= MAX_EXPR_NESTING) {
1534 return MAKE_TOKEN(_PyTokenizer_syntaxerror(tok,
1535 "%c-string: expressions nested too deeply", TOK_GET_STRING_PREFIX(tok)));
1536 }
1537 TOK_GET_MODE(tok)->kind = TOK_REGULAR_MODE;
1538 current_tok->in_format_spec = 0;
1539 p_start = tok->start;
1540 p_end = tok->cur;
1541 } else {
1542 p_start = tok->start;
1543 p_end = tok->cur - 1;
1544 }
1545 return MAKE_TOKEN(FTSTRING_MIDDLE(current_tok));
1546 } else if (c == '}') {
1547 if (unicode_escape) {
1548 p_start = tok->start;
1549 p_end = tok->cur;
1550 return MAKE_TOKEN(FTSTRING_MIDDLE(current_tok));
1551 }
1552 int peek = tok_nextc(tok);
1553
1554 // The tokenizer can only be in the format spec if we have already completed the expression
1555 // scanning (indicated by the end of the expression being set) and we are not at the top level
1556 // of the bracket stack (-1 is the top level). Since format specifiers can't legally use double
1557 // brackets, we can bypass it here.
1558 int cursor = current_tok->curly_bracket_depth;
1559 if (peek == '}' && !in_format_spec && cursor == 0) {
1560 p_start = tok->start;
1561 p_end = tok->cur - 1;
1562 } else {
1563 tok_backup(tok, peek);
1564 tok_backup(tok, c);
1565 TOK_GET_MODE(tok)->kind = TOK_REGULAR_MODE;
1566 current_tok->in_format_spec = 0;
1567 p_start = tok->start;
1568 p_end = tok->cur;
1569 }
1570 return MAKE_TOKEN(FTSTRING_MIDDLE(current_tok));
1571 } else if (c == '\\') {
1572 int peek = tok_nextc(tok);
1573 if (peek == '\r') {
1574 peek = tok_nextc(tok);
1575 }
1576 // Special case when the backslash is right before a curly
1577 // brace. We have to restore and return the control back
1578 // to the loop for the next iteration.
1579 if (peek == '{' || peek == '}') {
1580 if (!current_tok->raw) {
1581 if (_PyTokenizer_warn_invalid_escape_sequence(tok, peek)) {
1582 return MAKE_TOKEN(ERRORTOKEN);
1583 }
1584 }
1585 tok_backup(tok, peek);
1586 continue;
1587 }
1588
1589 if (!current_tok->raw) {
1590 if (peek == 'N') {
1591 /* Handle named unicode escapes (\N{BULLET}) */
1592 peek = tok_nextc(tok);
1593 if (peek == '{') {
1594 unicode_escape = 1;
1595 } else {
1596 tok_backup(tok, peek);
1597 }
1598 }
1599 } /* else {
1600 skip the escaped character
1601 }*/
1602 }
1603 }
1604
1605 // Backup the f-string quotes to emit a final FSTRING_MIDDLE and
1606 // add the quotes to the FSTRING_END in the next tokenizer iteration.
1607 for (int i = 0; i < current_tok->quote_size; i++) {
1608 tok_backup(tok, current_tok->quote);
1609 }
1610 p_start = tok->start;
1611 p_end = tok->cur;
1612 return MAKE_TOKEN(FTSTRING_MIDDLE(current_tok));
1613 }
1614
1615 static int
1616 tok_get(struct tok_state *tok, struct token *token)
1617 {
1618 tokenizer_mode *current_tok = TOK_GET_MODE(tok);
1619 if (current_tok->kind == TOK_REGULAR_MODE) {
1620 return tok_get_normal_mode(tok, current_tok, token);
1621 } else {
1622 return tok_get_fstring_mode(tok, current_tok, token);
1623 }
1624 }
1625
1626 int
1627 _PyTokenizer_Get(struct tok_state *tok, struct token *token)
1628 {
1629 int result = tok_get(tok, token);
1630 if (tok->decoding_erred) {
1631 result = ERRORTOKEN;
1632 tok->done = E_DECODE;
1633 }
1634 return result;
1635 }