| 1 |
#include "Python.h" |
| 2 |
#include "pycore_token.h" |
| 3 |
#include "pycore_unicodeobject.h" |
| 4 |
#include "errcode.h" |
| 5 |
|
| 6 |
#include "state.h" |
| 7 |
#include "../tokenizer/helpers.h" |
| 8 |
|
| 9 |
/* Alternate tab spacing */ |
| 10 |
#define ALTTABSIZE 1 |
| 11 |
|
| 12 |
#define is_potential_identifier_start(c) (\ |
| 13 |
(c >= 'a' && c <= 'z')\ |
| 14 |
|| (c >= 'A' && c <= 'Z')\ |
| 15 |
|| c == '_'\ |
| 16 |
|| (c >= 128)) |
| 17 |
|
| 18 |
#define is_potential_identifier_char(c) (\ |
| 19 |
(c >= 'a' && c <= 'z')\ |
| 20 |
|| (c >= 'A' && c <= 'Z')\ |
| 21 |
|| (c >= '0' && c <= '9')\ |
| 22 |
|| c == '_'\ |
| 23 |
|| (c >= 128)) |
| 24 |
|
| 25 |
#ifdef Py_DEBUG |
| 26 |
static inline tokenizer_mode* TOK_GET_MODE(struct tok_state* tok) { |
| 27 |
assert(tok->tok_mode_stack_index >= 0); |
| 28 |
assert(tok->tok_mode_stack_index < MAXFSTRINGLEVEL); |
| 29 |
return &(tok->tok_mode_stack[tok->tok_mode_stack_index]); |
| 30 |
} |
| 31 |
static inline tokenizer_mode* TOK_NEXT_MODE(struct tok_state* tok) { |
| 32 |
assert(tok->tok_mode_stack_index >= 0); |
| 33 |
assert(tok->tok_mode_stack_index + 1 < MAXFSTRINGLEVEL); |
| 34 |
return &(tok->tok_mode_stack[++tok->tok_mode_stack_index]); |
| 35 |
} |
| 36 |
#else |
| 37 |
#define TOK_GET_MODE(tok) (&(tok->tok_mode_stack[tok->tok_mode_stack_index])) |
| 38 |
#define TOK_NEXT_MODE(tok) (&(tok->tok_mode_stack[++tok->tok_mode_stack_index])) |
| 39 |
#endif |
| 40 |
|
| 41 |
#define FTSTRING_MIDDLE(tok_mode) (tok_mode->string_kind == TSTRING ? TSTRING_MIDDLE : FSTRING_MIDDLE) |
| 42 |
#define FTSTRING_END(tok_mode) (tok_mode->string_kind == TSTRING ? TSTRING_END : FSTRING_END) |
| 43 |
#define TOK_GET_STRING_PREFIX(tok) (TOK_GET_MODE(tok)->string_kind == TSTRING ? 't' : 'f') |
| 44 |
#define MAKE_TOKEN(token_type) _PyLexer_token_setup(tok, token, token_type, p_start, p_end) |
| 45 |
#define MAKE_TYPE_COMMENT_TOKEN(token_type, col_offset, end_col_offset) (\ |
| 46 |
_PyLexer_type_comment_token_setup(tok, token, token_type, col_offset, end_col_offset, p_start, p_end)) |
| 47 |
|
| 48 |
/* Spaces in this constant are treated as "zero or more spaces or tabs" when |
| 49 |
tokenizing. */ |
| 50 |
static const char* type_comment_prefix = "# type: "; |
| 51 |
|
| 52 |
static inline int |
| 53 |
contains_null_bytes(const char* str, size_t size) |
| 54 |
{ |
| 55 |
return memchr(str, 0, size) != NULL; |
| 56 |
} |
| 57 |
|
| 58 |
/* Get next char, updating state; error code goes into tok->done */ |
| 59 |
static int |
| 60 |
tok_nextc(struct tok_state *tok) |
| 61 |
{ |
| 62 |
int rc; |
| 63 |
for (;;) { |
| 64 |
if (tok->cur != tok->inp) { |
| 65 |
if ((unsigned int) tok->col_offset >= (unsigned int) INT_MAX) { |
| 66 |
tok->done = E_COLUMNOVERFLOW; |
| 67 |
return EOF; |
| 68 |
} |
| 69 |
tok->col_offset++; |
| 70 |
return Py_CHARMASK(*tok->cur++); /* Fast path */ |
| 71 |
} |
| 72 |
if (tok->done != E_OK) { |
| 73 |
return EOF; |
| 74 |
} |
| 75 |
rc = tok->underflow(tok); |
| 76 |
#if defined(Py_DEBUG) |
| 77 |
if (tok->debug) { |
| 78 |
fprintf(stderr, "line[%d] = ", tok->lineno); |
| 79 |
_PyTokenizer_print_escape(stderr, tok->cur, tok->inp - tok->cur); |
| 80 |
fprintf(stderr, " tok->done = %d\n", tok->done); |
| 81 |
} |
| 82 |
#endif |
| 83 |
if (!rc) { |
| 84 |
tok->cur = tok->inp; |
| 85 |
return EOF; |
| 86 |
} |
| 87 |
tok->line_start = tok->cur; |
| 88 |
|
| 89 |
if (contains_null_bytes(tok->line_start, tok->inp - tok->line_start)) { |
| 90 |
_PyTokenizer_syntaxerror(tok, "source code cannot contain null bytes"); |
| 91 |
tok->cur = tok->inp; |
| 92 |
return EOF; |
| 93 |
} |
| 94 |
} |
| 95 |
Py_UNREACHABLE(); |
| 96 |
} |
| 97 |
|
| 98 |
/* Back-up one character */ |
| 99 |
static void |
| 100 |
tok_backup(struct tok_state *tok, int c) |
| 101 |
{ |
| 102 |
if (c != EOF) { |
| 103 |
if (--tok->cur < tok->buf) { |
| 104 |
Py_FatalError("tokenizer beginning of buffer"); |
| 105 |
} |
| 106 |
if ((int)(unsigned char)*tok->cur != Py_CHARMASK(c)) { |
| 107 |
Py_FatalError("tok_backup: wrong character"); |
| 108 |
} |
| 109 |
tok->col_offset--; |
| 110 |
} |
| 111 |
} |
| 112 |
|
| 113 |
static int |
| 114 |
set_ftstring_expr(struct tok_state* tok, struct token *token, char c) { |
| 115 |
assert(token != NULL); |
| 116 |
assert(c == '}' || c == ':' || c == '!'); |
| 117 |
tokenizer_mode *tok_mode = TOK_GET_MODE(tok); |
| 118 |
|
| 119 |
if (!(tok_mode->in_debug || tok_mode->string_kind == TSTRING) || token->metadata) { |
| 120 |
return 0; |
| 121 |
} |
| 122 |
PyObject *res = NULL; |
| 123 |
|
| 124 |
// Look for a # character outside of string literals |
| 125 |
int hash_detected = 0; |
| 126 |
int in_string = 0; |
| 127 |
char quote_char = 0; |
| 128 |
|
| 129 |
for (Py_ssize_t i = 0; i < tok_mode->last_expr_size - tok_mode->last_expr_end; i++) { |
| 130 |
char ch = tok_mode->last_expr_buffer[i]; |
| 131 |
|
| 132 |
// Skip escaped characters |
| 133 |
if (ch == '\\') { |
| 134 |
i++; |
| 135 |
continue; |
| 136 |
} |
| 137 |
|
| 138 |
// Handle quotes |
| 139 |
if (ch == '"' || ch == '\'') { |
| 140 |
// The following if/else block works becase there is an off number |
| 141 |
// of quotes in STRING tokens and the lexer only ever reaches this |
| 142 |
// function with valid STRING tokens. |
| 143 |
// For example: """hello""" |
| 144 |
// First quote: in_string = 1 |
| 145 |
// Second quote: in_string = 0 |
| 146 |
// Third quote: in_string = 1 |
| 147 |
if (!in_string) { |
| 148 |
in_string = 1; |
| 149 |
quote_char = ch; |
| 150 |
} |
| 151 |
else if (ch == quote_char) { |
| 152 |
in_string = 0; |
| 153 |
} |
| 154 |
continue; |
| 155 |
} |
| 156 |
|
| 157 |
// Check for # outside strings |
| 158 |
if (ch == '#' && !in_string) { |
| 159 |
hash_detected = 1; |
| 160 |
break; |
| 161 |
} |
| 162 |
} |
| 163 |
// If we found a # character in the expression, we need to handle comments |
| 164 |
if (hash_detected) { |
| 165 |
// Allocate buffer for processed result |
| 166 |
char *result = (char *)PyMem_Malloc((tok_mode->last_expr_size - tok_mode->last_expr_end + 1) * sizeof(char)); |
| 167 |
if (!result) { |
| 168 |
return -1; |
| 169 |
} |
| 170 |
|
| 171 |
Py_ssize_t i = 0; // Input position |
| 172 |
Py_ssize_t j = 0; // Output position |
| 173 |
in_string = 0; // Whether we're in a string |
| 174 |
quote_char = 0; // Current string quote char |
| 175 |
|
| 176 |
// Process each character |
| 177 |
while (i < tok_mode->last_expr_size - tok_mode->last_expr_end) { |
| 178 |
char ch = tok_mode->last_expr_buffer[i]; |
| 179 |
|
| 180 |
// Handle string quotes |
| 181 |
if (ch == '"' || ch == '\'') { |
| 182 |
// See comment above to understand this part |
| 183 |
if (!in_string) { |
| 184 |
in_string = 1; |
| 185 |
quote_char = ch; |
| 186 |
} else if (ch == quote_char) { |
| 187 |
in_string = 0; |
| 188 |
} |
| 189 |
result[j++] = ch; |
| 190 |
} |
| 191 |
// Skip comments |
| 192 |
else if (ch == '#' && !in_string) { |
| 193 |
while (i < tok_mode->last_expr_size - tok_mode->last_expr_end && |
| 194 |
tok_mode->last_expr_buffer[i] != '\n') { |
| 195 |
i++; |
| 196 |
} |
| 197 |
if (i < tok_mode->last_expr_size - tok_mode->last_expr_end) { |
| 198 |
result[j++] = '\n'; |
| 199 |
} |
| 200 |
} |
| 201 |
// Copy other chars |
| 202 |
else { |
| 203 |
result[j++] = ch; |
| 204 |
} |
| 205 |
i++; |
| 206 |
} |
| 207 |
|
| 208 |
result[j] = '\0'; // Null-terminate the result string |
| 209 |
res = PyUnicode_DecodeUTF8(result, j, NULL); |
| 210 |
PyMem_Free(result); |
| 211 |
} else { |
| 212 |
res = PyUnicode_DecodeUTF8( |
| 213 |
tok_mode->last_expr_buffer, |
| 214 |
tok_mode->last_expr_size - tok_mode->last_expr_end, |
| 215 |
NULL |
| 216 |
); |
| 217 |
} |
| 218 |
|
| 219 |
if (!res) { |
| 220 |
return -1; |
| 221 |
} |
| 222 |
token->metadata = res; |
| 223 |
return 0; |
| 224 |
} |
| 225 |
|
| 226 |
int |
| 227 |
_PyLexer_update_ftstring_expr(struct tok_state *tok, char cur) |
| 228 |
{ |
| 229 |
assert(tok->cur != NULL); |
| 230 |
|
| 231 |
Py_ssize_t size = strlen(tok->cur); |
| 232 |
tokenizer_mode *tok_mode = TOK_GET_MODE(tok); |
| 233 |
|
| 234 |
switch (cur) { |
| 235 |
case 0: |
| 236 |
if (!tok_mode->last_expr_buffer || tok_mode->last_expr_end >= 0) { |
| 237 |
return 1; |
| 238 |
} |
| 239 |
char *new_buffer = PyMem_Realloc( |
| 240 |
tok_mode->last_expr_buffer, |
| 241 |
tok_mode->last_expr_size + size |
| 242 |
); |
| 243 |
if (new_buffer == NULL) { |
| 244 |
PyMem_Free(tok_mode->last_expr_buffer); |
| 245 |
goto error; |
| 246 |
} |
| 247 |
tok_mode->last_expr_buffer = new_buffer; |
| 248 |
strncpy(tok_mode->last_expr_buffer + tok_mode->last_expr_size, tok->cur, size); |
| 249 |
tok_mode->last_expr_size += size; |
| 250 |
break; |
| 251 |
case '{': |
| 252 |
if (tok_mode->last_expr_buffer != NULL) { |
| 253 |
PyMem_Free(tok_mode->last_expr_buffer); |
| 254 |
} |
| 255 |
tok_mode->last_expr_buffer = PyMem_Malloc(size); |
| 256 |
if (tok_mode->last_expr_buffer == NULL) { |
| 257 |
goto error; |
| 258 |
} |
| 259 |
tok_mode->last_expr_size = size; |
| 260 |
tok_mode->last_expr_end = -1; |
| 261 |
strncpy(tok_mode->last_expr_buffer, tok->cur, size); |
| 262 |
break; |
| 263 |
case '}': |
| 264 |
case '!': |
| 265 |
tok_mode->last_expr_end = strlen(tok->start); |
| 266 |
break; |
| 267 |
case ':': |
| 268 |
if (tok_mode->last_expr_end == -1) { |
| 269 |
tok_mode->last_expr_end = strlen(tok->start); |
| 270 |
} |
| 271 |
break; |
| 272 |
default: |
| 273 |
Py_UNREACHABLE(); |
| 274 |
} |
| 275 |
return 1; |
| 276 |
error: |
| 277 |
tok->done = E_NOMEM; |
| 278 |
return 0; |
| 279 |
} |
| 280 |
|
| 281 |
static int |
| 282 |
lookahead(struct tok_state *tok, const char *test) |
| 283 |
{ |
| 284 |
const char *s = test; |
| 285 |
int res = 0; |
| 286 |
while (1) { |
| 287 |
int c = tok_nextc(tok); |
| 288 |
if (*s == 0) { |
| 289 |
res = !is_potential_identifier_char(c); |
| 290 |
} |
| 291 |
else if (c == *s) { |
| 292 |
s++; |
| 293 |
continue; |
| 294 |
} |
| 295 |
|
| 296 |
tok_backup(tok, c); |
| 297 |
while (s != test) { |
| 298 |
tok_backup(tok, *--s); |
| 299 |
} |
| 300 |
return res; |
| 301 |
} |
| 302 |
} |
| 303 |
|
| 304 |
static int |
| 305 |
verify_end_of_number(struct tok_state *tok, int c, const char *kind) { |
| 306 |
if (tok->tok_extra_tokens) { |
| 307 |
// When we are parsing extra tokens, we don't want to emit warnings |
| 308 |
// about invalid literals, because we want to be a bit more liberal. |
| 309 |
return 1; |
| 310 |
} |
| 311 |
/* Emit a deprecation warning only if the numeric literal is immediately |
| 312 |
* followed by one of keywords which can occur after a numeric literal |
| 313 |
* in valid code: "and", "else", "for", "if", "in", "is" and "or". |
| 314 |
* It allows to gradually deprecate existing valid code without adding |
| 315 |
* warning before error in most cases of invalid numeric literal (which |
| 316 |
* would be confusing and break existing tests). |
| 317 |
* Raise a syntax error with slightly better message than plain |
| 318 |
* "invalid syntax" if the numeric literal is immediately followed by |
| 319 |
* other keyword or identifier. |
| 320 |
*/ |
| 321 |
int r = 0; |
| 322 |
if (c == 'a') { |
| 323 |
r = lookahead(tok, "nd"); |
| 324 |
} |
| 325 |
else if (c == 'e') { |
| 326 |
r = lookahead(tok, "lse"); |
| 327 |
} |
| 328 |
else if (c == 'f') { |
| 329 |
r = lookahead(tok, "or"); |
| 330 |
} |
| 331 |
else if (c == 'i') { |
| 332 |
int c2 = tok_nextc(tok); |
| 333 |
if (c2 == 'f' || c2 == 'n' || c2 == 's') { |
| 334 |
r = 1; |
| 335 |
} |
| 336 |
tok_backup(tok, c2); |
| 337 |
} |
| 338 |
else if (c == 'o') { |
| 339 |
r = lookahead(tok, "r"); |
| 340 |
} |
| 341 |
else if (c == 'n') { |
| 342 |
r = lookahead(tok, "ot"); |
| 343 |
} |
| 344 |
if (r) { |
| 345 |
tok_backup(tok, c); |
| 346 |
if (_PyTokenizer_parser_warn(tok, PyExc_SyntaxWarning, |
| 347 |
"invalid %s literal", kind)) |
| 348 |
{ |
| 349 |
return 0; |
| 350 |
} |
| 351 |
tok_nextc(tok); |
| 352 |
} |
| 353 |
else /* In future releases, only error will remain. */ |
| 354 |
if (c < 128 && is_potential_identifier_char(c)) { |
| 355 |
tok_backup(tok, c); |
| 356 |
_PyTokenizer_syntaxerror(tok, "invalid %s literal", kind); |
| 357 |
return 0; |
| 358 |
} |
| 359 |
return 1; |
| 360 |
} |
| 361 |
|
| 362 |
/* Verify that the identifier follows PEP 3131. */ |
| 363 |
static int |
| 364 |
verify_identifier(struct tok_state *tok) |
| 365 |
{ |
| 366 |
if (tok->tok_extra_tokens) { |
| 367 |
return 1; |
| 368 |
} |
| 369 |
PyObject *s; |
| 370 |
if (tok->decoding_erred) |
| 371 |
return 0; |
| 372 |
s = PyUnicode_DecodeUTF8(tok->start, tok->cur - tok->start, NULL); |
| 373 |
if (s == NULL) { |
| 374 |
if (PyErr_ExceptionMatches(PyExc_UnicodeDecodeError)) { |
| 375 |
tok->done = E_DECODE; |
| 376 |
} |
| 377 |
else { |
| 378 |
tok->done = E_ERROR; |
| 379 |
} |
| 380 |
return 0; |
| 381 |
} |
| 382 |
Py_ssize_t invalid = _PyUnicode_ScanIdentifier(s); |
| 383 |
assert(invalid >= 0); |
| 384 |
assert(PyUnicode_GET_LENGTH(s) > 0); |
| 385 |
if (invalid < PyUnicode_GET_LENGTH(s)) { |
| 386 |
Py_UCS4 ch = PyUnicode_READ_CHAR(s, invalid); |
| 387 |
if (invalid + 1 < PyUnicode_GET_LENGTH(s)) { |
| 388 |
/* Determine the offset in UTF-8 encoded input */ |
| 389 |
Py_SETREF(s, PyUnicode_Substring(s, 0, invalid + 1)); |
| 390 |
if (s != NULL) { |
| 391 |
Py_SETREF(s, PyUnicode_AsUTF8String(s)); |
| 392 |
} |
| 393 |
if (s == NULL) { |
| 394 |
tok->done = E_ERROR; |
| 395 |
return 0; |
| 396 |
} |
| 397 |
tok->cur = (char *)tok->start + PyBytes_GET_SIZE(s); |
| 398 |
} |
| 399 |
Py_DECREF(s); |
| 400 |
if (Py_UNICODE_ISPRINTABLE(ch)) { |
| 401 |
_PyTokenizer_syntaxerror(tok, "invalid character '%c' (U+%04X)", ch, ch); |
| 402 |
} |
| 403 |
else { |
| 404 |
_PyTokenizer_syntaxerror(tok, "invalid non-printable character U+%04X", ch); |
| 405 |
} |
| 406 |
return 0; |
| 407 |
} |
| 408 |
Py_DECREF(s); |
| 409 |
return 1; |
| 410 |
} |
| 411 |
|
| 412 |
static int |
| 413 |
tok_decimal_tail(struct tok_state *tok) |
| 414 |
{ |
| 415 |
int c; |
| 416 |
|
| 417 |
while (1) { |
| 418 |
do { |
| 419 |
c = tok_nextc(tok); |
| 420 |
} while (Py_ISDIGIT(c)); |
| 421 |
if (c != '_') { |
| 422 |
break; |
| 423 |
} |
| 424 |
c = tok_nextc(tok); |
| 425 |
if (!Py_ISDIGIT(c)) { |
| 426 |
tok_backup(tok, c); |
| 427 |
_PyTokenizer_syntaxerror(tok, "invalid decimal literal"); |
| 428 |
return 0; |
| 429 |
} |
| 430 |
} |
| 431 |
return c; |
| 432 |
} |
| 433 |
|
| 434 |
static inline int |
| 435 |
tok_continuation_line(struct tok_state *tok) { |
| 436 |
int c = tok_nextc(tok); |
| 437 |
if (c == '\r') { |
| 438 |
c = tok_nextc(tok); |
| 439 |
} |
| 440 |
if (c != '\n') { |
| 441 |
tok->done = E_LINECONT; |
| 442 |
return -1; |
| 443 |
} |
| 444 |
c = tok_nextc(tok); |
| 445 |
if (c == EOF) { |
| 446 |
tok->done = E_EOF; |
| 447 |
tok->cur = tok->inp; |
| 448 |
return -1; |
| 449 |
} else { |
| 450 |
tok_backup(tok, c); |
| 451 |
} |
| 452 |
return c; |
| 453 |
} |
| 454 |
|
| 455 |
static int |
| 456 |
maybe_raise_syntax_error_for_string_prefixes(struct tok_state *tok, |
| 457 |
int saw_b, int saw_r, int saw_u, |
| 458 |
int saw_f, int saw_t) { |
| 459 |
// Supported: rb, rf, rt (in any order) |
| 460 |
// Unsupported: ub, ur, uf, ut, bf, bt, ft (in any order) |
| 461 |
|
| 462 |
#define RETURN_SYNTAX_ERROR(PREFIX1, PREFIX2) \ |
| 463 |
do { \ |
| 464 |
(void)_PyTokenizer_syntaxerror_known_range( \ |
| 465 |
tok, (int)(tok->start + 1 - tok->line_start), \ |
| 466 |
(int)(tok->cur - tok->line_start), \ |
| 467 |
"'" PREFIX1 "' and '" PREFIX2 "' prefixes are incompatible"); \ |
| 468 |
return -1; \ |
| 469 |
} while (0) |
| 470 |
|
| 471 |
if (saw_u && saw_b) { |
| 472 |
RETURN_SYNTAX_ERROR("u", "b"); |
| 473 |
} |
| 474 |
if (saw_u && saw_r) { |
| 475 |
RETURN_SYNTAX_ERROR("u", "r"); |
| 476 |
} |
| 477 |
if (saw_u && saw_f) { |
| 478 |
RETURN_SYNTAX_ERROR("u", "f"); |
| 479 |
} |
| 480 |
if (saw_u && saw_t) { |
| 481 |
RETURN_SYNTAX_ERROR("u", "t"); |
| 482 |
} |
| 483 |
|
| 484 |
if (saw_b && saw_f) { |
| 485 |
RETURN_SYNTAX_ERROR("b", "f"); |
| 486 |
} |
| 487 |
if (saw_b && saw_t) { |
| 488 |
RETURN_SYNTAX_ERROR("b", "t"); |
| 489 |
} |
| 490 |
|
| 491 |
if (saw_f && saw_t) { |
| 492 |
RETURN_SYNTAX_ERROR("f", "t"); |
| 493 |
} |
| 494 |
|
| 495 |
#undef RETURN_SYNTAX_ERROR |
| 496 |
|
| 497 |
return 0; |
| 498 |
} |
| 499 |
|
| 500 |
static int |
| 501 |
tok_get_normal_mode(struct tok_state *tok, tokenizer_mode* current_tok, struct token *token) |
| 502 |
{ |
| 503 |
int c; |
| 504 |
int blankline, nonascii; |
| 505 |
|
| 506 |
const char *p_start = NULL; |
| 507 |
const char *p_end = NULL; |
| 508 |
nextline: |
| 509 |
tok->start = NULL; |
| 510 |
tok->starting_col_offset = -1; |
| 511 |
blankline = 0; |
| 512 |
|
| 513 |
|
| 514 |
/* Get indentation level */ |
| 515 |
if (tok->atbol) { |
| 516 |
int col = 0; |
| 517 |
int altcol = 0; |
| 518 |
tok->atbol = 0; |
| 519 |
int cont_line_col = 0; |
| 520 |
for (;;) { |
| 521 |
c = tok_nextc(tok); |
| 522 |
if (c == ' ') { |
| 523 |
col++, altcol++; |
| 524 |
} |
| 525 |
else if (c == '\t') { |
| 526 |
col = (col / tok->tabsize + 1) * tok->tabsize; |
| 527 |
altcol = (altcol / ALTTABSIZE + 1) * ALTTABSIZE; |
| 528 |
} |
| 529 |
else if (c == '\014') {/* Control-L (formfeed) */ |
| 530 |
col = altcol = 0; /* For Emacs users */ |
| 531 |
} |
| 532 |
else if (c == '\\') { |
| 533 |
// Indentation cannot be split over multiple physical lines |
| 534 |
// using backslashes. This means that if we found a backslash |
| 535 |
// preceded by whitespace, **the first one we find** determines |
| 536 |
// the level of indentation of whatever comes next. |
| 537 |
cont_line_col = cont_line_col ? cont_line_col : col; |
| 538 |
if ((c = tok_continuation_line(tok)) == -1) { |
| 539 |
return MAKE_TOKEN(ERRORTOKEN); |
| 540 |
} |
| 541 |
} |
| 542 |
else if (c == EOF && PyErr_Occurred()) { |
| 543 |
return MAKE_TOKEN(ERRORTOKEN); |
| 544 |
} |
| 545 |
else { |
| 546 |
break; |
| 547 |
} |
| 548 |
} |
| 549 |
tok_backup(tok, c); |
| 550 |
if (c == '#' || c == '\n' || c == '\r') { |
| 551 |
/* Lines with only whitespace and/or comments |
| 552 |
shouldn't affect the indentation and are |
| 553 |
not passed to the parser as NEWLINE tokens, |
| 554 |
except *totally* empty lines in interactive |
| 555 |
mode, which signal the end of a command group. */ |
| 556 |
if (col == 0 && c == '\n' && tok->prompt != NULL) { |
| 557 |
blankline = 0; /* Let it through */ |
| 558 |
} |
| 559 |
else if (tok->prompt != NULL && tok->lineno == 1) { |
| 560 |
/* In interactive mode, if the first line contains |
| 561 |
only spaces and/or a comment, let it through. */ |
| 562 |
blankline = 0; |
| 563 |
col = altcol = 0; |
| 564 |
} |
| 565 |
else { |
| 566 |
blankline = 1; /* Ignore completely */ |
| 567 |
} |
| 568 |
/* We can't jump back right here since we still |
| 569 |
may need to skip to the end of a comment */ |
| 570 |
} |
| 571 |
if (!blankline && tok->level == 0) { |
| 572 |
col = cont_line_col ? cont_line_col : col; |
| 573 |
altcol = cont_line_col ? cont_line_col : altcol; |
| 574 |
if (col == tok->indstack[tok->indent]) { |
| 575 |
/* No change */ |
| 576 |
if (altcol != tok->altindstack[tok->indent]) { |
| 577 |
return MAKE_TOKEN(_PyTokenizer_indenterror(tok)); |
| 578 |
} |
| 579 |
} |
| 580 |
else if (col > tok->indstack[tok->indent]) { |
| 581 |
/* Indent -- always one */ |
| 582 |
if (tok->indent+1 >= MAXINDENT) { |
| 583 |
tok->done = E_TOODEEP; |
| 584 |
tok->cur = tok->inp; |
| 585 |
return MAKE_TOKEN(ERRORTOKEN); |
| 586 |
} |
| 587 |
if (altcol <= tok->altindstack[tok->indent]) { |
| 588 |
return MAKE_TOKEN(_PyTokenizer_indenterror(tok)); |
| 589 |
} |
| 590 |
tok->pendin++; |
| 591 |
tok->indstack[++tok->indent] = col; |
| 592 |
tok->altindstack[tok->indent] = altcol; |
| 593 |
} |
| 594 |
else /* col < tok->indstack[tok->indent] */ { |
| 595 |
/* Dedent -- any number, must be consistent */ |
| 596 |
while (tok->indent > 0 && |
| 597 |
col < tok->indstack[tok->indent]) { |
| 598 |
tok->pendin--; |
| 599 |
tok->indent--; |
| 600 |
} |
| 601 |
if (col != tok->indstack[tok->indent]) { |
| 602 |
tok->done = E_DEDENT; |
| 603 |
tok->cur = tok->inp; |
| 604 |
return MAKE_TOKEN(ERRORTOKEN); |
| 605 |
} |
| 606 |
if (altcol != tok->altindstack[tok->indent]) { |
| 607 |
return MAKE_TOKEN(_PyTokenizer_indenterror(tok)); |
| 608 |
} |
| 609 |
} |
| 610 |
} |
| 611 |
} |
| 612 |
|
| 613 |
tok->start = tok->cur; |
| 614 |
tok->starting_col_offset = tok->col_offset; |
| 615 |
|
| 616 |
/* Return pending indents/dedents */ |
| 617 |
if (tok->pendin != 0) { |
| 618 |
if (tok->pendin < 0) { |
| 619 |
if (tok->tok_extra_tokens) { |
| 620 |
p_start = tok->cur; |
| 621 |
p_end = tok->cur; |
| 622 |
} |
| 623 |
tok->pendin++; |
| 624 |
return MAKE_TOKEN(DEDENT); |
| 625 |
} |
| 626 |
else { |
| 627 |
if (tok->tok_extra_tokens) { |
| 628 |
p_start = tok->buf; |
| 629 |
p_end = tok->cur; |
| 630 |
} |
| 631 |
tok->pendin--; |
| 632 |
return MAKE_TOKEN(INDENT); |
| 633 |
} |
| 634 |
} |
| 635 |
|
| 636 |
/* Peek ahead at the next character */ |
| 637 |
c = tok_nextc(tok); |
| 638 |
tok_backup(tok, c); |
| 639 |
|
| 640 |
again: |
| 641 |
tok->start = NULL; |
| 642 |
/* Skip spaces */ |
| 643 |
do { |
| 644 |
c = tok_nextc(tok); |
| 645 |
} while (c == ' ' || c == '\t' || c == '\014'); |
| 646 |
|
| 647 |
/* Set start of current token */ |
| 648 |
tok->start = tok->cur == NULL ? NULL : tok->cur - 1; |
| 649 |
tok->starting_col_offset = tok->col_offset - 1; |
| 650 |
|
| 651 |
/* Skip comment, unless it's a type comment */ |
| 652 |
if (c == '#') { |
| 653 |
|
| 654 |
const char* p = NULL; |
| 655 |
const char *prefix, *type_start; |
| 656 |
int current_starting_col_offset; |
| 657 |
|
| 658 |
while (c != EOF && c != '\n' && c != '\r') { |
| 659 |
c = tok_nextc(tok); |
| 660 |
} |
| 661 |
|
| 662 |
if (tok->tok_extra_tokens) { |
| 663 |
p = tok->start; |
| 664 |
} |
| 665 |
|
| 666 |
if (tok->type_comments) { |
| 667 |
p = tok->start; |
| 668 |
current_starting_col_offset = tok->starting_col_offset; |
| 669 |
prefix = type_comment_prefix; |
| 670 |
while (*prefix && p < tok->cur) { |
| 671 |
if (*prefix == ' ') { |
| 672 |
while (*p == ' ' || *p == '\t') { |
| 673 |
p++; |
| 674 |
current_starting_col_offset++; |
| 675 |
} |
| 676 |
} else if (*prefix == *p) { |
| 677 |
p++; |
| 678 |
current_starting_col_offset++; |
| 679 |
} else { |
| 680 |
break; |
| 681 |
} |
| 682 |
|
| 683 |
prefix++; |
| 684 |
} |
| 685 |
|
| 686 |
/* This is a type comment if we matched all of type_comment_prefix. */ |
| 687 |
if (!*prefix) { |
| 688 |
int is_type_ignore = 1; |
| 689 |
// +6 in order to skip the word 'ignore' |
| 690 |
const char *ignore_end = p + 6; |
| 691 |
const int ignore_end_col_offset = current_starting_col_offset + 6; |
| 692 |
tok_backup(tok, c); /* don't eat the newline or EOF */ |
| 693 |
|
| 694 |
type_start = p; |
| 695 |
|
| 696 |
/* A TYPE_IGNORE is "type: ignore" followed by the end of the token |
| 697 |
* or anything ASCII and non-alphanumeric. */ |
| 698 |
is_type_ignore = ( |
| 699 |
tok->cur >= ignore_end && memcmp(p, "ignore", 6) == 0 |
| 700 |
&& !(tok->cur > ignore_end |
| 701 |
&& ((unsigned char)ignore_end[0] >= 128 || Py_ISALNUM(ignore_end[0])))); |
| 702 |
|
| 703 |
if (is_type_ignore) { |
| 704 |
p_start = ignore_end; |
| 705 |
p_end = tok->cur; |
| 706 |
|
| 707 |
/* If this type ignore is the only thing on the line, consume the newline also. */ |
| 708 |
if (blankline) { |
| 709 |
tok_nextc(tok); |
| 710 |
tok->atbol = 1; |
| 711 |
} |
| 712 |
return MAKE_TYPE_COMMENT_TOKEN(TYPE_IGNORE, ignore_end_col_offset, tok->col_offset); |
| 713 |
} else { |
| 714 |
p_start = type_start; |
| 715 |
p_end = tok->cur; |
| 716 |
return MAKE_TYPE_COMMENT_TOKEN(TYPE_COMMENT, current_starting_col_offset, tok->col_offset); |
| 717 |
} |
| 718 |
} |
| 719 |
} |
| 720 |
if (tok->tok_extra_tokens) { |
| 721 |
tok_backup(tok, c); /* don't eat the newline or EOF */ |
| 722 |
p_start = p; |
| 723 |
p_end = tok->cur; |
| 724 |
tok->comment_newline = blankline; |
| 725 |
return MAKE_TOKEN(COMMENT); |
| 726 |
} |
| 727 |
} |
| 728 |
|
| 729 |
if (tok->done == E_INTERACT_STOP) { |
| 730 |
return MAKE_TOKEN(ENDMARKER); |
| 731 |
} |
| 732 |
|
| 733 |
/* Check for EOF and errors now */ |
| 734 |
if (c == EOF) { |
| 735 |
if (tok->level) { |
| 736 |
return MAKE_TOKEN(ERRORTOKEN); |
| 737 |
} |
| 738 |
return MAKE_TOKEN(tok->done == E_EOF ? ENDMARKER : ERRORTOKEN); |
| 739 |
} |
| 740 |
|
| 741 |
/* Identifier (most frequent token!) */ |
| 742 |
nonascii = 0; |
| 743 |
if (is_potential_identifier_start(c)) { |
| 744 |
/* Process the various legal combinations of b"", r"", u"", and f"". */ |
| 745 |
int saw_b = 0, saw_r = 0, saw_u = 0, saw_f = 0, saw_t = 0; |
| 746 |
while (1) { |
| 747 |
if (!saw_b && (c == 'b' || c == 'B')) { |
| 748 |
saw_b = 1; |
| 749 |
} |
| 750 |
/* Since this is a backwards compatibility support literal we don't |
| 751 |
want to support it in arbitrary order like byte literals. */ |
| 752 |
else if (!saw_u && (c == 'u'|| c == 'U')) { |
| 753 |
saw_u = 1; |
| 754 |
} |
| 755 |
/* ur"" and ru"" are not supported */ |
| 756 |
else if (!saw_r && (c == 'r' || c == 'R')) { |
| 757 |
saw_r = 1; |
| 758 |
} |
| 759 |
else if (!saw_f && (c == 'f' || c == 'F')) { |
| 760 |
saw_f = 1; |
| 761 |
} |
| 762 |
else if (!saw_t && (c == 't' || c == 'T')) { |
| 763 |
saw_t = 1; |
| 764 |
} |
| 765 |
else { |
| 766 |
break; |
| 767 |
} |
| 768 |
c = tok_nextc(tok); |
| 769 |
if (c == '"' || c == '\'') { |
| 770 |
// Raise error on incompatible string prefixes: |
| 771 |
int status = maybe_raise_syntax_error_for_string_prefixes( |
| 772 |
tok, saw_b, saw_r, saw_u, saw_f, saw_t); |
| 773 |
if (status < 0) { |
| 774 |
return MAKE_TOKEN(ERRORTOKEN); |
| 775 |
} |
| 776 |
|
| 777 |
// Handle valid f or t string creation: |
| 778 |
if (saw_f || saw_t) { |
| 779 |
goto f_string_quote; |
| 780 |
} |
| 781 |
goto letter_quote; |
| 782 |
} |
| 783 |
} |
| 784 |
while (is_potential_identifier_char(c)) { |
| 785 |
if (c >= 128) { |
| 786 |
nonascii = 1; |
| 787 |
} |
| 788 |
c = tok_nextc(tok); |
| 789 |
} |
| 790 |
tok_backup(tok, c); |
| 791 |
if (nonascii && !verify_identifier(tok)) { |
| 792 |
return MAKE_TOKEN(ERRORTOKEN); |
| 793 |
} |
| 794 |
|
| 795 |
p_start = tok->start; |
| 796 |
p_end = tok->cur; |
| 797 |
|
| 798 |
return MAKE_TOKEN(NAME); |
| 799 |
} |
| 800 |
|
| 801 |
if (c == '\r') { |
| 802 |
c = tok_nextc(tok); |
| 803 |
} |
| 804 |
|
| 805 |
/* Newline */ |
| 806 |
if (c == '\n') { |
| 807 |
tok->atbol = 1; |
| 808 |
if (blankline || tok->level > 0) { |
| 809 |
if (tok->tok_extra_tokens) { |
| 810 |
if (tok->comment_newline) { |
| 811 |
tok->comment_newline = 0; |
| 812 |
} |
| 813 |
p_start = tok->start; |
| 814 |
p_end = tok->cur; |
| 815 |
return MAKE_TOKEN(NL); |
| 816 |
} |
| 817 |
goto nextline; |
| 818 |
} |
| 819 |
if (tok->comment_newline && tok->tok_extra_tokens) { |
| 820 |
tok->comment_newline = 0; |
| 821 |
p_start = tok->start; |
| 822 |
p_end = tok->cur; |
| 823 |
return MAKE_TOKEN(NL); |
| 824 |
} |
| 825 |
p_start = tok->start; |
| 826 |
p_end = tok->cur - 1; /* Leave '\n' out of the string */ |
| 827 |
tok->cont_line = 0; |
| 828 |
return MAKE_TOKEN(NEWLINE); |
| 829 |
} |
| 830 |
|
| 831 |
/* Period or number starting with period? */ |
| 832 |
if (c == '.') { |
| 833 |
c = tok_nextc(tok); |
| 834 |
if (Py_ISDIGIT(c)) { |
| 835 |
goto fraction; |
| 836 |
} else if (c == '.') { |
| 837 |
c = tok_nextc(tok); |
| 838 |
if (c == '.') { |
| 839 |
p_start = tok->start; |
| 840 |
p_end = tok->cur; |
| 841 |
return MAKE_TOKEN(ELLIPSIS); |
| 842 |
} |
| 843 |
else { |
| 844 |
tok_backup(tok, c); |
| 845 |
} |
| 846 |
tok_backup(tok, '.'); |
| 847 |
} |
| 848 |
else { |
| 849 |
tok_backup(tok, c); |
| 850 |
} |
| 851 |
p_start = tok->start; |
| 852 |
p_end = tok->cur; |
| 853 |
return MAKE_TOKEN(DOT); |
| 854 |
} |
| 855 |
|
| 856 |
/* Number */ |
| 857 |
if (Py_ISDIGIT(c)) { |
| 858 |
if (c == '0') { |
| 859 |
/* Hex, octal or binary -- maybe. */ |
| 860 |
c = tok_nextc(tok); |
| 861 |
if (c == 'x' || c == 'X') { |
| 862 |
/* Hex */ |
| 863 |
c = tok_nextc(tok); |
| 864 |
do { |
| 865 |
if (c == '_') { |
| 866 |
c = tok_nextc(tok); |
| 867 |
} |
| 868 |
if (!Py_ISXDIGIT(c)) { |
| 869 |
tok_backup(tok, c); |
| 870 |
return MAKE_TOKEN(_PyTokenizer_syntaxerror(tok, "invalid hexadecimal literal")); |
| 871 |
} |
| 872 |
do { |
| 873 |
c = tok_nextc(tok); |
| 874 |
} while (Py_ISXDIGIT(c)); |
| 875 |
} while (c == '_'); |
| 876 |
if (!verify_end_of_number(tok, c, "hexadecimal")) { |
| 877 |
return MAKE_TOKEN(ERRORTOKEN); |
| 878 |
} |
| 879 |
} |
| 880 |
else if (c == 'o' || c == 'O') { |
| 881 |
/* Octal */ |
| 882 |
c = tok_nextc(tok); |
| 883 |
do { |
| 884 |
if (c == '_') { |
| 885 |
c = tok_nextc(tok); |
| 886 |
} |
| 887 |
if (c < '0' || c >= '8') { |
| 888 |
if (Py_ISDIGIT(c)) { |
| 889 |
return MAKE_TOKEN(_PyTokenizer_syntaxerror(tok, |
| 890 |
"invalid digit '%c' in octal literal", c)); |
| 891 |
} |
| 892 |
else { |
| 893 |
tok_backup(tok, c); |
| 894 |
return MAKE_TOKEN(_PyTokenizer_syntaxerror(tok, "invalid octal literal")); |
| 895 |
} |
| 896 |
} |
| 897 |
do { |
| 898 |
c = tok_nextc(tok); |
| 899 |
} while ('0' <= c && c < '8'); |
| 900 |
} while (c == '_'); |
| 901 |
if (Py_ISDIGIT(c)) { |
| 902 |
return MAKE_TOKEN(_PyTokenizer_syntaxerror(tok, |
| 903 |
"invalid digit '%c' in octal literal", c)); |
| 904 |
} |
| 905 |
if (!verify_end_of_number(tok, c, "octal")) { |
| 906 |
return MAKE_TOKEN(ERRORTOKEN); |
| 907 |
} |
| 908 |
} |
| 909 |
else if (c == 'b' || c == 'B') { |
| 910 |
/* Binary */ |
| 911 |
c = tok_nextc(tok); |
| 912 |
do { |
| 913 |
if (c == '_') { |
| 914 |
c = tok_nextc(tok); |
| 915 |
} |
| 916 |
if (c != '0' && c != '1') { |
| 917 |
if (Py_ISDIGIT(c)) { |
| 918 |
return MAKE_TOKEN(_PyTokenizer_syntaxerror(tok, "invalid digit '%c' in binary literal", c)); |
| 919 |
} |
| 920 |
else { |
| 921 |
tok_backup(tok, c); |
| 922 |
return MAKE_TOKEN(_PyTokenizer_syntaxerror(tok, "invalid binary literal")); |
| 923 |
} |
| 924 |
} |
| 925 |
do { |
| 926 |
c = tok_nextc(tok); |
| 927 |
} while (c == '0' || c == '1'); |
| 928 |
} while (c == '_'); |
| 929 |
if (Py_ISDIGIT(c)) { |
| 930 |
return MAKE_TOKEN(_PyTokenizer_syntaxerror(tok, "invalid digit '%c' in binary literal", c)); |
| 931 |
} |
| 932 |
if (!verify_end_of_number(tok, c, "binary")) { |
| 933 |
return MAKE_TOKEN(ERRORTOKEN); |
| 934 |
} |
| 935 |
} |
| 936 |
else { |
| 937 |
int nonzero = 0; |
| 938 |
/* maybe old-style octal; c is first char of it */ |
| 939 |
/* in any case, allow '0' as a literal */ |
| 940 |
while (1) { |
| 941 |
if (c == '_') { |
| 942 |
c = tok_nextc(tok); |
| 943 |
if (!Py_ISDIGIT(c)) { |
| 944 |
tok_backup(tok, c); |
| 945 |
return MAKE_TOKEN(_PyTokenizer_syntaxerror(tok, "invalid decimal literal")); |
| 946 |
} |
| 947 |
} |
| 948 |
if (c != '0') { |
| 949 |
break; |
| 950 |
} |
| 951 |
c = tok_nextc(tok); |
| 952 |
} |
| 953 |
char* zeros_end = tok->cur; |
| 954 |
if (Py_ISDIGIT(c)) { |
| 955 |
nonzero = 1; |
| 956 |
c = tok_decimal_tail(tok); |
| 957 |
if (c == 0) { |
| 958 |
return MAKE_TOKEN(ERRORTOKEN); |
| 959 |
} |
| 960 |
} |
| 961 |
if (c == '.') { |
| 962 |
c = tok_nextc(tok); |
| 963 |
goto fraction; |
| 964 |
} |
| 965 |
else if (c == 'e' || c == 'E') { |
| 966 |
goto exponent; |
| 967 |
} |
| 968 |
else if (c == 'j' || c == 'J') { |
| 969 |
goto imaginary; |
| 970 |
} |
| 971 |
else if (nonzero && !tok->tok_extra_tokens) { |
| 972 |
/* Old-style octal: now disallowed. */ |
| 973 |
tok_backup(tok, c); |
| 974 |
return MAKE_TOKEN(_PyTokenizer_syntaxerror_known_range( |
| 975 |
tok, (int)(tok->start + 1 - tok->line_start), |
| 976 |
(int)(zeros_end - tok->line_start), |
| 977 |
"leading zeros in decimal integer " |
| 978 |
"literals are not permitted; " |
| 979 |
"use an 0o prefix for octal integers")); |
| 980 |
} |
| 981 |
if (!verify_end_of_number(tok, c, "decimal")) { |
| 982 |
return MAKE_TOKEN(ERRORTOKEN); |
| 983 |
} |
| 984 |
} |
| 985 |
} |
| 986 |
else { |
| 987 |
/* Decimal */ |
| 988 |
c = tok_decimal_tail(tok); |
| 989 |
if (c == 0) { |
| 990 |
return MAKE_TOKEN(ERRORTOKEN); |
| 991 |
} |
| 992 |
{ |
| 993 |
/* Accept floating-point numbers. */ |
| 994 |
if (c == '.') { |
| 995 |
c = tok_nextc(tok); |
| 996 |
fraction: |
| 997 |
/* Fraction */ |
| 998 |
if (Py_ISDIGIT(c)) { |
| 999 |
c = tok_decimal_tail(tok); |
| 1000 |
if (c == 0) { |
| 1001 |
return MAKE_TOKEN(ERRORTOKEN); |
| 1002 |
} |
| 1003 |
} |
| 1004 |
} |
| 1005 |
if (c == 'e' || c == 'E') { |
| 1006 |
int e; |
| 1007 |
exponent: |
| 1008 |
e = c; |
| 1009 |
/* Exponent part */ |
| 1010 |
c = tok_nextc(tok); |
| 1011 |
if (c == '+' || c == '-') { |
| 1012 |
c = tok_nextc(tok); |
| 1013 |
if (!Py_ISDIGIT(c)) { |
| 1014 |
tok_backup(tok, c); |
| 1015 |
return MAKE_TOKEN(_PyTokenizer_syntaxerror(tok, "invalid decimal literal")); |
| 1016 |
} |
| 1017 |
} else if (!Py_ISDIGIT(c)) { |
| 1018 |
tok_backup(tok, c); |
| 1019 |
if (!verify_end_of_number(tok, e, "decimal")) { |
| 1020 |
return MAKE_TOKEN(ERRORTOKEN); |
| 1021 |
} |
| 1022 |
tok_backup(tok, e); |
| 1023 |
p_start = tok->start; |
| 1024 |
p_end = tok->cur; |
| 1025 |
return MAKE_TOKEN(NUMBER); |
| 1026 |
} |
| 1027 |
c = tok_decimal_tail(tok); |
| 1028 |
if (c == 0) { |
| 1029 |
return MAKE_TOKEN(ERRORTOKEN); |
| 1030 |
} |
| 1031 |
} |
| 1032 |
if (c == 'j' || c == 'J') { |
| 1033 |
/* Imaginary part */ |
| 1034 |
imaginary: |
| 1035 |
c = tok_nextc(tok); |
| 1036 |
if (!verify_end_of_number(tok, c, "imaginary")) { |
| 1037 |
return MAKE_TOKEN(ERRORTOKEN); |
| 1038 |
} |
| 1039 |
} |
| 1040 |
else if (!verify_end_of_number(tok, c, "decimal")) { |
| 1041 |
return MAKE_TOKEN(ERRORTOKEN); |
| 1042 |
} |
| 1043 |
} |
| 1044 |
} |
| 1045 |
tok_backup(tok, c); |
| 1046 |
p_start = tok->start; |
| 1047 |
p_end = tok->cur; |
| 1048 |
return MAKE_TOKEN(NUMBER); |
| 1049 |
} |
| 1050 |
|
| 1051 |
f_string_quote: |
| 1052 |
if (((Py_TOLOWER(*tok->start) == 'f' || Py_TOLOWER(*tok->start) == 'r' || Py_TOLOWER(*tok->start) == 't') |
| 1053 |
&& (c == '\'' || c == '"'))) { |
| 1054 |
|
| 1055 |
int quote = c; |
| 1056 |
int quote_size = 1; /* 1 or 3 */ |
| 1057 |
|
| 1058 |
/* Nodes of type STRING, especially multi line strings |
| 1059 |
must be handled differently in order to get both |
| 1060 |
the starting line number and the column offset right. |
| 1061 |
(cf. issue 16806) */ |
| 1062 |
tok->first_lineno = tok->lineno; |
| 1063 |
tok->multi_line_start = tok->line_start; |
| 1064 |
|
| 1065 |
/* Find the quote size and start of string */ |
| 1066 |
int after_quote = tok_nextc(tok); |
| 1067 |
if (after_quote == quote) { |
| 1068 |
int after_after_quote = tok_nextc(tok); |
| 1069 |
if (after_after_quote == quote) { |
| 1070 |
quote_size = 3; |
| 1071 |
} |
| 1072 |
else { |
| 1073 |
// TODO: Check this |
| 1074 |
tok_backup(tok, after_after_quote); |
| 1075 |
tok_backup(tok, after_quote); |
| 1076 |
} |
| 1077 |
} |
| 1078 |
if (after_quote != quote) { |
| 1079 |
tok_backup(tok, after_quote); |
| 1080 |
} |
| 1081 |
|
| 1082 |
|
| 1083 |
p_start = tok->start; |
| 1084 |
p_end = tok->cur; |
| 1085 |
if (tok->tok_mode_stack_index + 1 >= MAXFSTRINGLEVEL) { |
| 1086 |
return MAKE_TOKEN(_PyTokenizer_syntaxerror(tok, "too many nested f-strings or t-strings")); |
| 1087 |
} |
| 1088 |
tokenizer_mode *the_current_tok = TOK_NEXT_MODE(tok); |
| 1089 |
the_current_tok->kind = TOK_FSTRING_MODE; |
| 1090 |
the_current_tok->quote = quote; |
| 1091 |
the_current_tok->quote_size = quote_size; |
| 1092 |
the_current_tok->start = tok->start; |
| 1093 |
the_current_tok->multi_line_start = tok->line_start; |
| 1094 |
the_current_tok->first_line = tok->lineno; |
| 1095 |
the_current_tok->start_offset = -1; |
| 1096 |
the_current_tok->multi_line_start_offset = -1; |
| 1097 |
the_current_tok->last_expr_buffer = NULL; |
| 1098 |
the_current_tok->last_expr_size = 0; |
| 1099 |
the_current_tok->last_expr_end = -1; |
| 1100 |
the_current_tok->in_format_spec = 0; |
| 1101 |
the_current_tok->in_debug = 0; |
| 1102 |
|
| 1103 |
enum string_kind_t string_kind = FSTRING; |
| 1104 |
switch (*tok->start) { |
| 1105 |
case 'T': |
| 1106 |
case 't': |
| 1107 |
the_current_tok->raw = Py_TOLOWER(*(tok->start + 1)) == 'r'; |
| 1108 |
string_kind = TSTRING; |
| 1109 |
break; |
| 1110 |
case 'F': |
| 1111 |
case 'f': |
| 1112 |
the_current_tok->raw = Py_TOLOWER(*(tok->start + 1)) == 'r'; |
| 1113 |
break; |
| 1114 |
case 'R': |
| 1115 |
case 'r': |
| 1116 |
the_current_tok->raw = 1; |
| 1117 |
if (Py_TOLOWER(*(tok->start + 1)) == 't') { |
| 1118 |
string_kind = TSTRING; |
| 1119 |
} |
| 1120 |
break; |
| 1121 |
default: |
| 1122 |
Py_UNREACHABLE(); |
| 1123 |
} |
| 1124 |
|
| 1125 |
the_current_tok->string_kind = string_kind; |
| 1126 |
the_current_tok->curly_bracket_depth = 0; |
| 1127 |
the_current_tok->curly_bracket_expr_start_depth = -1; |
| 1128 |
return string_kind == TSTRING ? MAKE_TOKEN(TSTRING_START) : MAKE_TOKEN(FSTRING_START); |
| 1129 |
} |
| 1130 |
|
| 1131 |
letter_quote: |
| 1132 |
/* String */ |
| 1133 |
if (c == '\'' || c == '"') { |
| 1134 |
int quote = c; |
| 1135 |
int quote_size = 1; /* 1 or 3 */ |
| 1136 |
int end_quote_size = 0; |
| 1137 |
int has_escaped_quote = 0; |
| 1138 |
|
| 1139 |
/* Nodes of type STRING, especially multi line strings |
| 1140 |
must be handled differently in order to get both |
| 1141 |
the starting line number and the column offset right. |
| 1142 |
(cf. issue 16806) */ |
| 1143 |
tok->first_lineno = tok->lineno; |
| 1144 |
tok->multi_line_start = tok->line_start; |
| 1145 |
|
| 1146 |
/* Find the quote size and start of string */ |
| 1147 |
c = tok_nextc(tok); |
| 1148 |
if (c == quote) { |
| 1149 |
c = tok_nextc(tok); |
| 1150 |
if (c == quote) { |
| 1151 |
quote_size = 3; |
| 1152 |
} |
| 1153 |
else { |
| 1154 |
end_quote_size = 1; /* empty string found */ |
| 1155 |
} |
| 1156 |
} |
| 1157 |
if (c != quote) { |
| 1158 |
tok_backup(tok, c); |
| 1159 |
} |
| 1160 |
|
| 1161 |
/* Get rest of string */ |
| 1162 |
while (end_quote_size != quote_size) { |
| 1163 |
c = tok_nextc(tok); |
| 1164 |
if (tok->done == E_ERROR) { |
| 1165 |
return MAKE_TOKEN(ERRORTOKEN); |
| 1166 |
} |
| 1167 |
if (tok->done == E_DECODE) { |
| 1168 |
break; |
| 1169 |
} |
| 1170 |
if (c == EOF || (quote_size == 1 && c == '\n')) { |
| 1171 |
assert(tok->multi_line_start != NULL); |
| 1172 |
// shift the tok_state's location into |
| 1173 |
// the start of string, and report the error |
| 1174 |
// from the initial quote character |
| 1175 |
tok->cur = (char *)tok->start; |
| 1176 |
tok->cur++; |
| 1177 |
tok->line_start = tok->multi_line_start; |
| 1178 |
int start = tok->lineno; |
| 1179 |
tok->lineno = tok->first_lineno; |
| 1180 |
|
| 1181 |
if (INSIDE_FSTRING(tok)) { |
| 1182 |
/* When we are in an f-string, before raising the |
| 1183 |
* unterminated string literal error, check whether |
| 1184 |
* does the initial quote matches with f-strings quotes |
| 1185 |
* and if it is, then this must be a missing '}' token |
| 1186 |
* so raise the proper error */ |
| 1187 |
tokenizer_mode *the_current_tok = TOK_GET_MODE(tok); |
| 1188 |
if (the_current_tok->quote == quote && |
| 1189 |
the_current_tok->quote_size == quote_size) { |
| 1190 |
return MAKE_TOKEN(_PyTokenizer_syntaxerror(tok, |
| 1191 |
"%c-string: expecting '}'", TOK_GET_STRING_PREFIX(tok))); |
| 1192 |
} |
| 1193 |
} |
| 1194 |
|
| 1195 |
if (quote_size == 3) { |
| 1196 |
_PyTokenizer_syntaxerror(tok, "unterminated triple-quoted string literal" |
| 1197 |
" (detected at line %d)", start); |
| 1198 |
if (c != '\n') { |
| 1199 |
tok->done = E_EOFS; |
| 1200 |
} |
| 1201 |
return MAKE_TOKEN(ERRORTOKEN); |
| 1202 |
} |
| 1203 |
else { |
| 1204 |
if (has_escaped_quote) { |
| 1205 |
_PyTokenizer_syntaxerror( |
| 1206 |
tok, |
| 1207 |
"unterminated string literal (detected at line %d); " |
| 1208 |
"perhaps you escaped the end quote?", |
| 1209 |
start |
| 1210 |
); |
| 1211 |
} else { |
| 1212 |
_PyTokenizer_syntaxerror( |
| 1213 |
tok, "unterminated string literal (detected at line %d)", start |
| 1214 |
); |
| 1215 |
} |
| 1216 |
if (c != '\n') { |
| 1217 |
tok->done = E_EOLS; |
| 1218 |
} |
| 1219 |
return MAKE_TOKEN(ERRORTOKEN); |
| 1220 |
} |
| 1221 |
} |
| 1222 |
if (c == quote) { |
| 1223 |
end_quote_size += 1; |
| 1224 |
} |
| 1225 |
else { |
| 1226 |
end_quote_size = 0; |
| 1227 |
if (c == '\\') { |
| 1228 |
c = tok_nextc(tok); /* skip escaped char */ |
| 1229 |
if (c == quote) { /* but record whether the escaped char was a quote */ |
| 1230 |
has_escaped_quote = 1; |
| 1231 |
} |
| 1232 |
if (c == '\r') { |
| 1233 |
c = tok_nextc(tok); |
| 1234 |
} |
| 1235 |
} |
| 1236 |
} |
| 1237 |
} |
| 1238 |
|
| 1239 |
p_start = tok->start; |
| 1240 |
p_end = tok->cur; |
| 1241 |
return MAKE_TOKEN(STRING); |
| 1242 |
} |
| 1243 |
|
| 1244 |
/* Line continuation */ |
| 1245 |
if (c == '\\') { |
| 1246 |
if ((c = tok_continuation_line(tok)) == -1) { |
| 1247 |
return MAKE_TOKEN(ERRORTOKEN); |
| 1248 |
} |
| 1249 |
tok->cont_line = 1; |
| 1250 |
goto again; /* Read next line */ |
| 1251 |
} |
| 1252 |
|
| 1253 |
/* Punctuation character */ |
| 1254 |
int is_punctuation = (c == ':' || c == '}' || c == '!' || c == '{'); |
| 1255 |
if (is_punctuation && INSIDE_FSTRING(tok) && INSIDE_FSTRING_EXPR(current_tok)) { |
| 1256 |
/* This code block gets executed before the curly_bracket_depth is incremented |
| 1257 |
* by the `{` case, so for ensuring that we are on the 0th level, we need |
| 1258 |
* to adjust it manually */ |
| 1259 |
int cursor = current_tok->curly_bracket_depth - (c != '{'); |
| 1260 |
int in_format_spec = current_tok->in_format_spec; |
| 1261 |
int cursor_in_format_with_debug = |
| 1262 |
cursor == 1 && (current_tok->in_debug || in_format_spec); |
| 1263 |
int cursor_valid = cursor == 0 || cursor_in_format_with_debug; |
| 1264 |
if ((cursor_valid) && !_PyLexer_update_ftstring_expr(tok, c)) { |
| 1265 |
return MAKE_TOKEN(ENDMARKER); |
| 1266 |
} |
| 1267 |
if ((cursor_valid) && c != '{' && set_ftstring_expr(tok, token, c)) { |
| 1268 |
return MAKE_TOKEN(ERRORTOKEN); |
| 1269 |
} |
| 1270 |
|
| 1271 |
if (c == ':' && cursor == current_tok->curly_bracket_expr_start_depth) { |
| 1272 |
current_tok->kind = TOK_FSTRING_MODE; |
| 1273 |
current_tok->in_format_spec = 1; |
| 1274 |
p_start = tok->start; |
| 1275 |
p_end = tok->cur; |
| 1276 |
return MAKE_TOKEN(_PyToken_OneChar(c)); |
| 1277 |
} |
| 1278 |
} |
| 1279 |
|
| 1280 |
/* Check for two-character token */ |
| 1281 |
{ |
| 1282 |
int c2 = tok_nextc(tok); |
| 1283 |
int current_token = _PyToken_TwoChars(c, c2); |
| 1284 |
if (current_token != OP) { |
| 1285 |
int c3 = tok_nextc(tok); |
| 1286 |
int current_token3 = _PyToken_ThreeChars(c, c2, c3); |
| 1287 |
if (current_token3 != OP) { |
| 1288 |
current_token = current_token3; |
| 1289 |
} |
| 1290 |
else { |
| 1291 |
tok_backup(tok, c3); |
| 1292 |
} |
| 1293 |
p_start = tok->start; |
| 1294 |
p_end = tok->cur; |
| 1295 |
return MAKE_TOKEN(current_token); |
| 1296 |
} |
| 1297 |
tok_backup(tok, c2); |
| 1298 |
} |
| 1299 |
|
| 1300 |
/* Keep track of parentheses nesting level */ |
| 1301 |
switch (c) { |
| 1302 |
case '(': |
| 1303 |
case '[': |
| 1304 |
case '{': |
| 1305 |
if (tok->level >= MAXLEVEL) { |
| 1306 |
return MAKE_TOKEN(_PyTokenizer_syntaxerror(tok, "too many nested parentheses")); |
| 1307 |
} |
| 1308 |
tok->parenstack[tok->level] = c; |
| 1309 |
tok->parenlinenostack[tok->level] = tok->lineno; |
| 1310 |
tok->parencolstack[tok->level] = (int)(tok->start - tok->line_start); |
| 1311 |
tok->level++; |
| 1312 |
if (INSIDE_FSTRING(tok)) { |
| 1313 |
current_tok->curly_bracket_depth++; |
| 1314 |
} |
| 1315 |
break; |
| 1316 |
case ')': |
| 1317 |
case ']': |
| 1318 |
case '}': |
| 1319 |
if (INSIDE_FSTRING(tok) && !current_tok->curly_bracket_depth && c == '}') { |
| 1320 |
return MAKE_TOKEN(_PyTokenizer_syntaxerror(tok, |
| 1321 |
"%c-string: single '}' is not allowed", TOK_GET_STRING_PREFIX(tok))); |
| 1322 |
} |
| 1323 |
if (!tok->tok_extra_tokens && !tok->level) { |
| 1324 |
return MAKE_TOKEN(_PyTokenizer_syntaxerror(tok, "unmatched '%c'", c)); |
| 1325 |
} |
| 1326 |
if (tok->level > 0) { |
| 1327 |
tok->level--; |
| 1328 |
int opening = tok->parenstack[tok->level]; |
| 1329 |
if (!tok->tok_extra_tokens && !((opening == '(' && c == ')') || |
| 1330 |
(opening == '[' && c == ']') || |
| 1331 |
(opening == '{' && c == '}'))) { |
| 1332 |
/* If the opening bracket belongs to an f-string's expression |
| 1333 |
part (e.g. f"{)}") and the closing bracket is an arbitrary |
| 1334 |
nested expression, then instead of matching a different |
| 1335 |
syntactical construct with it; we'll throw an unmatched |
| 1336 |
parentheses error. */ |
| 1337 |
if (INSIDE_FSTRING(tok) && opening == '{') { |
| 1338 |
assert(current_tok->curly_bracket_depth >= 0); |
| 1339 |
int previous_bracket = current_tok->curly_bracket_depth - 1; |
| 1340 |
if (previous_bracket == current_tok->curly_bracket_expr_start_depth) { |
| 1341 |
return MAKE_TOKEN(_PyTokenizer_syntaxerror(tok, |
| 1342 |
"%c-string: unmatched '%c'", TOK_GET_STRING_PREFIX(tok), c)); |
| 1343 |
} |
| 1344 |
} |
| 1345 |
if (tok->parenlinenostack[tok->level] != tok->lineno) { |
| 1346 |
return MAKE_TOKEN(_PyTokenizer_syntaxerror(tok, |
| 1347 |
"closing parenthesis '%c' does not match " |
| 1348 |
"opening parenthesis '%c' on line %d", |
| 1349 |
c, opening, tok->parenlinenostack[tok->level])); |
| 1350 |
} |
| 1351 |
else { |
| 1352 |
return MAKE_TOKEN(_PyTokenizer_syntaxerror(tok, |
| 1353 |
"closing parenthesis '%c' does not match " |
| 1354 |
"opening parenthesis '%c'", |
| 1355 |
c, opening)); |
| 1356 |
} |
| 1357 |
} |
| 1358 |
} |
| 1359 |
|
| 1360 |
if (INSIDE_FSTRING(tok)) { |
| 1361 |
current_tok->curly_bracket_depth--; |
| 1362 |
if (current_tok->curly_bracket_depth < 0) { |
| 1363 |
return MAKE_TOKEN(_PyTokenizer_syntaxerror(tok, "%c-string: unmatched '%c'", |
| 1364 |
TOK_GET_STRING_PREFIX(tok), c)); |
| 1365 |
} |
| 1366 |
if (c == '}' && current_tok->curly_bracket_depth == current_tok->curly_bracket_expr_start_depth) { |
| 1367 |
current_tok->curly_bracket_expr_start_depth--; |
| 1368 |
current_tok->kind = TOK_FSTRING_MODE; |
| 1369 |
current_tok->in_format_spec = 0; |
| 1370 |
current_tok->in_debug = 0; |
| 1371 |
} |
| 1372 |
} |
| 1373 |
break; |
| 1374 |
default: |
| 1375 |
break; |
| 1376 |
} |
| 1377 |
|
| 1378 |
if (!Py_UNICODE_ISPRINTABLE(c)) { |
| 1379 |
return MAKE_TOKEN(_PyTokenizer_syntaxerror(tok, "invalid non-printable character U+%04X", c)); |
| 1380 |
} |
| 1381 |
|
| 1382 |
if( c == '=' && INSIDE_FSTRING_EXPR_AT_TOP(current_tok)) { |
| 1383 |
current_tok->in_debug = 1; |
| 1384 |
} |
| 1385 |
|
| 1386 |
/* Punctuation character */ |
| 1387 |
p_start = tok->start; |
| 1388 |
p_end = tok->cur; |
| 1389 |
return MAKE_TOKEN(_PyToken_OneChar(c)); |
| 1390 |
} |
| 1391 |
|
| 1392 |
static int |
| 1393 |
tok_get_fstring_mode(struct tok_state *tok, tokenizer_mode* current_tok, struct token *token) |
| 1394 |
{ |
| 1395 |
const char *p_start = NULL; |
| 1396 |
const char *p_end = NULL; |
| 1397 |
int end_quote_size = 0; |
| 1398 |
int unicode_escape = 0; |
| 1399 |
|
| 1400 |
tok->start = tok->cur; |
| 1401 |
tok->first_lineno = tok->lineno; |
| 1402 |
tok->starting_col_offset = tok->col_offset; |
| 1403 |
|
| 1404 |
// If we start with a bracket, we defer to the normal mode as there is nothing for us to tokenize |
| 1405 |
// before it. |
| 1406 |
int start_char = tok_nextc(tok); |
| 1407 |
if (start_char == '{') { |
| 1408 |
int peek1 = tok_nextc(tok); |
| 1409 |
tok_backup(tok, peek1); |
| 1410 |
tok_backup(tok, start_char); |
| 1411 |
if (peek1 != '{') { |
| 1412 |
current_tok->curly_bracket_expr_start_depth++; |
| 1413 |
if (current_tok->curly_bracket_expr_start_depth >= MAX_EXPR_NESTING) { |
| 1414 |
return MAKE_TOKEN(_PyTokenizer_syntaxerror(tok, |
| 1415 |
"%c-string: expressions nested too deeply", TOK_GET_STRING_PREFIX(tok))); |
| 1416 |
} |
| 1417 |
TOK_GET_MODE(tok)->kind = TOK_REGULAR_MODE; |
| 1418 |
return tok_get_normal_mode(tok, current_tok, token); |
| 1419 |
} |
| 1420 |
} |
| 1421 |
else { |
| 1422 |
tok_backup(tok, start_char); |
| 1423 |
} |
| 1424 |
|
| 1425 |
// Check if we are at the end of the string |
| 1426 |
for (int i = 0; i < current_tok->quote_size; i++) { |
| 1427 |
int quote = tok_nextc(tok); |
| 1428 |
if (quote != current_tok->quote) { |
| 1429 |
tok_backup(tok, quote); |
| 1430 |
goto f_string_middle; |
| 1431 |
} |
| 1432 |
} |
| 1433 |
|
| 1434 |
if (current_tok->last_expr_buffer != NULL) { |
| 1435 |
PyMem_Free(current_tok->last_expr_buffer); |
| 1436 |
current_tok->last_expr_buffer = NULL; |
| 1437 |
current_tok->last_expr_size = 0; |
| 1438 |
current_tok->last_expr_end = -1; |
| 1439 |
} |
| 1440 |
|
| 1441 |
p_start = tok->start; |
| 1442 |
p_end = tok->cur; |
| 1443 |
tok->tok_mode_stack_index--; |
| 1444 |
return MAKE_TOKEN(FTSTRING_END(current_tok)); |
| 1445 |
|
| 1446 |
f_string_middle: |
| 1447 |
|
| 1448 |
// TODO: This is a bit of a hack, but it works for now. We need to find a better way to handle |
| 1449 |
// this. |
| 1450 |
tok->multi_line_start = tok->line_start; |
| 1451 |
while (end_quote_size != current_tok->quote_size) { |
| 1452 |
int c = tok_nextc(tok); |
| 1453 |
if (tok->done == E_ERROR || tok->done == E_DECODE) { |
| 1454 |
return MAKE_TOKEN(ERRORTOKEN); |
| 1455 |
} |
| 1456 |
int in_format_spec = ( |
| 1457 |
current_tok->in_format_spec |
| 1458 |
&& |
| 1459 |
INSIDE_FSTRING_EXPR(current_tok) |
| 1460 |
); |
| 1461 |
|
| 1462 |
if (c == EOF || (current_tok->quote_size == 1 && c == '\n')) { |
| 1463 |
if (tok->decoding_erred) { |
| 1464 |
return MAKE_TOKEN(ERRORTOKEN); |
| 1465 |
} |
| 1466 |
|
| 1467 |
// If we are in a format spec and we found a newline, |
| 1468 |
// it means that the format spec ends here and we should |
| 1469 |
// return to the regular mode. |
| 1470 |
if (in_format_spec && c == '\n') { |
| 1471 |
if (current_tok->quote_size == 1) { |
| 1472 |
return MAKE_TOKEN( |
| 1473 |
_PyTokenizer_syntaxerror( |
| 1474 |
tok, |
| 1475 |
"%c-string: newlines are not allowed in format specifiers for single quoted %c-strings", |
| 1476 |
TOK_GET_STRING_PREFIX(tok), TOK_GET_STRING_PREFIX(tok) |
| 1477 |
) |
| 1478 |
); |
| 1479 |
} |
| 1480 |
tok_backup(tok, c); |
| 1481 |
TOK_GET_MODE(tok)->kind = TOK_REGULAR_MODE; |
| 1482 |
current_tok->in_format_spec = 0; |
| 1483 |
p_start = tok->start; |
| 1484 |
p_end = tok->cur; |
| 1485 |
return MAKE_TOKEN(FTSTRING_MIDDLE(current_tok)); |
| 1486 |
} |
| 1487 |
|
| 1488 |
assert(tok->multi_line_start != NULL); |
| 1489 |
// shift the tok_state's location into |
| 1490 |
// the start of string, and report the error |
| 1491 |
// from the initial quote character |
| 1492 |
tok->cur = (char *)current_tok->start; |
| 1493 |
tok->cur++; |
| 1494 |
tok->line_start = current_tok->multi_line_start; |
| 1495 |
int start = tok->lineno; |
| 1496 |
|
| 1497 |
tokenizer_mode *the_current_tok = TOK_GET_MODE(tok); |
| 1498 |
tok->lineno = the_current_tok->first_line; |
| 1499 |
|
| 1500 |
if (current_tok->quote_size == 3) { |
| 1501 |
_PyTokenizer_syntaxerror(tok, |
| 1502 |
"unterminated triple-quoted %c-string literal" |
| 1503 |
" (detected at line %d)", |
| 1504 |
TOK_GET_STRING_PREFIX(tok), start); |
| 1505 |
if (c != '\n') { |
| 1506 |
tok->done = E_EOFS; |
| 1507 |
} |
| 1508 |
return MAKE_TOKEN(ERRORTOKEN); |
| 1509 |
} |
| 1510 |
else { |
| 1511 |
return MAKE_TOKEN(_PyTokenizer_syntaxerror(tok, |
| 1512 |
"unterminated %c-string literal (detected at" |
| 1513 |
" line %d)", TOK_GET_STRING_PREFIX(tok), start)); |
| 1514 |
} |
| 1515 |
} |
| 1516 |
|
| 1517 |
if (c == current_tok->quote) { |
| 1518 |
end_quote_size += 1; |
| 1519 |
continue; |
| 1520 |
} else { |
| 1521 |
end_quote_size = 0; |
| 1522 |
} |
| 1523 |
|
| 1524 |
if (c == '{') { |
| 1525 |
if (!_PyLexer_update_ftstring_expr(tok, c)) { |
| 1526 |
return MAKE_TOKEN(ENDMARKER); |
| 1527 |
} |
| 1528 |
int peek = tok_nextc(tok); |
| 1529 |
if (peek != '{' || in_format_spec) { |
| 1530 |
tok_backup(tok, peek); |
| 1531 |
tok_backup(tok, c); |
| 1532 |
current_tok->curly_bracket_expr_start_depth++; |
| 1533 |
if (current_tok->curly_bracket_expr_start_depth >= MAX_EXPR_NESTING) { |
| 1534 |
return MAKE_TOKEN(_PyTokenizer_syntaxerror(tok, |
| 1535 |
"%c-string: expressions nested too deeply", TOK_GET_STRING_PREFIX(tok))); |
| 1536 |
} |
| 1537 |
TOK_GET_MODE(tok)->kind = TOK_REGULAR_MODE; |
| 1538 |
current_tok->in_format_spec = 0; |
| 1539 |
p_start = tok->start; |
| 1540 |
p_end = tok->cur; |
| 1541 |
} else { |
| 1542 |
p_start = tok->start; |
| 1543 |
p_end = tok->cur - 1; |
| 1544 |
} |
| 1545 |
return MAKE_TOKEN(FTSTRING_MIDDLE(current_tok)); |
| 1546 |
} else if (c == '}') { |
| 1547 |
if (unicode_escape) { |
| 1548 |
p_start = tok->start; |
| 1549 |
p_end = tok->cur; |
| 1550 |
return MAKE_TOKEN(FTSTRING_MIDDLE(current_tok)); |
| 1551 |
} |
| 1552 |
int peek = tok_nextc(tok); |
| 1553 |
|
| 1554 |
// The tokenizer can only be in the format spec if we have already completed the expression |
| 1555 |
// scanning (indicated by the end of the expression being set) and we are not at the top level |
| 1556 |
// of the bracket stack (-1 is the top level). Since format specifiers can't legally use double |
| 1557 |
// brackets, we can bypass it here. |
| 1558 |
int cursor = current_tok->curly_bracket_depth; |
| 1559 |
if (peek == '}' && !in_format_spec && cursor == 0) { |
| 1560 |
p_start = tok->start; |
| 1561 |
p_end = tok->cur - 1; |
| 1562 |
} else { |
| 1563 |
tok_backup(tok, peek); |
| 1564 |
tok_backup(tok, c); |
| 1565 |
TOK_GET_MODE(tok)->kind = TOK_REGULAR_MODE; |
| 1566 |
current_tok->in_format_spec = 0; |
| 1567 |
p_start = tok->start; |
| 1568 |
p_end = tok->cur; |
| 1569 |
} |
| 1570 |
return MAKE_TOKEN(FTSTRING_MIDDLE(current_tok)); |
| 1571 |
} else if (c == '\\') { |
| 1572 |
int peek = tok_nextc(tok); |
| 1573 |
if (peek == '\r') { |
| 1574 |
peek = tok_nextc(tok); |
| 1575 |
} |
| 1576 |
// Special case when the backslash is right before a curly |
| 1577 |
// brace. We have to restore and return the control back |
| 1578 |
// to the loop for the next iteration. |
| 1579 |
if (peek == '{' || peek == '}') { |
| 1580 |
if (!current_tok->raw) { |
| 1581 |
if (_PyTokenizer_warn_invalid_escape_sequence(tok, peek)) { |
| 1582 |
return MAKE_TOKEN(ERRORTOKEN); |
| 1583 |
} |
| 1584 |
} |
| 1585 |
tok_backup(tok, peek); |
| 1586 |
continue; |
| 1587 |
} |
| 1588 |
|
| 1589 |
if (!current_tok->raw) { |
| 1590 |
if (peek == 'N') { |
| 1591 |
/* Handle named unicode escapes (\N{BULLET}) */ |
| 1592 |
peek = tok_nextc(tok); |
| 1593 |
if (peek == '{') { |
| 1594 |
unicode_escape = 1; |
| 1595 |
} else { |
| 1596 |
tok_backup(tok, peek); |
| 1597 |
} |
| 1598 |
} |
| 1599 |
} /* else { |
| 1600 |
skip the escaped character |
| 1601 |
}*/ |
| 1602 |
} |
| 1603 |
} |
| 1604 |
|
| 1605 |
// Backup the f-string quotes to emit a final FSTRING_MIDDLE and |
| 1606 |
// add the quotes to the FSTRING_END in the next tokenizer iteration. |
| 1607 |
for (int i = 0; i < current_tok->quote_size; i++) { |
| 1608 |
tok_backup(tok, current_tok->quote); |
| 1609 |
} |
| 1610 |
p_start = tok->start; |
| 1611 |
p_end = tok->cur; |
| 1612 |
return MAKE_TOKEN(FTSTRING_MIDDLE(current_tok)); |
| 1613 |
} |
| 1614 |
|
| 1615 |
static int |
| 1616 |
tok_get(struct tok_state *tok, struct token *token) |
| 1617 |
{ |
| 1618 |
tokenizer_mode *current_tok = TOK_GET_MODE(tok); |
| 1619 |
if (current_tok->kind == TOK_REGULAR_MODE) { |
| 1620 |
return tok_get_normal_mode(tok, current_tok, token); |
| 1621 |
} else { |
| 1622 |
return tok_get_fstring_mode(tok, current_tok, token); |
| 1623 |
} |
| 1624 |
} |
| 1625 |
|
| 1626 |
int |
| 1627 |
_PyTokenizer_Get(struct tok_state *tok, struct token *token) |
| 1628 |
{ |
| 1629 |
int result = tok_get(tok, token); |
| 1630 |
if (tok->decoding_erred) { |
| 1631 |
result = ERRORTOKEN; |
| 1632 |
tok->done = E_DECODE; |
| 1633 |
} |
| 1634 |
return result; |
| 1635 |
} |