Debian Codegraph @

Parser/tokenizer/helpers.c

raw
1 #include "Python.h"
2 #include "errcode.h"
3 #include "pycore_token.h"
4
5 #include "../lexer/state.h"
6
7
8 /* ############## ERRORS ############## */
9
10 static int
11 _syntaxerror_range(struct tok_state *tok, const char *format,
12 int col_offset, int end_col_offset,
13 va_list vargs)
14 {
15 // In release builds, we don't want to overwrite a previous error, but in debug builds we
16 // want to fail if we are not doing it so we can fix it.
17 assert(tok->done != E_ERROR);
18 if (tok->done == E_ERROR) {
19 return ERRORTOKEN;
20 }
21 PyObject *errmsg, *errtext, *args;
22 errmsg = PyUnicode_FromFormatV(format, vargs);
23 if (!errmsg) {
24 goto error;
25 }
26
27 errtext = PyUnicode_DecodeUTF8(tok->line_start, tok->cur - tok->line_start,
28 "replace");
29 if (!errtext) {
30 goto error;
31 }
32
33 if (col_offset == -1) {
34 col_offset = (int)PyUnicode_GET_LENGTH(errtext);
35 }
36 if (end_col_offset == -1) {
37 end_col_offset = col_offset;
38 }
39
40 Py_ssize_t line_len = strcspn(tok->line_start, "\n");
41 if (line_len != tok->cur - tok->line_start) {
42 Py_DECREF(errtext);
43 errtext = PyUnicode_DecodeUTF8(tok->line_start, line_len,
44 "replace");
45 }
46 if (!errtext) {
47 goto error;
48 }
49
50 args = Py_BuildValue("(O(OiiNii))", errmsg, tok->filename, tok->lineno,
51 col_offset, errtext, tok->lineno, end_col_offset);
52 if (args) {
53 PyErr_SetObject(PyExc_SyntaxError, args);
54 Py_DECREF(args);
55 }
56
57 error:
58 Py_XDECREF(errmsg);
59 tok->done = E_ERROR;
60 return ERRORTOKEN;
61 }
62
63 int
64 _PyTokenizer_syntaxerror(struct tok_state *tok, const char *format, ...)
65 {
66 // This errors are cleaned on startup. Todo: Fix it.
67 va_list vargs;
68 va_start(vargs, format);
69 int ret = _syntaxerror_range(tok, format, -1, -1, vargs);
70 va_end(vargs);
71 return ret;
72 }
73
74 int
75 _PyTokenizer_syntaxerror_known_range(struct tok_state *tok,
76 int col_offset, int end_col_offset,
77 const char *format, ...)
78 {
79 va_list vargs;
80 va_start(vargs, format);
81 int ret = _syntaxerror_range(tok, format, col_offset, end_col_offset, vargs);
82 va_end(vargs);
83 return ret;
84 }
85
86 int
87 _PyTokenizer_indenterror(struct tok_state *tok)
88 {
89 tok->done = E_TABSPACE;
90 tok->cur = tok->inp;
91 return ERRORTOKEN;
92 }
93
94 char *
95 _PyTokenizer_error_ret(struct tok_state *tok) /* XXX */
96 {
97 tok->decoding_erred = 1;
98 if ((tok->fp != NULL || tok->readline != NULL) && tok->buf != NULL) {/* see _PyTokenizer_Free */
99 PyMem_Free(tok->buf);
100 }
101 tok->buf = tok->cur = tok->inp = NULL;
102 tok->start = NULL;
103 tok->end = NULL;
104 tok->done = E_DECODE;
105 return NULL; /* as if it were EOF */
106 }
107
108 int
109 _PyTokenizer_warn_invalid_escape_sequence(struct tok_state *tok, int first_invalid_escape_char)
110 {
111 if (!tok->report_warnings) {
112 return 0;
113 }
114
115 PyObject *msg = PyUnicode_FromFormat(
116 "invalid escape sequence '\\%c'",
117 (char) first_invalid_escape_char
118 );
119
120 if (msg == NULL) {
121 return -1;
122 }
123
124 if (PyErr_WarnExplicitObject(PyExc_SyntaxWarning, msg, tok->filename,
125 tok->lineno, NULL, NULL) < 0) {
126 Py_DECREF(msg);
127
128 if (PyErr_ExceptionMatches(PyExc_SyntaxWarning)) {
129 /* Replace the SyntaxWarning exception with a SyntaxError
130 to get a more accurate error report */
131 PyErr_Clear();
132 return _PyTokenizer_syntaxerror(tok, "invalid escape sequence '\\%c'", (char) first_invalid_escape_char);
133 }
134
135 return -1;
136 }
137
138 Py_DECREF(msg);
139 return 0;
140 }
141
142 int
143 _PyTokenizer_parser_warn(struct tok_state *tok, PyObject *category, const char *format, ...)
144 {
145 if (!tok->report_warnings) {
146 return 0;
147 }
148
149 PyObject *errmsg;
150 va_list vargs;
151 va_start(vargs, format);
152 errmsg = PyUnicode_FromFormatV(format, vargs);
153 va_end(vargs);
154 if (!errmsg) {
155 goto error;
156 }
157
158 if (PyErr_WarnExplicitObject(category, errmsg, tok->filename,
159 tok->lineno, NULL, NULL) < 0) {
160 if (PyErr_ExceptionMatches(category)) {
161 /* Replace the DeprecationWarning exception with a SyntaxError
162 to get a more accurate error report */
163 PyErr_Clear();
164 _PyTokenizer_syntaxerror(tok, "%U", errmsg);
165 }
166 goto error;
167 }
168 Py_DECREF(errmsg);
169 return 0;
170
171 error:
172 Py_XDECREF(errmsg);
173 tok->done = E_ERROR;
174 return -1;
175 }
176
177
178 /* ############## STRING MANIPULATION ############## */
179
180 char *
181 _PyTokenizer_new_string(const char *s, Py_ssize_t len, struct tok_state *tok)
182 {
183 char* result = (char *)PyMem_Malloc(len + 1);
184 if (!result) {
185 tok->done = E_NOMEM;
186 PyErr_NoMemory();
187 return NULL;
188 }
189 memcpy(result, s, len);
190 result[len] = '\0';
191 return result;
192 }
193
194 PyObject *
195 _PyTokenizer_translate_into_utf8(const char* str, const char* enc) {
196 PyObject *utf8;
197 PyObject* buf = PyUnicode_Decode(str, strlen(str), enc, NULL);
198 if (buf == NULL)
199 return NULL;
200 utf8 = PyUnicode_AsUTF8String(buf);
201 Py_DECREF(buf);
202 return utf8;
203 }
204
205 char *
206 _PyTokenizer_translate_newlines(const char *s, int exec_input, int preserve_crlf,
207 struct tok_state *tok) {
208 int skip_next_lf = 0;
209 size_t needed_length = strlen(s) + 2, final_length;
210 char *buf, *current;
211 char c = '\0';
212 buf = PyMem_Malloc(needed_length);
213 if (buf == NULL) {
214 tok->done = E_NOMEM;
215 PyErr_NoMemory();
216 return NULL;
217 }
218 for (current = buf; *s; s++, current++) {
219 c = *s;
220 if (skip_next_lf) {
221 skip_next_lf = 0;
222 if (c == '\n') {
223 c = *++s;
224 if (!c)
225 break;
226 }
227 }
228 if (!preserve_crlf && c == '\r') {
229 skip_next_lf = 1;
230 c = '\n';
231 }
232 *current = c;
233 }
234 /* If this is exec input, add a newline to the end of the string if
235 there isn't one already. */
236 if (exec_input && c != '\n' && c != '\0') {
237 *current = '\n';
238 current++;
239 }
240 *current = '\0';
241 final_length = current - buf + 1;
242 if (final_length < needed_length && final_length) {
243 /* should never fail */
244 char* result = PyMem_Realloc(buf, final_length);
245 if (result == NULL) {
246 PyMem_Free(buf);
247 }
248 buf = result;
249 }
250 return buf;
251 }
252
253 /* ############## ENCODING STUFF ############## */
254
255
256 /* See whether the file starts with a BOM. If it does,
257 invoke the set_readline function with the new encoding.
258 Return 1 on success, 0 on failure. */
259 int
260 _PyTokenizer_check_bom(int get_char(struct tok_state *),
261 void unget_char(int, struct tok_state *),
262 int set_readline(struct tok_state *, const char *),
263 struct tok_state *tok)
264 {
265 int ch1, ch2, ch3;
266 ch1 = get_char(tok);
267 tok->decoding_state = STATE_SEEK_CODING;
268 if (ch1 == EOF) {
269 return 1;
270 } else if (ch1 == 0xEF) {
271 ch2 = get_char(tok);
272 if (ch2 != 0xBB) {
273 unget_char(ch2, tok);
274 unget_char(ch1, tok);
275 return 1;
276 }
277 ch3 = get_char(tok);
278 if (ch3 != 0xBF) {
279 unget_char(ch3, tok);
280 unget_char(ch2, tok);
281 unget_char(ch1, tok);
282 return 1;
283 }
284 } else {
285 unget_char(ch1, tok);
286 return 1;
287 }
288 if (tok->encoding != NULL)
289 PyMem_Free(tok->encoding);
290 tok->encoding = _PyTokenizer_new_string("utf-8", 5, tok);
291 if (!tok->encoding)
292 return 0;
293 /* No need to set_readline: input is already utf-8 */
294 return 1;
295 }
296
297 static const char *
298 get_normal_name(const char *s) /* for utf-8 and latin-1 */
299 {
300 char buf[13];
301 int i;
302 for (i = 0; i < 12; i++) {
303 int c = s[i];
304 if (c == '\0')
305 break;
306 else if (c == '_')
307 buf[i] = '-';
308 else
309 buf[i] = Py_TOLOWER(c);
310 }
311 buf[i] = '\0';
312 if (strcmp(buf, "utf-8") == 0 ||
313 strncmp(buf, "utf-8-", 6) == 0)
314 return "utf-8";
315 else if (strcmp(buf, "latin-1") == 0 ||
316 strcmp(buf, "iso-8859-1") == 0 ||
317 strcmp(buf, "iso-latin-1") == 0 ||
318 strncmp(buf, "latin-1-", 8) == 0 ||
319 strncmp(buf, "iso-8859-1-", 11) == 0 ||
320 strncmp(buf, "iso-latin-1-", 12) == 0)
321 return "iso-8859-1";
322 else
323 return s;
324 }
325
326 /* Return the coding spec in S, or NULL if none is found. */
327 static int
328 get_coding_spec(const char *s, char **spec, Py_ssize_t size, struct tok_state *tok)
329 {
330 Py_ssize_t i;
331 *spec = NULL;
332 /* Coding spec must be in a comment, and that comment must be
333 * the only statement on the source code line. */
334 for (i = 0; i < size - 6; i++) {
335 if (s[i] == '#')
336 break;
337 if (s[i] != ' ' && s[i] != '\t' && s[i] != '\014')
338 return 1;
339 }
340 for (; i < size - 6; i++) { /* XXX inefficient search */
341 const char* t = s + i;
342 if (memcmp(t, "coding", 6) == 0) {
343 const char* begin = NULL;
344 t += 6;
345 if (t[0] != ':' && t[0] != '=')
346 continue;
347 do {
348 t++;
349 } while (t[0] == ' ' || t[0] == '\t');
350
351 begin = t;
352 while (Py_ISALNUM(t[0]) ||
353 t[0] == '-' || t[0] == '_' || t[0] == '.')
354 t++;
355
356 if (begin < t) {
357 char* r = _PyTokenizer_new_string(begin, t - begin, tok);
358 const char* q;
359 if (!r)
360 return 0;
361 q = get_normal_name(r);
362 if (r != q) {
363 PyMem_Free(r);
364 r = _PyTokenizer_new_string(q, strlen(q), tok);
365 if (!r)
366 return 0;
367 }
368 *spec = r;
369 break;
370 }
371 }
372 }
373 return 1;
374 }
375
376 /* Check whether the line contains a coding spec. If it does,
377 invoke the set_readline function for the new encoding.
378 This function receives the tok_state and the new encoding.
379 Return 1 on success, 0 on failure. */
380 int
381 _PyTokenizer_check_coding_spec(const char* line, Py_ssize_t size, struct tok_state *tok,
382 int set_readline(struct tok_state *, const char *))
383 {
384 char *cs;
385 if (tok->cont_line) {
386 /* It's a continuation line, so it can't be a coding spec. */
387 tok->decoding_state = STATE_NORMAL;
388 return 1;
389 }
390 if (!get_coding_spec(line, &cs, size, tok)) {
391 return 0;
392 }
393 if (!cs) {
394 Py_ssize_t i;
395 for (i = 0; i < size; i++) {
396 if (line[i] == '#' || line[i] == '\n' || line[i] == '\r')
397 break;
398 if (line[i] != ' ' && line[i] != '\t' && line[i] != '\014') {
399 /* Stop checking coding spec after a line containing
400 * anything except a comment. */
401 tok->decoding_state = STATE_NORMAL;
402 break;
403 }
404 }
405 return 1;
406 }
407 tok->decoding_state = STATE_NORMAL;
408 if (tok->encoding == NULL) {
409 assert(tok->decoding_readline == NULL);
410 if (strcmp(cs, "utf-8") != 0 && !set_readline(tok, cs)) {
411 _PyTokenizer_error_ret(tok);
412 PyErr_Format(PyExc_SyntaxError, "encoding problem: %s", cs);
413 PyMem_Free(cs);
414 return 0;
415 }
416 tok->encoding = cs;
417 } else { /* then, compare cs with BOM */
418 if (strcmp(tok->encoding, cs) != 0) {
419 _PyTokenizer_error_ret(tok);
420 PyErr_Format(PyExc_SyntaxError,
421 "encoding problem: %s with BOM", cs);
422 PyMem_Free(cs);
423 return 0;
424 }
425 PyMem_Free(cs);
426 }
427 return 1;
428 }
429
430 /* Check whether the characters at s start a valid
431 UTF-8 sequence. Return the number of characters forming
432 the sequence if yes, 0 if not. The special cases match
433 those in stringlib/codecs.h:utf8_decode.
434 */
435 static int
436 valid_utf8(const unsigned char* s)
437 {
438 int expected = 0;
439 int length;
440 if (*s < 0x80) {
441 /* single-byte code */
442 return 1;
443 }
444 else if (*s < 0xE0) {
445 /* \xC2\x80-\xDF\xBF -- 0080-07FF */
446 if (*s < 0xC2) {
447 /* invalid sequence
448 \x80-\xBF -- continuation byte
449 \xC0-\xC1 -- fake 0000-007F */
450 return 0;
451 }
452 expected = 1;
453 }
454 else if (*s < 0xF0) {
455 /* \xE0\xA0\x80-\xEF\xBF\xBF -- 0800-FFFF */
456 if (*s == 0xE0 && *(s + 1) < 0xA0) {
457 /* invalid sequence
458 \xE0\x80\x80-\xE0\x9F\xBF -- fake 0000-0800 */
459 return 0;
460 }
461 else if (*s == 0xED && *(s + 1) >= 0xA0) {
462 /* Decoding UTF-8 sequences in range \xED\xA0\x80-\xED\xBF\xBF
463 will result in surrogates in range D800-DFFF. Surrogates are
464 not valid UTF-8 so they are rejected.
465 See https://www.unicode.org/versions/Unicode5.2.0/ch03.pdf
466 (table 3-7) and http://www.rfc-editor.org/rfc/rfc3629.txt */
467 return 0;
468 }
469 expected = 2;
470 }
471 else if (*s < 0xF5) {
472 /* \xF0\x90\x80\x80-\xF4\x8F\xBF\xBF -- 10000-10FFFF */
473 if (*(s + 1) < 0x90 ? *s == 0xF0 : *s == 0xF4) {
474 /* invalid sequence -- one of:
475 \xF0\x80\x80\x80-\xF0\x8F\xBF\xBF -- fake 0000-FFFF
476 \xF4\x90\x80\x80- -- 110000- overflow */
477 return 0;
478 }
479 expected = 3;
480 }
481 else {
482 /* invalid start byte */
483 return 0;
484 }
485 length = expected + 1;
486 for (int i = 1; i <= expected; i++) {
487 if (s[i] < 0x80 || s[i] >= 0xC0) {
488 return 0;
489 }
490 }
491 return length;
492 }
493
494 int
495 _PyTokenizer_ensure_utf8(char *line, struct tok_state *tok)
496 {
497 int badchar = 0;
498 unsigned char *c;
499 int length;
500 for (c = (unsigned char *)line; *c; c += length) {
501 if (!(length = valid_utf8(c))) {
502 badchar = *c;
503 break;
504 }
505 }
506 if (badchar) {
507 PyErr_Format(PyExc_SyntaxError,
508 "Non-UTF-8 code starting with '\\x%.2x' "
509 "in file %U on line %i, "
510 "but no encoding declared; "
511 "see https://peps.python.org/pep-0263/ for details",
512 badchar, tok->filename, tok->lineno);
513 return 0;
514 }
515 return 1;
516 }
517
518
519 /* ############## DEBUGGING STUFF ############## */
520
521 #ifdef Py_DEBUG
522 void
523 _PyTokenizer_print_escape(FILE *f, const char *s, Py_ssize_t size)
524 {
525 if (s == NULL) {
526 fputs("NULL", f);
527 return;
528 }
529 putc('"', f);
530 while (size-- > 0) {
531 unsigned char c = *s++;
532 switch (c) {
533 case '\n': fputs("\\n", f); break;
534 case '\r': fputs("\\r", f); break;
535 case '\t': fputs("\\t", f); break;
536 case '\f': fputs("\\f", f); break;
537 case '\'': fputs("\\'", f); break;
538 case '"': fputs("\\\"", f); break;
539 default:
540 if (0x20 <= c && c <= 0x7f)
541 putc(c, f);
542 else
543 fprintf(f, "\\x%02x", c);
544 }
545 }
546 putc('"', f);
547 }
548
549 void
550 _PyTokenizer_tok_dump(int type, char *start, char *end)
551 {
552 fprintf(stderr, "%s", _PyParser_TokenNames[type]);
553 if (type == NAME || type == NUMBER || type == STRING || type == OP)
554 fprintf(stderr, "(%.*s)", (int)(end - start), start);
555 }
556 #endif