Debian Codegraph @

Parser/tokenizer/file_tokenizer.c

raw
1 #include "Python.h"
2 #include "pycore_call.h" // _PyObject_CallNoArgs()
3 #include "pycore_fileutils.h" // _Py_UniversalNewlineFgetsWithSize()
4 #include "pycore_runtime.h" // _Py_ID()
5
6 #include "errcode.h" // E_NOMEM
7
8 #ifdef HAVE_UNISTD_H
9 # include <unistd.h> // lseek(), read()
10 #endif
11
12 #include "helpers.h"
13 #include "../lexer/state.h"
14 #include "../lexer/lexer.h"
15 #include "../lexer/buffer.h"
16
17
18 static int
19 tok_concatenate_interactive_new_line(struct tok_state *tok, const char *line) {
20 assert(tok->fp_interactive);
21
22 if (!line) {
23 return 0;
24 }
25
26 Py_ssize_t current_size = tok->interactive_src_end - tok->interactive_src_start;
27 Py_ssize_t line_size = strlen(line);
28 char last_char = line[line_size > 0 ? line_size - 1 : line_size];
29 if (last_char != '\n') {
30 line_size += 1;
31 }
32 char* new_str = tok->interactive_src_start;
33
34 new_str = PyMem_Realloc(new_str, current_size + line_size + 1);
35 if (!new_str) {
36 if (tok->interactive_src_start) {
37 PyMem_Free(tok->interactive_src_start);
38 }
39 tok->interactive_src_start = NULL;
40 tok->interactive_src_end = NULL;
41 tok->done = E_NOMEM;
42 return -1;
43 }
44 strcpy(new_str + current_size, line);
45 tok->implicit_newline = 0;
46 if (last_char != '\n') {
47 /* Last line does not end in \n, fake one */
48 new_str[current_size + line_size - 1] = '\n';
49 new_str[current_size + line_size] = '\0';
50 tok->implicit_newline = 1;
51 }
52 tok->interactive_src_start = new_str;
53 tok->interactive_src_end = new_str + current_size + line_size;
54 return 0;
55 }
56
57 static int
58 tok_readline_raw(struct tok_state *tok)
59 {
60 do {
61 if (!_PyLexer_tok_reserve_buf(tok, BUFSIZ)) {
62 return 0;
63 }
64 int n_chars = (int)(tok->end - tok->inp);
65 size_t line_size = 0;
66 char *line = _Py_UniversalNewlineFgetsWithSize(tok->inp, n_chars, tok->fp, NULL, &line_size);
67 if (line == NULL) {
68 return 1;
69 }
70 if (tok->fp_interactive &&
71 tok_concatenate_interactive_new_line(tok, line) == -1) {
72 return 0;
73 }
74 tok->inp += line_size;
75 if (tok->inp == tok->buf) {
76 return 0;
77 }
78 } while (tok->inp[-1] != '\n');
79 return 1;
80 }
81
82 static int
83 tok_readline_recode(struct tok_state *tok) {
84 PyObject *line;
85 const char *buf;
86 Py_ssize_t buflen;
87 line = tok->decoding_buffer;
88 if (line == NULL) {
89 line = PyObject_CallNoArgs(tok->decoding_readline);
90 if (line == NULL) {
91 _PyTokenizer_error_ret(tok);
92 goto error;
93 }
94 }
95 else {
96 tok->decoding_buffer = NULL;
97 }
98 buf = PyUnicode_AsUTF8AndSize(line, &buflen);
99 if (buf == NULL) {
100 _PyTokenizer_error_ret(tok);
101 goto error;
102 }
103 // Make room for the null terminator *and* potentially
104 // an extra newline character that we may need to artificially
105 // add.
106 size_t buffer_size = buflen + 2;
107 if (!_PyLexer_tok_reserve_buf(tok, buffer_size)) {
108 goto error;
109 }
110 memcpy(tok->inp, buf, buflen);
111 tok->inp += buflen;
112 *tok->inp = '\0';
113 if (tok->fp_interactive &&
114 tok_concatenate_interactive_new_line(tok, buf) == -1) {
115 goto error;
116 }
117 Py_DECREF(line);
118 return 1;
119 error:
120 Py_XDECREF(line);
121 return 0;
122 }
123
124 /* Fetch the next byte from TOK. */
125 static int fp_getc(struct tok_state *tok) {
126 return getc(tok->fp);
127 }
128
129 /* Unfetch the last byte back into TOK. */
130 static void fp_ungetc(int c, struct tok_state *tok) {
131 ungetc(c, tok->fp);
132 }
133
134 /* Set the readline function for TOK to a StreamReader's
135 readline function. The StreamReader is named ENC.
136
137 This function is called from _PyTokenizer_check_bom and _PyTokenizer_check_coding_spec.
138
139 ENC is usually identical to the future value of tok->encoding,
140 except for the (currently unsupported) case of UTF-16.
141
142 Return 1 on success, 0 on failure. */
143 static int
144 fp_setreadl(struct tok_state *tok, const char* enc)
145 {
146 PyObject *readline, *open, *stream;
147 int fd;
148 long pos;
149
150 fd = fileno(tok->fp);
151 /* Due to buffering the file offset for fd can be different from the file
152 * position of tok->fp. If tok->fp was opened in text mode on Windows,
153 * its file position counts CRLF as one char and can't be directly mapped
154 * to the file offset for fd. Instead we step back one byte and read to
155 * the end of line.*/
156 pos = ftell(tok->fp);
157 if (pos == -1 ||
158 lseek(fd, (off_t)(pos > 0 ? pos - 1 : pos), SEEK_SET) == (off_t)-1) {
159 PyErr_SetFromErrnoWithFilename(PyExc_OSError, NULL);
160 return 0;
161 }
162
163 open = PyImport_ImportModuleAttrString("io", "open");
164 if (open == NULL) {
165 return 0;
166 }
167 stream = PyObject_CallFunction(open, "isisOOO",
168 fd, "r", -1, enc, Py_None, Py_None, Py_False);
169 Py_DECREF(open);
170 if (stream == NULL) {
171 return 0;
172 }
173
174 readline = PyObject_GetAttr(stream, &_Py_ID(readline));
175 Py_DECREF(stream);
176 if (readline == NULL) {
177 return 0;
178 }
179 Py_XSETREF(tok->decoding_readline, readline);
180
181 if (pos > 0) {
182 PyObject *bufobj = _PyObject_CallNoArgs(readline);
183 if (bufobj == NULL) {
184 return 0;
185 }
186 Py_DECREF(bufobj);
187 }
188
189 return 1;
190 }
191
192 static int
193 tok_underflow_interactive(struct tok_state *tok) {
194 if (tok->interactive_underflow == IUNDERFLOW_STOP) {
195 tok->done = E_INTERACT_STOP;
196 return 1;
197 }
198 char *newtok = PyOS_Readline(tok->fp ? tok->fp : stdin, stdout, tok->prompt);
199 if (newtok != NULL) {
200 char *translated = _PyTokenizer_translate_newlines(newtok, 0, 0, tok);
201 PyMem_Free(newtok);
202 if (translated == NULL) {
203 return 0;
204 }
205 newtok = translated;
206 }
207 if (tok->encoding && newtok && *newtok) {
208 /* Recode to UTF-8 */
209 Py_ssize_t buflen;
210 const char* buf;
211 PyObject *u = _PyTokenizer_translate_into_utf8(newtok, tok->encoding);
212 PyMem_Free(newtok);
213 if (u == NULL) {
214 tok->done = E_DECODE;
215 return 0;
216 }
217 buflen = PyBytes_GET_SIZE(u);
218 buf = PyBytes_AS_STRING(u);
219 newtok = PyMem_Malloc(buflen+1);
220 if (newtok == NULL) {
221 Py_DECREF(u);
222 tok->done = E_NOMEM;
223 return 0;
224 }
225 strcpy(newtok, buf);
226 Py_DECREF(u);
227 }
228 if (tok->fp_interactive &&
229 tok_concatenate_interactive_new_line(tok, newtok) == -1) {
230 PyMem_Free(newtok);
231 return 0;
232 }
233 if (tok->nextprompt != NULL) {
234 tok->prompt = tok->nextprompt;
235 }
236 if (newtok == NULL) {
237 tok->done = E_INTR;
238 }
239 else if (*newtok == '\0') {
240 PyMem_Free(newtok);
241 tok->done = E_EOF;
242 }
243 else if (tok->start != NULL) {
244 Py_ssize_t cur_multi_line_start = tok->multi_line_start - tok->buf;
245 _PyLexer_remember_fstring_buffers(tok);
246 size_t size = strlen(newtok);
247 ADVANCE_LINENO();
248 if (!_PyLexer_tok_reserve_buf(tok, size + 1)) {
249 PyMem_Free(tok->buf);
250 tok->buf = NULL;
251 PyMem_Free(newtok);
252 return 0;
253 }
254 memcpy(tok->cur, newtok, size + 1);
255 PyMem_Free(newtok);
256 tok->inp += size;
257 tok->multi_line_start = tok->buf + cur_multi_line_start;
258 _PyLexer_restore_fstring_buffers(tok);
259 }
260 else {
261 _PyLexer_remember_fstring_buffers(tok);
262 ADVANCE_LINENO();
263 PyMem_Free(tok->buf);
264 tok->buf = newtok;
265 tok->cur = tok->buf;
266 tok->line_start = tok->buf;
267 tok->inp = strchr(tok->buf, '\0');
268 tok->end = tok->inp + 1;
269 _PyLexer_restore_fstring_buffers(tok);
270 }
271 if (tok->done != E_OK) {
272 if (tok->prompt != NULL) {
273 PySys_WriteStderr("\n");
274 }
275 return 0;
276 }
277
278 if (tok->tok_mode_stack_index && !_PyLexer_update_ftstring_expr(tok, 0)) {
279 return 0;
280 }
281 return 1;
282 }
283
284 static int
285 tok_underflow_file(struct tok_state *tok)
286 {
287 if (tok->decoding_state == STATE_INIT) {
288 /* We have not yet determined the encoding.
289 If an encoding is found, use the file-pointer
290 reader functions from now on. */
291 if (!_PyTokenizer_check_bom(fp_getc, fp_ungetc, fp_setreadl, tok)) {
292 _PyTokenizer_error_ret(tok);
293 return 0;
294 }
295 assert(tok->decoding_state != STATE_INIT);
296 }
297 int raw = tok->decoding_readline == NULL;
298 if (raw && tok->decoding_state != STATE_NORMAL) {
299 /* Keep the first line in the buffer to validate it later if
300 * the encoding has not yet been determined. */
301 }
302 else if (tok->start == NULL && !INSIDE_FSTRING(tok)) {
303 tok->cur = tok->inp = tok->buf;
304 }
305 /* Read until '\n' or EOF */
306 if (!raw) {
307 /* We already have a codec associated with this input. */
308 if (!tok_readline_recode(tok)) {
309 return 0;
310 }
311 }
312 else {
313 /* We want a 'raw' read. */
314 if (!tok_readline_raw(tok)) {
315 return 0;
316 }
317 }
318 if (tok->inp == tok->cur) {
319 tok->done = E_EOF;
320 return 0;
321 }
322 tok->implicit_newline = 0;
323 if (tok->inp[-1] != '\n') {
324 assert(tok->inp + 1 < tok->end);
325 /* Last line does not end in \n, fake one */
326 *tok->inp++ = '\n';
327 *tok->inp = '\0';
328 tok->implicit_newline = 1;
329 }
330
331 if (tok->tok_mode_stack_index && !_PyLexer_update_ftstring_expr(tok, 0)) {
332 return 0;
333 }
334
335 ADVANCE_LINENO();
336 if (tok->decoding_state != STATE_NORMAL) {
337 if (!_PyTokenizer_check_coding_spec(tok->cur, strlen(tok->cur),
338 tok, fp_setreadl))
339 {
340 return 0;
341 }
342 if (tok->lineno >= 2) {
343 tok->decoding_state = STATE_NORMAL;
344 }
345 }
346 if (raw && tok->decoding_state == STATE_NORMAL) {
347 const char *line = tok->lineno <= 2 ? tok->buf : tok->cur;
348 int lineno = tok->lineno <= 2 ? 1 : tok->lineno;
349 if (!tok->encoding) {
350 /* The default encoding is UTF-8, so make sure we don't have any
351 non-UTF-8 sequences in it. */
352 if (!_PyTokenizer_ensure_utf8(line, tok, lineno)) {
353 _PyTokenizer_error_ret(tok);
354 return 0;
355 }
356 }
357 else {
358 PyObject *tmp = PyUnicode_Decode(line, strlen(line),
359 tok->encoding, NULL);
360 if (tmp == NULL) {
361 _PyTokenizer_error_ret(tok);
362 return 0;
363 }
364 Py_DECREF(tmp);
365 }
366 }
367 assert(tok->done == E_OK);
368 return tok->done == E_OK;
369 }
370
371 /* Set up tokenizer for file */
372 struct tok_state *
373 _PyTokenizer_FromFile(FILE *fp, const char* enc,
374 const char *ps1, const char *ps2)
375 {
376 struct tok_state *tok = _PyTokenizer_tok_new();
377 if (tok == NULL)
378 return NULL;
379 if ((tok->buf = (char *)PyMem_Malloc(BUFSIZ)) == NULL) {
380 _PyTokenizer_Free(tok);
381 PyErr_NoMemory();
382 return NULL;
383 }
384 tok->cur = tok->inp = tok->buf;
385 tok->end = tok->buf + BUFSIZ;
386 tok->fp = fp;
387 tok->prompt = ps1;
388 tok->nextprompt = ps2;
389 if (ps1 || ps2) {
390 tok->underflow = &tok_underflow_interactive;
391 } else {
392 tok->underflow = &tok_underflow_file;
393 }
394 if (enc != NULL) {
395 /* Must copy encoding declaration since it
396 gets copied into the parse tree. */
397 tok->encoding = _PyTokenizer_new_string(enc, strlen(enc), tok);
398 if (!tok->encoding) {
399 _PyTokenizer_Free(tok);
400 return NULL;
401 }
402 tok->decoding_state = STATE_NORMAL;
403 }
404 return tok;
405 }
406
407 #if defined(__wasi__) || (defined(__EMSCRIPTEN__) && (__EMSCRIPTEN_major__ >= 3))
408 // fdopen() with borrowed fd. WASI does not provide dup() and Emscripten's
409 // dup() emulation with open() is slow.
410 typedef union {
411 void *cookie;
412 int fd;
413 } borrowed;
414
415 static ssize_t
416 borrow_read(void *cookie, char *buf, size_t size)
417 {
418 borrowed b = {.cookie = cookie};
419 return read(b.fd, (void *)buf, size);
420 }
421
422 static FILE *
423 fdopen_borrow(int fd) {
424 // supports only reading. seek fails. close and write are no-ops.
425 cookie_io_functions_t io_cb = {borrow_read, NULL, NULL, NULL};
426 borrowed b = {.fd = fd};
427 return fopencookie(b.cookie, "r", io_cb);
428 }
429 #else
430 static FILE *
431 fdopen_borrow(int fd) {
432 fd = _Py_dup(fd);
433 if (fd < 0) {
434 return NULL;
435 }
436 return fdopen(fd, "r");
437 }
438 #endif
439
440 /* Get the encoding of a Python file. Check for the coding cookie and check if
441 the file starts with a BOM.
442
443 _PyTokenizer_FindEncodingFilename() returns NULL when it can't find the
444 encoding in the first or second line of the file (in which case the encoding
445 should be assumed to be UTF-8).
446
447 The char* returned is malloc'ed via PyMem_Malloc() and thus must be freed
448 by the caller. */
449 char *
450 _PyTokenizer_FindEncodingFilename(int fd, PyObject *filename)
451 {
452 struct tok_state *tok;
453 FILE *fp;
454 char *encoding = NULL;
455
456 fp = fdopen_borrow(fd);
457 if (fp == NULL) {
458 return NULL;
459 }
460 tok = _PyTokenizer_FromFile(fp, NULL, NULL, NULL);
461 if (tok == NULL) {
462 fclose(fp);
463 return NULL;
464 }
465 if (filename != NULL) {
466 tok->filename = Py_NewRef(filename);
467 }
468 else {
469 tok->filename = PyUnicode_FromString("<string>");
470 if (tok->filename == NULL) {
471 fclose(fp);
472 _PyTokenizer_Free(tok);
473 return encoding;
474 }
475 }
476 struct token token;
477 // We don't want to report warnings here because it could cause infinite recursion
478 // if fetching the encoding shows a warning.
479 tok->report_warnings = 0;
480 while (tok->lineno < 2 && tok->done == E_OK) {
481 _PyToken_Init(&token);
482 _PyTokenizer_Get(tok, &token);
483 _PyToken_Free(&token);
484 }
485 fclose(fp);
486 if (tok->encoding) {
487 encoding = (char *)PyMem_Malloc(strlen(tok->encoding) + 1);
488 if (encoding) {
489 strcpy(encoding, tok->encoding);
490 }
491 }
492 _PyTokenizer_Free(tok);
493 return encoding;
494 }