| 1 |
#include "Python.h" |
| 2 |
#include "errcode.h" |
| 3 |
|
| 4 |
#include "state.h" |
| 5 |
|
| 6 |
/* Traverse and remember all f-string buffers, in order to be able to restore |
| 7 |
them after reallocating tok->buf */ |
| 8 |
void |
| 9 |
_PyLexer_remember_fstring_buffers(struct tok_state *tok) |
| 10 |
{ |
| 11 |
int index; |
| 12 |
tokenizer_mode *mode; |
| 13 |
|
| 14 |
for (index = tok->tok_mode_stack_index; index >= 0; --index) { |
| 15 |
mode = &(tok->tok_mode_stack[index]); |
| 16 |
mode->start_offset = mode->start == NULL ? -1 : mode->start - tok->buf; |
| 17 |
mode->multi_line_start_offset = mode->multi_line_start == NULL ? -1 : mode->multi_line_start - tok->buf; |
| 18 |
} |
| 19 |
} |
| 20 |
|
| 21 |
/* Traverse and restore all f-string buffers after reallocating tok->buf */ |
| 22 |
void |
| 23 |
_PyLexer_restore_fstring_buffers(struct tok_state *tok) |
| 24 |
{ |
| 25 |
int index; |
| 26 |
tokenizer_mode *mode; |
| 27 |
|
| 28 |
for (index = tok->tok_mode_stack_index; index >= 0; --index) { |
| 29 |
mode = &(tok->tok_mode_stack[index]); |
| 30 |
mode->start = mode->start_offset < 0 ? NULL : tok->buf + mode->start_offset; |
| 31 |
mode->multi_line_start = mode->multi_line_start_offset < 0 ? NULL : tok->buf + mode->multi_line_start_offset; |
| 32 |
} |
| 33 |
} |
| 34 |
|
| 35 |
/* Read a line of text from TOK into S, using the stream in TOK. |
| 36 |
Return NULL on failure, else S. |
| 37 |
|
| 38 |
On entry, tok->decoding_buffer will be one of: |
| 39 |
1) NULL: need to call tok->decoding_readline to get a new line |
| 40 |
2) PyUnicodeObject *: decoding_feof has called tok->decoding_readline and |
| 41 |
stored the result in tok->decoding_buffer |
| 42 |
3) PyByteArrayObject *: previous call to tok_readline_recode did not have enough room |
| 43 |
(in the s buffer) to copy entire contents of the line read |
| 44 |
by tok->decoding_readline. tok->decoding_buffer has the overflow. |
| 45 |
In this case, tok_readline_recode is called in a loop (with an expanded buffer) |
| 46 |
until the buffer ends with a '\n' (or until the end of the file is |
| 47 |
reached): see tok_nextc and its calls to tok_reserve_buf. |
| 48 |
*/ |
| 49 |
int |
| 50 |
_PyLexer_tok_reserve_buf(struct tok_state *tok, Py_ssize_t size) |
| 51 |
{ |
| 52 |
Py_ssize_t cur = tok->cur - tok->buf; |
| 53 |
Py_ssize_t oldsize = tok->inp - tok->buf; |
| 54 |
Py_ssize_t newsize = oldsize + Py_MAX(size, oldsize >> 1); |
| 55 |
if (newsize > tok->end - tok->buf) { |
| 56 |
char *newbuf = tok->buf; |
| 57 |
Py_ssize_t start = tok->start == NULL ? -1 : tok->start - tok->buf; |
| 58 |
Py_ssize_t line_start = tok->start == NULL ? -1 : tok->line_start - tok->buf; |
| 59 |
Py_ssize_t multi_line_start = tok->multi_line_start - tok->buf; |
| 60 |
_PyLexer_remember_fstring_buffers(tok); |
| 61 |
newbuf = (char *)PyMem_Realloc(newbuf, newsize); |
| 62 |
if (newbuf == NULL) { |
| 63 |
tok->done = E_NOMEM; |
| 64 |
return 0; |
| 65 |
} |
| 66 |
tok->buf = newbuf; |
| 67 |
tok->cur = tok->buf + cur; |
| 68 |
tok->inp = tok->buf + oldsize; |
| 69 |
tok->end = tok->buf + newsize; |
| 70 |
tok->start = start < 0 ? NULL : tok->buf + start; |
| 71 |
tok->line_start = line_start < 0 ? NULL : tok->buf + line_start; |
| 72 |
tok->multi_line_start = multi_line_start < 0 ? NULL : tok->buf + multi_line_start; |
| 73 |
_PyLexer_restore_fstring_buffers(tok); |
| 74 |
} |
| 75 |
return 1; |
| 76 |
} |