Debian Codegraph @

Parser/lexer/buffer.c

raw
1 #include "Python.h"
2 #include "errcode.h"
3
4 #include "state.h"
5
6 /* Traverse and remember all f-string buffers, in order to be able to restore
7 them after reallocating tok->buf */
8 void
9 _PyLexer_remember_fstring_buffers(struct tok_state *tok)
10 {
11 int index;
12 tokenizer_mode *mode;
13
14 for (index = tok->tok_mode_stack_index; index >= 0; --index) {
15 mode = &(tok->tok_mode_stack[index]);
16 mode->start_offset = mode->start == NULL ? -1 : mode->start - tok->buf;
17 mode->multi_line_start_offset = mode->multi_line_start == NULL ? -1 : mode->multi_line_start - tok->buf;
18 }
19 }
20
21 /* Traverse and restore all f-string buffers after reallocating tok->buf */
22 void
23 _PyLexer_restore_fstring_buffers(struct tok_state *tok)
24 {
25 int index;
26 tokenizer_mode *mode;
27
28 for (index = tok->tok_mode_stack_index; index >= 0; --index) {
29 mode = &(tok->tok_mode_stack[index]);
30 mode->start = mode->start_offset < 0 ? NULL : tok->buf + mode->start_offset;
31 mode->multi_line_start = mode->multi_line_start_offset < 0 ? NULL : tok->buf + mode->multi_line_start_offset;
32 }
33 }
34
35 /* Read a line of text from TOK into S, using the stream in TOK.
36 Return NULL on failure, else S.
37
38 On entry, tok->decoding_buffer will be one of:
39 1) NULL: need to call tok->decoding_readline to get a new line
40 2) PyUnicodeObject *: decoding_feof has called tok->decoding_readline and
41 stored the result in tok->decoding_buffer
42 3) PyByteArrayObject *: previous call to tok_readline_recode did not have enough room
43 (in the s buffer) to copy entire contents of the line read
44 by tok->decoding_readline. tok->decoding_buffer has the overflow.
45 In this case, tok_readline_recode is called in a loop (with an expanded buffer)
46 until the buffer ends with a '\n' (or until the end of the file is
47 reached): see tok_nextc and its calls to tok_reserve_buf.
48 */
49 int
50 _PyLexer_tok_reserve_buf(struct tok_state *tok, Py_ssize_t size)
51 {
52 Py_ssize_t cur = tok->cur - tok->buf;
53 Py_ssize_t oldsize = tok->inp - tok->buf;
54 Py_ssize_t newsize = oldsize + Py_MAX(size, oldsize >> 1);
55 if (newsize > tok->end - tok->buf) {
56 char *newbuf = tok->buf;
57 Py_ssize_t start = tok->start == NULL ? -1 : tok->start - tok->buf;
58 Py_ssize_t line_start = tok->start == NULL ? -1 : tok->line_start - tok->buf;
59 Py_ssize_t multi_line_start = tok->multi_line_start - tok->buf;
60 _PyLexer_remember_fstring_buffers(tok);
61 newbuf = (char *)PyMem_Realloc(newbuf, newsize);
62 if (newbuf == NULL) {
63 tok->done = E_NOMEM;
64 return 0;
65 }
66 tok->buf = newbuf;
67 tok->cur = tok->buf + cur;
68 tok->inp = tok->buf + oldsize;
69 tok->end = tok->buf + newsize;
70 tok->start = start < 0 ? NULL : tok->buf + start;
71 tok->line_start = line_start < 0 ? NULL : tok->buf + line_start;
72 tok->multi_line_start = multi_line_start < 0 ? NULL : tok->buf + multi_line_start;
73 _PyLexer_restore_fstring_buffers(tok);
74 }
75 return 1;
76 }