Debian Codegraph @

Parser/string_parser.c

raw
1 #include <Python.h>
2 #include "pycore_bytesobject.h" // _PyBytes_DecodeEscape()
3 #include "pycore_unicodeobject.h" // _PyUnicode_DecodeUnicodeEscapeInternal()
4
5 #include "lexer/state.h"
6 #include "pegen.h"
7 #include "string_parser.h"
8
9 #include <stdbool.h>
10
11 //// STRING HANDLING FUNCTIONS ////
12
13 static int
14 warn_invalid_escape_sequence(Parser *p, const char* buffer, const char *first_invalid_escape, Token *t)
15 {
16 if (p->call_invalid_rules) {
17 // Do not report warnings if we are in the second pass of the parser
18 // to avoid showing the warning twice.
19 return 0;
20 }
21 unsigned char c = (unsigned char)*first_invalid_escape;
22 if ((t->type == FSTRING_MIDDLE || t->type == FSTRING_END) && (c == '{' || c == '}')) {
23 // in this case the tokenizer has already emitted a warning,
24 // see Parser/tokenizer/helpers.c:warn_invalid_escape_sequence
25 return 0;
26 }
27
28 int octal = ('4' <= c && c <= '7');
29 PyObject *msg =
30 octal
31 ? PyUnicode_FromFormat("invalid octal escape sequence '\\%.3s'",
32 first_invalid_escape)
33 : PyUnicode_FromFormat("invalid escape sequence '\\%c'", c);
34 if (msg == NULL) {
35 return -1;
36 }
37 PyObject *category;
38 if (p->feature_version >= 12) {
39 category = PyExc_SyntaxWarning;
40 }
41 else {
42 category = PyExc_DeprecationWarning;
43 }
44
45 // Calculate the lineno and the col_offset of the invalid escape sequence
46 const char *start = buffer;
47 const char *end = first_invalid_escape;
48 int lineno = t->lineno;
49 int col_offset = t->col_offset;
50 while (start < end) {
51 if (*start == '\n') {
52 lineno++;
53 col_offset = 0;
54 }
55 else {
56 col_offset++;
57 }
58 start++;
59 }
60
61 // Count the number of quotes in the token
62 char first_quote = 0;
63 if (lineno == t->lineno) {
64 int quote_count = 0;
65 char* tok = PyBytes_AsString(t->bytes);
66 for (int i = 0; i < PyBytes_Size(t->bytes); i++) {
67 if (tok[i] == '\'' || tok[i] == '\"') {
68 if (quote_count == 0) {
69 first_quote = tok[i];
70 }
71 if (tok[i] == first_quote) {
72 quote_count++;
73 }
74 } else {
75 break;
76 }
77 }
78
79 col_offset += quote_count;
80 }
81
82 if (PyErr_WarnExplicitObject(category, msg, p->tok->filename,
83 lineno, NULL, NULL) < 0) {
84 if (PyErr_ExceptionMatches(category)) {
85 /* Replace the Syntax/DeprecationWarning exception with a SyntaxError
86 to get a more accurate error report */
87 PyErr_Clear();
88
89 /* This is needed, in order for the SyntaxError to point to the token t,
90 since _PyPegen_raise_error uses p->tokens[p->fill - 1] for the
91 error location, if p->known_err_token is not set. */
92 p->known_err_token = t;
93 if (octal) {
94 RAISE_ERROR_KNOWN_LOCATION(p, PyExc_SyntaxError, lineno, col_offset-1, lineno, col_offset+1,
95 "invalid octal escape sequence '\\%.3s'", first_invalid_escape);
96 }
97 else {
98 RAISE_ERROR_KNOWN_LOCATION(p, PyExc_SyntaxError, lineno, col_offset-1, lineno, col_offset+1,
99 "invalid escape sequence '\\%c'", c);
100 }
101 }
102 Py_DECREF(msg);
103 return -1;
104 }
105 Py_DECREF(msg);
106 return 0;
107 }
108
109 static PyObject *
110 decode_utf8(const char **sPtr, const char *end)
111 {
112 const char *s;
113 const char *t;
114 t = s = *sPtr;
115 while (s < end && (*s & 0x80)) {
116 s++;
117 }
118 *sPtr = s;
119 return PyUnicode_DecodeUTF8(t, s - t, NULL);
120 }
121
122 static PyObject *
123 decode_unicode_with_escapes(Parser *parser, const char *s, size_t len, Token *t)
124 {
125 PyObject *v;
126 PyObject *u;
127 char *buf;
128 char *p;
129 const char *end;
130
131 /* check for integer overflow */
132 if (len > (size_t)PY_SSIZE_T_MAX / 6) {
133 return NULL;
134 }
135 /* "ä" (2 bytes) may become "\U000000E4" (10 bytes), or 1:5
136 "\ä" (3 bytes) may become "\u005c\U000000E4" (16 bytes), or ~1:6 */
137 u = PyBytes_FromStringAndSize((char *)NULL, (Py_ssize_t)len * 6);
138 if (u == NULL) {
139 return NULL;
140 }
141 p = buf = PyBytes_AsString(u);
142 if (p == NULL) {
143 return NULL;
144 }
145 end = s + len;
146 while (s < end) {
147 if (*s == '\\') {
148 *p++ = *s++;
149 if (s >= end || *s & 0x80) {
150 strcpy(p, "u005c");
151 p += 5;
152 if (s >= end) {
153 break;
154 }
155 }
156 }
157 if (*s & 0x80) {
158 PyObject *w;
159 int kind;
160 const void *data;
161 Py_ssize_t w_len;
162 Py_ssize_t i;
163 w = decode_utf8(&s, end);
164 if (w == NULL) {
165 Py_DECREF(u);
166 return NULL;
167 }
168 kind = PyUnicode_KIND(w);
169 data = PyUnicode_DATA(w);
170 w_len = PyUnicode_GET_LENGTH(w);
171 for (i = 0; i < w_len; i++) {
172 Py_UCS4 chr = PyUnicode_READ(kind, data, i);
173 sprintf(p, "\\U%08x", chr);
174 p += 10;
175 }
176 /* Should be impossible to overflow */
177 assert(p - buf <= PyBytes_GET_SIZE(u));
178 Py_DECREF(w);
179 }
180 else {
181 *p++ = *s++;
182 }
183 }
184 len = (size_t)(p - buf);
185 s = buf;
186
187 int first_invalid_escape_char;
188 const char *first_invalid_escape_ptr;
189 v = _PyUnicode_DecodeUnicodeEscapeInternal2(s, (Py_ssize_t)len, NULL, NULL,
190 &first_invalid_escape_char,
191 &first_invalid_escape_ptr);
192
193 // HACK: later we can simply pass the line no, since we don't preserve the tokens
194 // when we are decoding the string but we preserve the line numbers.
195 if (v != NULL && first_invalid_escape_ptr != NULL && t != NULL) {
196 if (warn_invalid_escape_sequence(parser, s, first_invalid_escape_ptr, t) < 0) {
197 /* We have not decref u before because first_invalid_escape_ptr
198 points inside u. */
199 Py_XDECREF(u);
200 Py_DECREF(v);
201 return NULL;
202 }
203 }
204 Py_XDECREF(u);
205 return v;
206 }
207
208 static PyObject *
209 decode_bytes_with_escapes(Parser *p, const char *s, Py_ssize_t len, Token *t)
210 {
211 int first_invalid_escape_char;
212 const char *first_invalid_escape_ptr;
213 PyObject *result = _PyBytes_DecodeEscape2(s, len, NULL,
214 &first_invalid_escape_char,
215 &first_invalid_escape_ptr);
216 if (result == NULL) {
217 return NULL;
218 }
219
220 if (first_invalid_escape_ptr != NULL) {
221 if (warn_invalid_escape_sequence(p, s, first_invalid_escape_ptr, t) < 0) {
222 Py_DECREF(result);
223 return NULL;
224 }
225 }
226 return result;
227 }
228
229 PyObject *
230 _PyPegen_decode_string(Parser *p, int raw, const char *s, size_t len, Token *t)
231 {
232 if (raw) {
233 return PyUnicode_DecodeUTF8Stateful(s, (Py_ssize_t)len, NULL, NULL);
234 }
235 return decode_unicode_with_escapes(p, s, len, t);
236 }
237
238 /* s must include the bracketing quote characters, and r, b &/or f prefixes
239 (if any), and embedded escape sequences (if any). (f-strings are handled by the parser)
240 _PyPegen_parse_string parses it, and returns the decoded Python string object. */
241 PyObject *
242 _PyPegen_parse_string(Parser *p, Token *t)
243 {
244 const char *s = PyBytes_AsString(t->bytes);
245 if (s == NULL) {
246 return NULL;
247 }
248
249 size_t len;
250 int quote = Py_CHARMASK(*s);
251 int bytesmode = 0;
252 int rawmode = 0;
253
254 if (Py_ISALPHA(quote)) {
255 while (!bytesmode || !rawmode) {
256 if (quote == 'b' || quote == 'B') {
257 quote =(unsigned char)*++s;
258 bytesmode = 1;
259 }
260 else if (quote == 'u' || quote == 'U') {
261 quote = (unsigned char)*++s;
262 }
263 else if (quote == 'r' || quote == 'R') {
264 quote = (unsigned char)*++s;
265 rawmode = 1;
266 }
267 else {
268 break;
269 }
270 }
271 }
272
273 if (quote != '\'' && quote != '\"') {
274 PyErr_BadInternalCall();
275 return NULL;
276 }
277
278 /* Skip the leading quote char. */
279 s++;
280 len = strlen(s);
281 // gh-120155: 's' contains at least the trailing quote,
282 // so the code '--len' below is safe.
283 assert(len >= 1);
284
285 if (len > INT_MAX) {
286 PyErr_SetString(PyExc_OverflowError, "string to parse is too long");
287 return NULL;
288 }
289 if (s[--len] != quote) {
290 /* Last quote char must match the first. */
291 PyErr_BadInternalCall();
292 return NULL;
293 }
294 if (len >= 4 && s[0] == quote && s[1] == quote) {
295 /* A triple quoted string. We've already skipped one quote at
296 the start and one at the end of the string. Now skip the
297 two at the start. */
298 s += 2;
299 len -= 2;
300 /* And check that the last two match. */
301 if (s[--len] != quote || s[--len] != quote) {
302 PyErr_BadInternalCall();
303 return NULL;
304 }
305 }
306
307 /* Avoid invoking escape decoding routines if possible. */
308 rawmode = rawmode || strchr(s, '\\') == NULL;
309 if (bytesmode) {
310 /* Disallow non-ASCII characters. */
311 const char *ch;
312 for (ch = s; *ch; ch++) {
313 if (Py_CHARMASK(*ch) >= 0x80) {
314 RAISE_SYNTAX_ERROR_KNOWN_LOCATION(
315 t,
316 "bytes can only contain ASCII "
317 "literal characters");
318 return NULL;
319 }
320 }
321 if (rawmode) {
322 return PyBytes_FromStringAndSize(s, (Py_ssize_t)len);
323 }
324 return decode_bytes_with_escapes(p, s, (Py_ssize_t)len, t);
325 }
326 return _PyPegen_decode_string(p, rawmode, s, len, t);
327 }