| 1 |
#include "Python.h" |
| 2 |
#include "errcode.h" |
| 3 |
|
| 4 |
#include "helpers.h" |
| 5 |
#include "../lexer/state.h" |
| 6 |
|
| 7 |
static int |
| 8 |
tok_underflow_string(struct tok_state *tok) { |
| 9 |
char *end = strchr(tok->inp, '\n'); |
| 10 |
if (end != NULL) { |
| 11 |
end++; |
| 12 |
} |
| 13 |
else { |
| 14 |
end = strchr(tok->inp, '\0'); |
| 15 |
if (end == tok->inp) { |
| 16 |
tok->done = E_EOF; |
| 17 |
return 0; |
| 18 |
} |
| 19 |
} |
| 20 |
if (tok->start == NULL) { |
| 21 |
tok->buf = tok->cur; |
| 22 |
} |
| 23 |
tok->line_start = tok->cur; |
| 24 |
ADVANCE_LINENO(); |
| 25 |
tok->inp = end; |
| 26 |
return 1; |
| 27 |
} |
| 28 |
|
| 29 |
/* Set up tokenizer for UTF-8 string */ |
| 30 |
struct tok_state * |
| 31 |
_PyTokenizer_FromUTF8(const char *str, int exec_input, int preserve_crlf) |
| 32 |
{ |
| 33 |
struct tok_state *tok = _PyTokenizer_tok_new(); |
| 34 |
char *translated; |
| 35 |
if (tok == NULL) |
| 36 |
return NULL; |
| 37 |
tok->input = translated = _PyTokenizer_translate_newlines(str, exec_input, preserve_crlf, tok); |
| 38 |
if (translated == NULL) { |
| 39 |
_PyTokenizer_Free(tok); |
| 40 |
return NULL; |
| 41 |
} |
| 42 |
tok->decoding_state = STATE_NORMAL; |
| 43 |
tok->enc = NULL; |
| 44 |
tok->str = translated; |
| 45 |
tok->encoding = _PyTokenizer_new_string("utf-8", 5, tok); |
| 46 |
if (!tok->encoding) { |
| 47 |
_PyTokenizer_Free(tok); |
| 48 |
return NULL; |
| 49 |
} |
| 50 |
|
| 51 |
tok->buf = tok->cur = tok->inp = translated; |
| 52 |
tok->end = translated; |
| 53 |
tok->underflow = &tok_underflow_string; |
| 54 |
return tok; |
| 55 |
} |