| 1 |
#ifndef _PY_TOKENIZER_HELPERS_H_ |
| 2 |
#define _PY_TOKENIZER_HELPERS_H_ |
| 3 |
|
| 4 |
#include "Python.h" |
| 5 |
|
| 6 |
#include "../lexer/state.h" |
| 7 |
|
| 8 |
#define ADVANCE_LINENO() \ |
| 9 |
tok->lineno++; \ |
| 10 |
tok->col_offset = 0; |
| 11 |
|
| 12 |
int _PyTokenizer_syntaxerror(struct tok_state *tok, const char *format, ...); |
| 13 |
int _PyTokenizer_syntaxerror_known_range(struct tok_state *tok, int col_offset, int end_col_offset, const char *format, ...); |
| 14 |
int _PyTokenizer_indenterror(struct tok_state *tok); |
| 15 |
int _PyTokenizer_warn_invalid_escape_sequence(struct tok_state *tok, int first_invalid_escape_char); |
| 16 |
int _PyTokenizer_parser_warn(struct tok_state *tok, PyObject *category, const char *format, ...); |
| 17 |
char *_PyTokenizer_error_ret(struct tok_state *tok); |
| 18 |
|
| 19 |
char *_PyTokenizer_new_string(const char *s, Py_ssize_t len, struct tok_state *tok); |
| 20 |
char *_PyTokenizer_translate_newlines(const char *s, int exec_input, int preserve_crlf, struct tok_state *tok); |
| 21 |
PyObject *_PyTokenizer_translate_into_utf8(const char* str, const char* enc); |
| 22 |
|
| 23 |
int _PyTokenizer_check_bom(int get_char(struct tok_state *), |
| 24 |
void unget_char(int, struct tok_state *), |
| 25 |
int set_readline(struct tok_state *, const char *), |
| 26 |
struct tok_state *tok); |
| 27 |
int _PyTokenizer_check_coding_spec(const char* line, Py_ssize_t size, struct tok_state *tok, |
| 28 |
int set_readline(struct tok_state *, const char *)); |
| 29 |
int _PyTokenizer_ensure_utf8(const char *line, struct tok_state *tok, int lineno); |
| 30 |
|
| 31 |
#ifdef Py_DEBUG |
| 32 |
void _PyTokenizer_print_escape(FILE *f, const char *s, Py_ssize_t size); |
| 33 |
void _PyTokenizer_tok_dump(int type, char *start, char *end); |
| 34 |
#endif |
| 35 |
|
| 36 |
|
| 37 |
#endif |