| 1 |
#ifndef PEGEN_H |
| 2 |
#define PEGEN_H |
| 3 |
|
| 4 |
#include <Python.h> |
| 5 |
#include <pycore_ast.h> |
| 6 |
#include <pycore_token.h> |
| 7 |
|
| 8 |
#if 0 |
| 9 |
#define PyPARSE_YIELD_IS_KEYWORD 0x0001 |
| 10 |
#endif |
| 11 |
|
| 12 |
#define PyPARSE_DONT_IMPLY_DEDENT 0x0002 |
| 13 |
|
| 14 |
#if 0 |
| 15 |
#define PyPARSE_WITH_IS_KEYWORD 0x0003 |
| 16 |
#define PyPARSE_PRINT_IS_FUNCTION 0x0004 |
| 17 |
#define PyPARSE_UNICODE_LITERALS 0x0008 |
| 18 |
#endif |
| 19 |
|
| 20 |
#define PyPARSE_IGNORE_COOKIE 0x0010 |
| 21 |
#define PyPARSE_BARRY_AS_BDFL 0x0020 |
| 22 |
#define PyPARSE_TYPE_COMMENTS 0x0040 |
| 23 |
#define PyPARSE_ALLOW_INCOMPLETE_INPUT 0x0100 |
| 24 |
|
| 25 |
#define CURRENT_POS (-5) |
| 26 |
|
| 27 |
typedef struct _memo { |
| 28 |
int type; |
| 29 |
void *node; |
| 30 |
int mark; |
| 31 |
struct _memo *next; |
| 32 |
} Memo; |
| 33 |
|
| 34 |
typedef struct { |
| 35 |
int type; |
| 36 |
PyObject *bytes; |
| 37 |
int level; |
| 38 |
int lineno, col_offset, end_lineno, end_col_offset; |
| 39 |
Memo *memo; |
| 40 |
PyObject *metadata; |
| 41 |
} Token; |
| 42 |
|
| 43 |
typedef struct { |
| 44 |
const char *str; |
| 45 |
int type; |
| 46 |
} KeywordToken; |
| 47 |
|
| 48 |
|
| 49 |
typedef struct { |
| 50 |
struct { |
| 51 |
int lineno; |
| 52 |
char *comment; // The " <tag>" in "# type: ignore <tag>" |
| 53 |
} *items; |
| 54 |
size_t size; |
| 55 |
size_t num_items; |
| 56 |
} growable_comment_array; |
| 57 |
|
| 58 |
typedef struct { |
| 59 |
struct tok_state *tok; |
| 60 |
Token **tokens; |
| 61 |
int mark; |
| 62 |
int fill, size; |
| 63 |
PyArena *arena; |
| 64 |
KeywordToken **keywords; |
| 65 |
char **soft_keywords; |
| 66 |
int n_keyword_lists; |
| 67 |
int start_rule; |
| 68 |
int *errcode; |
| 69 |
int parsing_started; |
| 70 |
PyObject* normalize; |
| 71 |
int starting_lineno; |
| 72 |
int starting_col_offset; |
| 73 |
int error_indicator; |
| 74 |
int flags; |
| 75 |
int feature_version; |
| 76 |
growable_comment_array type_ignore_comments; |
| 77 |
Token *known_err_token; |
| 78 |
int level; |
| 79 |
int call_invalid_rules; |
| 80 |
int debug; |
| 81 |
} Parser; |
| 82 |
|
| 83 |
typedef struct { |
| 84 |
cmpop_ty cmpop; |
| 85 |
expr_ty expr; |
| 86 |
} CmpopExprPair; |
| 87 |
|
| 88 |
typedef struct { |
| 89 |
expr_ty key; |
| 90 |
expr_ty value; |
| 91 |
} KeyValuePair; |
| 92 |
|
| 93 |
typedef struct { |
| 94 |
expr_ty key; |
| 95 |
pattern_ty pattern; |
| 96 |
} KeyPatternPair; |
| 97 |
|
| 98 |
typedef struct { |
| 99 |
arg_ty arg; |
| 100 |
expr_ty value; |
| 101 |
} NameDefaultPair; |
| 102 |
|
| 103 |
typedef struct { |
| 104 |
asdl_arg_seq *plain_names; |
| 105 |
asdl_seq *names_with_defaults; // asdl_seq* of NameDefaultsPair's |
| 106 |
} SlashWithDefault; |
| 107 |
|
| 108 |
typedef struct { |
| 109 |
arg_ty vararg; |
| 110 |
asdl_seq *kwonlyargs; // asdl_seq* of NameDefaultsPair's |
| 111 |
arg_ty kwarg; |
| 112 |
} StarEtc; |
| 113 |
|
| 114 |
typedef struct { operator_ty kind; } AugOperator; |
| 115 |
typedef struct { |
| 116 |
void *element; |
| 117 |
int is_keyword; |
| 118 |
} KeywordOrStarred; |
| 119 |
|
| 120 |
typedef struct { |
| 121 |
void *result; |
| 122 |
PyObject *metadata; |
| 123 |
} ResultTokenWithMetadata; |
| 124 |
|
| 125 |
// Internal parser functions |
| 126 |
#if defined(Py_DEBUG) |
| 127 |
void _PyPegen_clear_memo_statistics(void); |
| 128 |
PyObject *_PyPegen_get_memo_statistics(void); |
| 129 |
#endif |
| 130 |
|
| 131 |
int _PyPegen_insert_memo(Parser *p, int mark, int type, void *node); |
| 132 |
int _PyPegen_update_memo(Parser *p, int mark, int type, void *node); |
| 133 |
int _PyPegen_is_memoized(Parser *p, int type, void *pres); |
| 134 |
|
| 135 |
int _PyPegen_lookahead(int, void *(func)(Parser *), Parser *); |
| 136 |
int _PyPegen_lookahead_for_expr(int, expr_ty (func)(Parser *), Parser *); |
| 137 |
int _PyPegen_lookahead_for_stmt(int, stmt_ty (func)(Parser *), Parser *); |
| 138 |
int _PyPegen_lookahead_with_int(int, Token *(func)(Parser *, int), Parser *, int); |
| 139 |
int _PyPegen_lookahead_with_string(int, expr_ty (func)(Parser *, const char*), Parser *, const char*); |
| 140 |
|
| 141 |
Token *_PyPegen_expect_token(Parser *p, int type); |
| 142 |
void* _PyPegen_expect_forced_result(Parser *p, void* result, const char* expected); |
| 143 |
Token *_PyPegen_expect_forced_token(Parser *p, int type, const char* expected); |
| 144 |
expr_ty _PyPegen_expect_soft_keyword(Parser *p, const char *keyword); |
| 145 |
expr_ty _PyPegen_soft_keyword_token(Parser *p); |
| 146 |
expr_ty _PyPegen_fstring_middle_token(Parser* p); |
| 147 |
Token *_PyPegen_get_last_nonnwhitespace_token(Parser *); |
| 148 |
int _PyPegen_fill_token(Parser *p); |
| 149 |
expr_ty _PyPegen_name_token(Parser *p); |
| 150 |
expr_ty _PyPegen_number_token(Parser *p); |
| 151 |
void *_PyPegen_string_token(Parser *p); |
| 152 |
Py_ssize_t _PyPegen_byte_offset_to_character_offset_line(PyObject *line, Py_ssize_t col_offset, Py_ssize_t end_col_offset); |
| 153 |
Py_ssize_t _PyPegen_byte_offset_to_character_offset(PyObject *line, Py_ssize_t col_offset); |
| 154 |
Py_ssize_t _PyPegen_byte_offset_to_character_offset_raw(const char*, Py_ssize_t col_offset); |
| 155 |
|
| 156 |
// Error handling functions and APIs |
| 157 |
typedef enum { |
| 158 |
STAR_TARGETS, |
| 159 |
DEL_TARGETS, |
| 160 |
FOR_TARGETS |
| 161 |
} TARGETS_TYPE; |
| 162 |
|
| 163 |
int _Pypegen_raise_decode_error(Parser *p); |
| 164 |
void _PyPegen_raise_tokenizer_init_error(PyObject *filename); |
| 165 |
int _Pypegen_tokenizer_error(Parser *p); |
| 166 |
void *_PyPegen_raise_error(Parser *p, PyObject *errtype, int use_mark, const char *errmsg, ...); |
| 167 |
void *_PyPegen_raise_error_known_location(Parser *p, PyObject *errtype, |
| 168 |
Py_ssize_t lineno, Py_ssize_t col_offset, |
| 169 |
Py_ssize_t end_lineno, Py_ssize_t end_col_offset, |
| 170 |
const char *errmsg, va_list va); |
| 171 |
void _Pypegen_set_syntax_error(Parser* p, Token* last_token); |
| 172 |
void _Pypegen_stack_overflow(Parser *p); |
| 173 |
|
| 174 |
Py_LOCAL_INLINE(void *) |
| 175 |
RAISE_ERROR_KNOWN_LOCATION(Parser *p, PyObject *errtype, |
| 176 |
Py_ssize_t lineno, Py_ssize_t col_offset, |
| 177 |
Py_ssize_t end_lineno, Py_ssize_t end_col_offset, |
| 178 |
const char *errmsg, ...) |
| 179 |
{ |
| 180 |
va_list va; |
| 181 |
va_start(va, errmsg); |
| 182 |
Py_ssize_t _col_offset = (col_offset == CURRENT_POS ? CURRENT_POS : col_offset + 1); |
| 183 |
Py_ssize_t _end_col_offset = (end_col_offset == CURRENT_POS ? CURRENT_POS : end_col_offset + 1); |
| 184 |
_PyPegen_raise_error_known_location(p, errtype, lineno, _col_offset, end_lineno, _end_col_offset, errmsg, va); |
| 185 |
va_end(va); |
| 186 |
return NULL; |
| 187 |
} |
| 188 |
#define RAISE_SYNTAX_ERROR(msg, ...) _PyPegen_raise_error(p, PyExc_SyntaxError, 0, msg, ##__VA_ARGS__) |
| 189 |
#define RAISE_INDENTATION_ERROR(msg, ...) _PyPegen_raise_error(p, PyExc_IndentationError, 0, msg, ##__VA_ARGS__) |
| 190 |
#define RAISE_SYNTAX_ERROR_ON_NEXT_TOKEN(msg, ...) _PyPegen_raise_error(p, PyExc_SyntaxError, 1, msg, ##__VA_ARGS__) |
| 191 |
#define RAISE_SYNTAX_ERROR_KNOWN_RANGE(a, b, msg, ...) \ |
| 192 |
RAISE_ERROR_KNOWN_LOCATION(p, PyExc_SyntaxError, (a)->lineno, (a)->col_offset, (b)->end_lineno, (b)->end_col_offset, msg, ##__VA_ARGS__) |
| 193 |
#define RAISE_SYNTAX_ERROR_KNOWN_LOCATION(a, msg, ...) \ |
| 194 |
RAISE_ERROR_KNOWN_LOCATION(p, PyExc_SyntaxError, (a)->lineno, (a)->col_offset, (a)->end_lineno, (a)->end_col_offset, msg, ##__VA_ARGS__) |
| 195 |
#define RAISE_SYNTAX_ERROR_STARTING_FROM(a, msg, ...) \ |
| 196 |
RAISE_ERROR_KNOWN_LOCATION(p, PyExc_SyntaxError, (a)->lineno, (a)->col_offset, CURRENT_POS, CURRENT_POS, msg, ##__VA_ARGS__) |
| 197 |
#define RAISE_SYNTAX_ERROR_INVALID_TARGET(type, e) _RAISE_SYNTAX_ERROR_INVALID_TARGET(p, type, e) |
| 198 |
|
| 199 |
Py_LOCAL_INLINE(void *) |
| 200 |
CHECK_CALL(Parser *p, void *result) |
| 201 |
{ |
| 202 |
if (result == NULL) { |
| 203 |
assert(PyErr_Occurred()); |
| 204 |
p->error_indicator = 1; |
| 205 |
} |
| 206 |
return result; |
| 207 |
} |
| 208 |
|
| 209 |
/* This is needed for helper functions that are allowed to |
| 210 |
return NULL without an error. Example: _PyPegen_seq_extract_starred_exprs */ |
| 211 |
Py_LOCAL_INLINE(void *) |
| 212 |
CHECK_CALL_NULL_ALLOWED(Parser *p, void *result) |
| 213 |
{ |
| 214 |
if (result == NULL && PyErr_Occurred()) { |
| 215 |
p->error_indicator = 1; |
| 216 |
} |
| 217 |
return result; |
| 218 |
} |
| 219 |
|
| 220 |
#define CHECK(type, result) ((type) CHECK_CALL(p, result)) |
| 221 |
#define CHECK_NULL_ALLOWED(type, result) ((type) CHECK_CALL_NULL_ALLOWED(p, result)) |
| 222 |
|
| 223 |
expr_ty _PyPegen_get_invalid_target(expr_ty e, TARGETS_TYPE targets_type); |
| 224 |
const char *_PyPegen_get_expr_name(expr_ty); |
| 225 |
Py_LOCAL_INLINE(void *) |
| 226 |
_RAISE_SYNTAX_ERROR_INVALID_TARGET(Parser *p, TARGETS_TYPE type, void *e) |
| 227 |
{ |
| 228 |
expr_ty invalid_target = CHECK_NULL_ALLOWED(expr_ty, _PyPegen_get_invalid_target(e, type)); |
| 229 |
if (invalid_target != NULL) { |
| 230 |
const char *msg; |
| 231 |
if (type == STAR_TARGETS || type == FOR_TARGETS) { |
| 232 |
msg = "cannot assign to %s"; |
| 233 |
} |
| 234 |
else { |
| 235 |
msg = "cannot delete %s"; |
| 236 |
} |
| 237 |
return RAISE_SYNTAX_ERROR_KNOWN_LOCATION( |
| 238 |
invalid_target, |
| 239 |
msg, |
| 240 |
_PyPegen_get_expr_name(invalid_target) |
| 241 |
); |
| 242 |
return RAISE_SYNTAX_ERROR_KNOWN_LOCATION(invalid_target, "invalid syntax"); |
| 243 |
} |
| 244 |
return NULL; |
| 245 |
} |
| 246 |
|
| 247 |
// Action utility functions |
| 248 |
|
| 249 |
void *_PyPegen_dummy_name(Parser *p, ...); |
| 250 |
void * _PyPegen_seq_last_item(asdl_seq *seq); |
| 251 |
#define PyPegen_last_item(seq, type) ((type)_PyPegen_seq_last_item((asdl_seq*)seq)) |
| 252 |
void * _PyPegen_seq_first_item(asdl_seq *seq); |
| 253 |
#define PyPegen_first_item(seq, type) ((type)_PyPegen_seq_first_item((asdl_seq*)seq)) |
| 254 |
#define UNUSED(expr) do { (void)(expr); } while (0) |
| 255 |
#define EXTRA_EXPR(head, tail) head->lineno, (head)->col_offset, (tail)->end_lineno, (tail)->end_col_offset, p->arena |
| 256 |
#define EXTRA _start_lineno, _start_col_offset, _end_lineno, _end_col_offset, p->arena |
| 257 |
PyObject *_PyPegen_new_type_comment(Parser *, const char *); |
| 258 |
|
| 259 |
Py_LOCAL_INLINE(PyObject *) |
| 260 |
NEW_TYPE_COMMENT(Parser *p, Token *tc) |
| 261 |
{ |
| 262 |
if (tc == NULL) { |
| 263 |
return NULL; |
| 264 |
} |
| 265 |
const char *bytes = PyBytes_AsString(tc->bytes); |
| 266 |
if (bytes == NULL) { |
| 267 |
goto error; |
| 268 |
} |
| 269 |
PyObject *tco = _PyPegen_new_type_comment(p, bytes); |
| 270 |
if (tco == NULL) { |
| 271 |
goto error; |
| 272 |
} |
| 273 |
return tco; |
| 274 |
error: |
| 275 |
p->error_indicator = 1; // Inline CHECK_CALL |
| 276 |
return NULL; |
| 277 |
} |
| 278 |
|
| 279 |
Py_LOCAL_INLINE(void *) |
| 280 |
INVALID_VERSION_CHECK(Parser *p, int version, char *msg, void *node) |
| 281 |
{ |
| 282 |
if (node == NULL) { |
| 283 |
p->error_indicator = 1; // Inline CHECK_CALL |
| 284 |
return NULL; |
| 285 |
} |
| 286 |
if (p->feature_version < version) { |
| 287 |
p->error_indicator = 1; |
| 288 |
return RAISE_SYNTAX_ERROR("%s only supported in Python 3.%i and greater", |
| 289 |
msg, version); |
| 290 |
} |
| 291 |
return node; |
| 292 |
} |
| 293 |
|
| 294 |
#define CHECK_VERSION(type, version, msg, node) ((type) INVALID_VERSION_CHECK(p, version, msg, node)) |
| 295 |
|
| 296 |
arg_ty _PyPegen_add_type_comment_to_arg(Parser *, arg_ty, Token *); |
| 297 |
PyObject *_PyPegen_new_identifier(Parser *, const char *); |
| 298 |
asdl_seq *_PyPegen_singleton_seq(Parser *, void *); |
| 299 |
asdl_seq *_PyPegen_seq_insert_in_front(Parser *, void *, asdl_seq *); |
| 300 |
asdl_seq *_PyPegen_seq_append_to_end(Parser *, asdl_seq *, void *); |
| 301 |
asdl_seq *_PyPegen_seq_flatten(Parser *, asdl_seq *); |
| 302 |
expr_ty _PyPegen_join_names_with_dot(Parser *, expr_ty, expr_ty); |
| 303 |
int _PyPegen_seq_count_dots(asdl_seq *); |
| 304 |
alias_ty _PyPegen_alias_for_star(Parser *, int, int, int, int, PyArena *); |
| 305 |
asdl_identifier_seq *_PyPegen_map_names_to_ids(Parser *, asdl_expr_seq *); |
| 306 |
CmpopExprPair *_PyPegen_cmpop_expr_pair(Parser *, cmpop_ty, expr_ty); |
| 307 |
asdl_int_seq *_PyPegen_get_cmpops(Parser *p, asdl_seq *); |
| 308 |
asdl_expr_seq *_PyPegen_get_exprs(Parser *, asdl_seq *); |
| 309 |
expr_ty _PyPegen_set_expr_context(Parser *, expr_ty, expr_context_ty); |
| 310 |
KeyValuePair *_PyPegen_key_value_pair(Parser *, expr_ty, expr_ty); |
| 311 |
asdl_expr_seq *_PyPegen_get_keys(Parser *, asdl_seq *); |
| 312 |
asdl_expr_seq *_PyPegen_get_values(Parser *, asdl_seq *); |
| 313 |
KeyPatternPair *_PyPegen_key_pattern_pair(Parser *, expr_ty, pattern_ty); |
| 314 |
asdl_expr_seq *_PyPegen_get_pattern_keys(Parser *, asdl_seq *); |
| 315 |
asdl_pattern_seq *_PyPegen_get_patterns(Parser *, asdl_seq *); |
| 316 |
NameDefaultPair *_PyPegen_name_default_pair(Parser *, arg_ty, expr_ty, Token *); |
| 317 |
SlashWithDefault *_PyPegen_slash_with_default(Parser *, asdl_arg_seq *, asdl_seq *); |
| 318 |
StarEtc *_PyPegen_star_etc(Parser *, arg_ty, asdl_seq *, arg_ty); |
| 319 |
arguments_ty _PyPegen_make_arguments(Parser *, asdl_arg_seq *, SlashWithDefault *, |
| 320 |
asdl_arg_seq *, asdl_seq *, StarEtc *); |
| 321 |
arguments_ty _PyPegen_empty_arguments(Parser *); |
| 322 |
expr_ty _PyPegen_formatted_value(Parser *, expr_ty, Token *, ResultTokenWithMetadata *, ResultTokenWithMetadata *, Token *, |
| 323 |
int, int, int, int, PyArena *); |
| 324 |
AugOperator *_PyPegen_augoperator(Parser*, operator_ty type); |
| 325 |
stmt_ty _PyPegen_function_def_decorators(Parser *, asdl_expr_seq *, stmt_ty); |
| 326 |
stmt_ty _PyPegen_class_def_decorators(Parser *, asdl_expr_seq *, stmt_ty); |
| 327 |
KeywordOrStarred *_PyPegen_keyword_or_starred(Parser *, void *, int); |
| 328 |
asdl_expr_seq *_PyPegen_seq_extract_starred_exprs(Parser *, asdl_seq *); |
| 329 |
asdl_keyword_seq *_PyPegen_seq_delete_starred_exprs(Parser *, asdl_seq *); |
| 330 |
expr_ty _PyPegen_collect_call_seqs(Parser *, asdl_expr_seq *, asdl_seq *, |
| 331 |
int lineno, int col_offset, int end_lineno, |
| 332 |
int end_col_offset, PyArena *arena); |
| 333 |
expr_ty _PyPegen_constant_from_token(Parser* p, Token* tok); |
| 334 |
expr_ty _PyPegen_decoded_constant_from_token(Parser* p, Token* tok); |
| 335 |
expr_ty _PyPegen_constant_from_string(Parser* p, Token* tok); |
| 336 |
expr_ty _PyPegen_concatenate_strings(Parser *p, asdl_expr_seq *, int, int, int, int, PyArena *); |
| 337 |
expr_ty _PyPegen_FetchRawForm(Parser *p, int, int, int, int); |
| 338 |
expr_ty _PyPegen_ensure_imaginary(Parser *p, expr_ty); |
| 339 |
expr_ty _PyPegen_ensure_real(Parser *p, expr_ty); |
| 340 |
asdl_seq *_PyPegen_join_sequences(Parser *, asdl_seq *, asdl_seq *); |
| 341 |
int _PyPegen_check_barry_as_flufl(Parser *, Token *); |
| 342 |
int _PyPegen_check_legacy_stmt(Parser *p, expr_ty t); |
| 343 |
ResultTokenWithMetadata *_PyPegen_check_fstring_conversion(Parser *p, Token *, expr_ty t); |
| 344 |
ResultTokenWithMetadata *_PyPegen_setup_full_format_spec(Parser *, Token *, asdl_expr_seq *, int, int, |
| 345 |
int, int, PyArena *); |
| 346 |
mod_ty _PyPegen_make_module(Parser *, asdl_stmt_seq *); |
| 347 |
void *_PyPegen_arguments_parsing_error(Parser *, expr_ty); |
| 348 |
expr_ty _PyPegen_get_last_comprehension_item(comprehension_ty comprehension); |
| 349 |
void *_PyPegen_nonparen_genexp_in_call(Parser *p, expr_ty args, asdl_comprehension_seq *comprehensions); |
| 350 |
stmt_ty _PyPegen_checked_future_import(Parser *p, identifier module, asdl_alias_seq *, |
| 351 |
int , int, int , int , int , PyArena *); |
| 352 |
|
| 353 |
// Parser API |
| 354 |
|
| 355 |
Parser *_PyPegen_Parser_New(struct tok_state *, int, int, int, int *, PyArena *); |
| 356 |
void _PyPegen_Parser_Free(Parser *); |
| 357 |
mod_ty _PyPegen_run_parser_from_file_pointer(FILE *, int, PyObject *, const char *, |
| 358 |
const char *, const char *, PyCompilerFlags *, int *, PyObject **, |
| 359 |
PyArena *); |
| 360 |
void *_PyPegen_run_parser(Parser *); |
| 361 |
mod_ty _PyPegen_run_parser_from_string(const char *, int, PyObject *, PyCompilerFlags *, PyArena *); |
| 362 |
asdl_stmt_seq *_PyPegen_interactive_exit(Parser *); |
| 363 |
|
| 364 |
// TODO: move to the correct place in this file |
| 365 |
expr_ty _PyPegen_joined_str(Parser *p, Token* a, asdl_expr_seq* expr, Token*b); |
| 366 |
|
| 367 |
// Generated function in parse.c - function definition in python.gram |
| 368 |
void *_PyPegen_parse(Parser *); |
| 369 |
|
| 370 |
#endif |