| 1 |
/*[clinic input] |
| 2 |
preserve |
| 3 |
[clinic start generated code]*/ |
| 4 |
|
| 5 |
#if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) |
| 6 |
# include "pycore_gc.h" // PyGC_Head |
| 7 |
# include "pycore_runtime.h" // _Py_ID() |
| 8 |
#endif |
| 9 |
#include "pycore_modsupport.h" // _PyArg_UnpackKeywords() |
| 10 |
|
| 11 |
static PyObject * |
| 12 |
tokenizeriter_new_impl(PyTypeObject *type, PyObject *readline, |
| 13 |
int extra_tokens, const char *encoding); |
| 14 |
|
| 15 |
static PyObject * |
| 16 |
tokenizeriter_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) |
| 17 |
{ |
| 18 |
PyObject *return_value = NULL; |
| 19 |
#if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) |
| 20 |
|
| 21 |
#define NUM_KEYWORDS 2 |
| 22 |
static struct { |
| 23 |
PyGC_Head _this_is_not_used; |
| 24 |
PyObject_VAR_HEAD |
| 25 |
Py_hash_t ob_hash; |
| 26 |
PyObject *ob_item[NUM_KEYWORDS]; |
| 27 |
} _kwtuple = { |
| 28 |
.ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) |
| 29 |
.ob_hash = -1, |
| 30 |
.ob_item = { &_Py_ID(extra_tokens), &_Py_ID(encoding), }, |
| 31 |
}; |
| 32 |
#undef NUM_KEYWORDS |
| 33 |
#define KWTUPLE (&_kwtuple.ob_base.ob_base) |
| 34 |
|
| 35 |
#else // !Py_BUILD_CORE |
| 36 |
# define KWTUPLE NULL |
| 37 |
#endif // !Py_BUILD_CORE |
| 38 |
|
| 39 |
static const char * const _keywords[] = {"", "extra_tokens", "encoding", NULL}; |
| 40 |
static _PyArg_Parser _parser = { |
| 41 |
.keywords = _keywords, |
| 42 |
.fname = "tokenizeriter", |
| 43 |
.kwtuple = KWTUPLE, |
| 44 |
}; |
| 45 |
#undef KWTUPLE |
| 46 |
PyObject *argsbuf[3]; |
| 47 |
PyObject * const *fastargs; |
| 48 |
Py_ssize_t nargs = PyTuple_GET_SIZE(args); |
| 49 |
Py_ssize_t noptargs = nargs + (kwargs ? PyDict_GET_SIZE(kwargs) : 0) - 2; |
| 50 |
PyObject *readline; |
| 51 |
int extra_tokens; |
| 52 |
const char *encoding = NULL; |
| 53 |
|
| 54 |
fastargs = _PyArg_UnpackKeywords(_PyTuple_CAST(args)->ob_item, nargs, kwargs, NULL, &_parser, |
| 55 |
/*minpos*/ 1, /*maxpos*/ 1, /*minkw*/ 1, /*varpos*/ 0, argsbuf); |
| 56 |
if (!fastargs) { |
| 57 |
goto exit; |
| 58 |
} |
| 59 |
readline = fastargs[0]; |
| 60 |
extra_tokens = PyObject_IsTrue(fastargs[1]); |
| 61 |
if (extra_tokens < 0) { |
| 62 |
goto exit; |
| 63 |
} |
| 64 |
if (!noptargs) { |
| 65 |
goto skip_optional_kwonly; |
| 66 |
} |
| 67 |
if (!PyUnicode_Check(fastargs[2])) { |
| 68 |
_PyArg_BadArgument("tokenizeriter", "argument 'encoding'", "str", fastargs[2]); |
| 69 |
goto exit; |
| 70 |
} |
| 71 |
Py_ssize_t encoding_length; |
| 72 |
encoding = PyUnicode_AsUTF8AndSize(fastargs[2], &encoding_length); |
| 73 |
if (encoding == NULL) { |
| 74 |
goto exit; |
| 75 |
} |
| 76 |
if (strlen(encoding) != (size_t)encoding_length) { |
| 77 |
PyErr_SetString(PyExc_ValueError, "embedded null character"); |
| 78 |
goto exit; |
| 79 |
} |
| 80 |
skip_optional_kwonly: |
| 81 |
return_value = tokenizeriter_new_impl(type, readline, extra_tokens, encoding); |
| 82 |
|
| 83 |
exit: |
| 84 |
return return_value; |
| 85 |
} |
| 86 |
/*[clinic end generated code: output=4c448f34d9c835c0 input=a9049054013a1b77]*/ |