Debian Codegraph @

Python/clinic/Python-tokenize.c.h

raw
1 /*[clinic input]
2 preserve
3 [clinic start generated code]*/
4
5 #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE)
6 # include "pycore_gc.h" // PyGC_Head
7 # include "pycore_runtime.h" // _Py_ID()
8 #endif
9 #include "pycore_modsupport.h" // _PyArg_UnpackKeywords()
10
11 static PyObject *
12 tokenizeriter_new_impl(PyTypeObject *type, PyObject *readline,
13 int extra_tokens, const char *encoding);
14
15 static PyObject *
16 tokenizeriter_new(PyTypeObject *type, PyObject *args, PyObject *kwargs)
17 {
18 PyObject *return_value = NULL;
19 #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE)
20
21 #define NUM_KEYWORDS 2
22 static struct {
23 PyGC_Head _this_is_not_used;
24 PyObject_VAR_HEAD
25 Py_hash_t ob_hash;
26 PyObject *ob_item[NUM_KEYWORDS];
27 } _kwtuple = {
28 .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS)
29 .ob_hash = -1,
30 .ob_item = { &_Py_ID(extra_tokens), &_Py_ID(encoding), },
31 };
32 #undef NUM_KEYWORDS
33 #define KWTUPLE (&_kwtuple.ob_base.ob_base)
34
35 #else // !Py_BUILD_CORE
36 # define KWTUPLE NULL
37 #endif // !Py_BUILD_CORE
38
39 static const char * const _keywords[] = {"", "extra_tokens", "encoding", NULL};
40 static _PyArg_Parser _parser = {
41 .keywords = _keywords,
42 .fname = "tokenizeriter",
43 .kwtuple = KWTUPLE,
44 };
45 #undef KWTUPLE
46 PyObject *argsbuf[3];
47 PyObject * const *fastargs;
48 Py_ssize_t nargs = PyTuple_GET_SIZE(args);
49 Py_ssize_t noptargs = nargs + (kwargs ? PyDict_GET_SIZE(kwargs) : 0) - 2;
50 PyObject *readline;
51 int extra_tokens;
52 const char *encoding = NULL;
53
54 fastargs = _PyArg_UnpackKeywords(_PyTuple_CAST(args)->ob_item, nargs, kwargs, NULL, &_parser,
55 /*minpos*/ 1, /*maxpos*/ 1, /*minkw*/ 1, /*varpos*/ 0, argsbuf);
56 if (!fastargs) {
57 goto exit;
58 }
59 readline = fastargs[0];
60 extra_tokens = PyObject_IsTrue(fastargs[1]);
61 if (extra_tokens < 0) {
62 goto exit;
63 }
64 if (!noptargs) {
65 goto skip_optional_kwonly;
66 }
67 if (!PyUnicode_Check(fastargs[2])) {
68 _PyArg_BadArgument("tokenizeriter", "argument 'encoding'", "str", fastargs[2]);
69 goto exit;
70 }
71 Py_ssize_t encoding_length;
72 encoding = PyUnicode_AsUTF8AndSize(fastargs[2], &encoding_length);
73 if (encoding == NULL) {
74 goto exit;
75 }
76 if (strlen(encoding) != (size_t)encoding_length) {
77 PyErr_SetString(PyExc_ValueError, "embedded null character");
78 goto exit;
79 }
80 skip_optional_kwonly:
81 return_value = tokenizeriter_new_impl(type, readline, extra_tokens, encoding);
82
83 exit:
84 return return_value;
85 }
86 /*[clinic end generated code: output=4c448f34d9c835c0 input=a9049054013a1b77]*/