Debian Codegraph @

src/tinycss/tokenizer.c

raw
1 /*
2 * tokenizer.c
3 * Copyright (C) 2014 Kovid Goyal <kovid at kovidgoyal.net>
4 *
5 * Distributed under terms of the GPL3 license.
6 */
7
8 #define UNICODE
9 #define PY_SSIZE_T_CLEAN
10 #include <Python.h>
11 #include <structmember.h>
12
13 // Token type definition {{{
14 typedef struct {
15 PyObject_HEAD
16 // Type-specific fields go here.
17 PyObject *is_container;
18 PyObject *type;
19 PyObject *_as_css;
20 PyObject *value;
21 PyObject *unit;
22 PyObject *line;
23 PyObject *column;
24
25 } tokenizer_Token;
26
27 static void
28 tokenizer_Token_dealloc(tokenizer_Token *self) {
29 Py_XDECREF(self->is_container);
30 self->is_container = NULL;
31 Py_XDECREF(self->type);
32 self->type = NULL;
33 Py_XDECREF(self->_as_css);
34 self->_as_css = NULL;
35 Py_XDECREF(self->value);
36 self->value = NULL;
37 Py_XDECREF(self->unit);
38 self->unit = NULL;
39 Py_XDECREF(self->line);
40 self->line = NULL;
41 Py_XDECREF(self->column);
42 self->column = NULL;
43 Py_TYPE(self)->tp_free((PyObject *)self);
44 }
45
46
47 static PyObject *
48 tokenizer_Token_new(PyTypeObject *type, PyObject *args, PyObject *kwds) {
49 tokenizer_Token *self = NULL;
50 self = (tokenizer_Token *)type->tp_alloc(type, 0);
51 if (self == NULL) return PyErr_NoMemory();
52
53 if (!PyArg_ParseTuple(args, "OOOOOO", &(self->type), &(self->_as_css), &(self->value), &(self->unit), &(self->line), &(self->column))) {
54 Py_TYPE(self)->tp_free((PyObject *)self);
55 return NULL;
56 }
57 Py_INCREF(self->type);
58 Py_INCREF(self->_as_css);
59 Py_INCREF(self->value);
60 Py_INCREF(self->unit);
61 Py_INCREF(self->line);
62 Py_INCREF(self->column);
63 self->is_container = Py_False;
64 Py_INCREF(self->is_container);
65
66 return (PyObject *)self;
67 }
68
69 static PyObject *
70 tokenizer_Token_repr(tokenizer_Token *self) {
71 PyObject *type = NULL, *line = NULL, *column = NULL, *value = NULL, *ans = NULL, *unit = NULL;
72 if (!self->type || !self->line || !self->column || !self->value) return PyBytes_FromString("<Token NULL fields>");
73 type = PyObject_Str(self->type);
74 line = PyObject_Str(self->line);
75 column = PyObject_Str(self->column);
76 value = PyObject_Str(self->value);
77 if (type && line && column && value) {
78 if (self->unit != NULL && PyObject_IsTrue(self->unit)) {
79 unit = PyObject_Str(self->unit);
80 if (unit != NULL) ans = PyUnicode_FromFormat("<Token %U at %U:%U %U%U>", type, line, column, value, unit);
81 else PyErr_NoMemory();
82 } else ans = PyUnicode_FromFormat("<Token %U at %U:%U %U>", type, line, column, value);
83 } else PyErr_NoMemory();
84 Py_XDECREF(type);
85 Py_XDECREF(line);
86 Py_XDECREF(column);
87 Py_XDECREF(value);
88 Py_XDECREF(unit);
89 return ans;
90 }
91
92 static PyObject *
93 tokenizer_Token_as_css(tokenizer_Token *self, PyObject *args, PyObject *kwargs) {
94 if (!self->_as_css) { Py_RETURN_NONE; }
95 Py_INCREF(self->_as_css);
96 return self->_as_css;
97 }
98
99 static PyMemberDef tokenizer_Token_members[] = {
100 {"is_container", T_OBJECT_EX, offsetof(tokenizer_Token, is_container), 0, "False unless this token is a container for other tokens"},
101 {"type", T_OBJECT_EX, offsetof(tokenizer_Token, type), 0, "The token type"},
102 {"_as_css", T_OBJECT_EX, offsetof(tokenizer_Token, _as_css), 0, "Internal variable, use as_css() method instead."},
103 {"value", T_OBJECT_EX, offsetof(tokenizer_Token, value), 0, "The token value"},
104 {"unit", T_OBJECT_EX, offsetof(tokenizer_Token, unit), 0, "The token unit"},
105 {"line", T_OBJECT_EX, offsetof(tokenizer_Token, line), 0, "The token line number"},
106 {"column", T_OBJECT_EX, offsetof(tokenizer_Token, column), 0, "The token column number"},
107 {NULL} /* Sentinel */
108 };
109
110 static PyMethodDef tokenizer_Token_methods[] = {
111 {"as_css", (PyCFunction)tokenizer_Token_as_css, METH_VARARGS, "as_css() -> Return the CSS representation of this token"},
112
113 {NULL} /* Sentinel */
114 };
115
116 static PyTypeObject tokenizer_TokenType = {
117 // {{{
118 PyVarObject_HEAD_INIT(NULL, 0)
119 /* tp_name */ "tokenizer.Token",
120 /* tp_basicsize */ sizeof(tokenizer_Token),
121 /* tp_itemsize */ 0,
122 /* tp_dealloc */ (destructor)tokenizer_Token_dealloc,
123 /* tp_print */ 0,
124 /* tp_getattr */ 0,
125 /* tp_setattr */ 0,
126 /* tp_compare */ 0,
127 /* tp_repr */ (reprfunc)tokenizer_Token_repr,
128 /* tp_as_number */ 0,
129 /* tp_as_sequence */ 0,
130 /* tp_as_mapping */ 0,
131 /* tp_hash */ 0,
132 /* tp_call */ 0,
133 /* tp_str */ 0,
134 /* tp_getattro */ 0,
135 /* tp_setattro */ 0,
136 /* tp_as_buffer */ 0,
137 /* tp_flags */ Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE,
138 /* tp_doc */ "Token",
139 /* tp_traverse */ 0,
140 /* tp_clear */ 0,
141 /* tp_richcompare */ 0,
142 /* tp_weaklistoffset */ 0,
143 /* tp_iter */ 0,
144 /* tp_iternext */ 0,
145 /* tp_methods */ tokenizer_Token_methods,
146 /* tp_members */ tokenizer_Token_members,
147 /* tp_getset */ 0,
148 /* tp_base */ 0,
149 /* tp_dict */ 0,
150 /* tp_descr_get */ 0,
151 /* tp_descr_set */ 0,
152 /* tp_dictoffset */ 0,
153 /* tp_init */ 0,
154 /* tp_alloc */ 0,
155 /* tp_new */ tokenizer_Token_new,
156 }; // }}}
157 // }}}
158
159 static PyObject *COMPILED_TOKEN_REGEXPS = NULL, *UNICODE_UNESCAPE = NULL, *NEWLINE_UNESCAPE = NULL, *SIMPLE_UNESCAPE = NULL, *FIND_NEWLINES = NULL,
160 *TOKEN_DISPATCH = NULL;
161 static PyObject *COLON = NULL, *SCOLON = NULL, *LPAR = NULL, *RPAR = NULL, *LBRACE = NULL, *RBRACE = NULL, *LBOX = NULL, *RBOX = NULL, *DELIM_TOK = NULL,
162 *INTEGER = NULL, *STRING_TOK = NULL;
163
164 static Py_ssize_t BAD_COMMENT, BAD_STRING, PERCENTAGE, DIMENSION, ATKEYWORD, FUNCTION, COMMENT, NUMBER, STRING, IDENT, HASH, URI, DELIM = -1;
165
166 #define CLEANUP(x) \
167 Py_XDECREF((x)); \
168 x = NULL;
169
170 static PyObject *
171 tokenize_cleanup(PyObject *self, PyObject *args) {
172 CLEANUP(COMPILED_TOKEN_REGEXPS);
173 CLEANUP(UNICODE_UNESCAPE);
174 CLEANUP(NEWLINE_UNESCAPE);
175 CLEANUP(SIMPLE_UNESCAPE);
176 CLEANUP(FIND_NEWLINES);
177 CLEANUP(TOKEN_DISPATCH);
178 CLEANUP(COLON);
179 CLEANUP(SCOLON);
180 CLEANUP(LPAR);
181 CLEANUP(RPAR);
182 CLEANUP(LBRACE);
183 CLEANUP(RBRACE);
184 CLEANUP(LBOX);
185 CLEANUP(RBOX);
186 CLEANUP(DELIM_TOK);
187 CLEANUP(INTEGER);
188 CLEANUP(STRING_TOK);
189 Py_RETURN_NONE;
190 }
191
192 static PyObject *
193 tokenize_init(PyObject *self, PyObject *args) {
194 PyObject *cti = NULL;
195
196 if (COMPILED_TOKEN_REGEXPS != NULL) { tokenize_cleanup(NULL, NULL); }
197 if (!PyArg_ParseTuple(
198 args,
199 "OOOOOOOOOOOOOOOOOO",
200 &COMPILED_TOKEN_REGEXPS,
201 &UNICODE_UNESCAPE,
202 &NEWLINE_UNESCAPE,
203 &SIMPLE_UNESCAPE,
204 &FIND_NEWLINES,
205 &TOKEN_DISPATCH,
206 &cti,
207 &COLON,
208 &SCOLON,
209 &LPAR,
210 &RPAR,
211 &LBRACE,
212 &RBRACE,
213 &LBOX,
214 &RBOX,
215 &DELIM_TOK,
216 &INTEGER,
217 &STRING_TOK))
218 return NULL;
219 Py_INCREF(COMPILED_TOKEN_REGEXPS);
220 Py_INCREF(UNICODE_UNESCAPE);
221 Py_INCREF(NEWLINE_UNESCAPE);
222 Py_INCREF(SIMPLE_UNESCAPE);
223 Py_INCREF(FIND_NEWLINES);
224 Py_INCREF(TOKEN_DISPATCH);
225 Py_INCREF(COLON);
226 Py_INCREF(SCOLON);
227 Py_INCREF(LPAR);
228 Py_INCREF(RPAR);
229 Py_INCREF(LBRACE);
230 Py_INCREF(RBRACE);
231 Py_INCREF(LBOX);
232 Py_INCREF(RBOX);
233 Py_INCREF(DELIM_TOK);
234 Py_INCREF(INTEGER);
235 Py_INCREF(STRING_TOK);
236
237 #define SETCONST(x) \
238 do { \
239 (x) = PyNumber_AsSsize_t(PyDict_GetItemString(cti, #x), PyExc_OverflowError); \
240 if ((x) == -1 && PyErr_Occurred() != NULL) { return NULL; } \
241 } while (0)
242 SETCONST(BAD_COMMENT);
243 SETCONST(BAD_STRING);
244 SETCONST(PERCENTAGE);
245 SETCONST(DIMENSION);
246 SETCONST(ATKEYWORD);
247 SETCONST(FUNCTION);
248 SETCONST(COMMENT);
249 SETCONST(NUMBER);
250 SETCONST(STRING);
251 SETCONST(IDENT);
252 SETCONST(HASH);
253 SETCONST(URI);
254
255 Py_RETURN_NONE;
256 }
257
258 #if PY_VERSION_HEX >= 0x03030000
259 #define ITER_CODE_PTS(unicode_object) \
260 { \
261 int _kind = PyUnicode_KIND(unicode_object); \
262 void *_data = PyUnicode_DATA(unicode_object); \
263 for (Py_ssize_t iteridx = 0; iteridx < PyUnicode_GET_LENGTH(unicode_object); iteridx++) { \
264 Py_UCS4 ch = PyUnicode_READ(_kind, _data, iteridx);
265 #else
266 #define PyUnicode_GET_LENGTH PyUnicode_GET_SIZE
267 #define ITER_CODE_PTS(unicode_object) \
268 { \
269 Py_UNICODE *_data = PyUnicode_AS_UNICODE(unicode_object); \
270 Py_ssize_t iteridx; \
271 for (iteridx = 0; iteridx < PyUnicode_GET_LENGTH(unicode_object); iteridx++) { \
272 Py_UNICODE ch = _data[iteridx];
273 #endif
274
275 #define END_ITER_CODE_PTS \
276 } \
277 }
278
279 static PyObject *
280 unicode_to_number(PyObject *src) {
281 PyObject *ans = PyFloat_FromString(src);
282 double val = PyFloat_AsDouble(ans);
283 long lval = (long)val;
284 if (val - lval != 0) return ans;
285 Py_DECREF(ans);
286 return PyLong_FromLong(lval);
287 }
288
289
290 static void
291 lowercase(PyObject *x) {
292 ITER_CODE_PTS(x)
293 if ('A' <= ch && ch <= 'Z') {
294 #if PY_VERSION_HEX >= 0x03030000
295 PyUnicode_WRITE(_kind, _data, iteridx, ch + 32);
296 #else
297 _data[iteridx] += 32;
298 #endif
299 }
300 END_ITER_CODE_PTS
301 }
302
303 static PyObject *
304 clone_unicode(const PyObject *src, Py_ssize_t start_offset, Py_ssize_t end_offset) {
305 #if PY_VERSION_HEX >= 0x03030000
306 int kind = PyUnicode_KIND(src);
307 void *data;
308 switch (kind) {
309 case PyUnicode_1BYTE_KIND: data = PyUnicode_1BYTE_DATA(src) + start_offset; break;
310 case PyUnicode_2BYTE_KIND: data = PyUnicode_2BYTE_DATA(src) + start_offset; break;
311 case PyUnicode_4BYTE_KIND: data = PyUnicode_4BYTE_DATA(src) + start_offset; break;
312 default: PyErr_SetString(PyExc_RuntimeError, "Invalid byte kind for unicode object"); return NULL;
313 }
314 return PyUnicode_FromKindAndData(kind, data, PyUnicode_GET_LENGTH(src) - start_offset - end_offset);
315 #else
316 return PyUnicode_FromUnicode(PyUnicode_AS_UNICODE(src) + start_offset, PyUnicode_GET_LENGTH(src) - start_offset - end_offset);
317 #endif
318 }
319
320 static PyObject *
321 tokenize_flat(PyObject *self, PyObject *args) {
322 #if PY_VERSION_HEX >= 0x03030000
323 void *css_source = NULL;
324 int css_kind;
325 Py_UCS4 c = 0, codepoint = 0;
326 #define first_char(string) PyUnicode_READ_CHAR(string, 0)
327 #define unicode_from_data(data, sz) PyUnicode_FromKindAndData(css_kind, data, sz)
328 #else
329 Py_UNICODE *css_source = NULL, c = 0, codepoint = 0;
330 #define first_char(string) PyUnicode_AS_UNICODE(string)[0]
331 #define unicode_from_data(data, sz) PyUnicode_FromUnicode(data, sz)
332 #endif
333 PyObject *ic = NULL, *token = NULL, *tokens = NULL, *type_name = NULL, *css_value = NULL, *value = NULL, *unit = NULL, *tries = NULL, *match = NULL,
334 *match_func = NULL, *py_source = NULL, *item = NULL, *newlines = NULL;
335 int ignore_comments = 0;
336 Py_ssize_t pos = 0, line = 1, column = 1, i = 0;
337 Py_ssize_t length = 0, next_pos = 0, type_ = -1, source_len = 0;
338
339
340 if (COMPILED_TOKEN_REGEXPS == NULL) {
341 PyErr_SetString(PyExc_RuntimeError, "tokenizer module not initialized. You must call init() first.");
342 return NULL;
343 }
344
345 if (!PyArg_ParseTuple(args, "UO", &py_source, &ic)) return NULL;
346 if (PyObject_IsTrue(ic)) ignore_comments = 1;
347 #if PY_VERSION_HEX >= 0x03030000
348 if (PyUnicode_READY(py_source) != 0) return NULL;
349 css_source = PyUnicode_DATA(py_source);
350 css_kind = PyUnicode_KIND(py_source);
351 #else
352 css_source = PyUnicode_AS_UNICODE(py_source);
353 #endif
354 source_len = PyUnicode_GET_LENGTH(py_source);
355
356 tokens = PyList_New(0);
357 if (tokens == NULL) return PyErr_NoMemory();
358
359 #define UNESCAPE(x, func) \
360 item = PyObject_CallFunctionObjArgs(func, x, NULL); \
361 if (item == NULL) { goto error; } \
362 Py_DECREF(x); \
363 x = item; \
364 item = NULL;
365
366 #define TONUMBER(x) \
367 item = unicode_to_number(x); \
368 if (item == NULL) goto error; \
369 Py_DECREF(x); \
370 x = item; \
371 item = NULL;
372
373 #define SINGLE(x) \
374 { \
375 type_ = -1; \
376 type_name = x; \
377 Py_INCREF(type_name); \
378 css_value = x; \
379 Py_INCREF(css_value); \
380 }
381
382 while (pos < source_len) {
383 #if PY_VERSION_HEX >= 0x03030000
384 c = PyUnicode_READ(css_kind, css_source, pos);
385 #else
386 c = css_source[pos];
387 #endif
388
389 css_value = NULL;
390 type_name = NULL;
391 value = NULL;
392 unit = NULL;
393 match = NULL;
394
395 if (c == ':') SINGLE(COLON)
396 else if (c == ';') SINGLE(SCOLON)
397 else if (c == '(') SINGLE(LPAR)
398 else if (c == ')') SINGLE(RPAR)
399 else if (c == '{') SINGLE(LBRACE)
400 else if (c == '}') SINGLE(RBRACE)
401 else if (c == '[') SINGLE(LBOX)
402 else if (c == ']') SINGLE(RBOX)
403 else {
404 codepoint = (c > 160) ? 160 : c;
405 tries = PyList_GET_ITEM(TOKEN_DISPATCH, codepoint);
406 for (i = 0; i < PyList_Size(tries); i++) {
407 item = PyList_GET_ITEM(tries, i);
408 match_func = PyTuple_GET_ITEM(item, 2);
409 match = PyObject_CallFunction(match_func, "On", py_source, pos);
410 if (match == NULL) { goto error; }
411 if (match != Py_None) {
412 css_value = PyObject_CallMethod(match, "group", NULL);
413 if (css_value == NULL) { goto error; }
414 type_ = PyNumber_AsSsize_t(PyTuple_GET_ITEM(item, 0), PyExc_OverflowError);
415 if (type_ == -1 && PyErr_Occurred() != NULL) { goto error; }
416 type_name = PyTuple_GET_ITEM(item, 1);
417 Py_INCREF(type_name);
418 break;
419 }
420 }
421 if (css_value == NULL) { // No match
422 type_ = DELIM;
423 type_name = DELIM_TOK;
424 Py_INCREF(type_name);
425 css_value = unicode_from_data(&c, 1);
426 if (css_value == NULL) { goto error; }
427 }
428 }
429
430 length = PyUnicode_GET_LENGTH(css_value);
431 next_pos = pos + length;
432
433 // Now calculate the value and unit for this token (if any)
434 if (!(ignore_comments && (type_ == COMMENT || type_ == BAD_COMMENT))) {
435 if (type_ == DIMENSION) {
436 value = PyObject_CallMethod(match, "group", "I", 1);
437 if (value == NULL) { goto error; }
438 TONUMBER(value);
439 unit = PyObject_CallMethod(match, "group", "I", 2);
440 if (unit == NULL) { goto error; }
441 UNESCAPE(unit, SIMPLE_UNESCAPE);
442 UNESCAPE(unit, UNICODE_UNESCAPE);
443 lowercase(unit);
444 } else
445
446 if (type_ == PERCENTAGE) {
447 if (PyUnicode_GET_LENGTH(css_value) > 0) {
448 value = clone_unicode(css_value, 0, 1);
449 if (value == NULL) goto error;
450 } else {
451 value = css_value;
452 Py_INCREF(value);
453 }
454 if (value == NULL) goto error;
455 TONUMBER(value);
456 unit = PyUnicode_FromString("%");
457 if (unit == NULL) goto error;
458 } else
459
460 if (type_ == NUMBER) {
461 value = css_value;
462 Py_INCREF(value);
463 TONUMBER(value);
464 if (!PyFloat_Check(value)) {
465 Py_XDECREF(type_name);
466 type_name = INTEGER;
467 Py_INCREF(type_name);
468 }
469 } else
470
471 if (type_ == IDENT || type_ == ATKEYWORD || type_ == HASH || type_ == FUNCTION) {
472 value = PyObject_CallFunctionObjArgs(SIMPLE_UNESCAPE, css_value, NULL);
473 if (value == NULL) goto error;
474 UNESCAPE(value, UNICODE_UNESCAPE);
475 } else
476
477 if (type_ == URI) {
478 value = PyObject_CallMethod(match, "group", "I", 1);
479 if (value == NULL) { goto error; }
480 if (PyObject_IsTrue(value) && PyUnicode_GET_LENGTH(value) > 1 && (first_char(value) == '"' || first_char(value) == '\'')) {
481 item = clone_unicode(value, 1, 1);
482 if (item == NULL) goto error;
483 Py_DECREF(value);
484 value = item;
485 item = NULL;
486 UNESCAPE(value, NEWLINE_UNESCAPE);
487 }
488 UNESCAPE(value, SIMPLE_UNESCAPE);
489 UNESCAPE(value, UNICODE_UNESCAPE);
490 } else
491
492 if (type_ == STRING) {
493 if (PyObject_IsTrue(css_value) && PyUnicode_GET_LENGTH(css_value) > 1) { // remove quotes
494 value = clone_unicode(css_value, 1, 1);
495 } else {
496 value = css_value;
497 Py_INCREF(value);
498 }
499 UNESCAPE(value, NEWLINE_UNESCAPE);
500 UNESCAPE(value, SIMPLE_UNESCAPE);
501 UNESCAPE(value, UNICODE_UNESCAPE);
502 } else
503
504 if (type_ == BAD_STRING && next_pos == source_len) {
505 Py_XDECREF(type_name);
506 type_name = STRING_TOK;
507 Py_INCREF(type_name);
508 if (PyObject_IsTrue(css_value) && PyUnicode_GET_LENGTH(css_value) > 0) { // remove quote
509 value = clone_unicode(css_value, 1, 0);
510 } else {
511 value = css_value;
512 Py_INCREF(value);
513 }
514 UNESCAPE(value, NEWLINE_UNESCAPE);
515 UNESCAPE(value, SIMPLE_UNESCAPE);
516 UNESCAPE(value, UNICODE_UNESCAPE);
517 } else {
518 value = css_value;
519 Py_INCREF(value);
520 } // if(type_ == ...)
521
522 if (unit == NULL) {
523 unit = Py_None;
524 Py_INCREF(unit);
525 }
526 item = Py_BuildValue("OOOOnn", type_name, css_value, value, unit, line, column);
527 if (item == NULL) goto error;
528 token = PyObject_CallObject((PyObject *)&tokenizer_TokenType, item);
529 Py_DECREF(item);
530 item = NULL;
531 if (token == NULL) goto error;
532 if (PyList_Append(tokens, token) != 0) {
533 Py_DECREF(token);
534 token = NULL;
535 goto error;
536 }
537 Py_DECREF(token);
538
539 } // if(!(ignore_comments...
540
541 Py_XDECREF(match);
542 match = NULL;
543
544 pos = next_pos;
545 newlines = PyObject_CallFunctionObjArgs(FIND_NEWLINES, css_value, NULL);
546 if (newlines == NULL) goto error;
547 Py_XDECREF(css_value);
548 css_value = NULL;
549 Py_XDECREF(type_name);
550 type_name = NULL;
551 Py_XDECREF(value);
552 value = NULL;
553 Py_XDECREF(unit);
554 unit = NULL;
555 if (PyObject_IsTrue(newlines)) {
556 line += PyList_Size(newlines);
557 item = PyObject_CallMethod(PyList_GET_ITEM(newlines, PyList_Size(newlines) - 1), "end", NULL);
558 if (item == NULL) {
559 Py_DECREF(newlines);
560 newlines = NULL;
561 goto error;
562 }
563 column = PyNumber_AsSsize_t(item, PyExc_OverflowError);
564 if (column == -1 && PyErr_Occurred()) {
565 Py_DECREF(newlines);
566 newlines = NULL;
567 goto error;
568 }
569 column = length - column + 1;
570 Py_DECREF(item);
571 item = NULL;
572 } else column += length;
573 Py_DECREF(newlines);
574 newlines = NULL;
575
576 } // while (pos < ...)
577
578 return tokens;
579 error:
580 Py_XDECREF(tokens);
581 Py_XDECREF(css_value);
582 Py_XDECREF(type_name);
583 Py_XDECREF(value);
584 Py_XDECREF(unit);
585 Py_XDECREF(match);
586 return NULL;
587 #undef unicode_from_data
588 #undef first_char
589 }
590
591 static PyMethodDef tokenizer_methods[] = {
592 {"tokenize_flat", tokenize_flat, METH_VARARGS, "tokenize_flat(css_source, ignore_comments)\n\n Convert CSS source into a flat list of tokens"},
593
594 {"init", tokenize_init, METH_VARARGS, "init()\n\nInitialize the module."},
595
596 {"cleanup", tokenize_cleanup, METH_VARARGS, "cleanup()\n\nRelease resources allocated by init(). Safe to call multiple times."},
597
598 {NULL, NULL, 0, NULL}};
599
600 static int
601 exec_module(PyObject *mod) {
602 if (PyType_Ready(&tokenizer_TokenType) < 0) return -1;
603 Py_INCREF(&tokenizer_TokenType);
604 PyModule_AddObject(mod, "Token", (PyObject *)&tokenizer_TokenType);
605 return 0;
606 }
607 static PyModuleDef_Slot slots[] = {{Py_mod_exec, exec_module}, {0, NULL}};
608
609 static struct PyModuleDef module_def = {
610 .m_base = PyModuleDef_HEAD_INIT,
611 .m_name = "tokenizer",
612 .m_doc = "Implementation of tokenizer in C for speed.",
613 .m_methods = tokenizer_methods,
614 .m_slots = slots,
615 };
616
617 CALIBRE_MODINIT_FUNC
618 PyInit_tokenizer(void) { return PyModuleDef_Init(&module_def); }