| 1 |
/* |
| 2 |
* tokenizer.c |
| 3 |
* Copyright (C) 2014 Kovid Goyal <kovid at kovidgoyal.net> |
| 4 |
* |
| 5 |
* Distributed under terms of the GPL3 license. |
| 6 |
*/ |
| 7 |
|
| 8 |
#define UNICODE |
| 9 |
#define PY_SSIZE_T_CLEAN |
| 10 |
#include <Python.h> |
| 11 |
#include <structmember.h> |
| 12 |
|
| 13 |
// Token type definition {{{ |
| 14 |
typedef struct { |
| 15 |
PyObject_HEAD |
| 16 |
// Type-specific fields go here. |
| 17 |
PyObject *is_container; |
| 18 |
PyObject *type; |
| 19 |
PyObject *_as_css; |
| 20 |
PyObject *value; |
| 21 |
PyObject *unit; |
| 22 |
PyObject *line; |
| 23 |
PyObject *column; |
| 24 |
|
| 25 |
} tokenizer_Token; |
| 26 |
|
| 27 |
static void |
| 28 |
tokenizer_Token_dealloc(tokenizer_Token *self) { |
| 29 |
Py_XDECREF(self->is_container); |
| 30 |
self->is_container = NULL; |
| 31 |
Py_XDECREF(self->type); |
| 32 |
self->type = NULL; |
| 33 |
Py_XDECREF(self->_as_css); |
| 34 |
self->_as_css = NULL; |
| 35 |
Py_XDECREF(self->value); |
| 36 |
self->value = NULL; |
| 37 |
Py_XDECREF(self->unit); |
| 38 |
self->unit = NULL; |
| 39 |
Py_XDECREF(self->line); |
| 40 |
self->line = NULL; |
| 41 |
Py_XDECREF(self->column); |
| 42 |
self->column = NULL; |
| 43 |
Py_TYPE(self)->tp_free((PyObject *)self); |
| 44 |
} |
| 45 |
|
| 46 |
|
| 47 |
static PyObject * |
| 48 |
tokenizer_Token_new(PyTypeObject *type, PyObject *args, PyObject *kwds) { |
| 49 |
tokenizer_Token *self = NULL; |
| 50 |
self = (tokenizer_Token *)type->tp_alloc(type, 0); |
| 51 |
if (self == NULL) return PyErr_NoMemory(); |
| 52 |
|
| 53 |
if (!PyArg_ParseTuple(args, "OOOOOO", &(self->type), &(self->_as_css), &(self->value), &(self->unit), &(self->line), &(self->column))) { |
| 54 |
Py_TYPE(self)->tp_free((PyObject *)self); |
| 55 |
return NULL; |
| 56 |
} |
| 57 |
Py_INCREF(self->type); |
| 58 |
Py_INCREF(self->_as_css); |
| 59 |
Py_INCREF(self->value); |
| 60 |
Py_INCREF(self->unit); |
| 61 |
Py_INCREF(self->line); |
| 62 |
Py_INCREF(self->column); |
| 63 |
self->is_container = Py_False; |
| 64 |
Py_INCREF(self->is_container); |
| 65 |
|
| 66 |
return (PyObject *)self; |
| 67 |
} |
| 68 |
|
| 69 |
static PyObject * |
| 70 |
tokenizer_Token_repr(tokenizer_Token *self) { |
| 71 |
PyObject *type = NULL, *line = NULL, *column = NULL, *value = NULL, *ans = NULL, *unit = NULL; |
| 72 |
if (!self->type || !self->line || !self->column || !self->value) return PyBytes_FromString("<Token NULL fields>"); |
| 73 |
type = PyObject_Str(self->type); |
| 74 |
line = PyObject_Str(self->line); |
| 75 |
column = PyObject_Str(self->column); |
| 76 |
value = PyObject_Str(self->value); |
| 77 |
if (type && line && column && value) { |
| 78 |
if (self->unit != NULL && PyObject_IsTrue(self->unit)) { |
| 79 |
unit = PyObject_Str(self->unit); |
| 80 |
if (unit != NULL) ans = PyUnicode_FromFormat("<Token %U at %U:%U %U%U>", type, line, column, value, unit); |
| 81 |
else PyErr_NoMemory(); |
| 82 |
} else ans = PyUnicode_FromFormat("<Token %U at %U:%U %U>", type, line, column, value); |
| 83 |
} else PyErr_NoMemory(); |
| 84 |
Py_XDECREF(type); |
| 85 |
Py_XDECREF(line); |
| 86 |
Py_XDECREF(column); |
| 87 |
Py_XDECREF(value); |
| 88 |
Py_XDECREF(unit); |
| 89 |
return ans; |
| 90 |
} |
| 91 |
|
| 92 |
static PyObject * |
| 93 |
tokenizer_Token_as_css(tokenizer_Token *self, PyObject *args, PyObject *kwargs) { |
| 94 |
if (!self->_as_css) { Py_RETURN_NONE; } |
| 95 |
Py_INCREF(self->_as_css); |
| 96 |
return self->_as_css; |
| 97 |
} |
| 98 |
|
| 99 |
static PyMemberDef tokenizer_Token_members[] = { |
| 100 |
{"is_container", T_OBJECT_EX, offsetof(tokenizer_Token, is_container), 0, "False unless this token is a container for other tokens"}, |
| 101 |
{"type", T_OBJECT_EX, offsetof(tokenizer_Token, type), 0, "The token type"}, |
| 102 |
{"_as_css", T_OBJECT_EX, offsetof(tokenizer_Token, _as_css), 0, "Internal variable, use as_css() method instead."}, |
| 103 |
{"value", T_OBJECT_EX, offsetof(tokenizer_Token, value), 0, "The token value"}, |
| 104 |
{"unit", T_OBJECT_EX, offsetof(tokenizer_Token, unit), 0, "The token unit"}, |
| 105 |
{"line", T_OBJECT_EX, offsetof(tokenizer_Token, line), 0, "The token line number"}, |
| 106 |
{"column", T_OBJECT_EX, offsetof(tokenizer_Token, column), 0, "The token column number"}, |
| 107 |
{NULL} /* Sentinel */ |
| 108 |
}; |
| 109 |
|
| 110 |
static PyMethodDef tokenizer_Token_methods[] = { |
| 111 |
{"as_css", (PyCFunction)tokenizer_Token_as_css, METH_VARARGS, "as_css() -> Return the CSS representation of this token"}, |
| 112 |
|
| 113 |
{NULL} /* Sentinel */ |
| 114 |
}; |
| 115 |
|
| 116 |
static PyTypeObject tokenizer_TokenType = { |
| 117 |
// {{{ |
| 118 |
PyVarObject_HEAD_INIT(NULL, 0) |
| 119 |
/* tp_name */ "tokenizer.Token", |
| 120 |
/* tp_basicsize */ sizeof(tokenizer_Token), |
| 121 |
/* tp_itemsize */ 0, |
| 122 |
/* tp_dealloc */ (destructor)tokenizer_Token_dealloc, |
| 123 |
/* tp_print */ 0, |
| 124 |
/* tp_getattr */ 0, |
| 125 |
/* tp_setattr */ 0, |
| 126 |
/* tp_compare */ 0, |
| 127 |
/* tp_repr */ (reprfunc)tokenizer_Token_repr, |
| 128 |
/* tp_as_number */ 0, |
| 129 |
/* tp_as_sequence */ 0, |
| 130 |
/* tp_as_mapping */ 0, |
| 131 |
/* tp_hash */ 0, |
| 132 |
/* tp_call */ 0, |
| 133 |
/* tp_str */ 0, |
| 134 |
/* tp_getattro */ 0, |
| 135 |
/* tp_setattro */ 0, |
| 136 |
/* tp_as_buffer */ 0, |
| 137 |
/* tp_flags */ Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE, |
| 138 |
/* tp_doc */ "Token", |
| 139 |
/* tp_traverse */ 0, |
| 140 |
/* tp_clear */ 0, |
| 141 |
/* tp_richcompare */ 0, |
| 142 |
/* tp_weaklistoffset */ 0, |
| 143 |
/* tp_iter */ 0, |
| 144 |
/* tp_iternext */ 0, |
| 145 |
/* tp_methods */ tokenizer_Token_methods, |
| 146 |
/* tp_members */ tokenizer_Token_members, |
| 147 |
/* tp_getset */ 0, |
| 148 |
/* tp_base */ 0, |
| 149 |
/* tp_dict */ 0, |
| 150 |
/* tp_descr_get */ 0, |
| 151 |
/* tp_descr_set */ 0, |
| 152 |
/* tp_dictoffset */ 0, |
| 153 |
/* tp_init */ 0, |
| 154 |
/* tp_alloc */ 0, |
| 155 |
/* tp_new */ tokenizer_Token_new, |
| 156 |
}; // }}} |
| 157 |
// }}} |
| 158 |
|
| 159 |
static PyObject *COMPILED_TOKEN_REGEXPS = NULL, *UNICODE_UNESCAPE = NULL, *NEWLINE_UNESCAPE = NULL, *SIMPLE_UNESCAPE = NULL, *FIND_NEWLINES = NULL, |
| 160 |
*TOKEN_DISPATCH = NULL; |
| 161 |
static PyObject *COLON = NULL, *SCOLON = NULL, *LPAR = NULL, *RPAR = NULL, *LBRACE = NULL, *RBRACE = NULL, *LBOX = NULL, *RBOX = NULL, *DELIM_TOK = NULL, |
| 162 |
*INTEGER = NULL, *STRING_TOK = NULL; |
| 163 |
|
| 164 |
static Py_ssize_t BAD_COMMENT, BAD_STRING, PERCENTAGE, DIMENSION, ATKEYWORD, FUNCTION, COMMENT, NUMBER, STRING, IDENT, HASH, URI, DELIM = -1; |
| 165 |
|
| 166 |
#define CLEANUP(x) \ |
| 167 |
Py_XDECREF((x)); \ |
| 168 |
x = NULL; |
| 169 |
|
| 170 |
static PyObject * |
| 171 |
tokenize_cleanup(PyObject *self, PyObject *args) { |
| 172 |
CLEANUP(COMPILED_TOKEN_REGEXPS); |
| 173 |
CLEANUP(UNICODE_UNESCAPE); |
| 174 |
CLEANUP(NEWLINE_UNESCAPE); |
| 175 |
CLEANUP(SIMPLE_UNESCAPE); |
| 176 |
CLEANUP(FIND_NEWLINES); |
| 177 |
CLEANUP(TOKEN_DISPATCH); |
| 178 |
CLEANUP(COLON); |
| 179 |
CLEANUP(SCOLON); |
| 180 |
CLEANUP(LPAR); |
| 181 |
CLEANUP(RPAR); |
| 182 |
CLEANUP(LBRACE); |
| 183 |
CLEANUP(RBRACE); |
| 184 |
CLEANUP(LBOX); |
| 185 |
CLEANUP(RBOX); |
| 186 |
CLEANUP(DELIM_TOK); |
| 187 |
CLEANUP(INTEGER); |
| 188 |
CLEANUP(STRING_TOK); |
| 189 |
Py_RETURN_NONE; |
| 190 |
} |
| 191 |
|
| 192 |
static PyObject * |
| 193 |
tokenize_init(PyObject *self, PyObject *args) { |
| 194 |
PyObject *cti = NULL; |
| 195 |
|
| 196 |
if (COMPILED_TOKEN_REGEXPS != NULL) { tokenize_cleanup(NULL, NULL); } |
| 197 |
if (!PyArg_ParseTuple( |
| 198 |
args, |
| 199 |
"OOOOOOOOOOOOOOOOOO", |
| 200 |
&COMPILED_TOKEN_REGEXPS, |
| 201 |
&UNICODE_UNESCAPE, |
| 202 |
&NEWLINE_UNESCAPE, |
| 203 |
&SIMPLE_UNESCAPE, |
| 204 |
&FIND_NEWLINES, |
| 205 |
&TOKEN_DISPATCH, |
| 206 |
&cti, |
| 207 |
&COLON, |
| 208 |
&SCOLON, |
| 209 |
&LPAR, |
| 210 |
&RPAR, |
| 211 |
&LBRACE, |
| 212 |
&RBRACE, |
| 213 |
&LBOX, |
| 214 |
&RBOX, |
| 215 |
&DELIM_TOK, |
| 216 |
&INTEGER, |
| 217 |
&STRING_TOK)) |
| 218 |
return NULL; |
| 219 |
Py_INCREF(COMPILED_TOKEN_REGEXPS); |
| 220 |
Py_INCREF(UNICODE_UNESCAPE); |
| 221 |
Py_INCREF(NEWLINE_UNESCAPE); |
| 222 |
Py_INCREF(SIMPLE_UNESCAPE); |
| 223 |
Py_INCREF(FIND_NEWLINES); |
| 224 |
Py_INCREF(TOKEN_DISPATCH); |
| 225 |
Py_INCREF(COLON); |
| 226 |
Py_INCREF(SCOLON); |
| 227 |
Py_INCREF(LPAR); |
| 228 |
Py_INCREF(RPAR); |
| 229 |
Py_INCREF(LBRACE); |
| 230 |
Py_INCREF(RBRACE); |
| 231 |
Py_INCREF(LBOX); |
| 232 |
Py_INCREF(RBOX); |
| 233 |
Py_INCREF(DELIM_TOK); |
| 234 |
Py_INCREF(INTEGER); |
| 235 |
Py_INCREF(STRING_TOK); |
| 236 |
|
| 237 |
#define SETCONST(x) \ |
| 238 |
do { \ |
| 239 |
(x) = PyNumber_AsSsize_t(PyDict_GetItemString(cti, #x), PyExc_OverflowError); \ |
| 240 |
if ((x) == -1 && PyErr_Occurred() != NULL) { return NULL; } \ |
| 241 |
} while (0) |
| 242 |
SETCONST(BAD_COMMENT); |
| 243 |
SETCONST(BAD_STRING); |
| 244 |
SETCONST(PERCENTAGE); |
| 245 |
SETCONST(DIMENSION); |
| 246 |
SETCONST(ATKEYWORD); |
| 247 |
SETCONST(FUNCTION); |
| 248 |
SETCONST(COMMENT); |
| 249 |
SETCONST(NUMBER); |
| 250 |
SETCONST(STRING); |
| 251 |
SETCONST(IDENT); |
| 252 |
SETCONST(HASH); |
| 253 |
SETCONST(URI); |
| 254 |
|
| 255 |
Py_RETURN_NONE; |
| 256 |
} |
| 257 |
|
| 258 |
#if PY_VERSION_HEX >= 0x03030000 |
| 259 |
#define ITER_CODE_PTS(unicode_object) \ |
| 260 |
{ \ |
| 261 |
int _kind = PyUnicode_KIND(unicode_object); \ |
| 262 |
void *_data = PyUnicode_DATA(unicode_object); \ |
| 263 |
for (Py_ssize_t iteridx = 0; iteridx < PyUnicode_GET_LENGTH(unicode_object); iteridx++) { \ |
| 264 |
Py_UCS4 ch = PyUnicode_READ(_kind, _data, iteridx); |
| 265 |
#else |
| 266 |
#define PyUnicode_GET_LENGTH PyUnicode_GET_SIZE |
| 267 |
#define ITER_CODE_PTS(unicode_object) \ |
| 268 |
{ \ |
| 269 |
Py_UNICODE *_data = PyUnicode_AS_UNICODE(unicode_object); \ |
| 270 |
Py_ssize_t iteridx; \ |
| 271 |
for (iteridx = 0; iteridx < PyUnicode_GET_LENGTH(unicode_object); iteridx++) { \ |
| 272 |
Py_UNICODE ch = _data[iteridx]; |
| 273 |
#endif |
| 274 |
|
| 275 |
#define END_ITER_CODE_PTS \ |
| 276 |
} \ |
| 277 |
} |
| 278 |
|
| 279 |
static PyObject * |
| 280 |
unicode_to_number(PyObject *src) { |
| 281 |
PyObject *ans = PyFloat_FromString(src); |
| 282 |
double val = PyFloat_AsDouble(ans); |
| 283 |
long lval = (long)val; |
| 284 |
if (val - lval != 0) return ans; |
| 285 |
Py_DECREF(ans); |
| 286 |
return PyLong_FromLong(lval); |
| 287 |
} |
| 288 |
|
| 289 |
|
| 290 |
static void |
| 291 |
lowercase(PyObject *x) { |
| 292 |
ITER_CODE_PTS(x) |
| 293 |
if ('A' <= ch && ch <= 'Z') { |
| 294 |
#if PY_VERSION_HEX >= 0x03030000 |
| 295 |
PyUnicode_WRITE(_kind, _data, iteridx, ch + 32); |
| 296 |
#else |
| 297 |
_data[iteridx] += 32; |
| 298 |
#endif |
| 299 |
} |
| 300 |
END_ITER_CODE_PTS |
| 301 |
} |
| 302 |
|
| 303 |
static PyObject * |
| 304 |
clone_unicode(const PyObject *src, Py_ssize_t start_offset, Py_ssize_t end_offset) { |
| 305 |
#if PY_VERSION_HEX >= 0x03030000 |
| 306 |
int kind = PyUnicode_KIND(src); |
| 307 |
void *data; |
| 308 |
switch (kind) { |
| 309 |
case PyUnicode_1BYTE_KIND: data = PyUnicode_1BYTE_DATA(src) + start_offset; break; |
| 310 |
case PyUnicode_2BYTE_KIND: data = PyUnicode_2BYTE_DATA(src) + start_offset; break; |
| 311 |
case PyUnicode_4BYTE_KIND: data = PyUnicode_4BYTE_DATA(src) + start_offset; break; |
| 312 |
default: PyErr_SetString(PyExc_RuntimeError, "Invalid byte kind for unicode object"); return NULL; |
| 313 |
} |
| 314 |
return PyUnicode_FromKindAndData(kind, data, PyUnicode_GET_LENGTH(src) - start_offset - end_offset); |
| 315 |
#else |
| 316 |
return PyUnicode_FromUnicode(PyUnicode_AS_UNICODE(src) + start_offset, PyUnicode_GET_LENGTH(src) - start_offset - end_offset); |
| 317 |
#endif |
| 318 |
} |
| 319 |
|
| 320 |
static PyObject * |
| 321 |
tokenize_flat(PyObject *self, PyObject *args) { |
| 322 |
#if PY_VERSION_HEX >= 0x03030000 |
| 323 |
void *css_source = NULL; |
| 324 |
int css_kind; |
| 325 |
Py_UCS4 c = 0, codepoint = 0; |
| 326 |
#define first_char(string) PyUnicode_READ_CHAR(string, 0) |
| 327 |
#define unicode_from_data(data, sz) PyUnicode_FromKindAndData(css_kind, data, sz) |
| 328 |
#else |
| 329 |
Py_UNICODE *css_source = NULL, c = 0, codepoint = 0; |
| 330 |
#define first_char(string) PyUnicode_AS_UNICODE(string)[0] |
| 331 |
#define unicode_from_data(data, sz) PyUnicode_FromUnicode(data, sz) |
| 332 |
#endif |
| 333 |
PyObject *ic = NULL, *token = NULL, *tokens = NULL, *type_name = NULL, *css_value = NULL, *value = NULL, *unit = NULL, *tries = NULL, *match = NULL, |
| 334 |
*match_func = NULL, *py_source = NULL, *item = NULL, *newlines = NULL; |
| 335 |
int ignore_comments = 0; |
| 336 |
Py_ssize_t pos = 0, line = 1, column = 1, i = 0; |
| 337 |
Py_ssize_t length = 0, next_pos = 0, type_ = -1, source_len = 0; |
| 338 |
|
| 339 |
|
| 340 |
if (COMPILED_TOKEN_REGEXPS == NULL) { |
| 341 |
PyErr_SetString(PyExc_RuntimeError, "tokenizer module not initialized. You must call init() first."); |
| 342 |
return NULL; |
| 343 |
} |
| 344 |
|
| 345 |
if (!PyArg_ParseTuple(args, "UO", &py_source, &ic)) return NULL; |
| 346 |
if (PyObject_IsTrue(ic)) ignore_comments = 1; |
| 347 |
#if PY_VERSION_HEX >= 0x03030000 |
| 348 |
if (PyUnicode_READY(py_source) != 0) return NULL; |
| 349 |
css_source = PyUnicode_DATA(py_source); |
| 350 |
css_kind = PyUnicode_KIND(py_source); |
| 351 |
#else |
| 352 |
css_source = PyUnicode_AS_UNICODE(py_source); |
| 353 |
#endif |
| 354 |
source_len = PyUnicode_GET_LENGTH(py_source); |
| 355 |
|
| 356 |
tokens = PyList_New(0); |
| 357 |
if (tokens == NULL) return PyErr_NoMemory(); |
| 358 |
|
| 359 |
#define UNESCAPE(x, func) \ |
| 360 |
item = PyObject_CallFunctionObjArgs(func, x, NULL); \ |
| 361 |
if (item == NULL) { goto error; } \ |
| 362 |
Py_DECREF(x); \ |
| 363 |
x = item; \ |
| 364 |
item = NULL; |
| 365 |
|
| 366 |
#define TONUMBER(x) \ |
| 367 |
item = unicode_to_number(x); \ |
| 368 |
if (item == NULL) goto error; \ |
| 369 |
Py_DECREF(x); \ |
| 370 |
x = item; \ |
| 371 |
item = NULL; |
| 372 |
|
| 373 |
#define SINGLE(x) \ |
| 374 |
{ \ |
| 375 |
type_ = -1; \ |
| 376 |
type_name = x; \ |
| 377 |
Py_INCREF(type_name); \ |
| 378 |
css_value = x; \ |
| 379 |
Py_INCREF(css_value); \ |
| 380 |
} |
| 381 |
|
| 382 |
while (pos < source_len) { |
| 383 |
#if PY_VERSION_HEX >= 0x03030000 |
| 384 |
c = PyUnicode_READ(css_kind, css_source, pos); |
| 385 |
#else |
| 386 |
c = css_source[pos]; |
| 387 |
#endif |
| 388 |
|
| 389 |
css_value = NULL; |
| 390 |
type_name = NULL; |
| 391 |
value = NULL; |
| 392 |
unit = NULL; |
| 393 |
match = NULL; |
| 394 |
|
| 395 |
if (c == ':') SINGLE(COLON) |
| 396 |
else if (c == ';') SINGLE(SCOLON) |
| 397 |
else if (c == '(') SINGLE(LPAR) |
| 398 |
else if (c == ')') SINGLE(RPAR) |
| 399 |
else if (c == '{') SINGLE(LBRACE) |
| 400 |
else if (c == '}') SINGLE(RBRACE) |
| 401 |
else if (c == '[') SINGLE(LBOX) |
| 402 |
else if (c == ']') SINGLE(RBOX) |
| 403 |
else { |
| 404 |
codepoint = (c > 160) ? 160 : c; |
| 405 |
tries = PyList_GET_ITEM(TOKEN_DISPATCH, codepoint); |
| 406 |
for (i = 0; i < PyList_Size(tries); i++) { |
| 407 |
item = PyList_GET_ITEM(tries, i); |
| 408 |
match_func = PyTuple_GET_ITEM(item, 2); |
| 409 |
match = PyObject_CallFunction(match_func, "On", py_source, pos); |
| 410 |
if (match == NULL) { goto error; } |
| 411 |
if (match != Py_None) { |
| 412 |
css_value = PyObject_CallMethod(match, "group", NULL); |
| 413 |
if (css_value == NULL) { goto error; } |
| 414 |
type_ = PyNumber_AsSsize_t(PyTuple_GET_ITEM(item, 0), PyExc_OverflowError); |
| 415 |
if (type_ == -1 && PyErr_Occurred() != NULL) { goto error; } |
| 416 |
type_name = PyTuple_GET_ITEM(item, 1); |
| 417 |
Py_INCREF(type_name); |
| 418 |
break; |
| 419 |
} |
| 420 |
} |
| 421 |
if (css_value == NULL) { // No match |
| 422 |
type_ = DELIM; |
| 423 |
type_name = DELIM_TOK; |
| 424 |
Py_INCREF(type_name); |
| 425 |
css_value = unicode_from_data(&c, 1); |
| 426 |
if (css_value == NULL) { goto error; } |
| 427 |
} |
| 428 |
} |
| 429 |
|
| 430 |
length = PyUnicode_GET_LENGTH(css_value); |
| 431 |
next_pos = pos + length; |
| 432 |
|
| 433 |
// Now calculate the value and unit for this token (if any) |
| 434 |
if (!(ignore_comments && (type_ == COMMENT || type_ == BAD_COMMENT))) { |
| 435 |
if (type_ == DIMENSION) { |
| 436 |
value = PyObject_CallMethod(match, "group", "I", 1); |
| 437 |
if (value == NULL) { goto error; } |
| 438 |
TONUMBER(value); |
| 439 |
unit = PyObject_CallMethod(match, "group", "I", 2); |
| 440 |
if (unit == NULL) { goto error; } |
| 441 |
UNESCAPE(unit, SIMPLE_UNESCAPE); |
| 442 |
UNESCAPE(unit, UNICODE_UNESCAPE); |
| 443 |
lowercase(unit); |
| 444 |
} else |
| 445 |
|
| 446 |
if (type_ == PERCENTAGE) { |
| 447 |
if (PyUnicode_GET_LENGTH(css_value) > 0) { |
| 448 |
value = clone_unicode(css_value, 0, 1); |
| 449 |
if (value == NULL) goto error; |
| 450 |
} else { |
| 451 |
value = css_value; |
| 452 |
Py_INCREF(value); |
| 453 |
} |
| 454 |
if (value == NULL) goto error; |
| 455 |
TONUMBER(value); |
| 456 |
unit = PyUnicode_FromString("%"); |
| 457 |
if (unit == NULL) goto error; |
| 458 |
} else |
| 459 |
|
| 460 |
if (type_ == NUMBER) { |
| 461 |
value = css_value; |
| 462 |
Py_INCREF(value); |
| 463 |
TONUMBER(value); |
| 464 |
if (!PyFloat_Check(value)) { |
| 465 |
Py_XDECREF(type_name); |
| 466 |
type_name = INTEGER; |
| 467 |
Py_INCREF(type_name); |
| 468 |
} |
| 469 |
} else |
| 470 |
|
| 471 |
if (type_ == IDENT || type_ == ATKEYWORD || type_ == HASH || type_ == FUNCTION) { |
| 472 |
value = PyObject_CallFunctionObjArgs(SIMPLE_UNESCAPE, css_value, NULL); |
| 473 |
if (value == NULL) goto error; |
| 474 |
UNESCAPE(value, UNICODE_UNESCAPE); |
| 475 |
} else |
| 476 |
|
| 477 |
if (type_ == URI) { |
| 478 |
value = PyObject_CallMethod(match, "group", "I", 1); |
| 479 |
if (value == NULL) { goto error; } |
| 480 |
if (PyObject_IsTrue(value) && PyUnicode_GET_LENGTH(value) > 1 && (first_char(value) == '"' || first_char(value) == '\'')) { |
| 481 |
item = clone_unicode(value, 1, 1); |
| 482 |
if (item == NULL) goto error; |
| 483 |
Py_DECREF(value); |
| 484 |
value = item; |
| 485 |
item = NULL; |
| 486 |
UNESCAPE(value, NEWLINE_UNESCAPE); |
| 487 |
} |
| 488 |
UNESCAPE(value, SIMPLE_UNESCAPE); |
| 489 |
UNESCAPE(value, UNICODE_UNESCAPE); |
| 490 |
} else |
| 491 |
|
| 492 |
if (type_ == STRING) { |
| 493 |
if (PyObject_IsTrue(css_value) && PyUnicode_GET_LENGTH(css_value) > 1) { // remove quotes |
| 494 |
value = clone_unicode(css_value, 1, 1); |
| 495 |
} else { |
| 496 |
value = css_value; |
| 497 |
Py_INCREF(value); |
| 498 |
} |
| 499 |
UNESCAPE(value, NEWLINE_UNESCAPE); |
| 500 |
UNESCAPE(value, SIMPLE_UNESCAPE); |
| 501 |
UNESCAPE(value, UNICODE_UNESCAPE); |
| 502 |
} else |
| 503 |
|
| 504 |
if (type_ == BAD_STRING && next_pos == source_len) { |
| 505 |
Py_XDECREF(type_name); |
| 506 |
type_name = STRING_TOK; |
| 507 |
Py_INCREF(type_name); |
| 508 |
if (PyObject_IsTrue(css_value) && PyUnicode_GET_LENGTH(css_value) > 0) { // remove quote |
| 509 |
value = clone_unicode(css_value, 1, 0); |
| 510 |
} else { |
| 511 |
value = css_value; |
| 512 |
Py_INCREF(value); |
| 513 |
} |
| 514 |
UNESCAPE(value, NEWLINE_UNESCAPE); |
| 515 |
UNESCAPE(value, SIMPLE_UNESCAPE); |
| 516 |
UNESCAPE(value, UNICODE_UNESCAPE); |
| 517 |
} else { |
| 518 |
value = css_value; |
| 519 |
Py_INCREF(value); |
| 520 |
} // if(type_ == ...) |
| 521 |
|
| 522 |
if (unit == NULL) { |
| 523 |
unit = Py_None; |
| 524 |
Py_INCREF(unit); |
| 525 |
} |
| 526 |
item = Py_BuildValue("OOOOnn", type_name, css_value, value, unit, line, column); |
| 527 |
if (item == NULL) goto error; |
| 528 |
token = PyObject_CallObject((PyObject *)&tokenizer_TokenType, item); |
| 529 |
Py_DECREF(item); |
| 530 |
item = NULL; |
| 531 |
if (token == NULL) goto error; |
| 532 |
if (PyList_Append(tokens, token) != 0) { |
| 533 |
Py_DECREF(token); |
| 534 |
token = NULL; |
| 535 |
goto error; |
| 536 |
} |
| 537 |
Py_DECREF(token); |
| 538 |
|
| 539 |
} // if(!(ignore_comments... |
| 540 |
|
| 541 |
Py_XDECREF(match); |
| 542 |
match = NULL; |
| 543 |
|
| 544 |
pos = next_pos; |
| 545 |
newlines = PyObject_CallFunctionObjArgs(FIND_NEWLINES, css_value, NULL); |
| 546 |
if (newlines == NULL) goto error; |
| 547 |
Py_XDECREF(css_value); |
| 548 |
css_value = NULL; |
| 549 |
Py_XDECREF(type_name); |
| 550 |
type_name = NULL; |
| 551 |
Py_XDECREF(value); |
| 552 |
value = NULL; |
| 553 |
Py_XDECREF(unit); |
| 554 |
unit = NULL; |
| 555 |
if (PyObject_IsTrue(newlines)) { |
| 556 |
line += PyList_Size(newlines); |
| 557 |
item = PyObject_CallMethod(PyList_GET_ITEM(newlines, PyList_Size(newlines) - 1), "end", NULL); |
| 558 |
if (item == NULL) { |
| 559 |
Py_DECREF(newlines); |
| 560 |
newlines = NULL; |
| 561 |
goto error; |
| 562 |
} |
| 563 |
column = PyNumber_AsSsize_t(item, PyExc_OverflowError); |
| 564 |
if (column == -1 && PyErr_Occurred()) { |
| 565 |
Py_DECREF(newlines); |
| 566 |
newlines = NULL; |
| 567 |
goto error; |
| 568 |
} |
| 569 |
column = length - column + 1; |
| 570 |
Py_DECREF(item); |
| 571 |
item = NULL; |
| 572 |
} else column += length; |
| 573 |
Py_DECREF(newlines); |
| 574 |
newlines = NULL; |
| 575 |
|
| 576 |
} // while (pos < ...) |
| 577 |
|
| 578 |
return tokens; |
| 579 |
error: |
| 580 |
Py_XDECREF(tokens); |
| 581 |
Py_XDECREF(css_value); |
| 582 |
Py_XDECREF(type_name); |
| 583 |
Py_XDECREF(value); |
| 584 |
Py_XDECREF(unit); |
| 585 |
Py_XDECREF(match); |
| 586 |
return NULL; |
| 587 |
#undef unicode_from_data |
| 588 |
#undef first_char |
| 589 |
} |
| 590 |
|
| 591 |
static PyMethodDef tokenizer_methods[] = { |
| 592 |
{"tokenize_flat", tokenize_flat, METH_VARARGS, "tokenize_flat(css_source, ignore_comments)\n\n Convert CSS source into a flat list of tokens"}, |
| 593 |
|
| 594 |
{"init", tokenize_init, METH_VARARGS, "init()\n\nInitialize the module."}, |
| 595 |
|
| 596 |
{"cleanup", tokenize_cleanup, METH_VARARGS, "cleanup()\n\nRelease resources allocated by init(). Safe to call multiple times."}, |
| 597 |
|
| 598 |
{NULL, NULL, 0, NULL}}; |
| 599 |
|
| 600 |
static int |
| 601 |
exec_module(PyObject *mod) { |
| 602 |
if (PyType_Ready(&tokenizer_TokenType) < 0) return -1; |
| 603 |
Py_INCREF(&tokenizer_TokenType); |
| 604 |
PyModule_AddObject(mod, "Token", (PyObject *)&tokenizer_TokenType); |
| 605 |
return 0; |
| 606 |
} |
| 607 |
static PyModuleDef_Slot slots[] = {{Py_mod_exec, exec_module}, {0, NULL}}; |
| 608 |
|
| 609 |
static struct PyModuleDef module_def = { |
| 610 |
.m_base = PyModuleDef_HEAD_INIT, |
| 611 |
.m_name = "tokenizer", |
| 612 |
.m_doc = "Implementation of tokenizer in C for speed.", |
| 613 |
.m_methods = tokenizer_methods, |
| 614 |
.m_slots = slots, |
| 615 |
}; |
| 616 |
|
| 617 |
CALIBRE_MODINIT_FUNC |
| 618 |
PyInit_tokenizer(void) { return PyModuleDef_Init(&module_def); } |