| 1 |
# coding: utf8 |
| 2 |
""" |
| 3 |
tinycss.token_data |
| 4 |
------------------ |
| 5 |
|
| 6 |
Shared data for both implementations (Cython and Python) of the tokenizer. |
| 7 |
|
| 8 |
:copyright: (c) 2012 by Simon Sapin. |
| 9 |
:license: BSD, see LICENSE for more details. |
| 10 |
""" |
| 11 |
|
| 12 |
|
| 13 |
import functools |
| 14 |
import operator |
| 15 |
import re |
| 16 |
import string |
| 17 |
import sys |
| 18 |
|
| 19 |
# * Raw strings with the r'' notation are used so that \ do not need |
| 20 |
# to be escaped. |
| 21 |
# * Names and regexps are separated by a tabulation. |
| 22 |
# * Macros are re-ordered so that only previous definitions are needed. |
| 23 |
# * {} are used for macro substitution with ``string.Formatter``, |
| 24 |
# so other uses of { or } have been doubled. |
| 25 |
# * The syntax is otherwise compatible with re.compile. |
| 26 |
# * Some parentheses were added to add capturing groups. |
| 27 |
# (in unicode, DIMENSION and URI) |
| 28 |
|
| 29 |
# *** Willful violation: *** |
| 30 |
# Numbers can take a + or - sign, but the sign is a separate DELIM token. |
| 31 |
# Since comments are allowed anywhere between tokens, this makes |
| 32 |
# the following this is valid. It means 10 negative pixels: |
| 33 |
# margin-top: -/**/10px |
| 34 |
|
| 35 |
# This makes parsing numbers a pain, so instead we’ll do the same is Firefox |
| 36 |
# and make the sign part as of the 'num' macro. The above CSS will be invalid. |
| 37 |
# See discussion: |
| 38 |
# http://lists.w3.org/Archives/Public/www-style/2011Oct/0028.html |
| 39 |
MACROS = r''' |
| 40 |
nl \n|\r\n|\r|\f |
| 41 |
w [ \t\r\n\f]* |
| 42 |
nonascii [^\0-\237] |
| 43 |
unicode \\([0-9a-f]{{1,6}})(\r\n|[ \n\r\t\f])? |
| 44 |
simple_escape [^\n\r\f0-9a-f] |
| 45 |
escape {unicode}|\\{simple_escape} |
| 46 |
nmstart [_a-z]|{nonascii}|{escape} |
| 47 |
nmchar [_a-z0-9-]|{nonascii}|{escape} |
| 48 |
name {nmchar}+ |
| 49 |
ident [-]?{nmstart}{nmchar}* |
| 50 |
num [-+]?(?:[0-9]*\.[0-9]+|[0-9]+) |
| 51 |
string1 \"([^\n\r\f\\"]|\\{nl}|{escape})*\" |
| 52 |
string2 \'([^\n\r\f\\']|\\{nl}|{escape})*\' |
| 53 |
string {string1}|{string2} |
| 54 |
badstring1 \"([^\n\r\f\\"]|\\{nl}|{escape})*\\? |
| 55 |
badstring2 \'([^\n\r\f\\']|\\{nl}|{escape})*\\? |
| 56 |
badstring {badstring1}|{badstring2} |
| 57 |
badcomment1 \/\*[^*]*\*+([^/*][^*]*\*+)* |
| 58 |
badcomment2 \/\*[^*]*(\*+[^/*][^*]*)* |
| 59 |
badcomment {badcomment1}|{badcomment2} |
| 60 |
baduri1 url\({w}([!#$%&*-~]|{nonascii}|{escape})*{w} |
| 61 |
baduri2 url\({w}{string}{w} |
| 62 |
baduri3 url\({w}{badstring} |
| 63 |
baduri {baduri1}|{baduri2}|{baduri3} |
| 64 |
'''.replace(r'\0', '\0').replace(r'\237', '\237') |
| 65 |
|
| 66 |
# Removed these tokens. Instead, they’re tokenized as two DELIM each. |
| 67 |
# INCLUDES ~= |
| 68 |
# DASHMATCH |= |
| 69 |
# They are only used in selectors but selectors3 also have ^=, *= and $=. |
| 70 |
# We don’t actually parse selectors anyway |
| 71 |
|
| 72 |
# Re-ordered so that the longest match is always the first. |
| 73 |
# For example, "url('foo')" matches URI, BAD_URI, FUNCTION and IDENT, |
| 74 |
# but URI would always be a longer match than the others. |
| 75 |
TOKENS = r''' |
| 76 |
S [ \t\r\n\f]+ |
| 77 |
|
| 78 |
URI url\({w}({string}|([!#$%&*-\[\]-~]|{nonascii}|{escape})*){w}\) |
| 79 |
BAD_URI {baduri} |
| 80 |
FUNCTION {ident}\( |
| 81 |
UNICODE-RANGE u\+[0-9a-f?]{{1,6}}(-[0-9a-f]{{1,6}})? |
| 82 |
IDENT {ident} |
| 83 |
|
| 84 |
ATKEYWORD @{ident} |
| 85 |
HASH #{name} |
| 86 |
|
| 87 |
DIMENSION ({num})({ident}) |
| 88 |
PERCENTAGE {num}% |
| 89 |
NUMBER {num} |
| 90 |
|
| 91 |
STRING {string} |
| 92 |
BAD_STRING {badstring} |
| 93 |
|
| 94 |
COMMENT \/\*[^*]*\*+([^/*][^*]*\*+)*\/ |
| 95 |
BAD_COMMENT {badcomment} |
| 96 |
|
| 97 |
: : |
| 98 |
; ; |
| 99 |
{ \{{ |
| 100 |
} \}} |
| 101 |
( \( |
| 102 |
) \) |
| 103 |
[ \[ |
| 104 |
] \] |
| 105 |
CDO <!-- |
| 106 |
CDC --> |
| 107 |
''' |
| 108 |
|
| 109 |
|
| 110 |
# Strings with {macro} expanded |
| 111 |
COMPILED_MACROS = {} |
| 112 |
|
| 113 |
|
| 114 |
COMPILED_TOKEN_REGEXPS = [] # [(name, regexp.match)] ordered |
| 115 |
COMPILED_TOKEN_INDEXES = {} # {name: i} helper for the C speedups |
| 116 |
|
| 117 |
|
| 118 |
# Indexed by codepoint value of the first character of a token. |
| 119 |
# Codepoints >= 160 (aka nonascii) all use the index 160. |
| 120 |
# values are (i, name, regexp.match) |
| 121 |
TOKEN_DISPATCH = [] |
| 122 |
|
| 123 |
|
| 124 |
unichr = chr |
| 125 |
unicode = str |
| 126 |
|
| 127 |
|
| 128 |
def _init(): |
| 129 |
"""Import-time initialization.""" |
| 130 |
COMPILED_MACROS.clear() |
| 131 |
for line in MACROS.splitlines(): |
| 132 |
if line.strip(): |
| 133 |
name, value = line.split('\t') |
| 134 |
COMPILED_MACROS[name.strip()] = '(?:%s)' \ |
| 135 |
% value.format(**COMPILED_MACROS) |
| 136 |
|
| 137 |
COMPILED_TOKEN_REGEXPS[:] = ( |
| 138 |
( |
| 139 |
name.strip(), |
| 140 |
re.compile( |
| 141 |
value.format(**COMPILED_MACROS), |
| 142 |
# Case-insensitive when matching eg. uRL(foo) |
| 143 |
# but preserve the case in extracted groups |
| 144 |
re.I |
| 145 |
).match |
| 146 |
) |
| 147 |
for line in TOKENS.splitlines() |
| 148 |
if line.strip() |
| 149 |
for name, value in [line.split('\t')] |
| 150 |
) |
| 151 |
|
| 152 |
COMPILED_TOKEN_INDEXES.clear() |
| 153 |
for i, (name, regexp) in enumerate(COMPILED_TOKEN_REGEXPS): |
| 154 |
COMPILED_TOKEN_INDEXES[name] = i |
| 155 |
|
| 156 |
dispatch = [[] for i in range(161)] |
| 157 |
for chars, names in [ |
| 158 |
(' \t\r\n\f', ['S']), |
| 159 |
('uU', ['URI', 'BAD_URI', 'UNICODE-RANGE']), |
| 160 |
# \ is an escape outside of another token |
| 161 |
(string.ascii_letters + '\\_-' + unichr(160), ['FUNCTION', 'IDENT']), |
| 162 |
(string.digits + '.+-', ['DIMENSION', 'PERCENTAGE', 'NUMBER']), |
| 163 |
('@', ['ATKEYWORD']), |
| 164 |
('#', ['HASH']), |
| 165 |
('\'"', ['STRING', 'BAD_STRING']), |
| 166 |
('/', ['COMMENT', 'BAD_COMMENT']), |
| 167 |
('<', ['CDO']), |
| 168 |
('-', ['CDC']), |
| 169 |
]: |
| 170 |
for char in chars: |
| 171 |
dispatch[ord(char)].extend(names) |
| 172 |
for char in ':;{}()[]': |
| 173 |
dispatch[ord(char)] = [char] |
| 174 |
|
| 175 |
TOKEN_DISPATCH[:] = ( |
| 176 |
[ |
| 177 |
(index,) + COMPILED_TOKEN_REGEXPS[index] |
| 178 |
for name in names |
| 179 |
for index in [COMPILED_TOKEN_INDEXES[name]] |
| 180 |
] |
| 181 |
for names in dispatch |
| 182 |
) |
| 183 |
|
| 184 |
|
| 185 |
_init() |
| 186 |
|
| 187 |
|
| 188 |
def _unicode_replace(match, int=int, unichr=unichr, maxunicode=sys.maxunicode): |
| 189 |
codepoint = int(match.group(1), 16) |
| 190 |
if codepoint <= maxunicode: |
| 191 |
return unichr(codepoint) |
| 192 |
else: |
| 193 |
return '\N{REPLACEMENT CHARACTER}' # U+FFFD |
| 194 |
|
| 195 |
|
| 196 |
UNICODE_UNESCAPE = functools.partial( |
| 197 |
re.compile(COMPILED_MACROS['unicode'], re.I).sub, |
| 198 |
_unicode_replace) |
| 199 |
|
| 200 |
NEWLINE_UNESCAPE = functools.partial( |
| 201 |
re.compile(r'()\\' + COMPILED_MACROS['nl']).sub, |
| 202 |
'') |
| 203 |
|
| 204 |
|
| 205 |
SIMPLE_UNESCAPE = functools.partial( |
| 206 |
re.compile(r'\\(%s)' % COMPILED_MACROS['simple_escape'], re.I).sub, |
| 207 |
# Same as r'\1', but faster on CPython |
| 208 |
operator.methodcaller('group', 1)) |
| 209 |
|
| 210 |
|
| 211 |
def FIND_NEWLINES(x): |
| 212 |
return list(re.compile(COMPILED_MACROS['nl']).finditer(x)) |
| 213 |
|
| 214 |
|
| 215 |
class Token: |
| 216 |
r"""A single atomic token. |
| 217 |
|
| 218 |
.. attribute:: is_container |
| 219 |
|
| 220 |
Always ``False``. |
| 221 |
Helps to tell :class:`Token` apart from :class:`ContainerToken`. |
| 222 |
|
| 223 |
.. attribute:: type |
| 224 |
|
| 225 |
The type of token as a string: |
| 226 |
|
| 227 |
``S`` |
| 228 |
A sequence of white space |
| 229 |
|
| 230 |
``IDENT`` |
| 231 |
An identifier: a name that does not start with a digit. |
| 232 |
A name is a sequence of letters, digits, ``_``, ``-``, escaped |
| 233 |
characters and non-ASCII characters. Eg: ``margin-left`` |
| 234 |
|
| 235 |
``HASH`` |
| 236 |
``#`` followed immediately by a name. Eg: ``#ff8800`` |
| 237 |
|
| 238 |
``ATKEYWORD`` |
| 239 |
``@`` followed immediately by an identifier. Eg: ``@page`` |
| 240 |
|
| 241 |
``URI`` |
| 242 |
Eg: ``url(foo)`` The content may or may not be quoted. |
| 243 |
|
| 244 |
``UNICODE-RANGE`` |
| 245 |
``U+`` followed by one or two hexadecimal |
| 246 |
Unicode codepoints. Eg: ``U+20-00FF`` |
| 247 |
|
| 248 |
``INTEGER`` |
| 249 |
An integer with an optional ``+`` or ``-`` sign |
| 250 |
|
| 251 |
``NUMBER`` |
| 252 |
A non-integer number with an optional ``+`` or ``-`` sign |
| 253 |
|
| 254 |
``DIMENSION`` |
| 255 |
An integer or number followed immediately by an |
| 256 |
identifier (the unit). Eg: ``12px`` |
| 257 |
|
| 258 |
``PERCENTAGE`` |
| 259 |
An integer or number followed immediately by ``%`` |
| 260 |
|
| 261 |
``STRING`` |
| 262 |
A string, quoted with ``"`` or ``'`` |
| 263 |
|
| 264 |
``:`` or ``;`` |
| 265 |
That character. |
| 266 |
|
| 267 |
``DELIM`` |
| 268 |
A single character not matched in another token. Eg: ``,`` |
| 269 |
|
| 270 |
See the source of the :mod:`.token_data` module for the precise |
| 271 |
regular expressions that match various tokens. |
| 272 |
|
| 273 |
Note that other token types exist in the early tokenization steps, |
| 274 |
but these are ignored, are syntax errors, or are later transformed |
| 275 |
into :class:`ContainerToken` or :class:`FunctionToken`. |
| 276 |
|
| 277 |
.. attribute:: value |
| 278 |
|
| 279 |
The parsed value: |
| 280 |
|
| 281 |
* INTEGER, NUMBER, PERCENTAGE or DIMENSION tokens: the numeric value |
| 282 |
as an int or float. |
| 283 |
* STRING tokens: the unescaped string without quotes |
| 284 |
* URI tokens: the unescaped URI without quotes or |
| 285 |
``url(`` and ``)`` markers. |
| 286 |
* IDENT, ATKEYWORD or HASH tokens: the unescaped token, |
| 287 |
with ``@`` or ``#`` markers left as-is |
| 288 |
* Other tokens: same as :attr:`as_css` |
| 289 |
|
| 290 |
*Unescaped* refers to the various escaping methods based on the |
| 291 |
backslash ``\`` character in CSS syntax. |
| 292 |
|
| 293 |
.. attribute:: unit |
| 294 |
|
| 295 |
* DIMENSION tokens: the normalized (unescaped, lower-case) |
| 296 |
unit name as a string. eg. ``'px'`` |
| 297 |
* PERCENTAGE tokens: the string ``'%'`` |
| 298 |
* Other tokens: ``None`` |
| 299 |
|
| 300 |
.. attribute:: line |
| 301 |
|
| 302 |
The line number in the CSS source of the start of this token. |
| 303 |
|
| 304 |
.. attribute:: column |
| 305 |
|
| 306 |
The column number (inside a source line) of the start of this token. |
| 307 |
|
| 308 |
""" |
| 309 |
is_container = False |
| 310 |
__slots__ = 'type', '_as_css', 'value', 'unit', 'line', 'column' |
| 311 |
|
| 312 |
def __init__(self, type_, css_value, value, unit, line, column): |
| 313 |
self.type = type_ |
| 314 |
self._as_css = css_value |
| 315 |
self.value = value |
| 316 |
self.unit = unit |
| 317 |
self.line = line |
| 318 |
self.column = column |
| 319 |
|
| 320 |
def as_css(self): |
| 321 |
""" |
| 322 |
Return as an Unicode string the CSS representation of the token, |
| 323 |
as parsed in the source. |
| 324 |
""" |
| 325 |
return self._as_css |
| 326 |
|
| 327 |
def __repr__(self): |
| 328 |
return ('<Token {0.type} at {0.line}:{0.column} {0.value!r}{1}>' |
| 329 |
.format(self, self.unit or '')) |
| 330 |
|
| 331 |
|
| 332 |
class ContainerToken: |
| 333 |
"""A token that contains other (nested) tokens. |
| 334 |
|
| 335 |
.. attribute:: is_container |
| 336 |
|
| 337 |
Always ``True``. |
| 338 |
Helps to tell :class:`ContainerToken` apart from :class:`Token`. |
| 339 |
|
| 340 |
.. attribute:: type |
| 341 |
|
| 342 |
The type of token as a string. One of ``{``, ``(``, ``[`` or |
| 343 |
``FUNCTION``. For ``FUNCTION``, the object is actually a |
| 344 |
:class:`FunctionToken`. |
| 345 |
|
| 346 |
.. attribute:: unit |
| 347 |
|
| 348 |
Always ``None``. Included to make :class:`ContainerToken` behave |
| 349 |
more like :class:`Token`. |
| 350 |
|
| 351 |
.. attribute:: content |
| 352 |
|
| 353 |
A list of :class:`Token` or nested :class:`ContainerToken`, |
| 354 |
not including the opening or closing token. |
| 355 |
|
| 356 |
.. attribute:: line |
| 357 |
|
| 358 |
The line number in the CSS source of the start of this token. |
| 359 |
|
| 360 |
.. attribute:: column |
| 361 |
|
| 362 |
The column number (inside a source line) of the start of this token. |
| 363 |
|
| 364 |
""" |
| 365 |
is_container = True |
| 366 |
unit = None |
| 367 |
__slots__ = 'type', '_css_start', '_css_end', 'content', 'line', 'column' |
| 368 |
|
| 369 |
def __init__(self, type_, css_start, css_end, content, line, column): |
| 370 |
self.type = type_ |
| 371 |
self._css_start = css_start |
| 372 |
self._css_end = css_end |
| 373 |
self.content = content |
| 374 |
self.line = line |
| 375 |
self.column = column |
| 376 |
|
| 377 |
def as_css(self): |
| 378 |
""" |
| 379 |
Return as an Unicode string the CSS representation of the token, |
| 380 |
as parsed in the source. |
| 381 |
""" |
| 382 |
parts = [self._css_start] |
| 383 |
parts.extend(token.as_css() for token in self.content) |
| 384 |
parts.append(self._css_end) |
| 385 |
return ''.join(parts) |
| 386 |
|
| 387 |
format_string = '<ContainerToken {0.type} at {0.line}:{0.column}>' |
| 388 |
|
| 389 |
def __repr__(self): |
| 390 |
return (self.format_string + ' {0.content}').format(self) |
| 391 |
|
| 392 |
|
| 393 |
class FunctionToken(ContainerToken): |
| 394 |
"""A specialized :class:`ContainerToken` for a ``FUNCTION`` group. |
| 395 |
Has an additional attribute: |
| 396 |
|
| 397 |
.. attribute:: function_name |
| 398 |
|
| 399 |
The unescaped name of the function, with the ``(`` marker removed. |
| 400 |
|
| 401 |
""" |
| 402 |
__slots__ = 'function_name', |
| 403 |
|
| 404 |
def __init__(self, type_, css_start, css_end, function_name, content, |
| 405 |
line, column): |
| 406 |
super(FunctionToken, self).__init__( |
| 407 |
type_, css_start, css_end, content, line, column) |
| 408 |
# Remove the ( marker: |
| 409 |
self.function_name = function_name[:-1] |
| 410 |
|
| 411 |
format_string = ('<FunctionToken {0.function_name}() at ' |
| 412 |
'{0.line}:{0.column}>') |
| 413 |
|
| 414 |
|
| 415 |
class TokenList(list): |
| 416 |
""" |
| 417 |
A mixed list of :class:`~.token_data.Token` and |
| 418 |
:class:`~.token_data.ContainerToken` objects. |
| 419 |
|
| 420 |
This is a subclass of the builtin :class:`~builtins.list` type. |
| 421 |
It can be iterated, indexed and sliced as usual, but also has some |
| 422 |
additional API: |
| 423 |
|
| 424 |
""" |
| 425 |
@property |
| 426 |
def line(self): |
| 427 |
"""The line number in the CSS source of the first token.""" |
| 428 |
return self[0].line |
| 429 |
|
| 430 |
@property |
| 431 |
def column(self): |
| 432 |
"""The column number (inside a source line) of the first token.""" |
| 433 |
return self[0].column |
| 434 |
|
| 435 |
def as_css(self): |
| 436 |
""" |
| 437 |
Return as an Unicode string the CSS representation of the tokens, |
| 438 |
as parsed in the source. |
| 439 |
""" |
| 440 |
return ''.join(token.as_css() for token in self) |
| 441 |
|
| 442 |
|
| 443 |
def load_c_tokenizer(): |
| 444 |
from calibre_extensions import tokenizer |
| 445 |
tokens = list(':;(){}[]') + ['DELIM', 'INTEGER', 'STRING'] |
| 446 |
tokenizer.init( |
| 447 |
COMPILED_TOKEN_REGEXPS, UNICODE_UNESCAPE, NEWLINE_UNESCAPE, |
| 448 |
SIMPLE_UNESCAPE, FIND_NEWLINES, TOKEN_DISPATCH, COMPILED_TOKEN_INDEXES, |
| 449 |
*tokens) |
| 450 |
return tokenizer |