Debian Codegraph @

src/tinycss/token_data.py

raw
1 # coding: utf8
2 """
3 tinycss.token_data
4 ------------------
5
6 Shared data for both implementations (Cython and Python) of the tokenizer.
7
8 :copyright: (c) 2012 by Simon Sapin.
9 :license: BSD, see LICENSE for more details.
10 """
11
12
13 import functools
14 import operator
15 import re
16 import string
17 import sys
18
19 # * Raw strings with the r'' notation are used so that \ do not need
20 # to be escaped.
21 # * Names and regexps are separated by a tabulation.
22 # * Macros are re-ordered so that only previous definitions are needed.
23 # * {} are used for macro substitution with ``string.Formatter``,
24 # so other uses of { or } have been doubled.
25 # * The syntax is otherwise compatible with re.compile.
26 # * Some parentheses were added to add capturing groups.
27 # (in unicode, DIMENSION and URI)
28
29 # *** Willful violation: ***
30 # Numbers can take a + or - sign, but the sign is a separate DELIM token.
31 # Since comments are allowed anywhere between tokens, this makes
32 # the following this is valid. It means 10 negative pixels:
33 # margin-top: -/**/10px
34
35 # This makes parsing numbers a pain, so instead we’ll do the same is Firefox
36 # and make the sign part as of the 'num' macro. The above CSS will be invalid.
37 # See discussion:
38 # http://lists.w3.org/Archives/Public/www-style/2011Oct/0028.html
39 MACROS = r'''
40 nl \n|\r\n|\r|\f
41 w [ \t\r\n\f]*
42 nonascii [^\0-\237]
43 unicode \\([0-9a-f]{{1,6}})(\r\n|[ \n\r\t\f])?
44 simple_escape [^\n\r\f0-9a-f]
45 escape {unicode}|\\{simple_escape}
46 nmstart [_a-z]|{nonascii}|{escape}
47 nmchar [_a-z0-9-]|{nonascii}|{escape}
48 name {nmchar}+
49 ident [-]?{nmstart}{nmchar}*
50 num [-+]?(?:[0-9]*\.[0-9]+|[0-9]+)
51 string1 \"([^\n\r\f\\"]|\\{nl}|{escape})*\"
52 string2 \'([^\n\r\f\\']|\\{nl}|{escape})*\'
53 string {string1}|{string2}
54 badstring1 \"([^\n\r\f\\"]|\\{nl}|{escape})*\\?
55 badstring2 \'([^\n\r\f\\']|\\{nl}|{escape})*\\?
56 badstring {badstring1}|{badstring2}
57 badcomment1 \/\*[^*]*\*+([^/*][^*]*\*+)*
58 badcomment2 \/\*[^*]*(\*+[^/*][^*]*)*
59 badcomment {badcomment1}|{badcomment2}
60 baduri1 url\({w}([!#$%&*-~]|{nonascii}|{escape})*{w}
61 baduri2 url\({w}{string}{w}
62 baduri3 url\({w}{badstring}
63 baduri {baduri1}|{baduri2}|{baduri3}
64 '''.replace(r'\0', '\0').replace(r'\237', '\237')
65
66 # Removed these tokens. Instead, they’re tokenized as two DELIM each.
67 # INCLUDES ~=
68 # DASHMATCH |=
69 # They are only used in selectors but selectors3 also have ^=, *= and $=.
70 # We don’t actually parse selectors anyway
71
72 # Re-ordered so that the longest match is always the first.
73 # For example, "url('foo')" matches URI, BAD_URI, FUNCTION and IDENT,
74 # but URI would always be a longer match than the others.
75 TOKENS = r'''
76 S [ \t\r\n\f]+
77
78 URI url\({w}({string}|([!#$%&*-\[\]-~]|{nonascii}|{escape})*){w}\)
79 BAD_URI {baduri}
80 FUNCTION {ident}\(
81 UNICODE-RANGE u\+[0-9a-f?]{{1,6}}(-[0-9a-f]{{1,6}})?
82 IDENT {ident}
83
84 ATKEYWORD @{ident}
85 HASH #{name}
86
87 DIMENSION ({num})({ident})
88 PERCENTAGE {num}%
89 NUMBER {num}
90
91 STRING {string}
92 BAD_STRING {badstring}
93
94 COMMENT \/\*[^*]*\*+([^/*][^*]*\*+)*\/
95 BAD_COMMENT {badcomment}
96
97 : :
98 ; ;
99 { \{{
100 } \}}
101 ( \(
102 ) \)
103 [ \[
104 ] \]
105 CDO <!--
106 CDC -->
107 '''
108
109
110 # Strings with {macro} expanded
111 COMPILED_MACROS = {}
112
113
114 COMPILED_TOKEN_REGEXPS = [] # [(name, regexp.match)] ordered
115 COMPILED_TOKEN_INDEXES = {} # {name: i} helper for the C speedups
116
117
118 # Indexed by codepoint value of the first character of a token.
119 # Codepoints >= 160 (aka nonascii) all use the index 160.
120 # values are (i, name, regexp.match)
121 TOKEN_DISPATCH = []
122
123
124 unichr = chr
125 unicode = str
126
127
128 def _init():
129 """Import-time initialization."""
130 COMPILED_MACROS.clear()
131 for line in MACROS.splitlines():
132 if line.strip():
133 name, value = line.split('\t')
134 COMPILED_MACROS[name.strip()] = '(?:%s)' \
135 % value.format(**COMPILED_MACROS)
136
137 COMPILED_TOKEN_REGEXPS[:] = (
138 (
139 name.strip(),
140 re.compile(
141 value.format(**COMPILED_MACROS),
142 # Case-insensitive when matching eg. uRL(foo)
143 # but preserve the case in extracted groups
144 re.I
145 ).match
146 )
147 for line in TOKENS.splitlines()
148 if line.strip()
149 for name, value in [line.split('\t')]
150 )
151
152 COMPILED_TOKEN_INDEXES.clear()
153 for i, (name, regexp) in enumerate(COMPILED_TOKEN_REGEXPS):
154 COMPILED_TOKEN_INDEXES[name] = i
155
156 dispatch = [[] for i in range(161)]
157 for chars, names in [
158 (' \t\r\n\f', ['S']),
159 ('uU', ['URI', 'BAD_URI', 'UNICODE-RANGE']),
160 # \ is an escape outside of another token
161 (string.ascii_letters + '\\_-' + unichr(160), ['FUNCTION', 'IDENT']),
162 (string.digits + '.+-', ['DIMENSION', 'PERCENTAGE', 'NUMBER']),
163 ('@', ['ATKEYWORD']),
164 ('#', ['HASH']),
165 ('\'"', ['STRING', 'BAD_STRING']),
166 ('/', ['COMMENT', 'BAD_COMMENT']),
167 ('<', ['CDO']),
168 ('-', ['CDC']),
169 ]:
170 for char in chars:
171 dispatch[ord(char)].extend(names)
172 for char in ':;{}()[]':
173 dispatch[ord(char)] = [char]
174
175 TOKEN_DISPATCH[:] = (
176 [
177 (index,) + COMPILED_TOKEN_REGEXPS[index]
178 for name in names
179 for index in [COMPILED_TOKEN_INDEXES[name]]
180 ]
181 for names in dispatch
182 )
183
184
185 _init()
186
187
188 def _unicode_replace(match, int=int, unichr=unichr, maxunicode=sys.maxunicode):
189 codepoint = int(match.group(1), 16)
190 if codepoint <= maxunicode:
191 return unichr(codepoint)
192 else:
193 return '\N{REPLACEMENT CHARACTER}' # U+FFFD
194
195
196 UNICODE_UNESCAPE = functools.partial(
197 re.compile(COMPILED_MACROS['unicode'], re.I).sub,
198 _unicode_replace)
199
200 NEWLINE_UNESCAPE = functools.partial(
201 re.compile(r'()\\' + COMPILED_MACROS['nl']).sub,
202 '')
203
204
205 SIMPLE_UNESCAPE = functools.partial(
206 re.compile(r'\\(%s)' % COMPILED_MACROS['simple_escape'], re.I).sub,
207 # Same as r'\1', but faster on CPython
208 operator.methodcaller('group', 1))
209
210
211 def FIND_NEWLINES(x):
212 return list(re.compile(COMPILED_MACROS['nl']).finditer(x))
213
214
215 class Token:
216 r"""A single atomic token.
217
218 .. attribute:: is_container
219
220 Always ``False``.
221 Helps to tell :class:`Token` apart from :class:`ContainerToken`.
222
223 .. attribute:: type
224
225 The type of token as a string:
226
227 ``S``
228 A sequence of white space
229
230 ``IDENT``
231 An identifier: a name that does not start with a digit.
232 A name is a sequence of letters, digits, ``_``, ``-``, escaped
233 characters and non-ASCII characters. Eg: ``margin-left``
234
235 ``HASH``
236 ``#`` followed immediately by a name. Eg: ``#ff8800``
237
238 ``ATKEYWORD``
239 ``@`` followed immediately by an identifier. Eg: ``@page``
240
241 ``URI``
242 Eg: ``url(foo)`` The content may or may not be quoted.
243
244 ``UNICODE-RANGE``
245 ``U+`` followed by one or two hexadecimal
246 Unicode codepoints. Eg: ``U+20-00FF``
247
248 ``INTEGER``
249 An integer with an optional ``+`` or ``-`` sign
250
251 ``NUMBER``
252 A non-integer number with an optional ``+`` or ``-`` sign
253
254 ``DIMENSION``
255 An integer or number followed immediately by an
256 identifier (the unit). Eg: ``12px``
257
258 ``PERCENTAGE``
259 An integer or number followed immediately by ``%``
260
261 ``STRING``
262 A string, quoted with ``"`` or ``'``
263
264 ``:`` or ``;``
265 That character.
266
267 ``DELIM``
268 A single character not matched in another token. Eg: ``,``
269
270 See the source of the :mod:`.token_data` module for the precise
271 regular expressions that match various tokens.
272
273 Note that other token types exist in the early tokenization steps,
274 but these are ignored, are syntax errors, or are later transformed
275 into :class:`ContainerToken` or :class:`FunctionToken`.
276
277 .. attribute:: value
278
279 The parsed value:
280
281 * INTEGER, NUMBER, PERCENTAGE or DIMENSION tokens: the numeric value
282 as an int or float.
283 * STRING tokens: the unescaped string without quotes
284 * URI tokens: the unescaped URI without quotes or
285 ``url(`` and ``)`` markers.
286 * IDENT, ATKEYWORD or HASH tokens: the unescaped token,
287 with ``@`` or ``#`` markers left as-is
288 * Other tokens: same as :attr:`as_css`
289
290 *Unescaped* refers to the various escaping methods based on the
291 backslash ``\`` character in CSS syntax.
292
293 .. attribute:: unit
294
295 * DIMENSION tokens: the normalized (unescaped, lower-case)
296 unit name as a string. eg. ``'px'``
297 * PERCENTAGE tokens: the string ``'%'``
298 * Other tokens: ``None``
299
300 .. attribute:: line
301
302 The line number in the CSS source of the start of this token.
303
304 .. attribute:: column
305
306 The column number (inside a source line) of the start of this token.
307
308 """
309 is_container = False
310 __slots__ = 'type', '_as_css', 'value', 'unit', 'line', 'column'
311
312 def __init__(self, type_, css_value, value, unit, line, column):
313 self.type = type_
314 self._as_css = css_value
315 self.value = value
316 self.unit = unit
317 self.line = line
318 self.column = column
319
320 def as_css(self):
321 """
322 Return as an Unicode string the CSS representation of the token,
323 as parsed in the source.
324 """
325 return self._as_css
326
327 def __repr__(self):
328 return ('<Token {0.type} at {0.line}:{0.column} {0.value!r}{1}>'
329 .format(self, self.unit or ''))
330
331
332 class ContainerToken:
333 """A token that contains other (nested) tokens.
334
335 .. attribute:: is_container
336
337 Always ``True``.
338 Helps to tell :class:`ContainerToken` apart from :class:`Token`.
339
340 .. attribute:: type
341
342 The type of token as a string. One of ``{``, ``(``, ``[`` or
343 ``FUNCTION``. For ``FUNCTION``, the object is actually a
344 :class:`FunctionToken`.
345
346 .. attribute:: unit
347
348 Always ``None``. Included to make :class:`ContainerToken` behave
349 more like :class:`Token`.
350
351 .. attribute:: content
352
353 A list of :class:`Token` or nested :class:`ContainerToken`,
354 not including the opening or closing token.
355
356 .. attribute:: line
357
358 The line number in the CSS source of the start of this token.
359
360 .. attribute:: column
361
362 The column number (inside a source line) of the start of this token.
363
364 """
365 is_container = True
366 unit = None
367 __slots__ = 'type', '_css_start', '_css_end', 'content', 'line', 'column'
368
369 def __init__(self, type_, css_start, css_end, content, line, column):
370 self.type = type_
371 self._css_start = css_start
372 self._css_end = css_end
373 self.content = content
374 self.line = line
375 self.column = column
376
377 def as_css(self):
378 """
379 Return as an Unicode string the CSS representation of the token,
380 as parsed in the source.
381 """
382 parts = [self._css_start]
383 parts.extend(token.as_css() for token in self.content)
384 parts.append(self._css_end)
385 return ''.join(parts)
386
387 format_string = '<ContainerToken {0.type} at {0.line}:{0.column}>'
388
389 def __repr__(self):
390 return (self.format_string + ' {0.content}').format(self)
391
392
393 class FunctionToken(ContainerToken):
394 """A specialized :class:`ContainerToken` for a ``FUNCTION`` group.
395 Has an additional attribute:
396
397 .. attribute:: function_name
398
399 The unescaped name of the function, with the ``(`` marker removed.
400
401 """
402 __slots__ = 'function_name',
403
404 def __init__(self, type_, css_start, css_end, function_name, content,
405 line, column):
406 super(FunctionToken, self).__init__(
407 type_, css_start, css_end, content, line, column)
408 # Remove the ( marker:
409 self.function_name = function_name[:-1]
410
411 format_string = ('<FunctionToken {0.function_name}() at '
412 '{0.line}:{0.column}>')
413
414
415 class TokenList(list):
416 """
417 A mixed list of :class:`~.token_data.Token` and
418 :class:`~.token_data.ContainerToken` objects.
419
420 This is a subclass of the builtin :class:`~builtins.list` type.
421 It can be iterated, indexed and sliced as usual, but also has some
422 additional API:
423
424 """
425 @property
426 def line(self):
427 """The line number in the CSS source of the first token."""
428 return self[0].line
429
430 @property
431 def column(self):
432 """The column number (inside a source line) of the first token."""
433 return self[0].column
434
435 def as_css(self):
436 """
437 Return as an Unicode string the CSS representation of the tokens,
438 as parsed in the source.
439 """
440 return ''.join(token.as_css() for token in self)
441
442
443 def load_c_tokenizer():
444 from calibre_extensions import tokenizer
445 tokens = list(':;(){}[]') + ['DELIM', 'INTEGER', 'STRING']
446 tokenizer.init(
447 COMPILED_TOKEN_REGEXPS, UNICODE_UNESCAPE, NEWLINE_UNESCAPE,
448 SIMPLE_UNESCAPE, FIND_NEWLINES, TOKEN_DISPATCH, COMPILED_TOKEN_INDEXES,
449 *tokens)
450 return tokenizer