Debian Codegraph @

src/tinycss/tokenizer.py

raw
1 # coding: utf8
2 """
3 tinycss.tokenizer
4 -----------------
5
6 Tokenizer for the CSS core syntax:
7 http://www.w3.org/TR/CSS21/syndata.html#tokenization
8
9 This is the pure-python implementation. See also speedups.pyx
10
11 :copyright: (c) 2012 by Simon Sapin.
12 :license: BSD, see LICENSE for more details.
13 """
14
15
16 from tinycss import token_data
17
18
19 def tokenize_flat(css_source, ignore_comments=True,
20 # Make these local variable to avoid global lookups in the loop
21 tokens_dispatch=token_data.TOKEN_DISPATCH,
22 unicode_unescape=token_data.UNICODE_UNESCAPE,
23 newline_unescape=token_data.NEWLINE_UNESCAPE,
24 simple_unescape=token_data.SIMPLE_UNESCAPE,
25 find_newlines=token_data.FIND_NEWLINES,
26 Token=token_data.Token,
27 len=len,
28 int=int,
29 float=float,
30 list=list,
31 _None=None,
32 ):
33 """
34 :param css_source:
35 CSS as an unicode string
36 :param ignore_comments:
37 if true (the default) comments will not be included in the
38 return value
39 :return:
40 An iterator of :class:`Token`
41
42 """
43
44 pos = 0
45 line = 1
46 column = 1
47 source_len = len(css_source)
48 tokens = []
49 while pos < source_len:
50 char = css_source[pos]
51 if char in ':;{}()[]':
52 type_ = char
53 css_value = char
54 else:
55 codepoint = min(ord(char), 160)
56 for _index, type_, regexp in tokens_dispatch[codepoint]:
57 match = regexp(css_source, pos)
58 if match is not None:
59 # First match is the longest. See comments on TOKENS above.
60 css_value = match.group()
61 break
62 else:
63 # No match.
64 # "Any other character not matched by the above rules,
65 # and neither a single nor a double quote."
66 # ... but quotes at the start of a token are always matched
67 # by STRING or BAD_STRING. So DELIM is any single character.
68 type_ = 'DELIM'
69 css_value = char
70 length = len(css_value)
71 next_pos = pos + length
72
73 # A BAD_COMMENT is a comment at EOF. Ignore it too.
74 if not (ignore_comments and type_ in ('COMMENT', 'BAD_COMMENT')):
75 # Parse numbers, extract strings and URIs, unescape
76 unit = _None
77 if type_ == 'DIMENSION':
78 value = match.group(1)
79 value = float(value) if '.' in value else int(value)
80 unit = match.group(2)
81 unit = simple_unescape(unit)
82 unit = unicode_unescape(unit)
83 unit = unit.lower() # normalize
84 elif type_ == 'PERCENTAGE':
85 value = css_value[:-1]
86 value = float(value) if '.' in value else int(value)
87 unit = '%'
88 elif type_ == 'NUMBER':
89 value = css_value
90 if '.' in value:
91 value = float(value)
92 else:
93 value = int(value)
94 type_ = 'INTEGER'
95 elif type_ in ('IDENT', 'ATKEYWORD', 'HASH', 'FUNCTION'):
96 value = simple_unescape(css_value)
97 value = unicode_unescape(value)
98 elif type_ == 'URI':
99 value = match.group(1)
100 if value and value[0] in '"\'':
101 value = value[1:-1] # Remove quotes
102 value = newline_unescape(value)
103 value = simple_unescape(value)
104 value = unicode_unescape(value)
105 elif type_ == 'STRING':
106 value = css_value[1:-1] # Remove quotes
107 value = newline_unescape(value)
108 value = simple_unescape(value)
109 value = unicode_unescape(value)
110 # BAD_STRING can only be one of:
111 # * Unclosed string at the end of the stylesheet:
112 # Close the string, but this is not an error.
113 # Make it a "good" STRING token.
114 # * Unclosed string at the (unescaped) end of the line:
115 # Close the string, but this is an error.
116 # Leave it as a BAD_STRING, don’t bother parsing it.
117 # See http://www.w3.org/TR/CSS21/syndata.html#parsing-errors
118 elif type_ == 'BAD_STRING' and next_pos == source_len:
119 type_ = 'STRING'
120 value = css_value[1:] # Remove quote
121 value = newline_unescape(value)
122 value = simple_unescape(value)
123 value = unicode_unescape(value)
124 else:
125 value = css_value
126 tokens.append(Token(type_, css_value, value, unit, line, column))
127
128 pos = next_pos
129 newlines = find_newlines(css_value)
130 if newlines:
131 line += len(newlines)
132 # Add 1 to have lines start at column 1, not 0
133 column = length - newlines[-1].end() + 1
134 else:
135 column += length
136 return tokens
137
138
139 def regroup(tokens):
140 """
141 Match pairs of tokens: () [] {} function()
142 (Strings in "" or '' are taken care of by the tokenizer.)
143
144 Opening tokens are replaced by a :class:`ContainerToken`.
145 Closing tokens are removed. Unmatched closing tokens are invalid
146 but left as-is. All nested structures that are still open at
147 the end of the stylesheet are implicitly closed.
148
149 :param tokens:
150 a *flat* iterable of tokens, as returned by :func:`tokenize_flat`.
151 :return:
152 A tree of tokens.
153
154 """
155 # "global" objects for the inner recursion
156 pairs = {'FUNCTION': ')', '(': ')', '[': ']', '{': '}'}
157 tokens = iter(tokens)
158 eof = [False]
159
160 def _regroup_inner(stop_at=None,
161 tokens=tokens, pairs=pairs, eof=eof,
162 ContainerToken=token_data.ContainerToken,
163 FunctionToken=token_data.FunctionToken):
164 for token in tokens:
165 type_ = token.type
166 if type_ == stop_at:
167 return
168
169 end = pairs.get(type_)
170 if end is None:
171 yield token # Not a grouping token
172 else:
173 assert not isinstance(token, ContainerToken), (
174 'Token looks already grouped: {0}'.format(token))
175 content = list(_regroup_inner(end))
176 if eof[0]:
177 end = '' # Implicit end of structure at EOF.
178 if type_ == 'FUNCTION':
179 yield FunctionToken(token.type, token.as_css(), end,
180 token.value, content,
181 token.line, token.column)
182 else:
183 yield ContainerToken(token.type, token.as_css(), end,
184 content,
185 token.line, token.column)
186 else:
187 eof[0] = True # end of file/stylesheet
188 return _regroup_inner()
189
190
191 def tokenize_grouped(css_source, ignore_comments=True):
192 """
193 :param css_source:
194 CSS as an unicode string
195 :param ignore_comments:
196 if true (the default) comments will not be included in the
197 return value
198 :return:
199 An iterator of :class:`Token`
200
201 """
202 return regroup(tokenize_flat(css_source, ignore_comments))
203
204
205 # Optional Cython version of tokenize_flat
206 # Make both versions available with explicit names for tests.
207 python_tokenize_flat = tokenize_flat
208
209 try:
210 tok = token_data.load_c_tokenizer()
211 except (ImportError, RuntimeError):
212 c_tokenize_flat = None
213 else:
214 # Use the c tokenizer by default
215 c_tokenize_flat = tokenize_flat = lambda s, ignore_comments=False:tok.tokenize_flat(s, ignore_comments) # type: ignore