Debian Codegraph @

src/tinycss/tests/tokenizing.py

raw
1 #!/usr/bin/env python
2 # vim:fileencoding=utf-8
3
4
5 __license__ = 'GPL v3'
6 __copyright__ = '2014, Kovid Goyal <kovid at kovidgoyal.net>'
7
8 from tinycss.tests import BaseTest, jsonify
9 from tinycss.tokenizer import c_tokenize_flat, python_tokenize_flat, regroup
10
11 if c_tokenize_flat is None:
12 tokenizers = (python_tokenize_flat,)
13 else:
14 tokenizers = (python_tokenize_flat, c_tokenize_flat)
15
16
17 def token_api(self, tokenize):
18 for css_source in [
19 '(8, foo, [z])', '[8, foo, (z)]', '{8, foo, [z]}', 'func(8, foo, [z])'
20 ]:
21 tokens = list(regroup(tokenize(css_source)))
22 self.ae(len(tokens), 1)
23 self.ae(len(tokens[0].content), 7)
24
25
26 def token_serialize_css(self, tokenize):
27 for tokenize in tokenizers:
28 for css_source in [
29 r'''p[example="\
30 foo(int x) {\
31 this.x = x;\
32 }\
33 "]''',
34 '"Lorem\\26Ipsum\ndolor" sit',
35 '/* Lorem\nipsum */\fa {\n color: red;\tcontent: "dolor\\\fsit" }',
36 'not([[lorem]]{ipsum (42)})',
37 'a[b{d]e}',
38 'a[b{"d',
39 ]:
40 for _regroup in (regroup, lambda x: x):
41 tokens = _regroup(tokenize(css_source, ignore_comments=False))
42 result = ''.join(token.as_css() for token in tokens)
43 self.ae(result, css_source)
44
45
46 def comments(self, tokenize):
47 for ignore_comments, expected_tokens in [
48 (False, [
49 ('COMMENT', '/* lorem */'),
50 ('S', ' '),
51 ('IDENT', 'ipsum'),
52 ('[', [
53 ('IDENT', 'dolor'),
54 ('COMMENT', '/* sit */'),
55 ]),
56 ('BAD_COMMENT', '/* amet')
57 ]),
58 (True, [
59 ('S', ' '),
60 ('IDENT', 'ipsum'),
61 ('[', [
62 ('IDENT', 'dolor'),
63 ]),
64 ]),
65 ]:
66 css_source = '/* lorem */ ipsum[dolor/* sit */]/* amet'
67 tokens = regroup(tokenize(css_source, ignore_comments))
68 result = list(jsonify(tokens))
69 self.ae(result, expected_tokens)
70
71
72 def token_grouping(self, tokenize):
73 for css_source, expected_tokens in [
74 ('', []),
75 (r'Lorem\26 "i\psum"4px', [
76 ('IDENT', 'Lorem&'), ('STRING', 'ipsum'), ('DIMENSION', 4)]),
77
78 ('not([[lorem]]{ipsum (42)})', [
79 ('FUNCTION', 'not', [
80 ('[', [
81 ('[', [
82 ('IDENT', 'lorem'),
83 ]),
84 ]),
85 ('{', [
86 ('IDENT', 'ipsum'),
87 ('S', ' '),
88 ('(', [
89 ('INTEGER', 42),
90 ])
91 ])
92 ])]),
93
94 # Close everything at EOF, no error
95 ('a[b{"d', [
96 ('IDENT', 'a'),
97 ('[', [
98 ('IDENT', 'b'),
99 ('{', [
100 ('STRING', 'd'),
101 ]),
102 ]),
103 ]),
104
105 # Any remaining ), ] or } token is a nesting error
106 ('a[b{d]e}', [
107 ('IDENT', 'a'),
108 ('[', [
109 ('IDENT', 'b'),
110 ('{', [
111 ('IDENT', 'd'),
112 (']', ']'), # The error is visible here
113 ('IDENT', 'e'),
114 ]),
115 ]),
116 ]),
117 # ref:
118 ('a[b{d}e]', [
119 ('IDENT', 'a'),
120 ('[', [
121 ('IDENT', 'b'),
122 ('{', [
123 ('IDENT', 'd'),
124 ]),
125 ('IDENT', 'e'),
126 ]),
127 ]),
128 ]:
129 tokens = regroup(tokenize(css_source, ignore_comments=False))
130 result = list(jsonify(tokens))
131 self.ae(result, expected_tokens)
132
133
134 def positions(self, tokenize):
135 css = '/* Lorem\nipsum */\fa {\n color: red;\tcontent: "dolor\\\fsit" }'
136 tokens = tokenize(css, ignore_comments=False)
137 result = [(token.type, token.line, token.column) for token in tokens]
138 self.ae(result, [
139 ('COMMENT', 1, 1), ('S', 2, 9),
140 ('IDENT', 3, 1), ('S', 3, 2), ('{', 3, 3),
141 ('S', 3, 4), ('IDENT', 4, 5), (':', 4, 10),
142 ('S', 4, 11), ('IDENT', 4, 12), (';', 4, 15), ('S', 4, 16),
143 ('IDENT', 4, 17), (':', 4, 24), ('S', 4, 25), ('STRING', 4, 26),
144 ('S', 5, 5), ('}', 5, 6)])
145
146
147 def tokens(self, tokenize):
148 for css_source, expected_tokens in [
149 ('', []),
150 ('red -->',
151 [('IDENT', 'red'), ('S', ' '), ('CDC', '-->')]),
152 # Longest match rule: no CDC
153 ('red-->',
154 [('IDENT', 'red--'), ('DELIM', '>')]),
155
156 (r'''p[example="\
157 foo(int x) {\
158 this.x = x;\
159 }\
160 "]''', [
161 ('IDENT', 'p'),
162 ('[', '['),
163 ('IDENT', 'example'),
164 ('DELIM', '='),
165 ('STRING', 'foo(int x) { this.x = x;}'),
166 (']', ']')]),
167
168 # Numbers are parsed
169 ('42 .5 -4pX 1.25em 30%',
170 [('INTEGER', 42), ('S', ' '),
171 ('NUMBER', .5), ('S', ' '),
172 # units are normalized to lower-case:
173 ('DIMENSION', -4, 'px'), ('S', ' '),
174 ('DIMENSION', 1.25, 'em'), ('S', ' '),
175 ('PERCENTAGE', 30, '%')]),
176
177 # URLs are extracted
178 ('url(foo.png)', [('URI', 'foo.png')]),
179 ('url("foo.png")', [('URI', 'foo.png')]),
180
181 # Escaping
182
183 (r'/* Comment with a \ backslash */',
184 [('COMMENT', r'/* Comment with a \ backslash */')]), # Unchanged
185
186 # backslash followed by a newline in a string: ignored
187 ('"Lorem\\\nIpsum"', [('STRING', 'LoremIpsum')]),
188
189 # backslash followed by a newline outside a string: stands for itself
190 ('Lorem\\\nIpsum', [
191 ('IDENT', 'Lorem'), ('DELIM', '\\'),
192 ('S', '\n'), ('IDENT', 'Ipsum')]),
193
194 # Cancel the meaning of special characters
195 (r'"Lore\m Ipsum"', [('STRING', 'Lorem Ipsum')]), # or not special
196 (r'"Lorem \49psum"', [('STRING', 'Lorem Ipsum')]),
197 (r'"Lorem \49 psum"', [('STRING', 'Lorem Ipsum')]),
198 (r'"Lorem\"Ipsum"', [('STRING', 'Lorem"Ipsum')]),
199 (r'"Lorem\\Ipsum"', [('STRING', r'Lorem\Ipsum')]),
200 (r'"Lorem\5c Ipsum"', [('STRING', r'Lorem\Ipsum')]),
201 (r'Lorem\+Ipsum', [('IDENT', 'Lorem+Ipsum')]),
202 (r'Lorem+Ipsum', [('IDENT', 'Lorem'), ('DELIM', '+'), ('IDENT', 'Ipsum')]),
203 (r'url(foo\).png)', [('URI', 'foo).png')]),
204
205 # Unicode and backslash escaping
206 ('\\26 B', [('IDENT', '&B')]),
207 ('\\&B', [('IDENT', '&B')]),
208 ('@\\26\tB', [('ATKEYWORD', '@&B')]),
209 ('@\\&B', [('ATKEYWORD', '@&B')]),
210 ('#\\26\nB', [('HASH', '#&B')]),
211 ('#\\&B', [('HASH', '#&B')]),
212 ('\\26\r\nB(', [('FUNCTION', '&B(')]),
213 ('\\&B(', [('FUNCTION', '&B(')]),
214 (r'12.5\000026B', [('DIMENSION', 12.5, '&b')]),
215 (r'12.5\0000263B', [('DIMENSION', 12.5, '&3b')]), # max 6 digits
216 (r'12.5\&B', [('DIMENSION', 12.5, '&b')]),
217 (r'"\26 B"', [('STRING', '&B')]),
218 (r"'\000026B'", [('STRING', '&B')]),
219 (r'"\&B"', [('STRING', '&B')]),
220 (r'url("\26 B")', [('URI', '&B')]),
221 (r'url(\26 B)', [('URI', '&B')]),
222 (r'url("\&B")', [('URI', '&B')]),
223 (r'url(\&B)', [('URI', '&B')]),
224 (r'Lorem\110000Ipsum', [('IDENT', 'Lorem\uFFFDIpsum')]),
225
226 # Bad strings
227
228 # String ends at EOF without closing: no error, parsed
229 ('"Lorem\\26Ipsum', [('STRING', 'Lorem&Ipsum')]),
230 # Unescaped newline: ends the string, error, unparsed
231 ('"Lorem\\26Ipsum\n', [
232 ('BAD_STRING', r'"Lorem\26Ipsum'), ('S', '\n')]),
233 # Tokenization restarts after the newline, so the second " starts
234 # a new string (which ends at EOF without errors, as above.)
235 ('"Lorem\\26Ipsum\ndolor" sit', [
236 ('BAD_STRING', r'"Lorem\26Ipsum'), ('S', '\n'),
237 ('IDENT', 'dolor'), ('STRING', ' sit')]),
238
239 ]:
240 sources = [css_source]
241 for css_source in sources:
242 tokens = tokenize(css_source, ignore_comments=False)
243 result = [
244 (token.type, token.value) + (
245 () if token.unit is None else (token.unit,))
246 for token in tokens
247 ]
248 self.ae(result, expected_tokens)
249
250
251 class TestTokenizer(BaseTest):
252
253 def run_test(self, func):
254 for tokenize in tokenizers:
255 func(self, tokenize)
256
257 def test_token_api(self):
258 self.run_test(token_api)
259
260 def test_token_serialize_css(self):
261 self.run_test(token_serialize_css)
262
263 def test_comments(self):
264 self.run_test(comments)
265
266 def test_token_grouping(self):
267 self.run_test(token_grouping)
268
269 def test_positions(self):
270 """Test the reported line/column position of each token."""
271 self.run_test(positions)
272
273 def test_tokens(self):
274 self.run_test(tokens)