| 1 |
#!/usr/bin/env python |
| 2 |
# vim:fileencoding=utf-8 |
| 3 |
|
| 4 |
|
| 5 |
__license__ = 'GPL v3' |
| 6 |
__copyright__ = '2014, Kovid Goyal <kovid at kovidgoyal.net>' |
| 7 |
|
| 8 |
from tinycss.tests import BaseTest, jsonify |
| 9 |
from tinycss.tokenizer import c_tokenize_flat, python_tokenize_flat, regroup |
| 10 |
|
| 11 |
if c_tokenize_flat is None: |
| 12 |
tokenizers = (python_tokenize_flat,) |
| 13 |
else: |
| 14 |
tokenizers = (python_tokenize_flat, c_tokenize_flat) |
| 15 |
|
| 16 |
|
| 17 |
def token_api(self, tokenize): |
| 18 |
for css_source in [ |
| 19 |
'(8, foo, [z])', '[8, foo, (z)]', '{8, foo, [z]}', 'func(8, foo, [z])' |
| 20 |
]: |
| 21 |
tokens = list(regroup(tokenize(css_source))) |
| 22 |
self.ae(len(tokens), 1) |
| 23 |
self.ae(len(tokens[0].content), 7) |
| 24 |
|
| 25 |
|
| 26 |
def token_serialize_css(self, tokenize): |
| 27 |
for tokenize in tokenizers: |
| 28 |
for css_source in [ |
| 29 |
r'''p[example="\ |
| 30 |
foo(int x) {\ |
| 31 |
this.x = x;\ |
| 32 |
}\ |
| 33 |
"]''', |
| 34 |
'"Lorem\\26Ipsum\ndolor" sit', |
| 35 |
'/* Lorem\nipsum */\fa {\n color: red;\tcontent: "dolor\\\fsit" }', |
| 36 |
'not([[lorem]]{ipsum (42)})', |
| 37 |
'a[b{d]e}', |
| 38 |
'a[b{"d', |
| 39 |
]: |
| 40 |
for _regroup in (regroup, lambda x: x): |
| 41 |
tokens = _regroup(tokenize(css_source, ignore_comments=False)) |
| 42 |
result = ''.join(token.as_css() for token in tokens) |
| 43 |
self.ae(result, css_source) |
| 44 |
|
| 45 |
|
| 46 |
def comments(self, tokenize): |
| 47 |
for ignore_comments, expected_tokens in [ |
| 48 |
(False, [ |
| 49 |
('COMMENT', '/* lorem */'), |
| 50 |
('S', ' '), |
| 51 |
('IDENT', 'ipsum'), |
| 52 |
('[', [ |
| 53 |
('IDENT', 'dolor'), |
| 54 |
('COMMENT', '/* sit */'), |
| 55 |
]), |
| 56 |
('BAD_COMMENT', '/* amet') |
| 57 |
]), |
| 58 |
(True, [ |
| 59 |
('S', ' '), |
| 60 |
('IDENT', 'ipsum'), |
| 61 |
('[', [ |
| 62 |
('IDENT', 'dolor'), |
| 63 |
]), |
| 64 |
]), |
| 65 |
]: |
| 66 |
css_source = '/* lorem */ ipsum[dolor/* sit */]/* amet' |
| 67 |
tokens = regroup(tokenize(css_source, ignore_comments)) |
| 68 |
result = list(jsonify(tokens)) |
| 69 |
self.ae(result, expected_tokens) |
| 70 |
|
| 71 |
|
| 72 |
def token_grouping(self, tokenize): |
| 73 |
for css_source, expected_tokens in [ |
| 74 |
('', []), |
| 75 |
(r'Lorem\26 "i\psum"4px', [ |
| 76 |
('IDENT', 'Lorem&'), ('STRING', 'ipsum'), ('DIMENSION', 4)]), |
| 77 |
|
| 78 |
('not([[lorem]]{ipsum (42)})', [ |
| 79 |
('FUNCTION', 'not', [ |
| 80 |
('[', [ |
| 81 |
('[', [ |
| 82 |
('IDENT', 'lorem'), |
| 83 |
]), |
| 84 |
]), |
| 85 |
('{', [ |
| 86 |
('IDENT', 'ipsum'), |
| 87 |
('S', ' '), |
| 88 |
('(', [ |
| 89 |
('INTEGER', 42), |
| 90 |
]) |
| 91 |
]) |
| 92 |
])]), |
| 93 |
|
| 94 |
# Close everything at EOF, no error |
| 95 |
('a[b{"d', [ |
| 96 |
('IDENT', 'a'), |
| 97 |
('[', [ |
| 98 |
('IDENT', 'b'), |
| 99 |
('{', [ |
| 100 |
('STRING', 'd'), |
| 101 |
]), |
| 102 |
]), |
| 103 |
]), |
| 104 |
|
| 105 |
# Any remaining ), ] or } token is a nesting error |
| 106 |
('a[b{d]e}', [ |
| 107 |
('IDENT', 'a'), |
| 108 |
('[', [ |
| 109 |
('IDENT', 'b'), |
| 110 |
('{', [ |
| 111 |
('IDENT', 'd'), |
| 112 |
(']', ']'), # The error is visible here |
| 113 |
('IDENT', 'e'), |
| 114 |
]), |
| 115 |
]), |
| 116 |
]), |
| 117 |
# ref: |
| 118 |
('a[b{d}e]', [ |
| 119 |
('IDENT', 'a'), |
| 120 |
('[', [ |
| 121 |
('IDENT', 'b'), |
| 122 |
('{', [ |
| 123 |
('IDENT', 'd'), |
| 124 |
]), |
| 125 |
('IDENT', 'e'), |
| 126 |
]), |
| 127 |
]), |
| 128 |
]: |
| 129 |
tokens = regroup(tokenize(css_source, ignore_comments=False)) |
| 130 |
result = list(jsonify(tokens)) |
| 131 |
self.ae(result, expected_tokens) |
| 132 |
|
| 133 |
|
| 134 |
def positions(self, tokenize): |
| 135 |
css = '/* Lorem\nipsum */\fa {\n color: red;\tcontent: "dolor\\\fsit" }' |
| 136 |
tokens = tokenize(css, ignore_comments=False) |
| 137 |
result = [(token.type, token.line, token.column) for token in tokens] |
| 138 |
self.ae(result, [ |
| 139 |
('COMMENT', 1, 1), ('S', 2, 9), |
| 140 |
('IDENT', 3, 1), ('S', 3, 2), ('{', 3, 3), |
| 141 |
('S', 3, 4), ('IDENT', 4, 5), (':', 4, 10), |
| 142 |
('S', 4, 11), ('IDENT', 4, 12), (';', 4, 15), ('S', 4, 16), |
| 143 |
('IDENT', 4, 17), (':', 4, 24), ('S', 4, 25), ('STRING', 4, 26), |
| 144 |
('S', 5, 5), ('}', 5, 6)]) |
| 145 |
|
| 146 |
|
| 147 |
def tokens(self, tokenize): |
| 148 |
for css_source, expected_tokens in [ |
| 149 |
('', []), |
| 150 |
('red -->', |
| 151 |
[('IDENT', 'red'), ('S', ' '), ('CDC', '-->')]), |
| 152 |
# Longest match rule: no CDC |
| 153 |
('red-->', |
| 154 |
[('IDENT', 'red--'), ('DELIM', '>')]), |
| 155 |
|
| 156 |
(r'''p[example="\ |
| 157 |
foo(int x) {\ |
| 158 |
this.x = x;\ |
| 159 |
}\ |
| 160 |
"]''', [ |
| 161 |
('IDENT', 'p'), |
| 162 |
('[', '['), |
| 163 |
('IDENT', 'example'), |
| 164 |
('DELIM', '='), |
| 165 |
('STRING', 'foo(int x) { this.x = x;}'), |
| 166 |
(']', ']')]), |
| 167 |
|
| 168 |
# Numbers are parsed |
| 169 |
('42 .5 -4pX 1.25em 30%', |
| 170 |
[('INTEGER', 42), ('S', ' '), |
| 171 |
('NUMBER', .5), ('S', ' '), |
| 172 |
# units are normalized to lower-case: |
| 173 |
('DIMENSION', -4, 'px'), ('S', ' '), |
| 174 |
('DIMENSION', 1.25, 'em'), ('S', ' '), |
| 175 |
('PERCENTAGE', 30, '%')]), |
| 176 |
|
| 177 |
# URLs are extracted |
| 178 |
('url(foo.png)', [('URI', 'foo.png')]), |
| 179 |
('url("foo.png")', [('URI', 'foo.png')]), |
| 180 |
|
| 181 |
# Escaping |
| 182 |
|
| 183 |
(r'/* Comment with a \ backslash */', |
| 184 |
[('COMMENT', r'/* Comment with a \ backslash */')]), # Unchanged |
| 185 |
|
| 186 |
# backslash followed by a newline in a string: ignored |
| 187 |
('"Lorem\\\nIpsum"', [('STRING', 'LoremIpsum')]), |
| 188 |
|
| 189 |
# backslash followed by a newline outside a string: stands for itself |
| 190 |
('Lorem\\\nIpsum', [ |
| 191 |
('IDENT', 'Lorem'), ('DELIM', '\\'), |
| 192 |
('S', '\n'), ('IDENT', 'Ipsum')]), |
| 193 |
|
| 194 |
# Cancel the meaning of special characters |
| 195 |
(r'"Lore\m Ipsum"', [('STRING', 'Lorem Ipsum')]), # or not special |
| 196 |
(r'"Lorem \49psum"', [('STRING', 'Lorem Ipsum')]), |
| 197 |
(r'"Lorem \49 psum"', [('STRING', 'Lorem Ipsum')]), |
| 198 |
(r'"Lorem\"Ipsum"', [('STRING', 'Lorem"Ipsum')]), |
| 199 |
(r'"Lorem\\Ipsum"', [('STRING', r'Lorem\Ipsum')]), |
| 200 |
(r'"Lorem\5c Ipsum"', [('STRING', r'Lorem\Ipsum')]), |
| 201 |
(r'Lorem\+Ipsum', [('IDENT', 'Lorem+Ipsum')]), |
| 202 |
(r'Lorem+Ipsum', [('IDENT', 'Lorem'), ('DELIM', '+'), ('IDENT', 'Ipsum')]), |
| 203 |
(r'url(foo\).png)', [('URI', 'foo).png')]), |
| 204 |
|
| 205 |
# Unicode and backslash escaping |
| 206 |
('\\26 B', [('IDENT', '&B')]), |
| 207 |
('\\&B', [('IDENT', '&B')]), |
| 208 |
('@\\26\tB', [('ATKEYWORD', '@&B')]), |
| 209 |
('@\\&B', [('ATKEYWORD', '@&B')]), |
| 210 |
('#\\26\nB', [('HASH', '#&B')]), |
| 211 |
('#\\&B', [('HASH', '#&B')]), |
| 212 |
('\\26\r\nB(', [('FUNCTION', '&B(')]), |
| 213 |
('\\&B(', [('FUNCTION', '&B(')]), |
| 214 |
(r'12.5\000026B', [('DIMENSION', 12.5, '&b')]), |
| 215 |
(r'12.5\0000263B', [('DIMENSION', 12.5, '&3b')]), # max 6 digits |
| 216 |
(r'12.5\&B', [('DIMENSION', 12.5, '&b')]), |
| 217 |
(r'"\26 B"', [('STRING', '&B')]), |
| 218 |
(r"'\000026B'", [('STRING', '&B')]), |
| 219 |
(r'"\&B"', [('STRING', '&B')]), |
| 220 |
(r'url("\26 B")', [('URI', '&B')]), |
| 221 |
(r'url(\26 B)', [('URI', '&B')]), |
| 222 |
(r'url("\&B")', [('URI', '&B')]), |
| 223 |
(r'url(\&B)', [('URI', '&B')]), |
| 224 |
(r'Lorem\110000Ipsum', [('IDENT', 'Lorem\uFFFDIpsum')]), |
| 225 |
|
| 226 |
# Bad strings |
| 227 |
|
| 228 |
# String ends at EOF without closing: no error, parsed |
| 229 |
('"Lorem\\26Ipsum', [('STRING', 'Lorem&Ipsum')]), |
| 230 |
# Unescaped newline: ends the string, error, unparsed |
| 231 |
('"Lorem\\26Ipsum\n', [ |
| 232 |
('BAD_STRING', r'"Lorem\26Ipsum'), ('S', '\n')]), |
| 233 |
# Tokenization restarts after the newline, so the second " starts |
| 234 |
# a new string (which ends at EOF without errors, as above.) |
| 235 |
('"Lorem\\26Ipsum\ndolor" sit', [ |
| 236 |
('BAD_STRING', r'"Lorem\26Ipsum'), ('S', '\n'), |
| 237 |
('IDENT', 'dolor'), ('STRING', ' sit')]), |
| 238 |
|
| 239 |
]: |
| 240 |
sources = [css_source] |
| 241 |
for css_source in sources: |
| 242 |
tokens = tokenize(css_source, ignore_comments=False) |
| 243 |
result = [ |
| 244 |
(token.type, token.value) + ( |
| 245 |
() if token.unit is None else (token.unit,)) |
| 246 |
for token in tokens |
| 247 |
] |
| 248 |
self.ae(result, expected_tokens) |
| 249 |
|
| 250 |
|
| 251 |
class TestTokenizer(BaseTest): |
| 252 |
|
| 253 |
def run_test(self, func): |
| 254 |
for tokenize in tokenizers: |
| 255 |
func(self, tokenize) |
| 256 |
|
| 257 |
def test_token_api(self): |
| 258 |
self.run_test(token_api) |
| 259 |
|
| 260 |
def test_token_serialize_css(self): |
| 261 |
self.run_test(token_serialize_css) |
| 262 |
|
| 263 |
def test_comments(self): |
| 264 |
self.run_test(comments) |
| 265 |
|
| 266 |
def test_token_grouping(self): |
| 267 |
self.run_test(token_grouping) |
| 268 |
|
| 269 |
def test_positions(self): |
| 270 |
"""Test the reported line/column position of each token.""" |
| 271 |
self.run_test(positions) |
| 272 |
|
| 273 |
def test_tokens(self): |
| 274 |
self.run_test(tokens) |