| 1 |
#!/usr/bin/env python |
| 2 |
# vim:fileencoding=utf-8 |
| 3 |
""" |
| 4 |
Tokenizer, parser and parsed objects for CSS selectors. |
| 5 |
|
| 6 |
:copyright: (c) 2007-2012 Ian Bicking and contributors. |
| 7 |
See AUTHORS for more details. |
| 8 |
:license: BSD, see LICENSE for more details. |
| 9 |
|
| 10 |
""" |
| 11 |
|
| 12 |
import operator |
| 13 |
import re |
| 14 |
import string |
| 15 |
import sys |
| 16 |
|
| 17 |
from css_selectors.errors import ExpressionError, SelectorSyntaxError |
| 18 |
from polyglot.builtins import codepoint_to_chr, unicode_type |
| 19 |
|
| 20 |
utab = {c:c+32 for c in range(ord(u'A'), ord(u'Z')+1)} |
| 21 |
|
| 22 |
if sys.version_info.major < 3: |
| 23 |
tab = string.maketrans(string.ascii_uppercase, string.ascii_lowercase) |
| 24 |
|
| 25 |
def ascii_lower(string): |
| 26 |
"""Lower-case, but only in the ASCII range.""" |
| 27 |
return string.translate(utab if isinstance(string, unicode_type) else tab) |
| 28 |
|
| 29 |
def urepr(x): |
| 30 |
if isinstance(x, list): |
| 31 |
return '[%s]' % ', '.join((map(urepr, x))) |
| 32 |
ans = repr(x) |
| 33 |
if ans.startswith("u'") or ans.startswith('u"'): |
| 34 |
ans = ans[1:] |
| 35 |
return ans |
| 36 |
|
| 37 |
|
| 38 |
else: |
| 39 |
|
| 40 |
def ascii_lower(x): |
| 41 |
return x.translate(utab) |
| 42 |
|
| 43 |
urepr = repr |
| 44 |
|
| 45 |
|
| 46 |
# Parsed objects |
| 47 |
|
| 48 |
class Selector: |
| 49 |
|
| 50 |
""" |
| 51 |
Represents a parsed selector. |
| 52 |
""" |
| 53 |
|
| 54 |
def __init__(self, tree, pseudo_element=None): |
| 55 |
self.parsed_tree = tree |
| 56 |
if pseudo_element is not None and not isinstance( |
| 57 |
pseudo_element, FunctionalPseudoElement): |
| 58 |
pseudo_element = ascii_lower(pseudo_element) |
| 59 |
#: A :class:`FunctionalPseudoElement`, |
| 60 |
#: or the identifier for the pseudo-element as a string, |
| 61 |
# or ``None``. |
| 62 |
#: |
| 63 |
#: +-------------------------+----------------+--------------------------------+ |
| 64 |
#: | | Selector | Pseudo-element | |
| 65 |
#: +=========================+================+================================+ |
| 66 |
#: | CSS3 syntax | ``a::before`` | ``'before'`` | |
| 67 |
#: +-------------------------+----------------+--------------------------------+ |
| 68 |
#: | Older syntax | ``a:before`` | ``'before'`` | |
| 69 |
#: +-------------------------+----------------+--------------------------------+ |
| 70 |
#: | From the Lists3_ draft, | ``li::marker`` | ``'marker'`` | |
| 71 |
#: | not in Selectors3 | | | |
| 72 |
#: +-------------------------+----------------+--------------------------------+ |
| 73 |
#: | Invalid pseudo-class | ``li:marker`` | ``None`` | |
| 74 |
#: +-------------------------+----------------+--------------------------------+ |
| 75 |
#: | Functional | ``a::foo(2)`` | ``FunctionalPseudoElement(…)`` | |
| 76 |
#: +-------------------------+----------------+--------------------------------+ |
| 77 |
#: |
| 78 |
# : .. _Lists3: http://www.w3.org/TR/2011/WD-css3-lists-20110524/#marker-pseudoelement |
| 79 |
self.pseudo_element = pseudo_element |
| 80 |
|
| 81 |
def __repr__(self): |
| 82 |
if isinstance(self.pseudo_element, FunctionalPseudoElement): |
| 83 |
pseudo_element = repr(self.pseudo_element) |
| 84 |
if self.pseudo_element: |
| 85 |
pseudo_element = '::%s' % self.pseudo_element |
| 86 |
else: |
| 87 |
pseudo_element = '' |
| 88 |
return '%s[%r%s]' % ( |
| 89 |
self.__class__.__name__, self.parsed_tree, pseudo_element) |
| 90 |
|
| 91 |
def specificity(self): |
| 92 |
"""Return the specificity_ of this selector as a tuple of 3 integers. |
| 93 |
|
| 94 |
.. _specificity: http://www.w3.org/TR/selectors/#specificity |
| 95 |
|
| 96 |
""" |
| 97 |
a, b, c = self.parsed_tree.specificity() |
| 98 |
if self.pseudo_element: |
| 99 |
c += 1 |
| 100 |
return a, b, c |
| 101 |
|
| 102 |
|
| 103 |
class Class: |
| 104 |
|
| 105 |
""" |
| 106 |
Represents selector.class_name |
| 107 |
""" |
| 108 |
def __init__(self, selector, class_name): |
| 109 |
self.selector = selector |
| 110 |
self.class_name = class_name |
| 111 |
|
| 112 |
def __repr__(self): |
| 113 |
return '%s[%r.%s]' % ( |
| 114 |
self.__class__.__name__, self.selector, self.class_name) |
| 115 |
|
| 116 |
def specificity(self): |
| 117 |
a, b, c = self.selector.specificity() |
| 118 |
b += 1 |
| 119 |
return a, b, c |
| 120 |
|
| 121 |
|
| 122 |
class FunctionalPseudoElement: |
| 123 |
|
| 124 |
""" |
| 125 |
Represents selector::name(arguments) |
| 126 |
|
| 127 |
.. attribute:: name |
| 128 |
|
| 129 |
The name (identifier) of the pseudo-element, as a string. |
| 130 |
|
| 131 |
.. attribute:: arguments |
| 132 |
|
| 133 |
The arguments of the pseudo-element, as a list of tokens. |
| 134 |
|
| 135 |
**Note:** tokens are not part of the public API, |
| 136 |
and may change between versions. |
| 137 |
Use at your own risks. |
| 138 |
|
| 139 |
""" |
| 140 |
def __init__(self, name, arguments, selector): |
| 141 |
self.name = ascii_lower(name) |
| 142 |
self.arguments = arguments |
| 143 |
self.selector = selector |
| 144 |
|
| 145 |
def __repr__(self): |
| 146 |
return '%s[::%s(%s)]' % ( |
| 147 |
self.__class__.__name__, self.name, |
| 148 |
urepr([token.value for token in self.arguments])) |
| 149 |
|
| 150 |
def argument_types(self): |
| 151 |
return [token.type for token in self.arguments] |
| 152 |
|
| 153 |
def specificity(self): |
| 154 |
a, b, c = self.selector.specificity() |
| 155 |
b += 1 |
| 156 |
return a, b, c |
| 157 |
|
| 158 |
|
| 159 |
class Function: |
| 160 |
|
| 161 |
""" |
| 162 |
Represents selector:name(expr) |
| 163 |
""" |
| 164 |
def __init__(self, selector, name, arguments): |
| 165 |
self.selector = selector |
| 166 |
self.name = ascii_lower(name) |
| 167 |
self.arguments = arguments |
| 168 |
self._parsed_arguments = None |
| 169 |
|
| 170 |
def __repr__(self): |
| 171 |
return '%s[%r:%s(%s)]' % ( |
| 172 |
self.__class__.__name__, self.selector, self.name, |
| 173 |
urepr([token.value for token in self.arguments])) |
| 174 |
|
| 175 |
def argument_types(self): |
| 176 |
return [token.type for token in self.arguments] |
| 177 |
|
| 178 |
@property |
| 179 |
def parsed_arguments(self): |
| 180 |
if self._parsed_arguments is None: |
| 181 |
try: |
| 182 |
self._parsed_arguments = parse_series(self.arguments) |
| 183 |
except ValueError: |
| 184 |
raise ExpressionError("Invalid series: '%r'" % self.arguments) |
| 185 |
return self._parsed_arguments |
| 186 |
|
| 187 |
def parse_arguments(self): |
| 188 |
if not self.arguments_parsed: |
| 189 |
self.arguments_parsed = True |
| 190 |
|
| 191 |
def specificity(self): |
| 192 |
a, b, c = self.selector.specificity() |
| 193 |
b += 1 |
| 194 |
return a, b, c |
| 195 |
|
| 196 |
|
| 197 |
class Pseudo: |
| 198 |
|
| 199 |
""" |
| 200 |
Represents selector:ident |
| 201 |
""" |
| 202 |
def __init__(self, selector, ident): |
| 203 |
self.selector = selector |
| 204 |
self.ident = ascii_lower(ident) |
| 205 |
|
| 206 |
def __repr__(self): |
| 207 |
return '%s[%r:%s]' % ( |
| 208 |
self.__class__.__name__, self.selector, self.ident) |
| 209 |
|
| 210 |
def specificity(self): |
| 211 |
a, b, c = self.selector.specificity() |
| 212 |
b += 1 |
| 213 |
return a, b, c |
| 214 |
|
| 215 |
|
| 216 |
class Negation: |
| 217 |
|
| 218 |
""" |
| 219 |
Represents selector:not(subselector) |
| 220 |
""" |
| 221 |
def __init__(self, selector, subselector): |
| 222 |
self.selector = selector |
| 223 |
self.subselector = subselector |
| 224 |
|
| 225 |
def __repr__(self): |
| 226 |
return '%s[%r:not(%r)]' % ( |
| 227 |
self.__class__.__name__, self.selector, self.subselector) |
| 228 |
|
| 229 |
def specificity(self): |
| 230 |
a1, b1, c1 = self.selector.specificity() |
| 231 |
a2, b2, c2 = self.subselector.specificity() |
| 232 |
return a1 + a2, b1 + b2, c1 + c2 |
| 233 |
|
| 234 |
|
| 235 |
class Attrib: |
| 236 |
|
| 237 |
""" |
| 238 |
Represents selector[namespace|attrib operator value] |
| 239 |
""" |
| 240 |
def __init__(self, selector, namespace, attrib, operator, value): |
| 241 |
self.selector = selector |
| 242 |
self.namespace = namespace |
| 243 |
self.attrib = attrib |
| 244 |
self.operator = operator |
| 245 |
self.value = value |
| 246 |
|
| 247 |
def __repr__(self): |
| 248 |
if self.namespace: |
| 249 |
attrib = '%s|%s' % (self.namespace, self.attrib) |
| 250 |
else: |
| 251 |
attrib = self.attrib |
| 252 |
if self.operator == 'exists': |
| 253 |
return '%s[%r[%s]]' % ( |
| 254 |
self.__class__.__name__, self.selector, attrib) |
| 255 |
else: |
| 256 |
return '%s[%r[%s %s %s]]' % ( |
| 257 |
self.__class__.__name__, self.selector, attrib, |
| 258 |
self.operator, urepr(self.value)) |
| 259 |
|
| 260 |
def specificity(self): |
| 261 |
a, b, c = self.selector.specificity() |
| 262 |
b += 1 |
| 263 |
return a, b, c |
| 264 |
|
| 265 |
|
| 266 |
class Element: |
| 267 |
|
| 268 |
""" |
| 269 |
Represents namespace|element |
| 270 |
|
| 271 |
`None` is for the universal selector '*' |
| 272 |
|
| 273 |
""" |
| 274 |
def __init__(self, namespace=None, element=None): |
| 275 |
self.namespace = namespace |
| 276 |
self.element = element |
| 277 |
|
| 278 |
def __repr__(self): |
| 279 |
element = self.element or '*' |
| 280 |
if self.namespace: |
| 281 |
element = '%s|%s' % (self.namespace, element) |
| 282 |
return '%s[%s]' % (self.__class__.__name__, element) |
| 283 |
|
| 284 |
def specificity(self): |
| 285 |
if self.element: |
| 286 |
return 0, 0, 1 |
| 287 |
else: |
| 288 |
return 0, 0, 0 |
| 289 |
|
| 290 |
|
| 291 |
class Hash: |
| 292 |
|
| 293 |
""" |
| 294 |
Represents selector#id |
| 295 |
""" |
| 296 |
def __init__(self, selector, id): |
| 297 |
self.selector = selector |
| 298 |
self.id = id |
| 299 |
|
| 300 |
def __repr__(self): |
| 301 |
return '%s[%r#%s]' % ( |
| 302 |
self.__class__.__name__, self.selector, self.id) |
| 303 |
|
| 304 |
def specificity(self): |
| 305 |
a, b, c = self.selector.specificity() |
| 306 |
a += 1 |
| 307 |
return a, b, c |
| 308 |
|
| 309 |
|
| 310 |
class CombinedSelector: |
| 311 |
|
| 312 |
def __init__(self, selector, combinator, subselector): |
| 313 |
assert selector is not None |
| 314 |
self.selector = selector |
| 315 |
self.combinator = combinator |
| 316 |
self.subselector = subselector |
| 317 |
|
| 318 |
def __repr__(self): |
| 319 |
if self.combinator == ' ': |
| 320 |
comb = '<followed>' |
| 321 |
else: |
| 322 |
comb = self.combinator |
| 323 |
return '%s[%r %s %r]' % ( |
| 324 |
self.__class__.__name__, self.selector, comb, self.subselector) |
| 325 |
|
| 326 |
def specificity(self): |
| 327 |
a1, b1, c1 = self.selector.specificity() |
| 328 |
a2, b2, c2 = self.subselector.specificity() |
| 329 |
return a1 + a2, b1 + b2, c1 + c2 |
| 330 |
|
| 331 |
|
| 332 |
# Parser |
| 333 |
|
| 334 |
# foo |
| 335 |
_el_re = re.compile(r'^[ \t\r\n\f]*([a-zA-Z]+)[ \t\r\n\f]*$') |
| 336 |
|
| 337 |
# foo#bar or #bar |
| 338 |
_id_re = re.compile(r'^[ \t\r\n\f]*([a-zA-Z]*)#([a-zA-Z0-9_-]+)[ \t\r\n\f]*$') |
| 339 |
|
| 340 |
# foo.bar or .bar |
| 341 |
_class_re = re.compile( |
| 342 |
r'^[ \t\r\n\f]*([a-zA-Z]*)\.([a-zA-Z][a-zA-Z0-9_-]*)[ \t\r\n\f]*$') |
| 343 |
|
| 344 |
|
| 345 |
def parse(css): |
| 346 |
"""Parse a CSS *group of selectors*. |
| 347 |
|
| 348 |
:param css: |
| 349 |
A *group of selectors* as an Unicode string. |
| 350 |
:raises: |
| 351 |
:class:`SelectorSyntaxError` on invalid selectors. |
| 352 |
:returns: |
| 353 |
A list of parsed :class:`Selector` objects, one for each |
| 354 |
selector in the comma-separated group. |
| 355 |
|
| 356 |
""" |
| 357 |
# Fast path for simple cases |
| 358 |
match = _el_re.match(css) |
| 359 |
if match: |
| 360 |
return [Selector(Element(element=match.group(1)))] |
| 361 |
match = _id_re.match(css) |
| 362 |
if match is not None: |
| 363 |
return [Selector(Hash(Element(element=match.group(1) or None), |
| 364 |
match.group(2)))] |
| 365 |
match = _class_re.match(css) |
| 366 |
if match is not None: |
| 367 |
return [Selector(Class(Element(element=match.group(1) or None), |
| 368 |
match.group(2)))] |
| 369 |
|
| 370 |
stream = TokenStream(tokenize(css)) |
| 371 |
stream.source = css |
| 372 |
return list(parse_selector_group(stream)) |
| 373 |
# except SelectorSyntaxError: |
| 374 |
# e = sys.exc_info()[1] |
| 375 |
# message = "%s at %s -> %r" % ( |
| 376 |
# e, stream.used, stream.peek()) |
| 377 |
# e.msg = message |
| 378 |
# e.args = tuple([message]) |
| 379 |
# raise |
| 380 |
|
| 381 |
|
| 382 |
def parse_selector_group(stream): |
| 383 |
stream.skip_whitespace() |
| 384 |
while 1: |
| 385 |
yield Selector(*parse_selector(stream)) |
| 386 |
if stream.peek() == ('DELIM', ','): |
| 387 |
stream.next() |
| 388 |
stream.skip_whitespace() |
| 389 |
else: |
| 390 |
break |
| 391 |
|
| 392 |
|
| 393 |
def parse_selector(stream): |
| 394 |
result, pseudo_element = parse_simple_selector(stream) |
| 395 |
while 1: |
| 396 |
stream.skip_whitespace() |
| 397 |
peek = stream.peek() |
| 398 |
if peek in (('EOF', None), ('DELIM', ',')): |
| 399 |
break |
| 400 |
if pseudo_element: |
| 401 |
raise SelectorSyntaxError( |
| 402 |
'Got pseudo-element ::%s not at the end of a selector' |
| 403 |
% pseudo_element) |
| 404 |
if peek.is_delim('+', '>', '~'): |
| 405 |
# A combinator |
| 406 |
combinator = stream.next().value |
| 407 |
stream.skip_whitespace() |
| 408 |
else: |
| 409 |
# By exclusion, the last parse_simple_selector() ended |
| 410 |
# at peek == ' ' |
| 411 |
combinator = ' ' |
| 412 |
next_selector, pseudo_element = parse_simple_selector(stream) |
| 413 |
result = CombinedSelector(result, combinator, next_selector) |
| 414 |
return result, pseudo_element |
| 415 |
|
| 416 |
|
| 417 |
special_pseudo_elements = ( |
| 418 |
'first-line', 'first-letter', 'before', 'after') |
| 419 |
|
| 420 |
|
| 421 |
def parse_simple_selector(stream, inside_negation=False): |
| 422 |
stream.skip_whitespace() |
| 423 |
selector_start = len(stream.used) |
| 424 |
peek = stream.peek() |
| 425 |
if peek.type == 'IDENT' or peek == ('DELIM', '*'): |
| 426 |
if peek.type == 'IDENT': |
| 427 |
namespace = stream.next().value |
| 428 |
else: |
| 429 |
stream.next() |
| 430 |
namespace = None |
| 431 |
if stream.peek() == ('DELIM', '|'): |
| 432 |
stream.next() |
| 433 |
element = stream.next_ident_or_star() |
| 434 |
else: |
| 435 |
element = namespace |
| 436 |
namespace = None |
| 437 |
else: |
| 438 |
element = namespace = None |
| 439 |
result = Element(namespace, element) |
| 440 |
pseudo_element = None |
| 441 |
while 1: |
| 442 |
peek = stream.peek() |
| 443 |
if peek.type in ('S', 'EOF') or peek.is_delim(',', '+', '>', '~') or ( |
| 444 |
inside_negation and peek == ('DELIM', ')')): |
| 445 |
break |
| 446 |
if pseudo_element: |
| 447 |
raise SelectorSyntaxError( |
| 448 |
'Got pseudo-element ::%s not at the end of a selector' |
| 449 |
% pseudo_element) |
| 450 |
if peek.type == 'HASH': |
| 451 |
result = Hash(result, stream.next().value) |
| 452 |
elif peek == ('DELIM', '.'): |
| 453 |
stream.next() |
| 454 |
result = Class(result, stream.next_ident()) |
| 455 |
elif peek == ('DELIM', '['): |
| 456 |
stream.next() |
| 457 |
result = parse_attrib(result, stream) |
| 458 |
elif peek == ('DELIM', ':'): |
| 459 |
stream.next() |
| 460 |
if stream.peek() == ('DELIM', ':'): |
| 461 |
stream.next() |
| 462 |
pseudo_element = stream.next_ident() |
| 463 |
if stream.peek() == ('DELIM', '('): |
| 464 |
stream.next() |
| 465 |
pseudo_element = FunctionalPseudoElement( |
| 466 |
pseudo_element, parse_arguments(stream), result) |
| 467 |
continue |
| 468 |
ident = stream.next_ident() |
| 469 |
if ident.lower() in special_pseudo_elements: |
| 470 |
# Special case: CSS 2.1 pseudo-elements can have a single ':' |
| 471 |
# Any new pseudo-element must have two. |
| 472 |
pseudo_element = unicode_type(ident) |
| 473 |
continue |
| 474 |
if stream.peek() != ('DELIM', '('): |
| 475 |
result = Pseudo(result, ident) |
| 476 |
continue |
| 477 |
stream.next() |
| 478 |
stream.skip_whitespace() |
| 479 |
if ident.lower() == 'not': |
| 480 |
if inside_negation: |
| 481 |
raise SelectorSyntaxError('Got nested :not()') |
| 482 |
argument, argument_pseudo_element = parse_simple_selector( |
| 483 |
stream, inside_negation=True) |
| 484 |
next = stream.next() |
| 485 |
if argument_pseudo_element: |
| 486 |
raise SelectorSyntaxError( |
| 487 |
'Got pseudo-element ::%s inside :not() at %s' |
| 488 |
% (argument_pseudo_element, next.pos)) |
| 489 |
if next != ('DELIM', ')'): |
| 490 |
raise SelectorSyntaxError("Expected ')', got %s" % (next,)) |
| 491 |
result = Negation(result, argument) |
| 492 |
else: |
| 493 |
result = Function(result, ident, parse_arguments(stream)) |
| 494 |
else: |
| 495 |
raise SelectorSyntaxError( |
| 496 |
"Expected selector, got %s" % (peek,)) |
| 497 |
if len(stream.used) == selector_start: |
| 498 |
raise SelectorSyntaxError( |
| 499 |
"Expected selector, got %s" % (stream.peek(),)) |
| 500 |
return result, pseudo_element |
| 501 |
|
| 502 |
|
| 503 |
def parse_arguments(stream): |
| 504 |
arguments = [] |
| 505 |
while 1: |
| 506 |
stream.skip_whitespace() |
| 507 |
next = stream.next() |
| 508 |
if next.type in ('IDENT', 'STRING', 'NUMBER') or next in [ |
| 509 |
('DELIM', '+'), ('DELIM', '-')]: |
| 510 |
arguments.append(next) |
| 511 |
elif next == ('DELIM', ')'): |
| 512 |
return arguments |
| 513 |
else: |
| 514 |
raise SelectorSyntaxError( |
| 515 |
"Expected an argument, got %s" % (next,)) |
| 516 |
|
| 517 |
|
| 518 |
def parse_attrib(selector, stream): |
| 519 |
stream.skip_whitespace() |
| 520 |
attrib = stream.next_ident_or_star() |
| 521 |
if attrib is None and stream.peek() != ('DELIM', '|'): |
| 522 |
raise SelectorSyntaxError( |
| 523 |
"Expected '|', got %s" % (stream.peek(),)) |
| 524 |
if stream.peek() == ('DELIM', '|'): |
| 525 |
stream.next() |
| 526 |
if stream.peek() == ('DELIM', '='): |
| 527 |
namespace = None |
| 528 |
stream.next() |
| 529 |
op = '|=' |
| 530 |
else: |
| 531 |
namespace = attrib |
| 532 |
attrib = stream.next_ident() |
| 533 |
op = None |
| 534 |
else: |
| 535 |
namespace = op = None |
| 536 |
if op is None: |
| 537 |
stream.skip_whitespace() |
| 538 |
next = stream.next() |
| 539 |
if next == ('DELIM', ']'): |
| 540 |
return Attrib(selector, namespace, attrib, 'exists', None) |
| 541 |
elif next == ('DELIM', '='): |
| 542 |
op = '=' |
| 543 |
elif next.is_delim('^', '$', '*', '~', '|', '!') and ( |
| 544 |
stream.peek() == ('DELIM', '=')): |
| 545 |
op = next.value + '=' |
| 546 |
stream.next() |
| 547 |
else: |
| 548 |
raise SelectorSyntaxError( |
| 549 |
"Operator expected, got %s" % (next,)) |
| 550 |
stream.skip_whitespace() |
| 551 |
value = stream.next() |
| 552 |
if value.type not in ('IDENT', 'STRING'): |
| 553 |
raise SelectorSyntaxError( |
| 554 |
"Expected string or ident, got %s" % (value,)) |
| 555 |
stream.skip_whitespace() |
| 556 |
next = stream.next() |
| 557 |
if next != ('DELIM', ']'): |
| 558 |
raise SelectorSyntaxError( |
| 559 |
"Expected ']', got %s" % (next,)) |
| 560 |
return Attrib(selector, namespace, attrib, op, value.value) |
| 561 |
|
| 562 |
|
| 563 |
def parse_series(tokens): |
| 564 |
""" |
| 565 |
Parses the arguments for :nth-child() and friends. |
| 566 |
|
| 567 |
:raises: A list of tokens |
| 568 |
:returns: :``(a, b)`` |
| 569 |
|
| 570 |
""" |
| 571 |
for token in tokens: |
| 572 |
if token.type == 'STRING': |
| 573 |
raise ValueError('String tokens not allowed in series.') |
| 574 |
s = ''.join(token.value for token in tokens).strip() |
| 575 |
if s == 'odd': |
| 576 |
return (2, 1) |
| 577 |
elif s == 'even': |
| 578 |
return (2, 0) |
| 579 |
elif s == 'n': |
| 580 |
return (1, 0) |
| 581 |
if 'n' not in s: |
| 582 |
# Just b |
| 583 |
return (0, int(s)) |
| 584 |
a, b = s.split('n', 1) |
| 585 |
if not a: |
| 586 |
a = 1 |
| 587 |
elif a == '-' or a == '+': |
| 588 |
a = int(a+'1') |
| 589 |
else: |
| 590 |
a = int(a) |
| 591 |
if not b: |
| 592 |
b = 0 |
| 593 |
else: |
| 594 |
b = int(b) |
| 595 |
return (a, b) |
| 596 |
|
| 597 |
|
| 598 |
# Token objects |
| 599 |
|
| 600 |
class Token(tuple): |
| 601 |
|
| 602 |
pos: int |
| 603 |
|
| 604 |
def __new__(cls, type_, value, pos): |
| 605 |
obj = tuple.__new__(cls, (type_, value)) |
| 606 |
obj.pos = pos |
| 607 |
return obj |
| 608 |
|
| 609 |
def __repr__(self): |
| 610 |
return "<%s '%s' at %i>" % (self.type, self.value, self.pos) |
| 611 |
|
| 612 |
def is_delim(self, *values): |
| 613 |
return self.type == 'DELIM' and self.value in values |
| 614 |
|
| 615 |
type = property(operator.itemgetter(0)) |
| 616 |
value = property(operator.itemgetter(1)) |
| 617 |
|
| 618 |
|
| 619 |
class EOFToken(Token): |
| 620 |
|
| 621 |
def __new__(cls, pos): |
| 622 |
return Token.__new__(cls, 'EOF', None, pos) |
| 623 |
|
| 624 |
def __repr__(self): |
| 625 |
return '<%s at %i>' % (self.type, self.pos) |
| 626 |
|
| 627 |
|
| 628 |
# Tokenizer |
| 629 |
|
| 630 |
|
| 631 |
class TokenMacros: |
| 632 |
unicode_escape = r'\\([0-9a-f]{1,6})(?:\r\n|[ \n\r\t\f])?' |
| 633 |
escape = unicode_escape + r'|\\[^\n\r\f0-9a-f]' |
| 634 |
string_escape = r'\\(?:\n|\r\n|\r|\f)|' + escape |
| 635 |
nonascii = r'[^\0-\177]' |
| 636 |
nmchar = '[_a-z0-9-]|%s|%s' % (escape, nonascii) |
| 637 |
nmstart = '[_a-z]|%s|%s' % (escape, nonascii) |
| 638 |
|
| 639 |
|
| 640 |
def _compile(pattern): |
| 641 |
return re.compile(pattern % vars(TokenMacros), re.IGNORECASE).match |
| 642 |
|
| 643 |
|
| 644 |
_match_whitespace = _compile(r'[ \t\r\n\f]+') |
| 645 |
_match_number = _compile(r'[+-]?(?:[0-9]*\.[0-9]+|[0-9]+)') |
| 646 |
_match_hash = _compile('#(?:%(nmchar)s)+') |
| 647 |
_match_ident = _compile('-?(?:%(nmstart)s)(?:%(nmchar)s)*') |
| 648 |
_match_string_by_quote = { |
| 649 |
"'": _compile(r"([^\n\r\f\\']|%(string_escape)s)*"), |
| 650 |
'"': _compile(r'([^\n\r\f\\"]|%(string_escape)s)*'), |
| 651 |
} |
| 652 |
|
| 653 |
_sub_simple_escape = re.compile(r'\\(.)').sub |
| 654 |
_sub_unicode_escape = re.compile(TokenMacros.unicode_escape, re.I).sub |
| 655 |
_sub_newline_escape =re.compile(r'\\(?:\n|\r\n|\r|\f)').sub |
| 656 |
|
| 657 |
# Same as r'\1', but faster on CPython |
| 658 |
if hasattr(operator, 'methodcaller'): |
| 659 |
# Python 2.6+ |
| 660 |
_replace_simple = operator.methodcaller('group', 1) |
| 661 |
else: |
| 662 |
def _replace_simple(match): |
| 663 |
return match.group(1) |
| 664 |
|
| 665 |
|
| 666 |
def _replace_unicode(match): |
| 667 |
codepoint = int(match.group(1), 16) |
| 668 |
if codepoint > sys.maxunicode: |
| 669 |
codepoint = 0xFFFD |
| 670 |
return codepoint_to_chr(codepoint) |
| 671 |
|
| 672 |
|
| 673 |
def unescape_ident(value): |
| 674 |
value = _sub_unicode_escape(_replace_unicode, value) |
| 675 |
value = _sub_simple_escape(_replace_simple, value) |
| 676 |
return value |
| 677 |
|
| 678 |
|
| 679 |
def tokenize(s): |
| 680 |
pos = 0 |
| 681 |
len_s = len(s) |
| 682 |
while pos < len_s: |
| 683 |
match = _match_whitespace(s, pos=pos) |
| 684 |
if match: |
| 685 |
yield Token('S', ' ', pos) |
| 686 |
pos = match.end() |
| 687 |
continue |
| 688 |
|
| 689 |
match = _match_ident(s, pos=pos) |
| 690 |
if match: |
| 691 |
value = _sub_simple_escape(_replace_simple, |
| 692 |
_sub_unicode_escape(_replace_unicode, match.group())) |
| 693 |
yield Token('IDENT', value, pos) |
| 694 |
pos = match.end() |
| 695 |
continue |
| 696 |
|
| 697 |
match = _match_hash(s, pos=pos) |
| 698 |
if match: |
| 699 |
value = _sub_simple_escape(_replace_simple, |
| 700 |
_sub_unicode_escape(_replace_unicode, match.group()[1:])) |
| 701 |
yield Token('HASH', value, pos) |
| 702 |
pos = match.end() |
| 703 |
continue |
| 704 |
|
| 705 |
quote = s[pos] |
| 706 |
if quote in _match_string_by_quote: |
| 707 |
match = _match_string_by_quote[quote](s, pos=pos + 1) |
| 708 |
assert match, 'Should have found at least an empty match' |
| 709 |
end_pos = match.end() |
| 710 |
if end_pos == len_s: |
| 711 |
raise SelectorSyntaxError('Unclosed string at %s' % pos) |
| 712 |
if s[end_pos] != quote: |
| 713 |
raise SelectorSyntaxError('Invalid string at %s' % pos) |
| 714 |
value = _sub_simple_escape(_replace_simple, |
| 715 |
_sub_unicode_escape(_replace_unicode, |
| 716 |
_sub_newline_escape('', match.group()))) |
| 717 |
yield Token('STRING', value, pos) |
| 718 |
pos = end_pos + 1 |
| 719 |
continue |
| 720 |
|
| 721 |
match = _match_number(s, pos=pos) |
| 722 |
if match: |
| 723 |
value = match.group() |
| 724 |
yield Token('NUMBER', value, pos) |
| 725 |
pos = match.end() |
| 726 |
continue |
| 727 |
|
| 728 |
pos2 = pos + 2 |
| 729 |
if s[pos:pos2] == '/*': |
| 730 |
pos = s.find('*/', pos2) |
| 731 |
if pos == -1: |
| 732 |
pos = len_s |
| 733 |
else: |
| 734 |
pos += 2 |
| 735 |
continue |
| 736 |
|
| 737 |
yield Token('DELIM', s[pos], pos) |
| 738 |
pos += 1 |
| 739 |
|
| 740 |
assert pos == len_s |
| 741 |
yield EOFToken(pos) |
| 742 |
|
| 743 |
|
| 744 |
class TokenStream: |
| 745 |
|
| 746 |
def __init__(self, tokens, source=None): |
| 747 |
self.used = [] |
| 748 |
self.tokens = iter(tokens) |
| 749 |
self.source = source |
| 750 |
self.peeked = None |
| 751 |
self._peeking = False |
| 752 |
try: |
| 753 |
self.next_token = self.tokens.next |
| 754 |
except AttributeError: |
| 755 |
# Python 3 |
| 756 |
self.next_token = self.tokens.__next__ |
| 757 |
|
| 758 |
def next(self): |
| 759 |
if self._peeking: |
| 760 |
self._peeking = False |
| 761 |
self.used.append(self.peeked) |
| 762 |
return self.peeked |
| 763 |
else: |
| 764 |
next = self.next_token() |
| 765 |
self.used.append(next) |
| 766 |
return next |
| 767 |
|
| 768 |
def peek(self): |
| 769 |
if not self._peeking: |
| 770 |
self.peeked = self.next_token() |
| 771 |
self._peeking = True |
| 772 |
return self.peeked |
| 773 |
|
| 774 |
def next_ident(self): |
| 775 |
next = self.next() |
| 776 |
if next.type != 'IDENT': |
| 777 |
raise SelectorSyntaxError('Expected ident, got %s' % (next,)) |
| 778 |
return next.value |
| 779 |
|
| 780 |
def next_ident_or_star(self): |
| 781 |
next = self.next() |
| 782 |
if next.type == 'IDENT': |
| 783 |
return next.value |
| 784 |
elif next == ('DELIM', '*'): |
| 785 |
return None |
| 786 |
else: |
| 787 |
raise SelectorSyntaxError( |
| 788 |
"Expected ident or '*', got %s" % (next,)) |
| 789 |
|
| 790 |
def skip_whitespace(self): |
| 791 |
peek = self.peek() |
| 792 |
if peek.type == 'S': |
| 793 |
self.next() |