Debian Codegraph @

src/css_selectors/parser.py

raw
1 #!/usr/bin/env python
2 # vim:fileencoding=utf-8
3 """
4 Tokenizer, parser and parsed objects for CSS selectors.
5
6 :copyright: (c) 2007-2012 Ian Bicking and contributors.
7 See AUTHORS for more details.
8 :license: BSD, see LICENSE for more details.
9
10 """
11
12 import operator
13 import re
14 import string
15 import sys
16
17 from css_selectors.errors import ExpressionError, SelectorSyntaxError
18 from polyglot.builtins import codepoint_to_chr, unicode_type
19
20 utab = {c:c+32 for c in range(ord(u'A'), ord(u'Z')+1)}
21
22 if sys.version_info.major < 3:
23 tab = string.maketrans(string.ascii_uppercase, string.ascii_lowercase)
24
25 def ascii_lower(string):
26 """Lower-case, but only in the ASCII range."""
27 return string.translate(utab if isinstance(string, unicode_type) else tab)
28
29 def urepr(x):
30 if isinstance(x, list):
31 return '[%s]' % ', '.join((map(urepr, x)))
32 ans = repr(x)
33 if ans.startswith("u'") or ans.startswith('u"'):
34 ans = ans[1:]
35 return ans
36
37
38 else:
39
40 def ascii_lower(x):
41 return x.translate(utab)
42
43 urepr = repr
44
45
46 # Parsed objects
47
48 class Selector:
49
50 """
51 Represents a parsed selector.
52 """
53
54 def __init__(self, tree, pseudo_element=None):
55 self.parsed_tree = tree
56 if pseudo_element is not None and not isinstance(
57 pseudo_element, FunctionalPseudoElement):
58 pseudo_element = ascii_lower(pseudo_element)
59 #: A :class:`FunctionalPseudoElement`,
60 #: or the identifier for the pseudo-element as a string,
61 # or ``None``.
62 #:
63 #: +-------------------------+----------------+--------------------------------+
64 #: | | Selector | Pseudo-element |
65 #: +=========================+================+================================+
66 #: | CSS3 syntax | ``a::before`` | ``'before'`` |
67 #: +-------------------------+----------------+--------------------------------+
68 #: | Older syntax | ``a:before`` | ``'before'`` |
69 #: +-------------------------+----------------+--------------------------------+
70 #: | From the Lists3_ draft, | ``li::marker`` | ``'marker'`` |
71 #: | not in Selectors3 | | |
72 #: +-------------------------+----------------+--------------------------------+
73 #: | Invalid pseudo-class | ``li:marker`` | ``None`` |
74 #: +-------------------------+----------------+--------------------------------+
75 #: | Functional | ``a::foo(2)`` | ``FunctionalPseudoElement(…)`` |
76 #: +-------------------------+----------------+--------------------------------+
77 #:
78 # : .. _Lists3: http://www.w3.org/TR/2011/WD-css3-lists-20110524/#marker-pseudoelement
79 self.pseudo_element = pseudo_element
80
81 def __repr__(self):
82 if isinstance(self.pseudo_element, FunctionalPseudoElement):
83 pseudo_element = repr(self.pseudo_element)
84 if self.pseudo_element:
85 pseudo_element = '::%s' % self.pseudo_element
86 else:
87 pseudo_element = ''
88 return '%s[%r%s]' % (
89 self.__class__.__name__, self.parsed_tree, pseudo_element)
90
91 def specificity(self):
92 """Return the specificity_ of this selector as a tuple of 3 integers.
93
94 .. _specificity: http://www.w3.org/TR/selectors/#specificity
95
96 """
97 a, b, c = self.parsed_tree.specificity()
98 if self.pseudo_element:
99 c += 1
100 return a, b, c
101
102
103 class Class:
104
105 """
106 Represents selector.class_name
107 """
108 def __init__(self, selector, class_name):
109 self.selector = selector
110 self.class_name = class_name
111
112 def __repr__(self):
113 return '%s[%r.%s]' % (
114 self.__class__.__name__, self.selector, self.class_name)
115
116 def specificity(self):
117 a, b, c = self.selector.specificity()
118 b += 1
119 return a, b, c
120
121
122 class FunctionalPseudoElement:
123
124 """
125 Represents selector::name(arguments)
126
127 .. attribute:: name
128
129 The name (identifier) of the pseudo-element, as a string.
130
131 .. attribute:: arguments
132
133 The arguments of the pseudo-element, as a list of tokens.
134
135 **Note:** tokens are not part of the public API,
136 and may change between versions.
137 Use at your own risks.
138
139 """
140 def __init__(self, name, arguments, selector):
141 self.name = ascii_lower(name)
142 self.arguments = arguments
143 self.selector = selector
144
145 def __repr__(self):
146 return '%s[::%s(%s)]' % (
147 self.__class__.__name__, self.name,
148 urepr([token.value for token in self.arguments]))
149
150 def argument_types(self):
151 return [token.type for token in self.arguments]
152
153 def specificity(self):
154 a, b, c = self.selector.specificity()
155 b += 1
156 return a, b, c
157
158
159 class Function:
160
161 """
162 Represents selector:name(expr)
163 """
164 def __init__(self, selector, name, arguments):
165 self.selector = selector
166 self.name = ascii_lower(name)
167 self.arguments = arguments
168 self._parsed_arguments = None
169
170 def __repr__(self):
171 return '%s[%r:%s(%s)]' % (
172 self.__class__.__name__, self.selector, self.name,
173 urepr([token.value for token in self.arguments]))
174
175 def argument_types(self):
176 return [token.type for token in self.arguments]
177
178 @property
179 def parsed_arguments(self):
180 if self._parsed_arguments is None:
181 try:
182 self._parsed_arguments = parse_series(self.arguments)
183 except ValueError:
184 raise ExpressionError("Invalid series: '%r'" % self.arguments)
185 return self._parsed_arguments
186
187 def parse_arguments(self):
188 if not self.arguments_parsed:
189 self.arguments_parsed = True
190
191 def specificity(self):
192 a, b, c = self.selector.specificity()
193 b += 1
194 return a, b, c
195
196
197 class Pseudo:
198
199 """
200 Represents selector:ident
201 """
202 def __init__(self, selector, ident):
203 self.selector = selector
204 self.ident = ascii_lower(ident)
205
206 def __repr__(self):
207 return '%s[%r:%s]' % (
208 self.__class__.__name__, self.selector, self.ident)
209
210 def specificity(self):
211 a, b, c = self.selector.specificity()
212 b += 1
213 return a, b, c
214
215
216 class Negation:
217
218 """
219 Represents selector:not(subselector)
220 """
221 def __init__(self, selector, subselector):
222 self.selector = selector
223 self.subselector = subselector
224
225 def __repr__(self):
226 return '%s[%r:not(%r)]' % (
227 self.__class__.__name__, self.selector, self.subselector)
228
229 def specificity(self):
230 a1, b1, c1 = self.selector.specificity()
231 a2, b2, c2 = self.subselector.specificity()
232 return a1 + a2, b1 + b2, c1 + c2
233
234
235 class Attrib:
236
237 """
238 Represents selector[namespace|attrib operator value]
239 """
240 def __init__(self, selector, namespace, attrib, operator, value):
241 self.selector = selector
242 self.namespace = namespace
243 self.attrib = attrib
244 self.operator = operator
245 self.value = value
246
247 def __repr__(self):
248 if self.namespace:
249 attrib = '%s|%s' % (self.namespace, self.attrib)
250 else:
251 attrib = self.attrib
252 if self.operator == 'exists':
253 return '%s[%r[%s]]' % (
254 self.__class__.__name__, self.selector, attrib)
255 else:
256 return '%s[%r[%s %s %s]]' % (
257 self.__class__.__name__, self.selector, attrib,
258 self.operator, urepr(self.value))
259
260 def specificity(self):
261 a, b, c = self.selector.specificity()
262 b += 1
263 return a, b, c
264
265
266 class Element:
267
268 """
269 Represents namespace|element
270
271 `None` is for the universal selector '*'
272
273 """
274 def __init__(self, namespace=None, element=None):
275 self.namespace = namespace
276 self.element = element
277
278 def __repr__(self):
279 element = self.element or '*'
280 if self.namespace:
281 element = '%s|%s' % (self.namespace, element)
282 return '%s[%s]' % (self.__class__.__name__, element)
283
284 def specificity(self):
285 if self.element:
286 return 0, 0, 1
287 else:
288 return 0, 0, 0
289
290
291 class Hash:
292
293 """
294 Represents selector#id
295 """
296 def __init__(self, selector, id):
297 self.selector = selector
298 self.id = id
299
300 def __repr__(self):
301 return '%s[%r#%s]' % (
302 self.__class__.__name__, self.selector, self.id)
303
304 def specificity(self):
305 a, b, c = self.selector.specificity()
306 a += 1
307 return a, b, c
308
309
310 class CombinedSelector:
311
312 def __init__(self, selector, combinator, subselector):
313 assert selector is not None
314 self.selector = selector
315 self.combinator = combinator
316 self.subselector = subselector
317
318 def __repr__(self):
319 if self.combinator == ' ':
320 comb = '<followed>'
321 else:
322 comb = self.combinator
323 return '%s[%r %s %r]' % (
324 self.__class__.__name__, self.selector, comb, self.subselector)
325
326 def specificity(self):
327 a1, b1, c1 = self.selector.specificity()
328 a2, b2, c2 = self.subselector.specificity()
329 return a1 + a2, b1 + b2, c1 + c2
330
331
332 # Parser
333
334 # foo
335 _el_re = re.compile(r'^[ \t\r\n\f]*([a-zA-Z]+)[ \t\r\n\f]*$')
336
337 # foo#bar or #bar
338 _id_re = re.compile(r'^[ \t\r\n\f]*([a-zA-Z]*)#([a-zA-Z0-9_-]+)[ \t\r\n\f]*$')
339
340 # foo.bar or .bar
341 _class_re = re.compile(
342 r'^[ \t\r\n\f]*([a-zA-Z]*)\.([a-zA-Z][a-zA-Z0-9_-]*)[ \t\r\n\f]*$')
343
344
345 def parse(css):
346 """Parse a CSS *group of selectors*.
347
348 :param css:
349 A *group of selectors* as an Unicode string.
350 :raises:
351 :class:`SelectorSyntaxError` on invalid selectors.
352 :returns:
353 A list of parsed :class:`Selector` objects, one for each
354 selector in the comma-separated group.
355
356 """
357 # Fast path for simple cases
358 match = _el_re.match(css)
359 if match:
360 return [Selector(Element(element=match.group(1)))]
361 match = _id_re.match(css)
362 if match is not None:
363 return [Selector(Hash(Element(element=match.group(1) or None),
364 match.group(2)))]
365 match = _class_re.match(css)
366 if match is not None:
367 return [Selector(Class(Element(element=match.group(1) or None),
368 match.group(2)))]
369
370 stream = TokenStream(tokenize(css))
371 stream.source = css
372 return list(parse_selector_group(stream))
373 # except SelectorSyntaxError:
374 # e = sys.exc_info()[1]
375 # message = "%s at %s -> %r" % (
376 # e, stream.used, stream.peek())
377 # e.msg = message
378 # e.args = tuple([message])
379 # raise
380
381
382 def parse_selector_group(stream):
383 stream.skip_whitespace()
384 while 1:
385 yield Selector(*parse_selector(stream))
386 if stream.peek() == ('DELIM', ','):
387 stream.next()
388 stream.skip_whitespace()
389 else:
390 break
391
392
393 def parse_selector(stream):
394 result, pseudo_element = parse_simple_selector(stream)
395 while 1:
396 stream.skip_whitespace()
397 peek = stream.peek()
398 if peek in (('EOF', None), ('DELIM', ',')):
399 break
400 if pseudo_element:
401 raise SelectorSyntaxError(
402 'Got pseudo-element ::%s not at the end of a selector'
403 % pseudo_element)
404 if peek.is_delim('+', '>', '~'):
405 # A combinator
406 combinator = stream.next().value
407 stream.skip_whitespace()
408 else:
409 # By exclusion, the last parse_simple_selector() ended
410 # at peek == ' '
411 combinator = ' '
412 next_selector, pseudo_element = parse_simple_selector(stream)
413 result = CombinedSelector(result, combinator, next_selector)
414 return result, pseudo_element
415
416
417 special_pseudo_elements = (
418 'first-line', 'first-letter', 'before', 'after')
419
420
421 def parse_simple_selector(stream, inside_negation=False):
422 stream.skip_whitespace()
423 selector_start = len(stream.used)
424 peek = stream.peek()
425 if peek.type == 'IDENT' or peek == ('DELIM', '*'):
426 if peek.type == 'IDENT':
427 namespace = stream.next().value
428 else:
429 stream.next()
430 namespace = None
431 if stream.peek() == ('DELIM', '|'):
432 stream.next()
433 element = stream.next_ident_or_star()
434 else:
435 element = namespace
436 namespace = None
437 else:
438 element = namespace = None
439 result = Element(namespace, element)
440 pseudo_element = None
441 while 1:
442 peek = stream.peek()
443 if peek.type in ('S', 'EOF') or peek.is_delim(',', '+', '>', '~') or (
444 inside_negation and peek == ('DELIM', ')')):
445 break
446 if pseudo_element:
447 raise SelectorSyntaxError(
448 'Got pseudo-element ::%s not at the end of a selector'
449 % pseudo_element)
450 if peek.type == 'HASH':
451 result = Hash(result, stream.next().value)
452 elif peek == ('DELIM', '.'):
453 stream.next()
454 result = Class(result, stream.next_ident())
455 elif peek == ('DELIM', '['):
456 stream.next()
457 result = parse_attrib(result, stream)
458 elif peek == ('DELIM', ':'):
459 stream.next()
460 if stream.peek() == ('DELIM', ':'):
461 stream.next()
462 pseudo_element = stream.next_ident()
463 if stream.peek() == ('DELIM', '('):
464 stream.next()
465 pseudo_element = FunctionalPseudoElement(
466 pseudo_element, parse_arguments(stream), result)
467 continue
468 ident = stream.next_ident()
469 if ident.lower() in special_pseudo_elements:
470 # Special case: CSS 2.1 pseudo-elements can have a single ':'
471 # Any new pseudo-element must have two.
472 pseudo_element = unicode_type(ident)
473 continue
474 if stream.peek() != ('DELIM', '('):
475 result = Pseudo(result, ident)
476 continue
477 stream.next()
478 stream.skip_whitespace()
479 if ident.lower() == 'not':
480 if inside_negation:
481 raise SelectorSyntaxError('Got nested :not()')
482 argument, argument_pseudo_element = parse_simple_selector(
483 stream, inside_negation=True)
484 next = stream.next()
485 if argument_pseudo_element:
486 raise SelectorSyntaxError(
487 'Got pseudo-element ::%s inside :not() at %s'
488 % (argument_pseudo_element, next.pos))
489 if next != ('DELIM', ')'):
490 raise SelectorSyntaxError("Expected ')', got %s" % (next,))
491 result = Negation(result, argument)
492 else:
493 result = Function(result, ident, parse_arguments(stream))
494 else:
495 raise SelectorSyntaxError(
496 "Expected selector, got %s" % (peek,))
497 if len(stream.used) == selector_start:
498 raise SelectorSyntaxError(
499 "Expected selector, got %s" % (stream.peek(),))
500 return result, pseudo_element
501
502
503 def parse_arguments(stream):
504 arguments = []
505 while 1:
506 stream.skip_whitespace()
507 next = stream.next()
508 if next.type in ('IDENT', 'STRING', 'NUMBER') or next in [
509 ('DELIM', '+'), ('DELIM', '-')]:
510 arguments.append(next)
511 elif next == ('DELIM', ')'):
512 return arguments
513 else:
514 raise SelectorSyntaxError(
515 "Expected an argument, got %s" % (next,))
516
517
518 def parse_attrib(selector, stream):
519 stream.skip_whitespace()
520 attrib = stream.next_ident_or_star()
521 if attrib is None and stream.peek() != ('DELIM', '|'):
522 raise SelectorSyntaxError(
523 "Expected '|', got %s" % (stream.peek(),))
524 if stream.peek() == ('DELIM', '|'):
525 stream.next()
526 if stream.peek() == ('DELIM', '='):
527 namespace = None
528 stream.next()
529 op = '|='
530 else:
531 namespace = attrib
532 attrib = stream.next_ident()
533 op = None
534 else:
535 namespace = op = None
536 if op is None:
537 stream.skip_whitespace()
538 next = stream.next()
539 if next == ('DELIM', ']'):
540 return Attrib(selector, namespace, attrib, 'exists', None)
541 elif next == ('DELIM', '='):
542 op = '='
543 elif next.is_delim('^', '$', '*', '~', '|', '!') and (
544 stream.peek() == ('DELIM', '=')):
545 op = next.value + '='
546 stream.next()
547 else:
548 raise SelectorSyntaxError(
549 "Operator expected, got %s" % (next,))
550 stream.skip_whitespace()
551 value = stream.next()
552 if value.type not in ('IDENT', 'STRING'):
553 raise SelectorSyntaxError(
554 "Expected string or ident, got %s" % (value,))
555 stream.skip_whitespace()
556 next = stream.next()
557 if next != ('DELIM', ']'):
558 raise SelectorSyntaxError(
559 "Expected ']', got %s" % (next,))
560 return Attrib(selector, namespace, attrib, op, value.value)
561
562
563 def parse_series(tokens):
564 """
565 Parses the arguments for :nth-child() and friends.
566
567 :raises: A list of tokens
568 :returns: :``(a, b)``
569
570 """
571 for token in tokens:
572 if token.type == 'STRING':
573 raise ValueError('String tokens not allowed in series.')
574 s = ''.join(token.value for token in tokens).strip()
575 if s == 'odd':
576 return (2, 1)
577 elif s == 'even':
578 return (2, 0)
579 elif s == 'n':
580 return (1, 0)
581 if 'n' not in s:
582 # Just b
583 return (0, int(s))
584 a, b = s.split('n', 1)
585 if not a:
586 a = 1
587 elif a == '-' or a == '+':
588 a = int(a+'1')
589 else:
590 a = int(a)
591 if not b:
592 b = 0
593 else:
594 b = int(b)
595 return (a, b)
596
597
598 # Token objects
599
600 class Token(tuple):
601
602 pos: int
603
604 def __new__(cls, type_, value, pos):
605 obj = tuple.__new__(cls, (type_, value))
606 obj.pos = pos
607 return obj
608
609 def __repr__(self):
610 return "<%s '%s' at %i>" % (self.type, self.value, self.pos)
611
612 def is_delim(self, *values):
613 return self.type == 'DELIM' and self.value in values
614
615 type = property(operator.itemgetter(0))
616 value = property(operator.itemgetter(1))
617
618
619 class EOFToken(Token):
620
621 def __new__(cls, pos):
622 return Token.__new__(cls, 'EOF', None, pos)
623
624 def __repr__(self):
625 return '<%s at %i>' % (self.type, self.pos)
626
627
628 # Tokenizer
629
630
631 class TokenMacros:
632 unicode_escape = r'\\([0-9a-f]{1,6})(?:\r\n|[ \n\r\t\f])?'
633 escape = unicode_escape + r'|\\[^\n\r\f0-9a-f]'
634 string_escape = r'\\(?:\n|\r\n|\r|\f)|' + escape
635 nonascii = r'[^\0-\177]'
636 nmchar = '[_a-z0-9-]|%s|%s' % (escape, nonascii)
637 nmstart = '[_a-z]|%s|%s' % (escape, nonascii)
638
639
640 def _compile(pattern):
641 return re.compile(pattern % vars(TokenMacros), re.IGNORECASE).match
642
643
644 _match_whitespace = _compile(r'[ \t\r\n\f]+')
645 _match_number = _compile(r'[+-]?(?:[0-9]*\.[0-9]+|[0-9]+)')
646 _match_hash = _compile('#(?:%(nmchar)s)+')
647 _match_ident = _compile('-?(?:%(nmstart)s)(?:%(nmchar)s)*')
648 _match_string_by_quote = {
649 "'": _compile(r"([^\n\r\f\\']|%(string_escape)s)*"),
650 '"': _compile(r'([^\n\r\f\\"]|%(string_escape)s)*'),
651 }
652
653 _sub_simple_escape = re.compile(r'\\(.)').sub
654 _sub_unicode_escape = re.compile(TokenMacros.unicode_escape, re.I).sub
655 _sub_newline_escape =re.compile(r'\\(?:\n|\r\n|\r|\f)').sub
656
657 # Same as r'\1', but faster on CPython
658 if hasattr(operator, 'methodcaller'):
659 # Python 2.6+
660 _replace_simple = operator.methodcaller('group', 1)
661 else:
662 def _replace_simple(match):
663 return match.group(1)
664
665
666 def _replace_unicode(match):
667 codepoint = int(match.group(1), 16)
668 if codepoint > sys.maxunicode:
669 codepoint = 0xFFFD
670 return codepoint_to_chr(codepoint)
671
672
673 def unescape_ident(value):
674 value = _sub_unicode_escape(_replace_unicode, value)
675 value = _sub_simple_escape(_replace_simple, value)
676 return value
677
678
679 def tokenize(s):
680 pos = 0
681 len_s = len(s)
682 while pos < len_s:
683 match = _match_whitespace(s, pos=pos)
684 if match:
685 yield Token('S', ' ', pos)
686 pos = match.end()
687 continue
688
689 match = _match_ident(s, pos=pos)
690 if match:
691 value = _sub_simple_escape(_replace_simple,
692 _sub_unicode_escape(_replace_unicode, match.group()))
693 yield Token('IDENT', value, pos)
694 pos = match.end()
695 continue
696
697 match = _match_hash(s, pos=pos)
698 if match:
699 value = _sub_simple_escape(_replace_simple,
700 _sub_unicode_escape(_replace_unicode, match.group()[1:]))
701 yield Token('HASH', value, pos)
702 pos = match.end()
703 continue
704
705 quote = s[pos]
706 if quote in _match_string_by_quote:
707 match = _match_string_by_quote[quote](s, pos=pos + 1)
708 assert match, 'Should have found at least an empty match'
709 end_pos = match.end()
710 if end_pos == len_s:
711 raise SelectorSyntaxError('Unclosed string at %s' % pos)
712 if s[end_pos] != quote:
713 raise SelectorSyntaxError('Invalid string at %s' % pos)
714 value = _sub_simple_escape(_replace_simple,
715 _sub_unicode_escape(_replace_unicode,
716 _sub_newline_escape('', match.group())))
717 yield Token('STRING', value, pos)
718 pos = end_pos + 1
719 continue
720
721 match = _match_number(s, pos=pos)
722 if match:
723 value = match.group()
724 yield Token('NUMBER', value, pos)
725 pos = match.end()
726 continue
727
728 pos2 = pos + 2
729 if s[pos:pos2] == '/*':
730 pos = s.find('*/', pos2)
731 if pos == -1:
732 pos = len_s
733 else:
734 pos += 2
735 continue
736
737 yield Token('DELIM', s[pos], pos)
738 pos += 1
739
740 assert pos == len_s
741 yield EOFToken(pos)
742
743
744 class TokenStream:
745
746 def __init__(self, tokens, source=None):
747 self.used = []
748 self.tokens = iter(tokens)
749 self.source = source
750 self.peeked = None
751 self._peeking = False
752 try:
753 self.next_token = self.tokens.next
754 except AttributeError:
755 # Python 3
756 self.next_token = self.tokens.__next__
757
758 def next(self):
759 if self._peeking:
760 self._peeking = False
761 self.used.append(self.peeked)
762 return self.peeked
763 else:
764 next = self.next_token()
765 self.used.append(next)
766 return next
767
768 def peek(self):
769 if not self._peeking:
770 self.peeked = self.next_token()
771 self._peeking = True
772 return self.peeked
773
774 def next_ident(self):
775 next = self.next()
776 if next.type != 'IDENT':
777 raise SelectorSyntaxError('Expected ident, got %s' % (next,))
778 return next.value
779
780 def next_ident_or_star(self):
781 next = self.next()
782 if next.type == 'IDENT':
783 return next.value
784 elif next == ('DELIM', '*'):
785 return None
786 else:
787 raise SelectorSyntaxError(
788 "Expected ident or '*', got %s" % (next,))
789
790 def skip_whitespace(self):
791 peek = self.peek()
792 if peek.type == 'S':
793 self.next()