Debian Codegraph @

Lib/html/parser.py

raw
1 """A parser for HTML and XHTML."""
2
3 # This file is based on sgmllib.py, but the API is slightly different.
4
5 # XXX There should be a way to distinguish between PCDATA (parsed
6 # character data -- the normal case), RCDATA (replaceable character
7 # data -- only char and entity references and end tags are special)
8 # and CDATA (character data -- only end tags are special).
9
10
11 import re
12 import _markupbase
13
14 from html import unescape
15 from html.entities import html5 as html5_entities
16
17
18 __all__ = ['HTMLParser']
19
20 # Regular expressions used for parsing
21
22 interesting_normal = re.compile('[&<]')
23 incomplete = re.compile('&[a-zA-Z#]')
24
25 entityref = re.compile('&([a-zA-Z][-.a-zA-Z0-9]*)[^a-zA-Z0-9]')
26 charref = re.compile('&#(?:[0-9]+|[xX][0-9a-fA-F]+)[^0-9a-fA-F]')
27 incomplete_charref = re.compile('&#(?:[0-9]|[xX][0-9a-fA-F])')
28 attr_charref = re.compile(r'&(#[0-9]+|#[xX][0-9a-fA-F]+|[a-zA-Z][a-zA-Z0-9]*)[;=]?')
29
30 starttagopen = re.compile('<[a-zA-Z]')
31 endtagopen = re.compile('</[a-zA-Z]')
32 piclose = re.compile('>')
33 commentclose = re.compile(r'--!?>')
34 commentabruptclose = re.compile(r'-?>')
35 # Note:
36 # 1) if you change tagfind/attrfind remember to update locatetagend too;
37 # 2) if you change tagfind/attrfind and/or locatetagend the parser will
38 # explode, so don't do it.
39 # see the HTML5 specs section "13.2.5.6 Tag open state",
40 # "13.2.5.8 Tag name state" and "13.2.5.33 Attribute name state".
41 # https://html.spec.whatwg.org/multipage/parsing.html#tag-open-state
42 # https://html.spec.whatwg.org/multipage/parsing.html#tag-name-state
43 # https://html.spec.whatwg.org/multipage/parsing.html#attribute-name-state
44 tagfind_tolerant = re.compile(r'([a-zA-Z][^\t\n\r\f />]*)(?:[\t\n\r\f ]|/(?!>))*')
45 attrfind_tolerant = re.compile(r"""
46 (
47 (?<=['"\t\n\r\f /])[^\t\n\r\f />][^\t\n\r\f /=>]* # attribute name
48 )
49 ([\t\n\r\f ]*=[\t\n\r\f ]* # value indicator
50 ('[^']*' # LITA-enclosed value
51 |"[^"]*" # LIT-enclosed value
52 |(?!['"])[^>\t\n\r\f ]* # bare value
53 )
54 )?
55 (?:[\t\n\r\f ]|/(?!>))* # possibly followed by a space
56 """, re.VERBOSE)
57 locatetagend = re.compile(r"""
58 [a-zA-Z][^\t\n\r\f />]* # tag name
59 [\t\n\r\f /]* # optional whitespace before attribute name
60 (?:(?<=['"\t\n\r\f /])[^\t\n\r\f />][^\t\n\r\f /=>]* # attribute name
61 (?:[\t\n\r\f ]*=[\t\n\r\f ]* # value indicator
62 (?:'[^']*' # LITA-enclosed value
63 |"[^"]*" # LIT-enclosed value
64 |(?!['"])[^>\t\n\r\f ]* # bare value
65 )
66 )?
67 [\t\n\r\f /]* # possibly followed by a space
68 )*
69 >?
70 """, re.VERBOSE)
71 # The following variables are not used, but are temporarily left for
72 # backward compatibility.
73 locatestarttagend_tolerant = re.compile(r"""
74 <[a-zA-Z][^\t\n\r\f />\x00]* # tag name
75 (?:[\s/]* # optional whitespace before attribute name
76 (?:(?<=['"\s/])[^\s/>][^\s/=>]* # attribute name
77 (?:\s*=+\s* # value indicator
78 (?:'[^']*' # LITA-enclosed value
79 |"[^"]*" # LIT-enclosed value
80 |(?!['"])[^>\s]* # bare value
81 )
82 \s* # possibly followed by a space
83 )?(?:\s|/(?!>))*
84 )*
85 )?
86 \s* # trailing whitespace
87 """, re.VERBOSE)
88 endendtag = re.compile('>')
89 endtagfind = re.compile(r'</\s*([a-zA-Z][-.a-zA-Z0-9:_]*)\s*>')
90
91 # Character reference processing logic specific to attribute values
92 # See: https://html.spec.whatwg.org/multipage/parsing.html#named-character-reference-state
93 def _replace_attr_charref(match):
94 ref = match.group(0)
95 # Numeric / hex char refs must always be unescaped
96 if ref.startswith('&#'):
97 return unescape(ref)
98 # Named character / entity references must only be unescaped
99 # if they are an exact match, and they are not followed by an equals sign
100 if not ref.endswith('=') and ref[1:] in html5_entities:
101 return unescape(ref)
102 # Otherwise do not unescape
103 return ref
104
105 def _unescape_attrvalue(s):
106 return attr_charref.sub(_replace_attr_charref, s)
107
108
109 class HTMLParser(_markupbase.ParserBase):
110 """Find tags and other markup and call handler functions.
111
112 Usage:
113 p = HTMLParser()
114 p.feed(data)
115 ...
116 p.close()
117
118 Start tags are handled by calling self.handle_starttag() or
119 self.handle_startendtag(); end tags by self.handle_endtag(). The
120 data between tags is passed from the parser to the derived class
121 by calling self.handle_data() with the data as argument (the data
122 may be split up in arbitrary chunks). If convert_charrefs is
123 True the character references are converted automatically to the
124 corresponding Unicode character (and self.handle_data() is no
125 longer split in chunks), otherwise they are passed by calling
126 self.handle_entityref() or self.handle_charref() with the string
127 containing respectively the named or numeric reference as the
128 argument.
129 """
130
131 # See the HTML5 specs section "13.4 Parsing HTML fragments".
132 # https://html.spec.whatwg.org/multipage/parsing.html#parsing-html-fragments
133 # CDATA_CONTENT_ELEMENTS are parsed in RAWTEXT mode
134 CDATA_CONTENT_ELEMENTS = ("script", "style", "xmp", "iframe", "noembed", "noframes")
135 RCDATA_CONTENT_ELEMENTS = ("textarea", "title")
136
137 def __init__(self, *, convert_charrefs=True, scripting=False):
138 """Initialize and reset this instance.
139
140 If convert_charrefs is true (the default), all character references
141 are automatically converted to the corresponding Unicode characters.
142
143 If *scripting* is false (the default), the content of the
144 ``noscript`` element is parsed normally; if it's true,
145 it's returned as is without being parsed.
146 """
147 super().__init__()
148 self.convert_charrefs = convert_charrefs
149 self.scripting = scripting
150 self.reset()
151
152 def reset(self):
153 """Reset this instance. Loses all unprocessed data."""
154 self.rawdata = ''
155 self.lasttag = '???'
156 self.interesting = interesting_normal
157 self.cdata_elem = None
158 self._support_cdata = True
159 self._escapable = True
160 self._pending = []
161 self._pending_len = 0
162 self._parse_threshold = 1
163 super().reset()
164
165 def feed(self, data):
166 r"""Feed data to the parser.
167
168 Call this as often as you want, with as little or as much text
169 as you want (may include '\n').
170 """
171 # Accumulate new data in a list and only join and parse it once
172 # enough has piled up. Rescanning an unparsed buffer (e.g. an
173 # unterminated tag) and concatenating onto it on every call would
174 # both be quadratic in the input size.
175 self._pending_len += len(data)
176 if self._pending_len < self._parse_threshold:
177 self._pending.append(data)
178 else:
179 if not self._pending:
180 self.rawdata += data
181 else:
182 self._pending.append(data)
183 self.rawdata += ''.join(self._pending)
184 self._pending.clear()
185 self._pending_len = 0
186 n = len(self.rawdata)
187 self.goahead(0)
188 if len(self.rawdata) < n:
189 # Some data was parsed; resume on the next call.
190 self._parse_threshold = 1
191 else:
192 # Nothing was parsed; wait until the buffer doubles.
193 self._parse_threshold = len(self.rawdata)
194
195 def close(self):
196 """Handle any buffered data."""
197 if self._pending:
198 self.rawdata += ''.join(self._pending)
199 self._pending.clear()
200 self._pending_len = 0
201 self.goahead(1)
202
203 __starttag_text = None
204
205 def get_starttag_text(self):
206 """Return full source of start tag: '<...>'."""
207 return self.__starttag_text
208
209 def set_cdata_mode(self, elem, *, escapable=False):
210 self.cdata_elem = elem.lower()
211 self._escapable = escapable
212 if self.cdata_elem == 'plaintext':
213 self.interesting = re.compile(r'\Z')
214 elif escapable and not self.convert_charrefs:
215 self.interesting = re.compile(r'&|</%s(?=[\t\n\r\f />])' % self.cdata_elem,
216 re.IGNORECASE|re.ASCII)
217 else:
218 self.interesting = re.compile(r'</%s(?=[\t\n\r\f />])' % self.cdata_elem,
219 re.IGNORECASE|re.ASCII)
220
221 def clear_cdata_mode(self):
222 self.interesting = interesting_normal
223 self.cdata_elem = None
224 self._escapable = True
225
226 def _set_support_cdata(self, flag=True):
227 """Enable or disable support of the CDATA sections.
228 If enabled, "<[CDATA[" starts a CDATA section which ends with "]]>".
229 If disabled, "<[CDATA[" starts a bogus comments which ends with ">".
230
231 This method is not called by default. Its purpose is to be called
232 in custom handle_starttag() and handle_endtag() methods, with
233 value that depends on the adjusted current node.
234 See https://html.spec.whatwg.org/multipage/parsing.html#markup-declaration-open-state
235 for details.
236 """
237 self._support_cdata = flag
238
239 # Internal -- handle data as far as reasonable. May leave state
240 # and data to be processed by a subsequent call. If 'end' is
241 # true, force handling all data as if followed by EOF marker.
242 def goahead(self, end):
243 rawdata = self.rawdata
244 i = 0
245 n = len(rawdata)
246 while i < n:
247 if self.convert_charrefs and not self.cdata_elem:
248 j = rawdata.find('<', i)
249 if j < 0:
250 # if we can't find the next <, either we are at the end
251 # or there's more text incoming. If the latter is True,
252 # we can't pass the text to handle_data in case we have
253 # a charref cut in half at end. Try to determine if
254 # this is the case before proceeding by looking for an
255 # & near the end and see if it's followed by a space or ;.
256 amppos = rawdata.rfind('&', max(i, n-34))
257 if (amppos >= 0 and
258 not re.compile(r'[\t\n\r\f ;]').search(rawdata, amppos)):
259 break # wait till we get all the text
260 j = n
261 else:
262 match = self.interesting.search(rawdata, i) # < or &
263 if match:
264 j = match.start()
265 else:
266 if self.cdata_elem:
267 break
268 j = n
269 if i < j:
270 if self.convert_charrefs and self._escapable:
271 self.handle_data(unescape(rawdata[i:j]))
272 else:
273 self.handle_data(rawdata[i:j])
274 i = self.updatepos(i, j)
275 if i == n: break
276 startswith = rawdata.startswith
277 if startswith('<', i):
278 if starttagopen.match(rawdata, i): # < + letter
279 k = self.parse_starttag(i)
280 elif startswith("</", i):
281 k = self.parse_endtag(i)
282 elif startswith("<!--", i):
283 k = self.parse_comment(i)
284 elif startswith("<?", i):
285 k = self.parse_pi(i)
286 elif startswith("<!", i):
287 k = self.parse_html_declaration(i)
288 elif (i + 1) < n or end:
289 self.handle_data("<")
290 k = i + 1
291 else:
292 break
293 if k < 0:
294 if not end:
295 break
296 if starttagopen.match(rawdata, i): # < + letter
297 pass
298 elif startswith("</", i):
299 if i + 2 == n:
300 self.handle_data("</")
301 elif endtagopen.match(rawdata, i): # </ + letter
302 pass
303 else:
304 # bogus comment
305 self.handle_comment(rawdata[i+2:])
306 elif startswith("<!--", i):
307 j = n
308 for suffix in ("--!", "--", "-"):
309 if rawdata.endswith(suffix, i+4):
310 j -= len(suffix)
311 break
312 self.handle_comment(rawdata[i+4:j])
313 elif startswith("<![CDATA[", i) and self._support_cdata:
314 self.unknown_decl(rawdata[i+3:])
315 elif rawdata[i:i+9].lower() == '<!doctype':
316 self.handle_decl(rawdata[i+2:])
317 elif startswith("<!", i):
318 # bogus comment
319 self.handle_comment(rawdata[i+2:])
320 elif startswith("<?", i):
321 self.handle_pi(rawdata[i+2:])
322 else:
323 raise AssertionError("we should not get here!")
324 k = n
325 i = self.updatepos(i, k)
326 elif startswith("&#", i):
327 match = charref.match(rawdata, i)
328 if match:
329 name = match.group()[2:-1]
330 self.handle_charref(name)
331 k = match.end()
332 if not startswith(';', k-1):
333 k = k - 1
334 i = self.updatepos(i, k)
335 continue
336 match = incomplete_charref.match(rawdata, i)
337 if match:
338 if end:
339 self.handle_charref(rawdata[i+2:])
340 i = self.updatepos(i, n)
341 break
342 # incomplete
343 break
344 elif i + 3 < n: # larger than "&#x"
345 # not the end of the buffer, and can't be confused
346 # with some other construct
347 self.handle_data("&#")
348 i = self.updatepos(i, i + 2)
349 else:
350 break
351 elif startswith('&', i):
352 match = entityref.match(rawdata, i)
353 if match:
354 name = match.group(1)
355 self.handle_entityref(name)
356 k = match.end()
357 if not startswith(';', k-1):
358 k = k - 1
359 i = self.updatepos(i, k)
360 continue
361 match = incomplete.match(rawdata, i)
362 if match:
363 if end:
364 self.handle_entityref(rawdata[i+1:])
365 i = self.updatepos(i, n)
366 break
367 # incomplete
368 break
369 elif i + 1 < n:
370 # not the end of the buffer, and can't be confused
371 # with some other construct
372 self.handle_data("&")
373 i = self.updatepos(i, i + 1)
374 else:
375 break
376 else:
377 assert 0, "interesting.search() lied"
378 # end while
379 if end and i < n:
380 if self.convert_charrefs and self._escapable:
381 self.handle_data(unescape(rawdata[i:n]))
382 else:
383 self.handle_data(rawdata[i:n])
384 i = self.updatepos(i, n)
385 self.rawdata = rawdata[i:]
386
387 # Internal -- parse html declarations, return length or -1 if not terminated
388 # See w3.org/TR/html5/tokenization.html#markup-declaration-open-state
389 # See also parse_declaration in _markupbase
390 def parse_html_declaration(self, i):
391 rawdata = self.rawdata
392 assert rawdata[i:i+2] == '<!', ('unexpected call to '
393 'parse_html_declaration()')
394 if rawdata[i:i+4] == '<!--':
395 # this case is actually already handled in goahead()
396 return self.parse_comment(i)
397 elif rawdata[i:i+9] == '<![CDATA[' and self._support_cdata:
398 j = rawdata.find(']]>', i+9)
399 if j < 0:
400 return -1
401 self.unknown_decl(rawdata[i+3: j])
402 return j + 3
403 elif rawdata[i:i+9].lower() == '<!doctype':
404 # find the closing >
405 gtpos = rawdata.find('>', i+9)
406 if gtpos == -1:
407 return -1
408 self.handle_decl(rawdata[i+2:gtpos])
409 return gtpos+1
410 else:
411 return self.parse_bogus_comment(i)
412
413 # Internal -- parse comment, return length or -1 if not terminated
414 # see https://html.spec.whatwg.org/multipage/parsing.html#comment-start-state
415 def parse_comment(self, i, report=True):
416 rawdata = self.rawdata
417 assert rawdata.startswith('<!--', i), 'unexpected call to parse_comment()'
418 # An empty comment is abruptly closed by the first ">" or "->",
419 # taking priority over a later "-->" or "--!>" close.
420 match = commentabruptclose.match(rawdata, i+4)
421 if not match:
422 match = commentclose.search(rawdata, i+4)
423 if not match:
424 return -1
425 if report:
426 j = match.start()
427 self.handle_comment(rawdata[i+4: j])
428 return match.end()
429
430 # Internal -- parse bogus comment, return length or -1 if not terminated
431 # see https://html.spec.whatwg.org/multipage/parsing.html#bogus-comment-state
432 def parse_bogus_comment(self, i, report=1):
433 rawdata = self.rawdata
434 assert rawdata[i:i+2] in ('<!', '</'), ('unexpected call to '
435 'parse_bogus_comment()')
436 pos = rawdata.find('>', i+2)
437 if pos == -1:
438 return -1
439 if report:
440 self.handle_comment(rawdata[i+2:pos])
441 return pos + 1
442
443 # Internal -- parse processing instr, return end or -1 if not terminated
444 def parse_pi(self, i):
445 rawdata = self.rawdata
446 assert rawdata[i:i+2] == '<?', 'unexpected call to parse_pi()'
447 match = piclose.search(rawdata, i+2) # >
448 if not match:
449 return -1
450 j = match.start()
451 self.handle_pi(rawdata[i+2: j])
452 j = match.end()
453 return j
454
455 # Internal -- handle starttag, return end or -1 if not terminated
456 def parse_starttag(self, i):
457 # See the HTML5 specs section "13.2.5.8 Tag name state"
458 # https://html.spec.whatwg.org/multipage/parsing.html#tag-name-state
459 self.__starttag_text = None
460 endpos = self.check_for_whole_start_tag(i)
461 if endpos < 0:
462 return endpos
463 rawdata = self.rawdata
464 self.__starttag_text = rawdata[i:endpos]
465
466 # Now parse the data between i+1 and j into a tag and attrs
467 attrs = []
468 match = tagfind_tolerant.match(rawdata, i+1)
469 assert match, 'unexpected call to parse_starttag()'
470 k = match.end()
471 self.lasttag = tag = match.group(1).lower()
472 while k < endpos:
473 m = attrfind_tolerant.match(rawdata, k)
474 if not m:
475 break
476 attrname, rest, attrvalue = m.group(1, 2, 3)
477 if not rest:
478 attrvalue = None
479 elif attrvalue[:1] == '\'' == attrvalue[-1:] or \
480 attrvalue[:1] == '"' == attrvalue[-1:]:
481 attrvalue = attrvalue[1:-1]
482 if attrvalue:
483 attrvalue = _unescape_attrvalue(attrvalue)
484 attrs.append((attrname.lower(), attrvalue))
485 k = m.end()
486
487 end = rawdata[k:endpos].strip()
488 if end not in (">", "/>"):
489 self.handle_data(rawdata[i:endpos])
490 return endpos
491 if end.endswith('/>'):
492 # XHTML-style empty tag: <span attr="value" />
493 self.handle_startendtag(tag, attrs)
494 else:
495 self.handle_starttag(tag, attrs)
496 if (tag in self.CDATA_CONTENT_ELEMENTS or
497 (self.scripting and tag == "noscript") or
498 tag == "plaintext"):
499 self.set_cdata_mode(tag, escapable=False)
500 elif tag in self.RCDATA_CONTENT_ELEMENTS:
501 self.set_cdata_mode(tag, escapable=True)
502 return endpos
503
504 # Internal -- check to see if we have a complete starttag; return end
505 # or -1 if incomplete.
506 def check_for_whole_start_tag(self, i):
507 rawdata = self.rawdata
508 match = locatetagend.match(rawdata, i+1)
509 assert match
510 j = match.end()
511 if rawdata[j-1] != ">":
512 return -1
513 return j
514
515 # Internal -- parse endtag, return end or -1 if incomplete
516 def parse_endtag(self, i):
517 # See the HTML5 specs section "13.2.5.7 End tag open state"
518 # https://html.spec.whatwg.org/multipage/parsing.html#end-tag-open-state
519 rawdata = self.rawdata
520 assert rawdata[i:i+2] == "</", "unexpected call to parse_endtag"
521 if rawdata.find('>', i+2) < 0: # fast check
522 return -1
523 if not endtagopen.match(rawdata, i): # </ + letter
524 if rawdata[i+2:i+3] == '>': # </> is ignored
525 # "missing-end-tag-name" parser error
526 return i+3
527 else:
528 return self.parse_bogus_comment(i)
529
530 match = locatetagend.match(rawdata, i+2)
531 assert match
532 j = match.end()
533 if rawdata[j-1] != ">":
534 return -1
535
536 # find the name: "13.2.5.8 Tag name state"
537 # https://html.spec.whatwg.org/multipage/parsing.html#tag-name-state
538 match = tagfind_tolerant.match(rawdata, i+2)
539 assert match
540 tag = match.group(1).lower()
541 self.handle_endtag(tag)
542 self.clear_cdata_mode()
543 return j
544
545 # Overridable -- finish processing of start+end tag: <tag.../>
546 def handle_startendtag(self, tag, attrs):
547 self.handle_starttag(tag, attrs)
548 self.handle_endtag(tag)
549
550 # Overridable -- handle start tag
551 def handle_starttag(self, tag, attrs):
552 pass
553
554 # Overridable -- handle end tag
555 def handle_endtag(self, tag):
556 pass
557
558 # Overridable -- handle character reference
559 def handle_charref(self, name):
560 pass
561
562 # Overridable -- handle entity reference
563 def handle_entityref(self, name):
564 pass
565
566 # Overridable -- handle data
567 def handle_data(self, data):
568 pass
569
570 # Overridable -- handle comment
571 def handle_comment(self, data):
572 pass
573
574 # Overridable -- handle declaration
575 def handle_decl(self, decl):
576 pass
577
578 # Overridable -- handle processing instruction
579 def handle_pi(self, data):
580 pass
581
582 def unknown_decl(self, data):
583 pass