Debian Codegraph @

Lib/html/parser.py

raw
1 """A parser for HTML and XHTML."""
2
3 # This file is based on sgmllib.py, but the API is slightly different.
4
5 # XXX There should be a way to distinguish between PCDATA (parsed
6 # character data -- the normal case), RCDATA (replaceable character
7 # data -- only char and entity references and end tags are special)
8 # and CDATA (character data -- only end tags are special).
9
10
11 import re
12 import _markupbase
13
14 from html import unescape
15 from html.entities import html5 as html5_entities
16
17
18 __all__ = ['HTMLParser']
19
20 # Regular expressions used for parsing
21
22 interesting_normal = re.compile('[&<]')
23 incomplete = re.compile('&[a-zA-Z#]')
24
25 entityref = re.compile('&([a-zA-Z][-.a-zA-Z0-9]*)[^a-zA-Z0-9]')
26 charref = re.compile('&#(?:[0-9]+|[xX][0-9a-fA-F]+)[^0-9a-fA-F]')
27 incomplete_charref = re.compile('&#(?:[0-9]|[xX][0-9a-fA-F])')
28 attr_charref = re.compile(r'&(#[0-9]+|#[xX][0-9a-fA-F]+|[a-zA-Z][a-zA-Z0-9]*)[;=]?')
29
30 starttagopen = re.compile('<[a-zA-Z]')
31 endtagopen = re.compile('</[a-zA-Z]')
32 piclose = re.compile('>')
33 commentclose = re.compile(r'--!?>')
34 commentabruptclose = re.compile(r'-?>')
35 # Note:
36 # 1) if you change tagfind/attrfind remember to update locatetagend too;
37 # 2) if you change tagfind/attrfind and/or locatetagend the parser will
38 # explode, so don't do it.
39 # see the HTML5 specs section "13.2.5.6 Tag open state",
40 # "13.2.5.8 Tag name state" and "13.2.5.33 Attribute name state".
41 # https://html.spec.whatwg.org/multipage/parsing.html#tag-open-state
42 # https://html.spec.whatwg.org/multipage/parsing.html#tag-name-state
43 # https://html.spec.whatwg.org/multipage/parsing.html#attribute-name-state
44 tagfind_tolerant = re.compile(r'([a-zA-Z][^\t\n\r\f />]*)(?:[\t\n\r\f ]|/(?!>))*')
45 attrfind_tolerant = re.compile(r"""
46 (
47 (?<=['"\t\n\r\f /])[^\t\n\r\f />][^\t\n\r\f /=>]* # attribute name
48 )
49 ([\t\n\r\f ]*=[\t\n\r\f ]* # value indicator
50 ('[^']*' # LITA-enclosed value
51 |"[^"]*" # LIT-enclosed value
52 |(?!['"])[^>\t\n\r\f ]* # bare value
53 )
54 )?
55 (?:[\t\n\r\f ]|/(?!>))* # possibly followed by a space
56 """, re.VERBOSE)
57 locatetagend = re.compile(r"""
58 [a-zA-Z][^\t\n\r\f />]* # tag name
59 [\t\n\r\f /]* # optional whitespace before attribute name
60 (?:(?<=['"\t\n\r\f /])[^\t\n\r\f />][^\t\n\r\f /=>]* # attribute name
61 (?:[\t\n\r\f ]*=[\t\n\r\f ]* # value indicator
62 (?:'[^']*' # LITA-enclosed value
63 |"[^"]*" # LIT-enclosed value
64 |(?!['"])[^>\t\n\r\f ]* # bare value
65 )
66 )?
67 [\t\n\r\f /]* # possibly followed by a space
68 )*
69 >?
70 """, re.VERBOSE)
71 # The following variables are not used, but are temporarily left for
72 # backward compatibility.
73 locatestarttagend_tolerant = re.compile(r"""
74 <[a-zA-Z][^\t\n\r\f />\x00]* # tag name
75 (?:[\s/]* # optional whitespace before attribute name
76 (?:(?<=['"\s/])[^\s/>][^\s/=>]* # attribute name
77 (?:\s*=+\s* # value indicator
78 (?:'[^']*' # LITA-enclosed value
79 |"[^"]*" # LIT-enclosed value
80 |(?!['"])[^>\s]* # bare value
81 )
82 \s* # possibly followed by a space
83 )?(?:\s|/(?!>))*
84 )*
85 )?
86 \s* # trailing whitespace
87 """, re.VERBOSE)
88 endendtag = re.compile('>')
89 endtagfind = re.compile(r'</\s*([a-zA-Z][-.a-zA-Z0-9:_]*)\s*>')
90
91 # Character reference processing logic specific to attribute values
92 # See: https://html.spec.whatwg.org/multipage/parsing.html#named-character-reference-state
93 def _replace_attr_charref(match):
94 ref = match.group(0)
95 # Numeric / hex char refs must always be unescaped
96 if ref.startswith('&#'):
97 return unescape(ref)
98 # Named character / entity references must only be unescaped
99 # if they are an exact match, and they are not followed by an equals sign
100 if not ref.endswith('=') and ref[1:] in html5_entities:
101 return unescape(ref)
102 # Otherwise do not unescape
103 return ref
104
105 def _unescape_attrvalue(s):
106 return attr_charref.sub(_replace_attr_charref, s)
107
108
109 class HTMLParser(_markupbase.ParserBase):
110 """Find tags and other markup and call handler functions.
111
112 Usage:
113 p = HTMLParser()
114 p.feed(data)
115 ...
116 p.close()
117
118 Start tags are handled by calling self.handle_starttag() or
119 self.handle_startendtag(); end tags by self.handle_endtag(). The
120 data between tags is passed from the parser to the derived class
121 by calling self.handle_data() with the data as argument (the data
122 may be split up in arbitrary chunks). If convert_charrefs is
123 True the character references are converted automatically to the
124 corresponding Unicode character (and self.handle_data() is no
125 longer split in chunks), otherwise they are passed by calling
126 self.handle_entityref() or self.handle_charref() with the string
127 containing respectively the named or numeric reference as the
128 argument.
129 """
130
131 # See the HTML5 specs section "13.4 Parsing HTML fragments".
132 # https://html.spec.whatwg.org/multipage/parsing.html#parsing-html-fragments
133 # CDATA_CONTENT_ELEMENTS are parsed in RAWTEXT mode
134 CDATA_CONTENT_ELEMENTS = ("script", "style", "xmp", "iframe", "noembed", "noframes")
135 RCDATA_CONTENT_ELEMENTS = ("textarea", "title")
136
137 def __init__(self, *, convert_charrefs=True, scripting=False):
138 """Initialize and reset this instance.
139
140 If convert_charrefs is true (the default), all character references
141 are automatically converted to the corresponding Unicode characters.
142
143 If *scripting* is false (the default), the content of the
144 ``noscript`` element is parsed normally; if it's true,
145 it's returned as is without being parsed.
146 """
147 super().__init__()
148 self.convert_charrefs = convert_charrefs
149 self.scripting = scripting
150 self.reset()
151
152 def reset(self):
153 """Reset this instance. Loses all unprocessed data."""
154 self.rawdata = ''
155 self.lasttag = '???'
156 self.interesting = interesting_normal
157 self.cdata_elem = None
158 self._support_cdata = True
159 self._escapable = True
160 super().reset()
161
162 def feed(self, data):
163 r"""Feed data to the parser.
164
165 Call this as often as you want, with as little or as much text
166 as you want (may include '\n').
167 """
168 self.rawdata = self.rawdata + data
169 self.goahead(0)
170
171 def close(self):
172 """Handle any buffered data."""
173 self.goahead(1)
174
175 __starttag_text = None
176
177 def get_starttag_text(self):
178 """Return full source of start tag: '<...>'."""
179 return self.__starttag_text
180
181 def set_cdata_mode(self, elem, *, escapable=False):
182 self.cdata_elem = elem.lower()
183 self._escapable = escapable
184 if self.cdata_elem == 'plaintext':
185 self.interesting = re.compile(r'\z')
186 elif escapable and not self.convert_charrefs:
187 self.interesting = re.compile(r'&|</%s(?=[\t\n\r\f />])' % self.cdata_elem,
188 re.IGNORECASE|re.ASCII)
189 else:
190 self.interesting = re.compile(r'</%s(?=[\t\n\r\f />])' % self.cdata_elem,
191 re.IGNORECASE|re.ASCII)
192
193 def clear_cdata_mode(self):
194 self.interesting = interesting_normal
195 self.cdata_elem = None
196 self._escapable = True
197
198 def _set_support_cdata(self, flag=True):
199 """Enable or disable support of the CDATA sections.
200 If enabled, "<[CDATA[" starts a CDATA section which ends with "]]>".
201 If disabled, "<[CDATA[" starts a bogus comments which ends with ">".
202
203 This method is not called by default. Its purpose is to be called
204 in custom handle_starttag() and handle_endtag() methods, with
205 value that depends on the adjusted current node.
206 See https://html.spec.whatwg.org/multipage/parsing.html#markup-declaration-open-state
207 for details.
208 """
209 self._support_cdata = flag
210
211 # Internal -- handle data as far as reasonable. May leave state
212 # and data to be processed by a subsequent call. If 'end' is
213 # true, force handling all data as if followed by EOF marker.
214 def goahead(self, end):
215 rawdata = self.rawdata
216 i = 0
217 n = len(rawdata)
218 while i < n:
219 if self.convert_charrefs and not self.cdata_elem:
220 j = rawdata.find('<', i)
221 if j < 0:
222 # if we can't find the next <, either we are at the end
223 # or there's more text incoming. If the latter is True,
224 # we can't pass the text to handle_data in case we have
225 # a charref cut in half at end. Try to determine if
226 # this is the case before proceeding by looking for an
227 # & near the end and see if it's followed by a space or ;.
228 amppos = rawdata.rfind('&', max(i, n-34))
229 if (amppos >= 0 and
230 not re.compile(r'[\t\n\r\f ;]').search(rawdata, amppos)):
231 break # wait till we get all the text
232 j = n
233 else:
234 match = self.interesting.search(rawdata, i) # < or &
235 if match:
236 j = match.start()
237 else:
238 if self.cdata_elem:
239 break
240 j = n
241 if i < j:
242 if self.convert_charrefs and self._escapable:
243 self.handle_data(unescape(rawdata[i:j]))
244 else:
245 self.handle_data(rawdata[i:j])
246 i = self.updatepos(i, j)
247 if i == n: break
248 startswith = rawdata.startswith
249 if startswith('<', i):
250 if starttagopen.match(rawdata, i): # < + letter
251 k = self.parse_starttag(i)
252 elif startswith("</", i):
253 k = self.parse_endtag(i)
254 elif startswith("<!--", i):
255 k = self.parse_comment(i)
256 elif startswith("<?", i):
257 k = self.parse_pi(i)
258 elif startswith("<!", i):
259 k = self.parse_html_declaration(i)
260 elif (i + 1) < n or end:
261 self.handle_data("<")
262 k = i + 1
263 else:
264 break
265 if k < 0:
266 if not end:
267 break
268 if starttagopen.match(rawdata, i): # < + letter
269 pass
270 elif startswith("</", i):
271 if i + 2 == n:
272 self.handle_data("</")
273 elif endtagopen.match(rawdata, i): # </ + letter
274 pass
275 else:
276 # bogus comment
277 self.handle_comment(rawdata[i+2:])
278 elif startswith("<!--", i):
279 j = n
280 for suffix in ("--!", "--", "-"):
281 if rawdata.endswith(suffix, i+4):
282 j -= len(suffix)
283 break
284 self.handle_comment(rawdata[i+4:j])
285 elif startswith("<![CDATA[", i) and self._support_cdata:
286 self.unknown_decl(rawdata[i+3:])
287 elif rawdata[i:i+9].lower() == '<!doctype':
288 self.handle_decl(rawdata[i+2:])
289 elif startswith("<!", i):
290 # bogus comment
291 self.handle_comment(rawdata[i+2:])
292 elif startswith("<?", i):
293 self.handle_pi(rawdata[i+2:])
294 else:
295 raise AssertionError("we should not get here!")
296 k = n
297 i = self.updatepos(i, k)
298 elif startswith("&#", i):
299 match = charref.match(rawdata, i)
300 if match:
301 name = match.group()[2:-1]
302 self.handle_charref(name)
303 k = match.end()
304 if not startswith(';', k-1):
305 k = k - 1
306 i = self.updatepos(i, k)
307 continue
308 match = incomplete_charref.match(rawdata, i)
309 if match:
310 if end:
311 self.handle_charref(rawdata[i+2:])
312 i = self.updatepos(i, n)
313 break
314 # incomplete
315 break
316 elif i + 3 < n: # larger than "&#x"
317 # not the end of the buffer, and can't be confused
318 # with some other construct
319 self.handle_data("&#")
320 i = self.updatepos(i, i + 2)
321 else:
322 break
323 elif startswith('&', i):
324 match = entityref.match(rawdata, i)
325 if match:
326 name = match.group(1)
327 self.handle_entityref(name)
328 k = match.end()
329 if not startswith(';', k-1):
330 k = k - 1
331 i = self.updatepos(i, k)
332 continue
333 match = incomplete.match(rawdata, i)
334 if match:
335 if end:
336 self.handle_entityref(rawdata[i+1:])
337 i = self.updatepos(i, n)
338 break
339 # incomplete
340 break
341 elif i + 1 < n:
342 # not the end of the buffer, and can't be confused
343 # with some other construct
344 self.handle_data("&")
345 i = self.updatepos(i, i + 1)
346 else:
347 break
348 else:
349 assert 0, "interesting.search() lied"
350 # end while
351 if end and i < n:
352 if self.convert_charrefs and self._escapable:
353 self.handle_data(unescape(rawdata[i:n]))
354 else:
355 self.handle_data(rawdata[i:n])
356 i = self.updatepos(i, n)
357 self.rawdata = rawdata[i:]
358
359 # Internal -- parse html declarations, return length or -1 if not terminated
360 # See w3.org/TR/html5/tokenization.html#markup-declaration-open-state
361 # See also parse_declaration in _markupbase
362 def parse_html_declaration(self, i):
363 rawdata = self.rawdata
364 assert rawdata[i:i+2] == '<!', ('unexpected call to '
365 'parse_html_declaration()')
366 if rawdata[i:i+4] == '<!--':
367 # this case is actually already handled in goahead()
368 return self.parse_comment(i)
369 elif rawdata[i:i+9] == '<![CDATA[' and self._support_cdata:
370 j = rawdata.find(']]>', i+9)
371 if j < 0:
372 return -1
373 self.unknown_decl(rawdata[i+3: j])
374 return j + 3
375 elif rawdata[i:i+9].lower() == '<!doctype':
376 # find the closing >
377 gtpos = rawdata.find('>', i+9)
378 if gtpos == -1:
379 return -1
380 self.handle_decl(rawdata[i+2:gtpos])
381 return gtpos+1
382 else:
383 return self.parse_bogus_comment(i)
384
385 # Internal -- parse comment, return length or -1 if not terminated
386 # see https://html.spec.whatwg.org/multipage/parsing.html#comment-start-state
387 def parse_comment(self, i, report=True):
388 rawdata = self.rawdata
389 assert rawdata.startswith('<!--', i), 'unexpected call to parse_comment()'
390 match = commentclose.search(rawdata, i+4)
391 if not match:
392 match = commentabruptclose.match(rawdata, i+4)
393 if not match:
394 return -1
395 if report:
396 j = match.start()
397 self.handle_comment(rawdata[i+4: j])
398 return match.end()
399
400 # Internal -- parse bogus comment, return length or -1 if not terminated
401 # see https://html.spec.whatwg.org/multipage/parsing.html#bogus-comment-state
402 def parse_bogus_comment(self, i, report=1):
403 rawdata = self.rawdata
404 assert rawdata[i:i+2] in ('<!', '</'), ('unexpected call to '
405 'parse_bogus_comment()')
406 pos = rawdata.find('>', i+2)
407 if pos == -1:
408 return -1
409 if report:
410 self.handle_comment(rawdata[i+2:pos])
411 return pos + 1
412
413 # Internal -- parse processing instr, return end or -1 if not terminated
414 def parse_pi(self, i):
415 rawdata = self.rawdata
416 assert rawdata[i:i+2] == '<?', 'unexpected call to parse_pi()'
417 match = piclose.search(rawdata, i+2) # >
418 if not match:
419 return -1
420 j = match.start()
421 self.handle_pi(rawdata[i+2: j])
422 j = match.end()
423 return j
424
425 # Internal -- handle starttag, return end or -1 if not terminated
426 def parse_starttag(self, i):
427 # See the HTML5 specs section "13.2.5.8 Tag name state"
428 # https://html.spec.whatwg.org/multipage/parsing.html#tag-name-state
429 self.__starttag_text = None
430 endpos = self.check_for_whole_start_tag(i)
431 if endpos < 0:
432 return endpos
433 rawdata = self.rawdata
434 self.__starttag_text = rawdata[i:endpos]
435
436 # Now parse the data between i+1 and j into a tag and attrs
437 attrs = []
438 match = tagfind_tolerant.match(rawdata, i+1)
439 assert match, 'unexpected call to parse_starttag()'
440 k = match.end()
441 self.lasttag = tag = match.group(1).lower()
442 while k < endpos:
443 m = attrfind_tolerant.match(rawdata, k)
444 if not m:
445 break
446 attrname, rest, attrvalue = m.group(1, 2, 3)
447 if not rest:
448 attrvalue = None
449 elif attrvalue[:1] == '\'' == attrvalue[-1:] or \
450 attrvalue[:1] == '"' == attrvalue[-1:]:
451 attrvalue = attrvalue[1:-1]
452 if attrvalue:
453 attrvalue = _unescape_attrvalue(attrvalue)
454 attrs.append((attrname.lower(), attrvalue))
455 k = m.end()
456
457 end = rawdata[k:endpos].strip()
458 if end not in (">", "/>"):
459 self.handle_data(rawdata[i:endpos])
460 return endpos
461 if end.endswith('/>'):
462 # XHTML-style empty tag: <span attr="value" />
463 self.handle_startendtag(tag, attrs)
464 else:
465 self.handle_starttag(tag, attrs)
466 if (tag in self.CDATA_CONTENT_ELEMENTS or
467 (self.scripting and tag == "noscript") or
468 tag == "plaintext"):
469 self.set_cdata_mode(tag, escapable=False)
470 elif tag in self.RCDATA_CONTENT_ELEMENTS:
471 self.set_cdata_mode(tag, escapable=True)
472 return endpos
473
474 # Internal -- check to see if we have a complete starttag; return end
475 # or -1 if incomplete.
476 def check_for_whole_start_tag(self, i):
477 rawdata = self.rawdata
478 match = locatetagend.match(rawdata, i+1)
479 assert match
480 j = match.end()
481 if rawdata[j-1] != ">":
482 return -1
483 return j
484
485 # Internal -- parse endtag, return end or -1 if incomplete
486 def parse_endtag(self, i):
487 # See the HTML5 specs section "13.2.5.7 End tag open state"
488 # https://html.spec.whatwg.org/multipage/parsing.html#end-tag-open-state
489 rawdata = self.rawdata
490 assert rawdata[i:i+2] == "</", "unexpected call to parse_endtag"
491 if rawdata.find('>', i+2) < 0: # fast check
492 return -1
493 if not endtagopen.match(rawdata, i): # </ + letter
494 if rawdata[i+2:i+3] == '>': # </> is ignored
495 # "missing-end-tag-name" parser error
496 return i+3
497 else:
498 return self.parse_bogus_comment(i)
499
500 match = locatetagend.match(rawdata, i+2)
501 assert match
502 j = match.end()
503 if rawdata[j-1] != ">":
504 return -1
505
506 # find the name: "13.2.5.8 Tag name state"
507 # https://html.spec.whatwg.org/multipage/parsing.html#tag-name-state
508 match = tagfind_tolerant.match(rawdata, i+2)
509 assert match
510 tag = match.group(1).lower()
511 self.handle_endtag(tag)
512 self.clear_cdata_mode()
513 return j
514
515 # Overridable -- finish processing of start+end tag: <tag.../>
516 def handle_startendtag(self, tag, attrs):
517 self.handle_starttag(tag, attrs)
518 self.handle_endtag(tag)
519
520 # Overridable -- handle start tag
521 def handle_starttag(self, tag, attrs):
522 pass
523
524 # Overridable -- handle end tag
525 def handle_endtag(self, tag):
526 pass
527
528 # Overridable -- handle character reference
529 def handle_charref(self, name):
530 pass
531
532 # Overridable -- handle entity reference
533 def handle_entityref(self, name):
534 pass
535
536 # Overridable -- handle data
537 def handle_data(self, data):
538 pass
539
540 # Overridable -- handle comment
541 def handle_comment(self, data):
542 pass
543
544 # Overridable -- handle declaration
545 def handle_decl(self, decl):
546 pass
547
548 # Overridable -- handle processing instruction
549 def handle_pi(self, data):
550 pass
551
552 def unknown_decl(self, data):
553 pass