| 1 |
"""Compare two HTML documents.""" |
| 2 |
|
| 3 |
import html |
| 4 |
from html.parser import HTMLParser |
| 5 |
|
| 6 |
from django.utils.html import VOID_ELEMENTS |
| 7 |
from django.utils.regex_helper import _lazy_re_compile |
| 8 |
|
| 9 |
# ASCII whitespace is U+0009 TAB, U+000A LF, U+000C FF, U+000D CR, or U+0020 |
| 10 |
# SPACE. |
| 11 |
# https://infra.spec.whatwg.org/#ascii-whitespace |
| 12 |
ASCII_WHITESPACE = _lazy_re_compile(r"[\t\n\f\r ]+") |
| 13 |
|
| 14 |
# https://html.spec.whatwg.org/#attributes-3 |
| 15 |
BOOLEAN_ATTRIBUTES = { |
| 16 |
"allowfullscreen", |
| 17 |
"async", |
| 18 |
"autofocus", |
| 19 |
"autoplay", |
| 20 |
"checked", |
| 21 |
"controls", |
| 22 |
"default", |
| 23 |
"defer ", |
| 24 |
"disabled", |
| 25 |
"formnovalidate", |
| 26 |
"hidden", |
| 27 |
"ismap", |
| 28 |
"itemscope", |
| 29 |
"loop", |
| 30 |
"multiple", |
| 31 |
"muted", |
| 32 |
"nomodule", |
| 33 |
"novalidate", |
| 34 |
"open", |
| 35 |
"playsinline", |
| 36 |
"readonly", |
| 37 |
"required", |
| 38 |
"reversed", |
| 39 |
"selected", |
| 40 |
# Attributes for deprecated tags. |
| 41 |
"truespeed", |
| 42 |
} |
| 43 |
|
| 44 |
|
| 45 |
def normalize_whitespace(string): |
| 46 |
return ASCII_WHITESPACE.sub(" ", string) |
| 47 |
|
| 48 |
|
| 49 |
def normalize_attributes(attributes): |
| 50 |
normalized = [] |
| 51 |
for name, value in attributes: |
| 52 |
if name == "class" and value: |
| 53 |
# Special case handling of 'class' attribute, so that comparisons |
| 54 |
# of DOM instances are not sensitive to ordering of classes. |
| 55 |
value = " ".join( |
| 56 |
sorted(value for value in ASCII_WHITESPACE.split(value) if value) |
| 57 |
) |
| 58 |
# Boolean attributes without a value is same as attribute with value |
| 59 |
# that equals the attributes name. For example: |
| 60 |
# <input checked> == <input checked="checked"> |
| 61 |
if name in BOOLEAN_ATTRIBUTES: |
| 62 |
if not value or value == name: |
| 63 |
value = None |
| 64 |
elif value is None: |
| 65 |
value = "" |
| 66 |
normalized.append((name, value)) |
| 67 |
return normalized |
| 68 |
|
| 69 |
|
| 70 |
class Element: |
| 71 |
def __init__(self, name, attributes): |
| 72 |
self.name = name |
| 73 |
self.attributes = sorted(attributes) |
| 74 |
self.children = [] |
| 75 |
|
| 76 |
def append(self, element): |
| 77 |
if isinstance(element, str): |
| 78 |
element = normalize_whitespace(element) |
| 79 |
if self.children and isinstance(self.children[-1], str): |
| 80 |
self.children[-1] += element |
| 81 |
self.children[-1] = normalize_whitespace(self.children[-1]) |
| 82 |
return |
| 83 |
elif self.children: |
| 84 |
# removing last children if it is only whitespace |
| 85 |
# this can result in incorrect dom representations since |
| 86 |
# whitespace between inline tags like <span> is significant |
| 87 |
if isinstance(self.children[-1], str) and self.children[-1].isspace(): |
| 88 |
self.children.pop() |
| 89 |
if element: |
| 90 |
self.children.append(element) |
| 91 |
|
| 92 |
def finalize(self): |
| 93 |
def rstrip_last_element(children): |
| 94 |
if children and isinstance(children[-1], str): |
| 95 |
children[-1] = children[-1].rstrip() |
| 96 |
if not children[-1]: |
| 97 |
children.pop() |
| 98 |
children = rstrip_last_element(children) |
| 99 |
return children |
| 100 |
|
| 101 |
rstrip_last_element(self.children) |
| 102 |
for i, child in enumerate(self.children): |
| 103 |
if isinstance(child, str): |
| 104 |
self.children[i] = child.strip() |
| 105 |
elif hasattr(child, "finalize"): |
| 106 |
child.finalize() |
| 107 |
|
| 108 |
def __eq__(self, element): |
| 109 |
if not hasattr(element, "name") or self.name != element.name: |
| 110 |
return False |
| 111 |
if self.attributes != element.attributes: |
| 112 |
return False |
| 113 |
return self.children == element.children |
| 114 |
|
| 115 |
def __hash__(self): |
| 116 |
return hash((self.name, *self.attributes)) |
| 117 |
|
| 118 |
def _count(self, element, count=True): |
| 119 |
if not isinstance(element, str) and self == element: |
| 120 |
return 1 |
| 121 |
if isinstance(element, RootElement) and self.children == element.children: |
| 122 |
return 1 |
| 123 |
i = 0 |
| 124 |
elem_child_idx = 0 |
| 125 |
for child in self.children: |
| 126 |
# child is text content and element is also text content, then |
| 127 |
# make a simple "text" in "text" |
| 128 |
if isinstance(child, str): |
| 129 |
if isinstance(element, str): |
| 130 |
if count: |
| 131 |
i += child.count(element) |
| 132 |
elif element in child: |
| 133 |
return 1 |
| 134 |
else: |
| 135 |
# Look for element wholly within this child. |
| 136 |
i += child._count(element, count=count) |
| 137 |
if not count and i: |
| 138 |
return i |
| 139 |
# Also look for a sequence of element's children among self's |
| 140 |
# children. self.children == element.children is tested above, |
| 141 |
# but will fail if self has additional children. Ex: '<a/><b/>' |
| 142 |
# is contained in '<a/><b/><c/>'. |
| 143 |
if isinstance(element, RootElement) and element.children: |
| 144 |
elem_child = element.children[elem_child_idx] |
| 145 |
# Start or continue match, advance index. |
| 146 |
if elem_child == child: |
| 147 |
elem_child_idx += 1 |
| 148 |
# Match found, reset index. |
| 149 |
if elem_child_idx == len(element.children): |
| 150 |
i += 1 |
| 151 |
elem_child_idx = 0 |
| 152 |
# No match, reset index. |
| 153 |
else: |
| 154 |
elem_child_idx = 0 |
| 155 |
return i |
| 156 |
|
| 157 |
def __contains__(self, element): |
| 158 |
return self._count(element, count=False) > 0 |
| 159 |
|
| 160 |
def count(self, element): |
| 161 |
return self._count(element, count=True) |
| 162 |
|
| 163 |
def __getitem__(self, key): |
| 164 |
return self.children[key] |
| 165 |
|
| 166 |
def __str__(self): |
| 167 |
output = "<%s" % self.name |
| 168 |
for key, value in self.attributes: |
| 169 |
if value is not None: |
| 170 |
output += ' %s="%s"' % (key, value) |
| 171 |
else: |
| 172 |
output += " %s" % key |
| 173 |
if self.children: |
| 174 |
output += ">\n" |
| 175 |
output += "".join( |
| 176 |
[ |
| 177 |
html.escape(c) if isinstance(c, str) else str(c) |
| 178 |
for c in self.children |
| 179 |
] |
| 180 |
) |
| 181 |
output += "\n</%s>" % self.name |
| 182 |
else: |
| 183 |
output += ">" |
| 184 |
return output |
| 185 |
|
| 186 |
def __repr__(self): |
| 187 |
return str(self) |
| 188 |
|
| 189 |
|
| 190 |
class RootElement(Element): |
| 191 |
def __init__(self): |
| 192 |
super().__init__(None, ()) |
| 193 |
|
| 194 |
def __str__(self): |
| 195 |
return "".join( |
| 196 |
[html.escape(c) if isinstance(c, str) else str(c) for c in self.children] |
| 197 |
) |
| 198 |
|
| 199 |
|
| 200 |
class HTMLParseError(Exception): |
| 201 |
pass |
| 202 |
|
| 203 |
|
| 204 |
class Parser(HTMLParser): |
| 205 |
def __init__(self): |
| 206 |
super().__init__() |
| 207 |
self.root = RootElement() |
| 208 |
self.open_tags = [] |
| 209 |
self.element_positions = {} |
| 210 |
|
| 211 |
def error(self, msg): |
| 212 |
raise HTMLParseError(msg, self.getpos()) |
| 213 |
|
| 214 |
def format_position(self, position=None, element=None): |
| 215 |
if not position and element: |
| 216 |
position = self.element_positions[element] |
| 217 |
if position is None: |
| 218 |
position = self.getpos() |
| 219 |
if hasattr(position, "lineno"): |
| 220 |
position = position.lineno, position.offset |
| 221 |
return "Line %d, Column %d" % position |
| 222 |
|
| 223 |
@property |
| 224 |
def current(self): |
| 225 |
if self.open_tags: |
| 226 |
return self.open_tags[-1] |
| 227 |
else: |
| 228 |
return self.root |
| 229 |
|
| 230 |
def handle_startendtag(self, tag, attrs): |
| 231 |
self.handle_starttag(tag, attrs) |
| 232 |
if tag not in VOID_ELEMENTS: |
| 233 |
self.handle_endtag(tag) |
| 234 |
|
| 235 |
def handle_starttag(self, tag, attrs): |
| 236 |
attrs = normalize_attributes(attrs) |
| 237 |
element = Element(tag, attrs) |
| 238 |
self.current.append(element) |
| 239 |
if tag not in VOID_ELEMENTS: |
| 240 |
self.open_tags.append(element) |
| 241 |
self.element_positions[element] = self.getpos() |
| 242 |
|
| 243 |
def handle_endtag(self, tag): |
| 244 |
if not self.open_tags: |
| 245 |
self.error("Unexpected end tag `%s` (%s)" % (tag, self.format_position())) |
| 246 |
element = self.open_tags.pop() |
| 247 |
while element.name != tag: |
| 248 |
if not self.open_tags: |
| 249 |
self.error( |
| 250 |
"Unexpected end tag `%s` (%s)" % (tag, self.format_position()) |
| 251 |
) |
| 252 |
element = self.open_tags.pop() |
| 253 |
|
| 254 |
def handle_data(self, data): |
| 255 |
self.current.append(data) |
| 256 |
|
| 257 |
|
| 258 |
def parse_html(html): |
| 259 |
""" |
| 260 |
Take a string that contains HTML and turn it into a Python object structure |
| 261 |
that can be easily compared against other HTML on semantic equivalence. |
| 262 |
Syntactical differences like which quotation is used on arguments will be |
| 263 |
ignored. |
| 264 |
""" |
| 265 |
parser = Parser() |
| 266 |
parser.feed(html) |
| 267 |
parser.close() |
| 268 |
document = parser.root |
| 269 |
document.finalize() |
| 270 |
# Removing ROOT element if it's not necessary |
| 271 |
if len(document.children) == 1 and not isinstance(document.children[0], str): |
| 272 |
document = document.children[0] |
| 273 |
return document |