Debian Codegraph @

django/utils/html.py

raw
1 """HTML utilities suitable for global use."""
2
3 import html
4 import json
5 import re
6 import warnings
7 from collections.abc import Mapping
8 from html.parser import HTMLParser
9 from urllib.parse import parse_qsl, quote, unquote, urlencode, urlsplit, urlunsplit
10
11 from django.core.exceptions import SuspiciousOperation, ValidationError
12 from django.core.validators import EmailValidator
13 from django.utils.deprecation import RemovedInDjango60Warning
14 from django.utils.functional import Promise, cached_property, keep_lazy, keep_lazy_text
15 from django.utils.http import MAX_URL_LENGTH, RFC3986_GENDELIMS, RFC3986_SUBDELIMS
16 from django.utils.regex_helper import _lazy_re_compile
17 from django.utils.safestring import SafeData, SafeString, mark_safe
18 from django.utils.text import normalize_newlines
19
20 # https://html.spec.whatwg.org/#void-elements
21 VOID_ELEMENTS = frozenset(
22 (
23 "area",
24 "base",
25 "br",
26 "col",
27 "embed",
28 "hr",
29 "img",
30 "input",
31 "link",
32 "meta",
33 "param",
34 "source",
35 "track",
36 "wbr",
37 # Deprecated tags.
38 "frame",
39 "spacer",
40 )
41 )
42
43 MAX_STRIP_TAGS_DEPTH = 50
44
45 # HTML tag that opens but has no closing ">" after 1k+ chars.
46 long_open_tag_without_closing_re = _lazy_re_compile(r"<[a-zA-Z][^>]{1000,}")
47
48
49 @keep_lazy(SafeString)
50 def escape(text):
51 """
52 Return the given text with ampersands, quotes and angle brackets encoded
53 for use in HTML.
54
55 Always escape input, even if it's already escaped and marked as such.
56 This may result in double-escaping. If this is a concern, use
57 conditional_escape() instead.
58 """
59 return SafeString(html.escape(str(text)))
60
61
62 _js_escapes = {
63 ord("\\"): "\\u005C",
64 ord("'"): "\\u0027",
65 ord('"'): "\\u0022",
66 ord(">"): "\\u003E",
67 ord("<"): "\\u003C",
68 ord("&"): "\\u0026",
69 ord("="): "\\u003D",
70 ord("-"): "\\u002D",
71 ord(";"): "\\u003B",
72 ord("`"): "\\u0060",
73 ord("\u2028"): "\\u2028",
74 ord("\u2029"): "\\u2029",
75 }
76
77 # Escape every ASCII character with a value less than 32.
78 _js_escapes.update((ord("%c" % z), "\\u%04X" % z) for z in range(32))
79
80
81 @keep_lazy(SafeString)
82 def escapejs(value):
83 """Hex encode characters for use in JavaScript strings."""
84 return mark_safe(str(value).translate(_js_escapes))
85
86
87 _json_script_escapes = {
88 ord(">"): "\\u003E",
89 ord("<"): "\\u003C",
90 ord("&"): "\\u0026",
91 }
92
93
94 def json_script(value, element_id=None, encoder=None):
95 """
96 Escape all the HTML/XML special characters with their unicode escapes, so
97 value is safe to be output anywhere except for inside a tag attribute. Wrap
98 the escaped JSON in a script tag.
99 """
100 from django.core.serializers.json import DjangoJSONEncoder
101
102 json_str = json.dumps(value, cls=encoder or DjangoJSONEncoder).translate(
103 _json_script_escapes
104 )
105 if element_id:
106 template = '<script id="{}" type="application/json">{}</script>'
107 args = (element_id, mark_safe(json_str))
108 else:
109 template = '<script type="application/json">{}</script>'
110 args = (mark_safe(json_str),)
111 return format_html(template, *args)
112
113
114 def conditional_escape(text):
115 """
116 Similar to escape(), except that it doesn't operate on pre-escaped strings.
117
118 This function relies on the __html__ convention used both by Django's
119 SafeData class and by third-party libraries like markupsafe.
120 """
121 if isinstance(text, Promise):
122 text = str(text)
123 if hasattr(text, "__html__"):
124 return text.__html__()
125 else:
126 return escape(text)
127
128
129 def format_html(format_string, *args, **kwargs):
130 """
131 Similar to str.format, but pass all arguments through conditional_escape(),
132 and call mark_safe() on the result. This function should be used instead
133 of str.format or % interpolation to build up small HTML fragments.
134 """
135 if not (args or kwargs):
136 # RemovedInDjango60Warning: when the deprecation ends, replace with:
137 # raise TypeError("args or kwargs must be provided.")
138 warnings.warn(
139 "Calling format_html() without passing args or kwargs is deprecated.",
140 RemovedInDjango60Warning,
141 stacklevel=2,
142 )
143 args_safe = map(conditional_escape, args)
144 kwargs_safe = {k: conditional_escape(v) for (k, v) in kwargs.items()}
145 return mark_safe(format_string.format(*args_safe, **kwargs_safe))
146
147
148 def format_html_join(sep, format_string, args_generator):
149 """
150 A wrapper of format_html, for the common case of a group of arguments that
151 need to be formatted using the same format string, and then joined using
152 'sep'. 'sep' is also passed through conditional_escape.
153
154 'args_generator' should be an iterator that returns the sequence of 'args'
155 that will be passed to format_html.
156
157 Example:
158
159 format_html_join('\n', "<li>{} {}</li>", ((u.first_name, u.last_name)
160 for u in users))
161 """
162 return mark_safe(
163 conditional_escape(sep).join(
164 (
165 format_html(format_string, **args)
166 if isinstance(args, Mapping)
167 else format_html(format_string, *args)
168 )
169 for args in args_generator
170 )
171 )
172
173
174 @keep_lazy_text
175 def linebreaks(value, autoescape=False):
176 """Convert newlines into <p> and <br>s."""
177 value = normalize_newlines(value)
178 paras = re.split("\n{2,}", str(value))
179 if autoescape:
180 paras = ["<p>%s</p>" % escape(p).replace("\n", "<br>") for p in paras]
181 else:
182 paras = ["<p>%s</p>" % p.replace("\n", "<br>") for p in paras]
183 return "\n\n".join(paras)
184
185
186 class MLStripper(HTMLParser):
187 def __init__(self):
188 super().__init__(convert_charrefs=False)
189 self.reset()
190 self.fed = []
191
192 def handle_data(self, d):
193 self.fed.append(d)
194
195 def handle_entityref(self, name):
196 self.fed.append("&%s;" % name)
197
198 def handle_charref(self, name):
199 self.fed.append("&#%s;" % name)
200
201 def get_data(self):
202 return "".join(self.fed)
203
204
205 def _strip_once(value):
206 """
207 Internal tag stripping utility used by strip_tags.
208 """
209 s = MLStripper()
210 s.feed(value)
211 s.close()
212 return s.get_data()
213
214
215 @keep_lazy_text
216 def strip_tags(value):
217 """Return the given HTML with all tags stripped."""
218 value = str(value)
219 for long_open_tag in long_open_tag_without_closing_re.finditer(value):
220 if long_open_tag.group().count("<") >= MAX_STRIP_TAGS_DEPTH:
221 raise SuspiciousOperation
222 # Note: in typical case this loop executes _strip_once twice (the second
223 # execution does not remove any more tags).
224 strip_tags_depth = 0
225 while "<" in value and ">" in value:
226 if strip_tags_depth >= MAX_STRIP_TAGS_DEPTH:
227 raise SuspiciousOperation
228 new_value = _strip_once(value)
229 if value.count("<") == new_value.count("<"):
230 # _strip_once wasn't able to detect more tags.
231 break
232 value = new_value
233 strip_tags_depth += 1
234 return value
235
236
237 @keep_lazy_text
238 def strip_spaces_between_tags(value):
239 """Return the given HTML with spaces between tags removed."""
240 return re.sub(r">\s+<", "><", str(value))
241
242
243 def smart_urlquote(url):
244 """Quote a URL if it isn't already quoted."""
245
246 def unquote_quote(segment):
247 segment = unquote(segment)
248 # Tilde is part of RFC 3986 Section 2.3 Unreserved Characters,
249 # see also https://bugs.python.org/issue16285
250 return quote(segment, safe=RFC3986_SUBDELIMS + RFC3986_GENDELIMS + "~")
251
252 try:
253 scheme, netloc, path, query, fragment = urlsplit(url)
254 except ValueError:
255 # invalid IPv6 URL (normally square brackets in hostname part).
256 return unquote_quote(url)
257
258 # Handle IDN as percent-encoded UTF-8 octets, per WHATWG URL Specification
259 # section 3.5 and RFC 3986 section 3.2.2. Defer any IDNA to the user agent.
260 # See #36013.
261 netloc = unquote_quote(netloc)
262
263 if query:
264 # Separately unquoting key/value, so as to not mix querystring separators
265 # included in query values. See #22267.
266 query_parts = [
267 (unquote(q[0]), unquote(q[1]))
268 for q in parse_qsl(query, keep_blank_values=True)
269 ]
270 # urlencode will take care of quoting
271 query = urlencode(query_parts)
272
273 path = unquote_quote(path)
274 fragment = unquote_quote(fragment)
275
276 return urlunsplit((scheme, netloc, path, query, fragment))
277
278
279 class CountsDict(dict):
280 def __init__(self, *args, word, **kwargs):
281 super().__init__(*args, *kwargs)
282 self.word = word
283
284 def __missing__(self, key):
285 self[key] = self.word.count(key)
286 return self[key]
287
288
289 class Urlizer:
290 """
291 Convert any URLs in text into clickable links.
292
293 Work on http://, https://, www. links, and also on links ending in one of
294 the original seven gTLDs (.com, .edu, .gov, .int, .mil, .net, and .org).
295 Links can have trailing punctuation (periods, commas, close-parens) and
296 leading punctuation (opening parens) and it'll still do the right thing.
297 """
298
299 trailing_punctuation_chars = ".,:;!"
300 wrapping_punctuation = [("(", ")"), ("[", "]")]
301
302 simple_url_re = _lazy_re_compile(r"^https?://\[?\w", re.IGNORECASE)
303 simple_url_2_re = _lazy_re_compile(
304 r"^www\.|^(?!http)\w[^@]+\.(com|edu|gov|int|mil|net|org)($|/.*)$", re.IGNORECASE
305 )
306 word_split_re = _lazy_re_compile(r"""([\s<>"']+)""")
307
308 mailto_template = "mailto:{local}@{domain}"
309 url_template = '<a href="{href}"{attrs}>{url}</a>'
310
311 def __call__(self, text, trim_url_limit=None, nofollow=False, autoescape=False):
312 """
313 If trim_url_limit is not None, truncate the URLs in the link text
314 longer than this limit to trim_url_limit - 1 characters and append an
315 ellipsis.
316
317 If nofollow is True, give the links a rel="nofollow" attribute.
318
319 If autoescape is True, autoescape the link text and URLs.
320 """
321 safe_input = isinstance(text, SafeData)
322
323 words = self.word_split_re.split(str(text))
324 local_cache = {}
325 urlized_words = []
326 for word in words:
327 if (urlized_word := local_cache.get(word)) is None:
328 urlized_word = self.handle_word(
329 word,
330 safe_input=safe_input,
331 trim_url_limit=trim_url_limit,
332 nofollow=nofollow,
333 autoescape=autoescape,
334 )
335 local_cache[word] = urlized_word
336 urlized_words.append(urlized_word)
337 return "".join(urlized_words)
338
339 def handle_word(
340 self,
341 word,
342 *,
343 safe_input,
344 trim_url_limit=None,
345 nofollow=False,
346 autoescape=False,
347 ):
348 if "." in word or "@" in word or ":" in word:
349 # lead: Punctuation trimmed from the beginning of the word.
350 # middle: State of the word.
351 # trail: Punctuation trimmed from the end of the word.
352 lead, middle, trail = self.trim_punctuation(word)
353 # Make URL we want to point to.
354 url = None
355 nofollow_attr = ' rel="nofollow"' if nofollow else ""
356 if len(middle) <= MAX_URL_LENGTH and self.simple_url_re.match(middle):
357 url = smart_urlquote(html.unescape(middle))
358 elif len(middle) <= MAX_URL_LENGTH and self.simple_url_2_re.match(middle):
359 url = smart_urlquote("http://%s" % html.unescape(middle))
360 elif ":" not in middle and self.is_email_simple(middle):
361 local, domain = middle.rsplit("@", 1)
362 # Encode per RFC 6068 Section 2 (items 1, 4, 5). Defer any IDNA
363 # to the user agent. See #36013.
364 local = quote(local, safe="")
365 domain = quote(domain, safe="")
366 url = self.mailto_template.format(local=local, domain=domain)
367 nofollow_attr = ""
368 # Make link.
369 if url:
370 trimmed = self.trim_url(middle, limit=trim_url_limit)
371 if autoescape and not safe_input:
372 lead, trail = escape(lead), escape(trail)
373 trimmed = escape(trimmed)
374 middle = self.url_template.format(
375 href=escape(url),
376 attrs=nofollow_attr,
377 url=trimmed,
378 )
379 return mark_safe(f"{lead}{middle}{trail}")
380 else:
381 if safe_input:
382 return mark_safe(word)
383 elif autoescape:
384 return escape(word)
385 elif safe_input:
386 return mark_safe(word)
387 elif autoescape:
388 return escape(word)
389 return word
390
391 def trim_url(self, x, *, limit):
392 if limit is None or len(x) <= limit:
393 return x
394 return "%s…" % x[: max(0, limit - 1)]
395
396 @cached_property
397 def wrapping_punctuation_openings(self):
398 return "".join(dict(self.wrapping_punctuation).keys())
399
400 @cached_property
401 def trailing_punctuation_chars_no_semicolon(self):
402 return self.trailing_punctuation_chars.replace(";", "")
403
404 @cached_property
405 def trailing_punctuation_chars_has_semicolon(self):
406 return ";" in self.trailing_punctuation_chars
407
408 def trim_punctuation(self, word):
409 """
410 Trim trailing and wrapping punctuation from `word`. Return the items of
411 the new state.
412 """
413 # Strip all opening wrapping punctuation.
414 middle = word.lstrip(self.wrapping_punctuation_openings)
415 lead = word[: len(word) - len(middle)]
416 trail = ""
417
418 # Continue trimming until middle remains unchanged.
419 trimmed_something = True
420 counts = CountsDict(word=middle)
421 while trimmed_something and middle:
422 trimmed_something = False
423 # Trim wrapping punctuation.
424 for opening, closing in self.wrapping_punctuation:
425 if counts[opening] < counts[closing]:
426 rstripped = middle.rstrip(closing)
427 if rstripped != middle:
428 strip = counts[closing] - counts[opening]
429 trail = middle[-strip:]
430 middle = middle[:-strip]
431 trimmed_something = True
432 counts[closing] -= strip
433
434 amp = middle.rfind("&")
435 if amp == -1:
436 rstripped = middle.rstrip(self.trailing_punctuation_chars)
437 else:
438 rstripped = middle.rstrip(self.trailing_punctuation_chars_no_semicolon)
439 if rstripped != middle:
440 trail = middle[len(rstripped) :] + trail
441 middle = rstripped
442 trimmed_something = True
443
444 if self.trailing_punctuation_chars_has_semicolon and middle.endswith(";"):
445 # Only strip if not part of an HTML entity.
446 potential_entity = middle[amp:]
447 escaped = html.unescape(potential_entity)
448 if escaped == potential_entity or escaped.endswith(";"):
449 rstripped = middle.rstrip(self.trailing_punctuation_chars)
450 trail_start = len(rstripped)
451 amount_trailing_semicolons = len(middle) - len(middle.rstrip(";"))
452 if amp > -1 and amount_trailing_semicolons > 1:
453 # Leave up to most recent semicolon as might be an entity.
454 recent_semicolon = middle[trail_start:].index(";")
455 middle_semicolon_index = recent_semicolon + trail_start + 1
456 trail = middle[middle_semicolon_index:] + trail
457 middle = rstripped + middle[trail_start:middle_semicolon_index]
458 else:
459 trail = middle[trail_start:] + trail
460 middle = rstripped
461 trimmed_something = True
462
463 return lead, middle, trail
464
465 @staticmethod
466 def is_email_simple(value):
467 """Return True if value looks like an email address."""
468 try:
469 EmailValidator(allowlist=[])(value)
470 except ValidationError:
471 return False
472 return True
473
474
475 urlizer = Urlizer()
476
477
478 @keep_lazy_text
479 def urlize(text, trim_url_limit=None, nofollow=False, autoescape=False):
480 return urlizer(
481 text, trim_url_limit=trim_url_limit, nofollow=nofollow, autoescape=autoescape
482 )
483
484
485 def avoid_wrapping(value):
486 """
487 Avoid text wrapping in the middle of a phrase by adding non-breaking
488 spaces where there previously were normal spaces.
489 """
490 return value.replace(" ", "\xa0")
491
492
493 def html_safe(klass):
494 """
495 A decorator that defines the __html__ method. This helps non-Django
496 templates to detect classes whose __str__ methods return SafeString.
497 """
498 if "__html__" in klass.__dict__:
499 raise ValueError(
500 "can't apply @html_safe to %s because it defines "
501 "__html__()." % klass.__name__
502 )
503 if "__str__" not in klass.__dict__:
504 raise ValueError(
505 "can't apply @html_safe to %s because it doesn't "
506 "define __str__()." % klass.__name__
507 )
508 klass_str = klass.__str__
509 klass.__str__ = lambda self: mark_safe(klass_str(self))
510 klass.__html__ = lambda self: str(self)
511 return klass