Debian Codegraph @

django/utils/http.py

raw
1 import base64
2 import re
3 import unicodedata
4 from binascii import Error as BinasciiError
5 from datetime import datetime, timezone
6 from email.utils import formatdate
7 from urllib.parse import quote, unquote
8 from urllib.parse import urlencode as original_urlencode
9 from urllib.parse import urlsplit
10
11 from django.utils.datastructures import MultiValueDict
12 from django.utils.regex_helper import _lazy_re_compile
13
14 # Based on RFC 9110 Appendix A.
15 ETAG_MATCH = _lazy_re_compile(
16 r"""
17 \A( # start of string and capture group
18 (?:W/)? # optional weak indicator
19 " # opening quote
20 [^"]* # any sequence of non-quote characters
21 " # end quote
22 )\Z # end of string and capture group
23 """,
24 re.X,
25 )
26
27 MONTHS = "jan feb mar apr may jun jul aug sep oct nov dec".split()
28 __D = r"(?P<day>[0-9]{2})"
29 __D2 = r"(?P<day>[ 0-9][0-9])"
30 __M = r"(?P<mon>\w{3})"
31 __Y = r"(?P<year>[0-9]{4})"
32 __Y2 = r"(?P<year>[0-9]{2})"
33 __T = r"(?P<hour>[0-9]{2}):(?P<min>[0-9]{2}):(?P<sec>[0-9]{2})"
34 RFC1123_DATE = _lazy_re_compile(r"^\w{3}, %s %s %s %s GMT$" % (__D, __M, __Y, __T))
35 RFC850_DATE = _lazy_re_compile(r"^\w{6,9}, %s-%s-%s %s GMT$" % (__D, __M, __Y2, __T))
36 ASCTIME_DATE = _lazy_re_compile(r"^\w{3} %s %s %s %s$" % (__M, __D2, __T, __Y))
37
38 RFC3986_GENDELIMS = ":/?#[]@"
39 RFC3986_SUBDELIMS = "!$&'()*+,;="
40 MAX_URL_LENGTH = 2048
41 MAX_URL_REDIRECT_LENGTH = 16384
42
43
44 def urlencode(query, doseq=False):
45 """
46 A version of Python's urllib.parse.urlencode() function that can operate on
47 MultiValueDict and non-string values.
48 """
49 if isinstance(query, MultiValueDict):
50 query = query.lists()
51 elif hasattr(query, "items"):
52 query = query.items()
53 query_params = []
54 for key, value in query:
55 if value is None:
56 raise TypeError(
57 "Cannot encode None for key '%s' in a query string. Did you "
58 "mean to pass an empty string or omit the value?" % key
59 )
60 elif not doseq or isinstance(value, (str, bytes)):
61 query_val = value
62 else:
63 try:
64 itr = iter(value)
65 except TypeError:
66 query_val = value
67 else:
68 # Consume generators and iterators, when doseq=True, to
69 # work around https://bugs.python.org/issue31706.
70 query_val = []
71 for item in itr:
72 if item is None:
73 raise TypeError(
74 "Cannot encode None for key '%s' in a query "
75 "string. Did you mean to pass an empty string or "
76 "omit the value?" % key
77 )
78 elif not isinstance(item, bytes):
79 item = str(item)
80 query_val.append(item)
81 query_params.append((key, query_val))
82 return original_urlencode(query_params, doseq)
83
84
85 def http_date(epoch_seconds=None):
86 """
87 Format the time to match the RFC 5322 date format as specified by RFC 9110
88 Section 5.6.7.
89
90 `epoch_seconds` is a floating point number expressed in seconds since the
91 epoch, in UTC - such as that outputted by time.time(). If set to None, it
92 defaults to the current time.
93
94 Output a string in the format 'Wdy, DD Mon YYYY HH:MM:SS GMT'.
95 """
96 return formatdate(epoch_seconds, usegmt=True)
97
98
99 def parse_http_date(date):
100 """
101 Parse a date format as specified by HTTP RFC 9110 Section 5.6.7.
102
103 The three formats allowed by the RFC are accepted, even if only the first
104 one is still in widespread use.
105
106 Return an integer expressed in seconds since the epoch, in UTC.
107 """
108 # email.utils.parsedate() does the job for RFC 1123 dates; unfortunately
109 # RFC 9110 makes it mandatory to support RFC 850 dates too. So we roll
110 # our own RFC-compliant parsing.
111 for regex in RFC1123_DATE, RFC850_DATE, ASCTIME_DATE:
112 m = regex.match(date)
113 if m is not None:
114 break
115 else:
116 raise ValueError("%r is not in a valid HTTP date format" % date)
117 try:
118 year = int(m["year"])
119 if year < 100:
120 current_year = datetime.now(tz=timezone.utc).year
121 current_century = current_year - (current_year % 100)
122 if year - (current_year % 100) > 50:
123 # year that appears to be more than 50 years in the future are
124 # interpreted as representing the past.
125 year += current_century - 100
126 else:
127 year += current_century
128 month = MONTHS.index(m["mon"].lower()) + 1
129 day = int(m["day"])
130 hour = int(m["hour"])
131 min = int(m["min"])
132 sec = int(m["sec"])
133 result = datetime(year, month, day, hour, min, sec, tzinfo=timezone.utc)
134 return int(result.timestamp())
135 except Exception as exc:
136 raise ValueError("%r is not a valid date" % date) from exc
137
138
139 def parse_http_date_safe(date):
140 """
141 Same as parse_http_date, but return None if the input is invalid.
142 """
143 try:
144 return parse_http_date(date)
145 except Exception:
146 pass
147
148
149 # Base 36 functions: useful for generating compact URLs
150
151
152 def base36_to_int(s):
153 """
154 Convert a base 36 string to an int. Raise ValueError if the input won't fit
155 into an int.
156 """
157 # To prevent overconsumption of server resources, reject any
158 # base36 string that is longer than 13 base36 digits (13 digits
159 # is sufficient to base36-encode any 64-bit integer)
160 if len(s) > 13:
161 raise ValueError("Base36 input too large")
162 return int(s, 36)
163
164
165 def int_to_base36(i):
166 """Convert an integer to a base36 string."""
167 char_set = "0123456789abcdefghijklmnopqrstuvwxyz"
168 if i < 0:
169 raise ValueError("Negative base36 conversion input.")
170 if i < 36:
171 return char_set[i]
172 b36 = ""
173 while i != 0:
174 i, n = divmod(i, 36)
175 b36 = char_set[n] + b36
176 return b36
177
178
179 def urlsafe_base64_encode(s):
180 """
181 Encode a bytestring to a base64 string for use in URLs. Strip any trailing
182 equal signs.
183 """
184 return base64.urlsafe_b64encode(s).rstrip(b"\n=").decode("ascii")
185
186
187 def urlsafe_base64_decode(s):
188 """
189 Decode a base64 encoded string. Add back any trailing equal signs that
190 might have been stripped.
191 """
192 s = s.encode()
193 try:
194 return base64.urlsafe_b64decode(s.ljust(len(s) + len(s) % 4, b"="))
195 except (LookupError, BinasciiError) as e:
196 raise ValueError(e)
197
198
199 def parse_etags(etag_str):
200 """
201 Parse a string of ETags given in an If-None-Match or If-Match header as
202 defined by RFC 9110. Return a list of quoted ETags, or ['*'] if all ETags
203 should be matched.
204 """
205 if etag_str.strip() == "*":
206 return ["*"]
207 else:
208 # Parse each ETag individually, and return any that are valid.
209 etag_matches = (ETAG_MATCH.match(etag.strip()) for etag in etag_str.split(","))
210 return [match[1] for match in etag_matches if match]
211
212
213 def quote_etag(etag_str):
214 """
215 If the provided string is already a quoted ETag, return it. Otherwise, wrap
216 the string in quotes, making it a strong ETag.
217 """
218 if ETAG_MATCH.match(etag_str):
219 return etag_str
220 else:
221 return '"%s"' % etag_str
222
223
224 def is_same_domain(host, pattern):
225 """
226 Return ``True`` if the host is either an exact match or a match
227 to the wildcard pattern.
228
229 Any pattern beginning with a period matches a domain and all of its
230 subdomains. (e.g. ``.example.com`` matches ``example.com`` and
231 ``foo.example.com``). Anything else is an exact string match.
232 """
233 if not pattern:
234 return False
235
236 pattern = pattern.lower()
237 return (
238 pattern[0] == "."
239 and (host.endswith(pattern) or host == pattern[1:])
240 or pattern == host
241 )
242
243
244 def url_has_allowed_host_and_scheme(url, allowed_hosts, require_https=False):
245 """
246 Return ``True`` if the url uses an allowed host and a safe scheme.
247
248 Always return ``False`` on an empty url.
249
250 If ``require_https`` is ``True``, only 'https' will be considered a valid
251 scheme, as opposed to 'http' and 'https' with the default, ``False``.
252
253 Note: "True" doesn't entail that a URL is "safe". It may still be e.g.
254 quoted incorrectly. Ensure to also use django.utils.encoding.iri_to_uri()
255 on the path component of untrusted URLs.
256 """
257 if url is not None:
258 url = url.strip()
259 if not url:
260 return False
261 if allowed_hosts is None:
262 allowed_hosts = set()
263 elif isinstance(allowed_hosts, str):
264 allowed_hosts = {allowed_hosts}
265 # Chrome treats \ completely as / in paths but it could be part of some
266 # basic auth credentials so we need to check both URLs.
267 return _url_has_allowed_host_and_scheme(
268 url, allowed_hosts, require_https=require_https
269 ) and _url_has_allowed_host_and_scheme(
270 url.replace("\\", "/"), allowed_hosts, require_https=require_https
271 )
272
273
274 def _url_has_allowed_host_and_scheme(url, allowed_hosts, require_https=False):
275 # Chrome considers any URL with more than two slashes to be absolute, but
276 # urlsplit is not so flexible. Treat any url with three slashes as unsafe.
277 if url.startswith("///") or len(url) > MAX_URL_LENGTH:
278 # urlsplit does not perform validation of inputs. Unicode normalization
279 # is very slow on Windows and can be a DoS attack vector.
280 # https://docs.python.org/3/library/urllib.parse.html#url-parsing-security
281 return False
282 try:
283 url_info = urlsplit(url)
284 except ValueError: # e.g. invalid IPv6 addresses
285 return False
286 # Forbid URLs like http:///example.com - with a scheme, but without a hostname.
287 # In that URL, example.com is not the hostname but, a path component. However,
288 # Chrome will still consider example.com to be the hostname, so we must not
289 # allow this syntax.
290 if not url_info.netloc and url_info.scheme:
291 return False
292 # Forbid URLs that start with control characters. Some browsers (like
293 # Chrome) ignore quite a few control characters at the start of a
294 # URL and might consider the URL as scheme relative.
295 if unicodedata.category(url[0])[0] == "C":
296 return False
297 scheme = url_info.scheme
298 # Consider URLs without a scheme (e.g. //example.com/p) to be http.
299 if not url_info.scheme and url_info.netloc:
300 scheme = "http"
301 valid_schemes = ["https"] if require_https else ["http", "https"]
302 return (not url_info.netloc or url_info.netloc in allowed_hosts) and (
303 not scheme or scheme in valid_schemes
304 )
305
306
307 def escape_leading_slashes(url):
308 """
309 If redirecting to an absolute path (two leading slashes), a slash must be
310 escaped to prevent browsers from handling the path as schemaless and
311 redirecting to another host.
312 """
313 if url.startswith("//"):
314 url = "/%2F{}".format(url.removeprefix("//"))
315 return url
316
317
318 def _parseparam(s):
319 while s[:1] == ";":
320 s = s[1:]
321 end = s.find(";")
322 while end > 0 and (s.count('"', 0, end) - s.count('\\"', 0, end)) % 2:
323 end = s.find(";", end + 1)
324 if end < 0:
325 end = len(s)
326 f = s[:end]
327 yield f.strip()
328 s = s[end:]
329
330
331 def parse_header_parameters(line):
332 """
333 Parse a Content-type like header.
334 Return the main content-type and a dictionary of options.
335 """
336 parts = _parseparam(";" + line)
337 key = parts.__next__().lower()
338 pdict = {}
339 for p in parts:
340 i = p.find("=")
341 if i >= 0:
342 has_encoding = False
343 name = p[:i].strip().lower()
344 if name.endswith("*"):
345 # Lang/encoding embedded in the value (like "filename*=UTF-8''file.ext")
346 # https://tools.ietf.org/html/rfc2231#section-4
347 name = name[:-1]
348 if p.count("'") == 2:
349 has_encoding = True
350 value = p[i + 1 :].strip()
351 if len(value) >= 2 and value[0] == value[-1] == '"':
352 value = value[1:-1]
353 value = value.replace("\\\\", "\\").replace('\\"', '"')
354 if has_encoding:
355 encoding, lang, value = value.split("'")
356 value = unquote(value, encoding=encoding)
357 pdict[name] = value
358 return key, pdict
359
360
361 def content_disposition_header(as_attachment, filename):
362 """
363 Construct a Content-Disposition HTTP header value from the given filename
364 as specified by RFC 6266.
365 """
366 if filename:
367 disposition = "attachment" if as_attachment else "inline"
368 try:
369 filename.encode("ascii")
370 is_ascii = True
371 except UnicodeEncodeError:
372 is_ascii = False
373 # Quoted strings can contain horizontal tabs, space characters, and
374 # characters from 0x21 to 0x7e, except 0x22 (`"`) and 0x5C (`\`) which
375 # can still be expressed but must be escaped with their own `\`.
376 # https://datatracker.ietf.org/doc/html/rfc9110#name-quoted-strings
377 quotable_characters = r"^[\t \x21-\x7e]*$"
378 if is_ascii and re.match(quotable_characters, filename):
379 file_expr = 'filename="{}"'.format(
380 filename.replace("\\", "\\\\").replace('"', r"\"")
381 )
382 else:
383 file_expr = "filename*=utf-8''{}".format(quote(filename))
384 return f"{disposition}; {file_expr}"
385 elif as_attachment:
386 return "attachment"
387 else:
388 return None