| 1 |
import base64 |
| 2 |
import re |
| 3 |
import unicodedata |
| 4 |
from binascii import Error as BinasciiError |
| 5 |
from datetime import datetime, timezone |
| 6 |
from email.utils import formatdate |
| 7 |
from urllib.parse import quote, unquote |
| 8 |
from urllib.parse import urlencode as original_urlencode |
| 9 |
from urllib.parse import urlsplit |
| 10 |
|
| 11 |
from django.utils.datastructures import MultiValueDict |
| 12 |
from django.utils.regex_helper import _lazy_re_compile |
| 13 |
|
| 14 |
# Based on RFC 9110 Appendix A. |
| 15 |
ETAG_MATCH = _lazy_re_compile( |
| 16 |
r""" |
| 17 |
\A( # start of string and capture group |
| 18 |
(?:W/)? # optional weak indicator |
| 19 |
" # opening quote |
| 20 |
[^"]* # any sequence of non-quote characters |
| 21 |
" # end quote |
| 22 |
)\Z # end of string and capture group |
| 23 |
""", |
| 24 |
re.X, |
| 25 |
) |
| 26 |
|
| 27 |
MONTHS = "jan feb mar apr may jun jul aug sep oct nov dec".split() |
| 28 |
__D = r"(?P<day>[0-9]{2})" |
| 29 |
__D2 = r"(?P<day>[ 0-9][0-9])" |
| 30 |
__M = r"(?P<mon>\w{3})" |
| 31 |
__Y = r"(?P<year>[0-9]{4})" |
| 32 |
__Y2 = r"(?P<year>[0-9]{2})" |
| 33 |
__T = r"(?P<hour>[0-9]{2}):(?P<min>[0-9]{2}):(?P<sec>[0-9]{2})" |
| 34 |
RFC1123_DATE = _lazy_re_compile(r"^\w{3}, %s %s %s %s GMT$" % (__D, __M, __Y, __T)) |
| 35 |
RFC850_DATE = _lazy_re_compile(r"^\w{6,9}, %s-%s-%s %s GMT$" % (__D, __M, __Y2, __T)) |
| 36 |
ASCTIME_DATE = _lazy_re_compile(r"^\w{3} %s %s %s %s$" % (__M, __D2, __T, __Y)) |
| 37 |
|
| 38 |
RFC3986_GENDELIMS = ":/?#[]@" |
| 39 |
RFC3986_SUBDELIMS = "!$&'()*+,;=" |
| 40 |
MAX_URL_LENGTH = 2048 |
| 41 |
MAX_URL_REDIRECT_LENGTH = 16384 |
| 42 |
|
| 43 |
|
| 44 |
def urlencode(query, doseq=False): |
| 45 |
""" |
| 46 |
A version of Python's urllib.parse.urlencode() function that can operate on |
| 47 |
MultiValueDict and non-string values. |
| 48 |
""" |
| 49 |
if isinstance(query, MultiValueDict): |
| 50 |
query = query.lists() |
| 51 |
elif hasattr(query, "items"): |
| 52 |
query = query.items() |
| 53 |
query_params = [] |
| 54 |
for key, value in query: |
| 55 |
if value is None: |
| 56 |
raise TypeError( |
| 57 |
"Cannot encode None for key '%s' in a query string. Did you " |
| 58 |
"mean to pass an empty string or omit the value?" % key |
| 59 |
) |
| 60 |
elif not doseq or isinstance(value, (str, bytes)): |
| 61 |
query_val = value |
| 62 |
else: |
| 63 |
try: |
| 64 |
itr = iter(value) |
| 65 |
except TypeError: |
| 66 |
query_val = value |
| 67 |
else: |
| 68 |
# Consume generators and iterators, when doseq=True, to |
| 69 |
# work around https://bugs.python.org/issue31706. |
| 70 |
query_val = [] |
| 71 |
for item in itr: |
| 72 |
if item is None: |
| 73 |
raise TypeError( |
| 74 |
"Cannot encode None for key '%s' in a query " |
| 75 |
"string. Did you mean to pass an empty string or " |
| 76 |
"omit the value?" % key |
| 77 |
) |
| 78 |
elif not isinstance(item, bytes): |
| 79 |
item = str(item) |
| 80 |
query_val.append(item) |
| 81 |
query_params.append((key, query_val)) |
| 82 |
return original_urlencode(query_params, doseq) |
| 83 |
|
| 84 |
|
| 85 |
def http_date(epoch_seconds=None): |
| 86 |
""" |
| 87 |
Format the time to match the RFC 5322 date format as specified by RFC 9110 |
| 88 |
Section 5.6.7. |
| 89 |
|
| 90 |
`epoch_seconds` is a floating point number expressed in seconds since the |
| 91 |
epoch, in UTC - such as that outputted by time.time(). If set to None, it |
| 92 |
defaults to the current time. |
| 93 |
|
| 94 |
Output a string in the format 'Wdy, DD Mon YYYY HH:MM:SS GMT'. |
| 95 |
""" |
| 96 |
return formatdate(epoch_seconds, usegmt=True) |
| 97 |
|
| 98 |
|
| 99 |
def parse_http_date(date): |
| 100 |
""" |
| 101 |
Parse a date format as specified by HTTP RFC 9110 Section 5.6.7. |
| 102 |
|
| 103 |
The three formats allowed by the RFC are accepted, even if only the first |
| 104 |
one is still in widespread use. |
| 105 |
|
| 106 |
Return an integer expressed in seconds since the epoch, in UTC. |
| 107 |
""" |
| 108 |
# email.utils.parsedate() does the job for RFC 1123 dates; unfortunately |
| 109 |
# RFC 9110 makes it mandatory to support RFC 850 dates too. So we roll |
| 110 |
# our own RFC-compliant parsing. |
| 111 |
for regex in RFC1123_DATE, RFC850_DATE, ASCTIME_DATE: |
| 112 |
m = regex.match(date) |
| 113 |
if m is not None: |
| 114 |
break |
| 115 |
else: |
| 116 |
raise ValueError("%r is not in a valid HTTP date format" % date) |
| 117 |
try: |
| 118 |
year = int(m["year"]) |
| 119 |
if year < 100: |
| 120 |
current_year = datetime.now(tz=timezone.utc).year |
| 121 |
current_century = current_year - (current_year % 100) |
| 122 |
if year - (current_year % 100) > 50: |
| 123 |
# year that appears to be more than 50 years in the future are |
| 124 |
# interpreted as representing the past. |
| 125 |
year += current_century - 100 |
| 126 |
else: |
| 127 |
year += current_century |
| 128 |
month = MONTHS.index(m["mon"].lower()) + 1 |
| 129 |
day = int(m["day"]) |
| 130 |
hour = int(m["hour"]) |
| 131 |
min = int(m["min"]) |
| 132 |
sec = int(m["sec"]) |
| 133 |
result = datetime(year, month, day, hour, min, sec, tzinfo=timezone.utc) |
| 134 |
return int(result.timestamp()) |
| 135 |
except Exception as exc: |
| 136 |
raise ValueError("%r is not a valid date" % date) from exc |
| 137 |
|
| 138 |
|
| 139 |
def parse_http_date_safe(date): |
| 140 |
""" |
| 141 |
Same as parse_http_date, but return None if the input is invalid. |
| 142 |
""" |
| 143 |
try: |
| 144 |
return parse_http_date(date) |
| 145 |
except Exception: |
| 146 |
pass |
| 147 |
|
| 148 |
|
| 149 |
# Base 36 functions: useful for generating compact URLs |
| 150 |
|
| 151 |
|
| 152 |
def base36_to_int(s): |
| 153 |
""" |
| 154 |
Convert a base 36 string to an int. Raise ValueError if the input won't fit |
| 155 |
into an int. |
| 156 |
""" |
| 157 |
# To prevent overconsumption of server resources, reject any |
| 158 |
# base36 string that is longer than 13 base36 digits (13 digits |
| 159 |
# is sufficient to base36-encode any 64-bit integer) |
| 160 |
if len(s) > 13: |
| 161 |
raise ValueError("Base36 input too large") |
| 162 |
return int(s, 36) |
| 163 |
|
| 164 |
|
| 165 |
def int_to_base36(i): |
| 166 |
"""Convert an integer to a base36 string.""" |
| 167 |
char_set = "0123456789abcdefghijklmnopqrstuvwxyz" |
| 168 |
if i < 0: |
| 169 |
raise ValueError("Negative base36 conversion input.") |
| 170 |
if i < 36: |
| 171 |
return char_set[i] |
| 172 |
b36 = "" |
| 173 |
while i != 0: |
| 174 |
i, n = divmod(i, 36) |
| 175 |
b36 = char_set[n] + b36 |
| 176 |
return b36 |
| 177 |
|
| 178 |
|
| 179 |
def urlsafe_base64_encode(s): |
| 180 |
""" |
| 181 |
Encode a bytestring to a base64 string for use in URLs. Strip any trailing |
| 182 |
equal signs. |
| 183 |
""" |
| 184 |
return base64.urlsafe_b64encode(s).rstrip(b"\n=").decode("ascii") |
| 185 |
|
| 186 |
|
| 187 |
def urlsafe_base64_decode(s): |
| 188 |
""" |
| 189 |
Decode a base64 encoded string. Add back any trailing equal signs that |
| 190 |
might have been stripped. |
| 191 |
""" |
| 192 |
s = s.encode() |
| 193 |
try: |
| 194 |
return base64.urlsafe_b64decode(s.ljust(len(s) + len(s) % 4, b"=")) |
| 195 |
except (LookupError, BinasciiError) as e: |
| 196 |
raise ValueError(e) |
| 197 |
|
| 198 |
|
| 199 |
def parse_etags(etag_str): |
| 200 |
""" |
| 201 |
Parse a string of ETags given in an If-None-Match or If-Match header as |
| 202 |
defined by RFC 9110. Return a list of quoted ETags, or ['*'] if all ETags |
| 203 |
should be matched. |
| 204 |
""" |
| 205 |
if etag_str.strip() == "*": |
| 206 |
return ["*"] |
| 207 |
else: |
| 208 |
# Parse each ETag individually, and return any that are valid. |
| 209 |
etag_matches = (ETAG_MATCH.match(etag.strip()) for etag in etag_str.split(",")) |
| 210 |
return [match[1] for match in etag_matches if match] |
| 211 |
|
| 212 |
|
| 213 |
def quote_etag(etag_str): |
| 214 |
""" |
| 215 |
If the provided string is already a quoted ETag, return it. Otherwise, wrap |
| 216 |
the string in quotes, making it a strong ETag. |
| 217 |
""" |
| 218 |
if ETAG_MATCH.match(etag_str): |
| 219 |
return etag_str |
| 220 |
else: |
| 221 |
return '"%s"' % etag_str |
| 222 |
|
| 223 |
|
| 224 |
def is_same_domain(host, pattern): |
| 225 |
""" |
| 226 |
Return ``True`` if the host is either an exact match or a match |
| 227 |
to the wildcard pattern. |
| 228 |
|
| 229 |
Any pattern beginning with a period matches a domain and all of its |
| 230 |
subdomains. (e.g. ``.example.com`` matches ``example.com`` and |
| 231 |
``foo.example.com``). Anything else is an exact string match. |
| 232 |
""" |
| 233 |
if not pattern: |
| 234 |
return False |
| 235 |
|
| 236 |
pattern = pattern.lower() |
| 237 |
return ( |
| 238 |
pattern[0] == "." |
| 239 |
and (host.endswith(pattern) or host == pattern[1:]) |
| 240 |
or pattern == host |
| 241 |
) |
| 242 |
|
| 243 |
|
| 244 |
def url_has_allowed_host_and_scheme(url, allowed_hosts, require_https=False): |
| 245 |
""" |
| 246 |
Return ``True`` if the url uses an allowed host and a safe scheme. |
| 247 |
|
| 248 |
Always return ``False`` on an empty url. |
| 249 |
|
| 250 |
If ``require_https`` is ``True``, only 'https' will be considered a valid |
| 251 |
scheme, as opposed to 'http' and 'https' with the default, ``False``. |
| 252 |
|
| 253 |
Note: "True" doesn't entail that a URL is "safe". It may still be e.g. |
| 254 |
quoted incorrectly. Ensure to also use django.utils.encoding.iri_to_uri() |
| 255 |
on the path component of untrusted URLs. |
| 256 |
""" |
| 257 |
if url is not None: |
| 258 |
url = url.strip() |
| 259 |
if not url: |
| 260 |
return False |
| 261 |
if allowed_hosts is None: |
| 262 |
allowed_hosts = set() |
| 263 |
elif isinstance(allowed_hosts, str): |
| 264 |
allowed_hosts = {allowed_hosts} |
| 265 |
# Chrome treats \ completely as / in paths but it could be part of some |
| 266 |
# basic auth credentials so we need to check both URLs. |
| 267 |
return _url_has_allowed_host_and_scheme( |
| 268 |
url, allowed_hosts, require_https=require_https |
| 269 |
) and _url_has_allowed_host_and_scheme( |
| 270 |
url.replace("\\", "/"), allowed_hosts, require_https=require_https |
| 271 |
) |
| 272 |
|
| 273 |
|
| 274 |
def _url_has_allowed_host_and_scheme(url, allowed_hosts, require_https=False): |
| 275 |
# Chrome considers any URL with more than two slashes to be absolute, but |
| 276 |
# urlsplit is not so flexible. Treat any url with three slashes as unsafe. |
| 277 |
if url.startswith("///") or len(url) > MAX_URL_LENGTH: |
| 278 |
# urlsplit does not perform validation of inputs. Unicode normalization |
| 279 |
# is very slow on Windows and can be a DoS attack vector. |
| 280 |
# https://docs.python.org/3/library/urllib.parse.html#url-parsing-security |
| 281 |
return False |
| 282 |
try: |
| 283 |
url_info = urlsplit(url) |
| 284 |
except ValueError: # e.g. invalid IPv6 addresses |
| 285 |
return False |
| 286 |
# Forbid URLs like http:///example.com - with a scheme, but without a hostname. |
| 287 |
# In that URL, example.com is not the hostname but, a path component. However, |
| 288 |
# Chrome will still consider example.com to be the hostname, so we must not |
| 289 |
# allow this syntax. |
| 290 |
if not url_info.netloc and url_info.scheme: |
| 291 |
return False |
| 292 |
# Forbid URLs that start with control characters. Some browsers (like |
| 293 |
# Chrome) ignore quite a few control characters at the start of a |
| 294 |
# URL and might consider the URL as scheme relative. |
| 295 |
if unicodedata.category(url[0])[0] == "C": |
| 296 |
return False |
| 297 |
scheme = url_info.scheme |
| 298 |
# Consider URLs without a scheme (e.g. //example.com/p) to be http. |
| 299 |
if not url_info.scheme and url_info.netloc: |
| 300 |
scheme = "http" |
| 301 |
valid_schemes = ["https"] if require_https else ["http", "https"] |
| 302 |
return (not url_info.netloc or url_info.netloc in allowed_hosts) and ( |
| 303 |
not scheme or scheme in valid_schemes |
| 304 |
) |
| 305 |
|
| 306 |
|
| 307 |
def escape_leading_slashes(url): |
| 308 |
""" |
| 309 |
If redirecting to an absolute path (two leading slashes), a slash must be |
| 310 |
escaped to prevent browsers from handling the path as schemaless and |
| 311 |
redirecting to another host. |
| 312 |
""" |
| 313 |
if url.startswith("//"): |
| 314 |
url = "/%2F{}".format(url.removeprefix("//")) |
| 315 |
return url |
| 316 |
|
| 317 |
|
| 318 |
def _parseparam(s): |
| 319 |
while s[:1] == ";": |
| 320 |
s = s[1:] |
| 321 |
end = s.find(";") |
| 322 |
while end > 0 and (s.count('"', 0, end) - s.count('\\"', 0, end)) % 2: |
| 323 |
end = s.find(";", end + 1) |
| 324 |
if end < 0: |
| 325 |
end = len(s) |
| 326 |
f = s[:end] |
| 327 |
yield f.strip() |
| 328 |
s = s[end:] |
| 329 |
|
| 330 |
|
| 331 |
def parse_header_parameters(line): |
| 332 |
""" |
| 333 |
Parse a Content-type like header. |
| 334 |
Return the main content-type and a dictionary of options. |
| 335 |
""" |
| 336 |
parts = _parseparam(";" + line) |
| 337 |
key = parts.__next__().lower() |
| 338 |
pdict = {} |
| 339 |
for p in parts: |
| 340 |
i = p.find("=") |
| 341 |
if i >= 0: |
| 342 |
has_encoding = False |
| 343 |
name = p[:i].strip().lower() |
| 344 |
if name.endswith("*"): |
| 345 |
# Lang/encoding embedded in the value (like "filename*=UTF-8''file.ext") |
| 346 |
# https://tools.ietf.org/html/rfc2231#section-4 |
| 347 |
name = name[:-1] |
| 348 |
if p.count("'") == 2: |
| 349 |
has_encoding = True |
| 350 |
value = p[i + 1 :].strip() |
| 351 |
if len(value) >= 2 and value[0] == value[-1] == '"': |
| 352 |
value = value[1:-1] |
| 353 |
value = value.replace("\\\\", "\\").replace('\\"', '"') |
| 354 |
if has_encoding: |
| 355 |
encoding, lang, value = value.split("'") |
| 356 |
value = unquote(value, encoding=encoding) |
| 357 |
pdict[name] = value |
| 358 |
return key, pdict |
| 359 |
|
| 360 |
|
| 361 |
def content_disposition_header(as_attachment, filename): |
| 362 |
""" |
| 363 |
Construct a Content-Disposition HTTP header value from the given filename |
| 364 |
as specified by RFC 6266. |
| 365 |
""" |
| 366 |
if filename: |
| 367 |
disposition = "attachment" if as_attachment else "inline" |
| 368 |
try: |
| 369 |
filename.encode("ascii") |
| 370 |
is_ascii = True |
| 371 |
except UnicodeEncodeError: |
| 372 |
is_ascii = False |
| 373 |
# Quoted strings can contain horizontal tabs, space characters, and |
| 374 |
# characters from 0x21 to 0x7e, except 0x22 (`"`) and 0x5C (`\`) which |
| 375 |
# can still be expressed but must be escaped with their own `\`. |
| 376 |
# https://datatracker.ietf.org/doc/html/rfc9110#name-quoted-strings |
| 377 |
quotable_characters = r"^[\t \x21-\x7e]*$" |
| 378 |
if is_ascii and re.match(quotable_characters, filename): |
| 379 |
file_expr = 'filename="{}"'.format( |
| 380 |
filename.replace("\\", "\\\\").replace('"', r"\"") |
| 381 |
) |
| 382 |
else: |
| 383 |
file_expr = "filename*=utf-8''{}".format(quote(filename)) |
| 384 |
return f"{disposition}; {file_expr}" |
| 385 |
elif as_attachment: |
| 386 |
return "attachment" |
| 387 |
else: |
| 388 |
return None |