| 1 |
import codecs |
| 2 |
import datetime |
| 3 |
import locale |
| 4 |
from decimal import Decimal |
| 5 |
from types import NoneType |
| 6 |
from urllib.parse import quote |
| 7 |
|
| 8 |
from django.utils.functional import Promise |
| 9 |
|
| 10 |
|
| 11 |
class DjangoUnicodeDecodeError(UnicodeDecodeError): |
| 12 |
def __str__(self): |
| 13 |
return "%s. You passed in %r (%s)" % ( |
| 14 |
super().__str__(), |
| 15 |
self.object, |
| 16 |
type(self.object), |
| 17 |
) |
| 18 |
|
| 19 |
|
| 20 |
def smart_str(s, encoding="utf-8", strings_only=False, errors="strict"): |
| 21 |
""" |
| 22 |
Return a string representing 's'. Treat bytestrings using the 'encoding' |
| 23 |
codec. |
| 24 |
|
| 25 |
If strings_only is True, don't convert (some) non-string-like objects. |
| 26 |
""" |
| 27 |
if isinstance(s, Promise): |
| 28 |
# The input is the result of a gettext_lazy() call. |
| 29 |
return s |
| 30 |
return force_str(s, encoding, strings_only, errors) |
| 31 |
|
| 32 |
|
| 33 |
_PROTECTED_TYPES = ( |
| 34 |
NoneType, |
| 35 |
int, |
| 36 |
float, |
| 37 |
Decimal, |
| 38 |
datetime.datetime, |
| 39 |
datetime.date, |
| 40 |
datetime.time, |
| 41 |
) |
| 42 |
|
| 43 |
|
| 44 |
def is_protected_type(obj): |
| 45 |
"""Determine if the object instance is of a protected type. |
| 46 |
|
| 47 |
Objects of protected types are preserved as-is when passed to |
| 48 |
force_str(strings_only=True). |
| 49 |
""" |
| 50 |
return isinstance(obj, _PROTECTED_TYPES) |
| 51 |
|
| 52 |
|
| 53 |
def force_str(s, encoding="utf-8", strings_only=False, errors="strict"): |
| 54 |
""" |
| 55 |
Similar to smart_str(), except that lazy instances are resolved to |
| 56 |
strings, rather than kept as lazy objects. |
| 57 |
|
| 58 |
If strings_only is True, don't convert (some) non-string-like objects. |
| 59 |
""" |
| 60 |
# Handle the common case first for performance reasons. |
| 61 |
if issubclass(type(s), str): |
| 62 |
return s |
| 63 |
if strings_only and is_protected_type(s): |
| 64 |
return s |
| 65 |
try: |
| 66 |
if isinstance(s, bytes): |
| 67 |
s = str(s, encoding, errors) |
| 68 |
else: |
| 69 |
s = str(s) |
| 70 |
except UnicodeDecodeError as e: |
| 71 |
raise DjangoUnicodeDecodeError(*e.args) from None |
| 72 |
return s |
| 73 |
|
| 74 |
|
| 75 |
def smart_bytes(s, encoding="utf-8", strings_only=False, errors="strict"): |
| 76 |
""" |
| 77 |
Return a bytestring version of 's', encoded as specified in 'encoding'. |
| 78 |
|
| 79 |
If strings_only is True, don't convert (some) non-string-like objects. |
| 80 |
""" |
| 81 |
if isinstance(s, Promise): |
| 82 |
# The input is the result of a gettext_lazy() call. |
| 83 |
return s |
| 84 |
return force_bytes(s, encoding, strings_only, errors) |
| 85 |
|
| 86 |
|
| 87 |
def force_bytes(s, encoding="utf-8", strings_only=False, errors="strict"): |
| 88 |
""" |
| 89 |
Similar to smart_bytes, except that lazy instances are resolved to |
| 90 |
strings, rather than kept as lazy objects. |
| 91 |
|
| 92 |
If strings_only is True, don't convert (some) non-string-like objects. |
| 93 |
""" |
| 94 |
# Handle the common case first for performance reasons. |
| 95 |
if isinstance(s, bytes): |
| 96 |
if encoding == "utf-8": |
| 97 |
return s |
| 98 |
else: |
| 99 |
return s.decode("utf-8", errors).encode(encoding, errors) |
| 100 |
if strings_only and is_protected_type(s): |
| 101 |
return s |
| 102 |
if isinstance(s, memoryview): |
| 103 |
return bytes(s) |
| 104 |
return str(s).encode(encoding, errors) |
| 105 |
|
| 106 |
|
| 107 |
def iri_to_uri(iri): |
| 108 |
""" |
| 109 |
Convert an Internationalized Resource Identifier (IRI) portion to a URI |
| 110 |
portion that is suitable for inclusion in a URL. |
| 111 |
|
| 112 |
This is the algorithm from RFC 3987 Section 3.1, slightly simplified since |
| 113 |
the input is assumed to be a string rather than an arbitrary byte stream. |
| 114 |
|
| 115 |
Take an IRI (string or UTF-8 bytes, e.g. '/I ♥ Django/' or |
| 116 |
b'/I \xe2\x99\xa5 Django/') and return a string containing the encoded |
| 117 |
result with ASCII chars only (e.g. '/I%20%E2%99%A5%20Django/'). |
| 118 |
""" |
| 119 |
# The list of safe characters here is constructed from the "reserved" and |
| 120 |
# "unreserved" characters specified in RFC 3986 Sections 2.2 and 2.3: |
| 121 |
# reserved = gen-delims / sub-delims |
| 122 |
# gen-delims = ":" / "/" / "?" / "#" / "[" / "]" / "@" |
| 123 |
# sub-delims = "!" / "$" / "&" / "'" / "(" / ")" |
| 124 |
# / "*" / "+" / "," / ";" / "=" |
| 125 |
# unreserved = ALPHA / DIGIT / "-" / "." / "_" / "~" |
| 126 |
# Of the unreserved characters, urllib.parse.quote() already considers all |
| 127 |
# but the ~ safe. |
| 128 |
# The % character is also added to the list of safe characters here, as the |
| 129 |
# end of RFC 3987 Section 3.1 specifically mentions that % must not be |
| 130 |
# converted. |
| 131 |
if iri is None: |
| 132 |
return iri |
| 133 |
elif isinstance(iri, Promise): |
| 134 |
iri = str(iri) |
| 135 |
return quote(iri, safe="/#%[]=:;$&()+,!?*@'~") |
| 136 |
|
| 137 |
|
| 138 |
# List of byte values that uri_to_iri() decodes from percent encoding. |
| 139 |
# First, the unreserved characters from RFC 3986: |
| 140 |
_ascii_ranges = [[45, 46, 95, 126], range(65, 91), range(97, 123)] |
| 141 |
_hextobyte = { |
| 142 |
(fmt % char).encode(): bytes((char,)) |
| 143 |
for ascii_range in _ascii_ranges |
| 144 |
for char in ascii_range |
| 145 |
for fmt in ["%02x", "%02X"] |
| 146 |
} |
| 147 |
# And then everything above 128, because bytes ≥ 128 are part of multibyte |
| 148 |
# Unicode characters. |
| 149 |
_hexdig = "0123456789ABCDEFabcdef" |
| 150 |
_hextobyte.update( |
| 151 |
{(a + b).encode(): bytes.fromhex(a + b) for a in _hexdig[8:] for b in _hexdig} |
| 152 |
) |
| 153 |
|
| 154 |
|
| 155 |
def uri_to_iri(uri): |
| 156 |
""" |
| 157 |
Convert a Uniform Resource Identifier(URI) into an Internationalized |
| 158 |
Resource Identifier(IRI). |
| 159 |
|
| 160 |
This is the algorithm from RFC 3987 Section 3.2, excluding step 4. |
| 161 |
|
| 162 |
Take an URI in ASCII bytes (e.g. '/I%20%E2%99%A5%20Django/') and return |
| 163 |
a string containing the encoded result (e.g. '/I%20♥%20Django/'). |
| 164 |
""" |
| 165 |
if uri is None: |
| 166 |
return uri |
| 167 |
uri = force_bytes(uri) |
| 168 |
# Fast selective unquote: First, split on '%' and then starting with the |
| 169 |
# second block, decode the first 2 bytes if they represent a hex code to |
| 170 |
# decode. The rest of the block is the part after '%AB', not containing |
| 171 |
# any '%'. Add that to the output without further processing. |
| 172 |
bits = uri.split(b"%") |
| 173 |
if len(bits) == 1: |
| 174 |
iri = uri |
| 175 |
else: |
| 176 |
parts = [bits[0]] |
| 177 |
append = parts.append |
| 178 |
hextobyte = _hextobyte |
| 179 |
for item in bits[1:]: |
| 180 |
hex = item[:2] |
| 181 |
if hex in hextobyte: |
| 182 |
append(hextobyte[item[:2]]) |
| 183 |
append(item[2:]) |
| 184 |
else: |
| 185 |
append(b"%") |
| 186 |
append(item) |
| 187 |
iri = b"".join(parts) |
| 188 |
return repercent_broken_unicode(iri).decode() |
| 189 |
|
| 190 |
|
| 191 |
def escape_uri_path(path): |
| 192 |
""" |
| 193 |
Escape the unsafe characters from the path portion of a Uniform Resource |
| 194 |
Identifier (URI). |
| 195 |
""" |
| 196 |
# These are the "reserved" and "unreserved" characters specified in RFC |
| 197 |
# 3986 Sections 2.2 and 2.3: |
| 198 |
# reserved = ";" | "/" | "?" | ":" | "@" | "&" | "=" | "+" | "$" | "," |
| 199 |
# unreserved = alphanum | mark |
| 200 |
# mark = "-" | "_" | "." | "!" | "~" | "*" | "'" | "(" | ")" |
| 201 |
# The list of safe characters here is constructed subtracting ";", "=", |
| 202 |
# and "?" according to RFC 3986 Section 3.3. |
| 203 |
# The reason for not subtracting and escaping "/" is that we are escaping |
| 204 |
# the entire path, not a path segment. |
| 205 |
return quote(path, safe="/:@&+$,-_.!~*'()") |
| 206 |
|
| 207 |
|
| 208 |
def punycode(domain): |
| 209 |
"""Return the Punycode of the given domain if it's non-ASCII.""" |
| 210 |
return domain.encode("idna").decode("ascii") |
| 211 |
|
| 212 |
|
| 213 |
def repercent_broken_unicode(path): |
| 214 |
""" |
| 215 |
As per RFC 3987 Section 3.2, step three of converting a URI into an IRI, |
| 216 |
repercent-encode any octet produced that is not part of a strictly legal |
| 217 |
UTF-8 octet sequence. |
| 218 |
""" |
| 219 |
changed_parts = [] |
| 220 |
while True: |
| 221 |
try: |
| 222 |
path.decode() |
| 223 |
except UnicodeDecodeError as e: |
| 224 |
# CVE-2019-14235: A recursion shouldn't be used since the exception |
| 225 |
# handling uses massive amounts of memory |
| 226 |
repercent = quote(path[e.start : e.end], safe=b"/#%[]=:;$&()+,!?*@'~") |
| 227 |
changed_parts.append(path[: e.start] + repercent.encode()) |
| 228 |
path = path[e.end :] |
| 229 |
else: |
| 230 |
return b"".join(changed_parts) + path |
| 231 |
|
| 232 |
|
| 233 |
def filepath_to_uri(path): |
| 234 |
"""Convert a file system path to a URI portion that is suitable for |
| 235 |
inclusion in a URL. |
| 236 |
|
| 237 |
Encode certain chars that would normally be recognized as special chars |
| 238 |
for URIs. Do not encode the ' character, as it is a valid character |
| 239 |
within URIs. See the encodeURIComponent() JavaScript function for details. |
| 240 |
""" |
| 241 |
if path is None: |
| 242 |
return path |
| 243 |
# I know about `os.sep` and `os.altsep` but I want to leave |
| 244 |
# some flexibility for hardcoding separators. |
| 245 |
return quote(str(path).replace("\\", "/"), safe="/~!*()'") |
| 246 |
|
| 247 |
|
| 248 |
def get_system_encoding(): |
| 249 |
""" |
| 250 |
The encoding for the character type functions. Fallback to 'ascii' if the |
| 251 |
#encoding is unsupported by Python or could not be determined. See tickets |
| 252 |
#10335 and #5846. |
| 253 |
""" |
| 254 |
try: |
| 255 |
encoding = locale.getlocale()[1] or "ascii" |
| 256 |
codecs.lookup(encoding) |
| 257 |
except Exception: |
| 258 |
encoding = "ascii" |
| 259 |
return encoding |
| 260 |
|
| 261 |
|
| 262 |
DEFAULT_LOCALE_ENCODING = get_system_encoding() |