| 1 |
"""Parse (absolute and relative) URLs. |
| 2 |
|
| 3 |
urllib.parse module is based upon the following RFC specifications. |
| 4 |
|
| 5 |
RFC 3986 (STD66): "Uniform Resource Identifiers" by T. Berners-Lee, R. Fielding |
| 6 |
and L. Masinter, January 2005. |
| 7 |
|
| 8 |
RFC 2732 : "Format for Literal IPv6 Addresses in URL's by R.Hinden, B.Carpenter |
| 9 |
and L.Masinter, December 1999. |
| 10 |
|
| 11 |
RFC 2396: "Uniform Resource Identifiers (URI)": Generic Syntax by T. |
| 12 |
Berners-Lee, R. Fielding, and L. Masinter, August 1998. |
| 13 |
|
| 14 |
RFC 2368: "The mailto URL scheme", by P.Hoffman , L Masinter, J. Zawinski, July 1998. |
| 15 |
|
| 16 |
RFC 1808: "Relative Uniform Resource Locators", by R. Fielding, UC Irvine, June |
| 17 |
1995. |
| 18 |
|
| 19 |
RFC 1738: "Uniform Resource Locators (URL)" by T. Berners-Lee, L. Masinter, M. |
| 20 |
McCahill, December 1994 |
| 21 |
|
| 22 |
RFC 3986 is considered the current standard and any future changes to |
| 23 |
urllib.parse module should conform with it. The urllib.parse module is |
| 24 |
currently not entirely compliant with this RFC due to defacto |
| 25 |
scenarios for parsing, and for backward compatibility purposes, some |
| 26 |
parsing quirks from older RFCs are retained. The testcases in |
| 27 |
test_urlparse.py provides a good indicator of parsing behavior. |
| 28 |
|
| 29 |
The WHATWG URL Parser spec should also be considered. We are not compliant with |
| 30 |
it either due to existing user code API behavior expectations (Hyrum's Law). |
| 31 |
It serves as a useful guide when making changes. |
| 32 |
""" |
| 33 |
|
| 34 |
from collections import namedtuple |
| 35 |
import functools |
| 36 |
import math |
| 37 |
import re |
| 38 |
import types |
| 39 |
import warnings |
| 40 |
import ipaddress |
| 41 |
|
| 42 |
__all__ = ["urlparse", "urlunparse", "urljoin", "urldefrag", |
| 43 |
"urlsplit", "urlunsplit", "urlencode", "parse_qs", |
| 44 |
"parse_qsl", "quote", "quote_plus", "quote_from_bytes", |
| 45 |
"unquote", "unquote_plus", "unquote_to_bytes", |
| 46 |
"DefragResult", "ParseResult", "SplitResult", |
| 47 |
"DefragResultBytes", "ParseResultBytes", "SplitResultBytes"] |
| 48 |
|
| 49 |
# A classification of schemes. |
| 50 |
# The empty string classifies URLs with no scheme specified, |
| 51 |
# being the default value returned by “urlsplit” and “urlparse”. |
| 52 |
|
| 53 |
uses_relative = ['', 'ftp', 'http', 'gopher', 'nntp', 'imap', |
| 54 |
'wais', 'file', 'https', 'shttp', 'mms', |
| 55 |
'prospero', 'rtsp', 'rtsps', 'rtspu', 'sftp', |
| 56 |
'svn', 'svn+ssh', 'ws', 'wss'] |
| 57 |
|
| 58 |
uses_netloc = ['', 'ftp', 'http', 'gopher', 'nntp', 'telnet', |
| 59 |
'imap', 'wais', 'file', 'mms', 'https', 'shttp', |
| 60 |
'snews', 'prospero', 'rtsp', 'rtsps', 'rtspu', 'rsync', |
| 61 |
'svn', 'svn+ssh', 'sftp', 'nfs', 'git', 'git+ssh', |
| 62 |
'ws', 'wss', 'itms-services'] |
| 63 |
|
| 64 |
uses_params = ['', 'ftp', 'hdl', 'prospero', 'http', 'imap', |
| 65 |
'https', 'shttp', 'rtsp', 'rtsps', 'rtspu', 'sip', |
| 66 |
'sips', 'mms', 'sftp', 'tel'] |
| 67 |
|
| 68 |
# These are not actually used anymore, but should stay for backwards |
| 69 |
# compatibility. (They are undocumented, but have a public-looking name.) |
| 70 |
|
| 71 |
non_hierarchical = ['gopher', 'hdl', 'mailto', 'news', |
| 72 |
'telnet', 'wais', 'imap', 'snews', 'sip', 'sips'] |
| 73 |
|
| 74 |
uses_query = ['', 'http', 'wais', 'imap', 'https', 'shttp', 'mms', |
| 75 |
'gopher', 'rtsp', 'rtsps', 'rtspu', 'sip', 'sips'] |
| 76 |
|
| 77 |
uses_fragment = ['', 'ftp', 'hdl', 'http', 'gopher', 'news', |
| 78 |
'nntp', 'wais', 'https', 'shttp', 'snews', |
| 79 |
'file', 'prospero'] |
| 80 |
|
| 81 |
# Characters valid in scheme names |
| 82 |
scheme_chars = ('abcdefghijklmnopqrstuvwxyz' |
| 83 |
'ABCDEFGHIJKLMNOPQRSTUVWXYZ' |
| 84 |
'0123456789' |
| 85 |
'+-.') |
| 86 |
|
| 87 |
# Leading and trailing C0 control and space to be stripped per WHATWG spec. |
| 88 |
# == "".join([chr(i) for i in range(0, 0x20 + 1)]) |
| 89 |
_WHATWG_C0_CONTROL_OR_SPACE = '\x00\x01\x02\x03\x04\x05\x06\x07\x08\t\n\x0b\x0c\r\x0e\x0f\x10\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f ' |
| 90 |
|
| 91 |
# Unsafe bytes to be removed per WHATWG spec |
| 92 |
_UNSAFE_URL_BYTES_TO_REMOVE = ['\t', '\r', '\n'] |
| 93 |
|
| 94 |
def clear_cache(): |
| 95 |
"""Clear internal performance caches. Undocumented; some tests want it.""" |
| 96 |
urlsplit.cache_clear() |
| 97 |
_byte_quoter_factory.cache_clear() |
| 98 |
|
| 99 |
# Helpers for bytes handling |
| 100 |
# For 3.2, we deliberately require applications that |
| 101 |
# handle improperly quoted URLs to do their own |
| 102 |
# decoding and encoding. If valid use cases are |
| 103 |
# presented, we may relax this by using latin-1 |
| 104 |
# decoding internally for 3.3 |
| 105 |
_implicit_encoding = 'ascii' |
| 106 |
_implicit_errors = 'strict' |
| 107 |
|
| 108 |
def _noop(obj): |
| 109 |
return obj |
| 110 |
|
| 111 |
def _encode_result(obj, encoding=_implicit_encoding, |
| 112 |
errors=_implicit_errors): |
| 113 |
return obj.encode(encoding, errors) |
| 114 |
|
| 115 |
def _decode_args(args, encoding=_implicit_encoding, |
| 116 |
errors=_implicit_errors): |
| 117 |
return tuple(x.decode(encoding, errors) if x else '' for x in args) |
| 118 |
|
| 119 |
def _coerce_args(*args): |
| 120 |
# Invokes decode if necessary to create str args |
| 121 |
# and returns the coerced inputs along with |
| 122 |
# an appropriate result coercion function |
| 123 |
# - noop for str inputs |
| 124 |
# - encoding function otherwise |
| 125 |
str_input = isinstance(args[0], str) |
| 126 |
for arg in args[1:]: |
| 127 |
# We special-case the empty string to support the |
| 128 |
# "scheme=''" default argument to some functions |
| 129 |
if arg and isinstance(arg, str) != str_input: |
| 130 |
raise TypeError("Cannot mix str and non-str arguments") |
| 131 |
if str_input: |
| 132 |
return args + (_noop,) |
| 133 |
return _decode_args(args) + (_encode_result,) |
| 134 |
|
| 135 |
# Result objects are more helpful than simple tuples |
| 136 |
class _ResultMixinStr(object): |
| 137 |
"""Standard approach to encoding parsed results from str to bytes""" |
| 138 |
__slots__ = () |
| 139 |
|
| 140 |
def encode(self, encoding='ascii', errors='strict'): |
| 141 |
return self._encoded_counterpart(*(x.encode(encoding, errors) for x in self)) |
| 142 |
|
| 143 |
|
| 144 |
class _ResultMixinBytes(object): |
| 145 |
"""Standard approach to decoding parsed results from bytes to str""" |
| 146 |
__slots__ = () |
| 147 |
|
| 148 |
def decode(self, encoding='ascii', errors='strict'): |
| 149 |
return self._decoded_counterpart(*(x.decode(encoding, errors) for x in self)) |
| 150 |
|
| 151 |
|
| 152 |
class _NetlocResultMixinBase(object): |
| 153 |
"""Shared methods for the parsed result objects containing a netloc element""" |
| 154 |
__slots__ = () |
| 155 |
|
| 156 |
@property |
| 157 |
def username(self): |
| 158 |
return self._userinfo[0] |
| 159 |
|
| 160 |
@property |
| 161 |
def password(self): |
| 162 |
return self._userinfo[1] |
| 163 |
|
| 164 |
@property |
| 165 |
def hostname(self): |
| 166 |
hostname = self._hostinfo[0] |
| 167 |
if not hostname: |
| 168 |
return None |
| 169 |
# Scoped IPv6 address may have zone info, which must not be lowercased |
| 170 |
# like http://[fe80::822a:a8ff:fe49:470c%tESt]:1234/keys |
| 171 |
separator = '%' if isinstance(hostname, str) else b'%' |
| 172 |
hostname, percent, zone = hostname.partition(separator) |
| 173 |
return hostname.lower() + percent + zone |
| 174 |
|
| 175 |
@property |
| 176 |
def port(self): |
| 177 |
port = self._hostinfo[1] |
| 178 |
if port is not None: |
| 179 |
if port.isdigit() and port.isascii(): |
| 180 |
port = int(port) |
| 181 |
else: |
| 182 |
raise ValueError(f"Port could not be cast to integer value as {port!r}") |
| 183 |
if not (0 <= port <= 65535): |
| 184 |
raise ValueError("Port out of range 0-65535") |
| 185 |
return port |
| 186 |
|
| 187 |
__class_getitem__ = classmethod(types.GenericAlias) |
| 188 |
|
| 189 |
|
| 190 |
class _NetlocResultMixinStr(_NetlocResultMixinBase, _ResultMixinStr): |
| 191 |
__slots__ = () |
| 192 |
|
| 193 |
@property |
| 194 |
def _userinfo(self): |
| 195 |
netloc = self.netloc |
| 196 |
userinfo, have_info, hostinfo = netloc.rpartition('@') |
| 197 |
if have_info: |
| 198 |
username, have_password, password = userinfo.partition(':') |
| 199 |
if not have_password: |
| 200 |
password = None |
| 201 |
else: |
| 202 |
username = password = None |
| 203 |
return username, password |
| 204 |
|
| 205 |
@property |
| 206 |
def _hostinfo(self): |
| 207 |
netloc = self.netloc |
| 208 |
_, _, hostinfo = netloc.rpartition('@') |
| 209 |
_, have_open_br, bracketed = hostinfo.partition('[') |
| 210 |
if have_open_br: |
| 211 |
hostname, _, port = bracketed.partition(']') |
| 212 |
_, _, port = port.partition(':') |
| 213 |
else: |
| 214 |
hostname, _, port = hostinfo.partition(':') |
| 215 |
if not port: |
| 216 |
port = None |
| 217 |
return hostname, port |
| 218 |
|
| 219 |
|
| 220 |
class _NetlocResultMixinBytes(_NetlocResultMixinBase, _ResultMixinBytes): |
| 221 |
__slots__ = () |
| 222 |
|
| 223 |
@property |
| 224 |
def _userinfo(self): |
| 225 |
netloc = self.netloc |
| 226 |
userinfo, have_info, hostinfo = netloc.rpartition(b'@') |
| 227 |
if have_info: |
| 228 |
username, have_password, password = userinfo.partition(b':') |
| 229 |
if not have_password: |
| 230 |
password = None |
| 231 |
else: |
| 232 |
username = password = None |
| 233 |
return username, password |
| 234 |
|
| 235 |
@property |
| 236 |
def _hostinfo(self): |
| 237 |
netloc = self.netloc |
| 238 |
_, _, hostinfo = netloc.rpartition(b'@') |
| 239 |
_, have_open_br, bracketed = hostinfo.partition(b'[') |
| 240 |
if have_open_br: |
| 241 |
hostname, _, port = bracketed.partition(b']') |
| 242 |
_, _, port = port.partition(b':') |
| 243 |
else: |
| 244 |
hostname, _, port = hostinfo.partition(b':') |
| 245 |
if not port: |
| 246 |
port = None |
| 247 |
return hostname, port |
| 248 |
|
| 249 |
|
| 250 |
_DefragResultBase = namedtuple('_DefragResultBase', 'url fragment') |
| 251 |
_SplitResultBase = namedtuple( |
| 252 |
'_SplitResultBase', 'scheme netloc path query fragment') |
| 253 |
_ParseResultBase = namedtuple( |
| 254 |
'_ParseResultBase', 'scheme netloc path params query fragment') |
| 255 |
|
| 256 |
_DefragResultBase.__doc__ = """ |
| 257 |
DefragResult(url, fragment) |
| 258 |
|
| 259 |
A 2-tuple that contains the url without fragment identifier and the fragment |
| 260 |
identifier as a separate argument. |
| 261 |
""" |
| 262 |
|
| 263 |
_DefragResultBase.url.__doc__ = """The URL with no fragment identifier.""" |
| 264 |
|
| 265 |
_DefragResultBase.fragment.__doc__ = """ |
| 266 |
Fragment identifier separated from URL, that allows indirect identification of a |
| 267 |
secondary resource by reference to a primary resource and additional identifying |
| 268 |
information. |
| 269 |
""" |
| 270 |
|
| 271 |
_SplitResultBase.__doc__ = """ |
| 272 |
SplitResult(scheme, netloc, path, query, fragment) |
| 273 |
|
| 274 |
A 5-tuple that contains the different components of a URL. Similar to |
| 275 |
ParseResult, but does not split params. |
| 276 |
""" |
| 277 |
|
| 278 |
_SplitResultBase.scheme.__doc__ = """Specifies URL scheme for the request.""" |
| 279 |
|
| 280 |
_SplitResultBase.netloc.__doc__ = """ |
| 281 |
Network location where the request is made to. |
| 282 |
""" |
| 283 |
|
| 284 |
_SplitResultBase.path.__doc__ = """ |
| 285 |
The hierarchical path, such as the path to a file to download. |
| 286 |
""" |
| 287 |
|
| 288 |
_SplitResultBase.query.__doc__ = """ |
| 289 |
The query component, that contains non-hierarchical data, that along with data |
| 290 |
in path component, identifies a resource in the scope of URI's scheme and |
| 291 |
network location. |
| 292 |
""" |
| 293 |
|
| 294 |
_SplitResultBase.fragment.__doc__ = """ |
| 295 |
Fragment identifier, that allows indirect identification of a secondary resource |
| 296 |
by reference to a primary resource and additional identifying information. |
| 297 |
""" |
| 298 |
|
| 299 |
_ParseResultBase.__doc__ = """ |
| 300 |
ParseResult(scheme, netloc, path, params, query, fragment) |
| 301 |
|
| 302 |
A 6-tuple that contains components of a parsed URL. |
| 303 |
""" |
| 304 |
|
| 305 |
_ParseResultBase.scheme.__doc__ = _SplitResultBase.scheme.__doc__ |
| 306 |
_ParseResultBase.netloc.__doc__ = _SplitResultBase.netloc.__doc__ |
| 307 |
_ParseResultBase.path.__doc__ = _SplitResultBase.path.__doc__ |
| 308 |
_ParseResultBase.params.__doc__ = """ |
| 309 |
Parameters for last path element used to dereference the URI in order to provide |
| 310 |
access to perform some operation on the resource. |
| 311 |
""" |
| 312 |
|
| 313 |
_ParseResultBase.query.__doc__ = _SplitResultBase.query.__doc__ |
| 314 |
_ParseResultBase.fragment.__doc__ = _SplitResultBase.fragment.__doc__ |
| 315 |
|
| 316 |
|
| 317 |
# For backwards compatibility, alias _NetlocResultMixinStr |
| 318 |
# ResultBase is no longer part of the documented API, but it is |
| 319 |
# retained since deprecating it isn't worth the hassle |
| 320 |
ResultBase = _NetlocResultMixinStr |
| 321 |
|
| 322 |
# Structured result objects for string data |
| 323 |
class DefragResult(_DefragResultBase, _ResultMixinStr): |
| 324 |
__slots__ = () |
| 325 |
def geturl(self): |
| 326 |
if self.fragment: |
| 327 |
return self.url + '#' + self.fragment |
| 328 |
else: |
| 329 |
return self.url |
| 330 |
|
| 331 |
class SplitResult(_SplitResultBase, _NetlocResultMixinStr): |
| 332 |
__slots__ = () |
| 333 |
def geturl(self): |
| 334 |
return urlunsplit(self) |
| 335 |
|
| 336 |
class ParseResult(_ParseResultBase, _NetlocResultMixinStr): |
| 337 |
__slots__ = () |
| 338 |
def geturl(self): |
| 339 |
return urlunparse(self) |
| 340 |
|
| 341 |
# Structured result objects for bytes data |
| 342 |
class DefragResultBytes(_DefragResultBase, _ResultMixinBytes): |
| 343 |
__slots__ = () |
| 344 |
def geturl(self): |
| 345 |
if self.fragment: |
| 346 |
return self.url + b'#' + self.fragment |
| 347 |
else: |
| 348 |
return self.url |
| 349 |
|
| 350 |
class SplitResultBytes(_SplitResultBase, _NetlocResultMixinBytes): |
| 351 |
__slots__ = () |
| 352 |
def geturl(self): |
| 353 |
return urlunsplit(self) |
| 354 |
|
| 355 |
class ParseResultBytes(_ParseResultBase, _NetlocResultMixinBytes): |
| 356 |
__slots__ = () |
| 357 |
def geturl(self): |
| 358 |
return urlunparse(self) |
| 359 |
|
| 360 |
# Set up the encode/decode result pairs |
| 361 |
def _fix_result_transcoding(): |
| 362 |
_result_pairs = ( |
| 363 |
(DefragResult, DefragResultBytes), |
| 364 |
(SplitResult, SplitResultBytes), |
| 365 |
(ParseResult, ParseResultBytes), |
| 366 |
) |
| 367 |
for _decoded, _encoded in _result_pairs: |
| 368 |
_decoded._encoded_counterpart = _encoded |
| 369 |
_encoded._decoded_counterpart = _decoded |
| 370 |
|
| 371 |
_fix_result_transcoding() |
| 372 |
del _fix_result_transcoding |
| 373 |
|
| 374 |
def urlparse(url, scheme='', allow_fragments=True): |
| 375 |
"""Parse a URL into 6 components: |
| 376 |
<scheme>://<netloc>/<path>;<params>?<query>#<fragment> |
| 377 |
|
| 378 |
The result is a named 6-tuple with fields corresponding to the |
| 379 |
above. It is either a ParseResult or ParseResultBytes object, |
| 380 |
depending on the type of the url parameter. |
| 381 |
|
| 382 |
The username, password, hostname, and port sub-components of netloc |
| 383 |
can also be accessed as attributes of the returned object. |
| 384 |
|
| 385 |
The scheme argument provides the default value of the scheme |
| 386 |
component when no scheme is found in url. |
| 387 |
|
| 388 |
If allow_fragments is False, no attempt is made to separate the |
| 389 |
fragment component from the previous component, which can be either |
| 390 |
path or query. |
| 391 |
|
| 392 |
Note that % escapes are not expanded. |
| 393 |
|
| 394 |
urlsplit() should generally be used instead of urlparse(). |
| 395 |
""" |
| 396 |
url, scheme, _coerce_result = _coerce_args(url, scheme) |
| 397 |
scheme, netloc, url, params, query, fragment = _urlparse(url, scheme, allow_fragments) |
| 398 |
result = ParseResult(scheme or '', netloc or '', url, params or '', query or '', fragment or '') |
| 399 |
return _coerce_result(result) |
| 400 |
|
| 401 |
def _urlparse(url, scheme=None, allow_fragments=True): |
| 402 |
scheme, netloc, url, query, fragment = _urlsplit(url, scheme, allow_fragments) |
| 403 |
if (scheme or '') in uses_params and ';' in url: |
| 404 |
url, params = _splitparams(url, allow_none=True) |
| 405 |
else: |
| 406 |
params = None |
| 407 |
return (scheme, netloc, url, params, query, fragment) |
| 408 |
|
| 409 |
def _splitparams(url, allow_none=False): |
| 410 |
if '/' in url: |
| 411 |
i = url.find(';', url.rfind('/')) |
| 412 |
if i < 0: |
| 413 |
return url, None if allow_none else '' |
| 414 |
else: |
| 415 |
i = url.find(';') |
| 416 |
return url[:i], url[i+1:] |
| 417 |
|
| 418 |
def _splitnetloc(url, start=0): |
| 419 |
delim = len(url) # position of end of domain part of url, default is end |
| 420 |
for c in '/?#': # look for delimiters; the order is NOT important |
| 421 |
wdelim = url.find(c, start) # find first of this delim |
| 422 |
if wdelim >= 0: # if found |
| 423 |
delim = min(delim, wdelim) # use earliest delim position |
| 424 |
return url[start:delim], url[delim:] # return (domain, rest) |
| 425 |
|
| 426 |
def _checknetloc(netloc): |
| 427 |
if not netloc or netloc.isascii(): |
| 428 |
return |
| 429 |
# looking for characters like \u2100 that expand to 'a/c' |
| 430 |
# IDNA uses NFKC equivalence, so normalize for this check |
| 431 |
import unicodedata |
| 432 |
n = netloc.replace('@', '') # ignore characters already included |
| 433 |
n = n.replace(':', '') # but not the surrounding text |
| 434 |
n = n.replace('#', '') |
| 435 |
n = n.replace('?', '') |
| 436 |
netloc2 = unicodedata.normalize('NFKC', n) |
| 437 |
if n == netloc2: |
| 438 |
return |
| 439 |
for c in '/?#@:': |
| 440 |
if c in netloc2: |
| 441 |
raise ValueError("netloc '" + netloc + "' contains invalid " + |
| 442 |
"characters under NFKC normalization") |
| 443 |
|
| 444 |
def _check_bracketed_netloc(netloc): |
| 445 |
# Note that this function must mirror the splitting |
| 446 |
# done in NetlocResultMixins._hostinfo(). |
| 447 |
hostname_and_port = netloc.rpartition('@')[2] |
| 448 |
before_bracket, have_open_br, bracketed = hostname_and_port.partition('[') |
| 449 |
if have_open_br: |
| 450 |
# No data is allowed before a bracket. |
| 451 |
if before_bracket: |
| 452 |
raise ValueError("Invalid IPv6 URL") |
| 453 |
hostname, _, port = bracketed.partition(']') |
| 454 |
# No data is allowed after the bracket but before the port delimiter. |
| 455 |
if port and not port.startswith(":"): |
| 456 |
raise ValueError("Invalid IPv6 URL") |
| 457 |
else: |
| 458 |
hostname, _, port = hostname_and_port.partition(':') |
| 459 |
_check_bracketed_host(hostname) |
| 460 |
|
| 461 |
# Valid bracketed hosts are defined in |
| 462 |
# https://www.rfc-editor.org/rfc/rfc3986#page-49 and https://url.spec.whatwg.org/ |
| 463 |
def _check_bracketed_host(hostname): |
| 464 |
if hostname.startswith('v'): |
| 465 |
if not re.match(r"\Av[a-fA-F0-9]+\..+\z", hostname): |
| 466 |
raise ValueError(f"IPvFuture address is invalid") |
| 467 |
else: |
| 468 |
ip = ipaddress.ip_address(hostname) # Throws Value Error if not IPv6 or IPv4 |
| 469 |
if isinstance(ip, ipaddress.IPv4Address): |
| 470 |
raise ValueError(f"An IPv4 address cannot be in brackets") |
| 471 |
|
| 472 |
# typed=True avoids BytesWarnings being emitted during cache key |
| 473 |
# comparison since this API supports both bytes and str input. |
| 474 |
@functools.lru_cache(typed=True) |
| 475 |
def urlsplit(url, scheme='', allow_fragments=True): |
| 476 |
"""Parse a URL into 5 components: |
| 477 |
<scheme>://<netloc>/<path>?<query>#<fragment> |
| 478 |
|
| 479 |
The result is a named 5-tuple with fields corresponding to the |
| 480 |
above. It is either a SplitResult or SplitResultBytes object, |
| 481 |
depending on the type of the url parameter. |
| 482 |
|
| 483 |
The username, password, hostname, and port sub-components of netloc |
| 484 |
can also be accessed as attributes of the returned object. |
| 485 |
|
| 486 |
The scheme argument provides the default value of the scheme |
| 487 |
component when no scheme is found in url. |
| 488 |
|
| 489 |
If allow_fragments is False, no attempt is made to separate the |
| 490 |
fragment component from the previous component, which can be either |
| 491 |
path or query. |
| 492 |
|
| 493 |
Note that % escapes are not expanded. |
| 494 |
""" |
| 495 |
|
| 496 |
url, scheme, _coerce_result = _coerce_args(url, scheme) |
| 497 |
scheme, netloc, url, query, fragment = _urlsplit(url, scheme, allow_fragments) |
| 498 |
v = SplitResult(scheme or '', netloc or '', url, query or '', fragment or '') |
| 499 |
return _coerce_result(v) |
| 500 |
|
| 501 |
def _urlsplit(url, scheme=None, allow_fragments=True): |
| 502 |
# Only lstrip url as some applications rely on preserving trailing space. |
| 503 |
# (https://url.spec.whatwg.org/#concept-basic-url-parser would strip both) |
| 504 |
url = url.lstrip(_WHATWG_C0_CONTROL_OR_SPACE) |
| 505 |
for b in _UNSAFE_URL_BYTES_TO_REMOVE: |
| 506 |
url = url.replace(b, "") |
| 507 |
if scheme is not None: |
| 508 |
scheme = scheme.strip(_WHATWG_C0_CONTROL_OR_SPACE) |
| 509 |
for b in _UNSAFE_URL_BYTES_TO_REMOVE: |
| 510 |
scheme = scheme.replace(b, "") |
| 511 |
|
| 512 |
allow_fragments = bool(allow_fragments) |
| 513 |
netloc = query = fragment = None |
| 514 |
i = url.find(':') |
| 515 |
if i > 0 and url[0].isascii() and url[0].isalpha(): |
| 516 |
for c in url[:i]: |
| 517 |
if c not in scheme_chars: |
| 518 |
break |
| 519 |
else: |
| 520 |
scheme, url = url[:i].lower(), url[i+1:] |
| 521 |
if url[:2] == '//': |
| 522 |
netloc, url = _splitnetloc(url, 2) |
| 523 |
if (('[' in netloc and ']' not in netloc) or |
| 524 |
(']' in netloc and '[' not in netloc)): |
| 525 |
raise ValueError("Invalid IPv6 URL") |
| 526 |
if '[' in netloc and ']' in netloc: |
| 527 |
_check_bracketed_netloc(netloc) |
| 528 |
if allow_fragments and '#' in url: |
| 529 |
url, fragment = url.split('#', 1) |
| 530 |
if '?' in url: |
| 531 |
url, query = url.split('?', 1) |
| 532 |
_checknetloc(netloc) |
| 533 |
return (scheme, netloc, url, query, fragment) |
| 534 |
|
| 535 |
def urlunparse(components): |
| 536 |
"""Put a parsed URL back together again. This may result in a |
| 537 |
slightly different, but equivalent URL, if the URL that was parsed |
| 538 |
originally had redundant delimiters, e.g. a ? with an empty query |
| 539 |
(the draft states that these are equivalent).""" |
| 540 |
scheme, netloc, url, params, query, fragment, _coerce_result = ( |
| 541 |
_coerce_args(*components)) |
| 542 |
if not netloc: |
| 543 |
if scheme and scheme in uses_netloc and (not url or url[:1] == '/'): |
| 544 |
netloc = '' |
| 545 |
else: |
| 546 |
netloc = None |
| 547 |
if params: |
| 548 |
url = "%s;%s" % (url, params) |
| 549 |
return _coerce_result(_urlunsplit(scheme or None, netloc, url, |
| 550 |
query or None, fragment or None)) |
| 551 |
|
| 552 |
def urlunsplit(components): |
| 553 |
"""Combine the elements of a tuple as returned by urlsplit() into a |
| 554 |
complete URL as a string. The data argument can be any five-item iterable. |
| 555 |
This may result in a slightly different, but equivalent URL, if the URL that |
| 556 |
was parsed originally had unnecessary delimiters (for example, a ? with an |
| 557 |
empty query; the RFC states that these are equivalent).""" |
| 558 |
scheme, netloc, url, query, fragment, _coerce_result = ( |
| 559 |
_coerce_args(*components)) |
| 560 |
if not netloc: |
| 561 |
if scheme and scheme in uses_netloc and (not url or url[:1] == '/'): |
| 562 |
netloc = '' |
| 563 |
else: |
| 564 |
netloc = None |
| 565 |
return _coerce_result(_urlunsplit(scheme or None, netloc, url, |
| 566 |
query or None, fragment or None)) |
| 567 |
|
| 568 |
def _urlunsplit(scheme, netloc, url, query, fragment): |
| 569 |
if netloc is not None: |
| 570 |
if url and url[:1] != '/': url = '/' + url |
| 571 |
url = '//' + netloc + url |
| 572 |
elif url[:2] == '//': |
| 573 |
url = '//' + url |
| 574 |
if scheme: |
| 575 |
url = scheme + ':' + url |
| 576 |
if query is not None: |
| 577 |
url = url + '?' + query |
| 578 |
if fragment is not None: |
| 579 |
url = url + '#' + fragment |
| 580 |
return url |
| 581 |
|
| 582 |
def urljoin(base, url, allow_fragments=True): |
| 583 |
"""Join a base URL and a possibly relative URL to form an absolute |
| 584 |
interpretation of the latter.""" |
| 585 |
if not base: |
| 586 |
return url |
| 587 |
if not url: |
| 588 |
return base |
| 589 |
|
| 590 |
base, url, _coerce_result = _coerce_args(base, url) |
| 591 |
bscheme, bnetloc, bpath, bquery, bfragment = \ |
| 592 |
_urlsplit(base, None, allow_fragments) |
| 593 |
scheme, netloc, path, query, fragment = \ |
| 594 |
_urlsplit(url, None, allow_fragments) |
| 595 |
|
| 596 |
if scheme is None: |
| 597 |
scheme = bscheme |
| 598 |
if scheme != bscheme or (scheme and scheme not in uses_relative): |
| 599 |
return _coerce_result(url) |
| 600 |
if not scheme or scheme in uses_netloc: |
| 601 |
if netloc: |
| 602 |
return _coerce_result(_urlunsplit(scheme, netloc, path, |
| 603 |
query, fragment)) |
| 604 |
netloc = bnetloc |
| 605 |
|
| 606 |
if not path: |
| 607 |
path = bpath |
| 608 |
if query is None: |
| 609 |
query = bquery |
| 610 |
if fragment is None: |
| 611 |
fragment = bfragment |
| 612 |
return _coerce_result(_urlunsplit(scheme, netloc, path, |
| 613 |
query, fragment)) |
| 614 |
|
| 615 |
base_parts = bpath.split('/') |
| 616 |
if base_parts[-1] != '': |
| 617 |
# the last item is not a directory, so will not be taken into account |
| 618 |
# in resolving the relative path |
| 619 |
del base_parts[-1] |
| 620 |
|
| 621 |
# for rfc3986, ignore all base path should the first character be root. |
| 622 |
if path[:1] == '/': |
| 623 |
segments = path.split('/') |
| 624 |
else: |
| 625 |
segments = base_parts + path.split('/') |
| 626 |
# filter out elements that would cause redundant slashes on re-joining |
| 627 |
# the resolved_path |
| 628 |
segments[1:-1] = filter(None, segments[1:-1]) |
| 629 |
|
| 630 |
resolved_path = [] |
| 631 |
|
| 632 |
for seg in segments: |
| 633 |
if seg == '..': |
| 634 |
try: |
| 635 |
resolved_path.pop() |
| 636 |
except IndexError: |
| 637 |
# ignore any .. segments that would otherwise cause an IndexError |
| 638 |
# when popped from resolved_path if resolving for rfc3986 |
| 639 |
pass |
| 640 |
elif seg == '.': |
| 641 |
continue |
| 642 |
else: |
| 643 |
resolved_path.append(seg) |
| 644 |
|
| 645 |
if segments[-1] in ('.', '..'): |
| 646 |
# do some post-processing here. if the last segment was a relative dir, |
| 647 |
# then we need to append the trailing '/' |
| 648 |
resolved_path.append('') |
| 649 |
|
| 650 |
return _coerce_result(_urlunsplit(scheme, netloc, '/'.join( |
| 651 |
resolved_path) or '/', query, fragment)) |
| 652 |
|
| 653 |
|
| 654 |
def urldefrag(url): |
| 655 |
"""Removes any existing fragment from URL. |
| 656 |
|
| 657 |
Returns a tuple of the defragmented URL and the fragment. If |
| 658 |
the URL contained no fragments, the second element is the |
| 659 |
empty string. |
| 660 |
""" |
| 661 |
url, _coerce_result = _coerce_args(url) |
| 662 |
if '#' in url: |
| 663 |
s, n, p, q, frag = _urlsplit(url) |
| 664 |
defrag = _urlunsplit(s, n, p, q, None) |
| 665 |
else: |
| 666 |
frag = '' |
| 667 |
defrag = url |
| 668 |
return _coerce_result(DefragResult(defrag, frag or '')) |
| 669 |
|
| 670 |
_hexdig = '0123456789ABCDEFabcdef' |
| 671 |
_hextobyte = None |
| 672 |
|
| 673 |
def unquote_to_bytes(string): |
| 674 |
"""unquote_to_bytes('abc%20def') -> b'abc def'.""" |
| 675 |
return bytes(_unquote_impl(string)) |
| 676 |
|
| 677 |
def _unquote_impl(string: bytes | bytearray | str) -> bytes | bytearray: |
| 678 |
# Note: strings are encoded as UTF-8. This is only an issue if it contains |
| 679 |
# unescaped non-ASCII characters, which URIs should not. |
| 680 |
if not string: |
| 681 |
# Is it a string-like object? |
| 682 |
string.split |
| 683 |
return b'' |
| 684 |
if isinstance(string, str): |
| 685 |
string = string.encode('utf-8') |
| 686 |
bits = string.split(b'%') |
| 687 |
if len(bits) == 1: |
| 688 |
return string |
| 689 |
res = bytearray(bits[0]) |
| 690 |
append = res.extend |
| 691 |
# Delay the initialization of the table to not waste memory |
| 692 |
# if the function is never called |
| 693 |
global _hextobyte |
| 694 |
if _hextobyte is None: |
| 695 |
_hextobyte = {(a + b).encode(): bytes.fromhex(a + b) |
| 696 |
for a in _hexdig for b in _hexdig} |
| 697 |
for item in bits[1:]: |
| 698 |
try: |
| 699 |
append(_hextobyte[item[:2]]) |
| 700 |
append(item[2:]) |
| 701 |
except KeyError: |
| 702 |
append(b'%') |
| 703 |
append(item) |
| 704 |
return res |
| 705 |
|
| 706 |
_asciire = re.compile('([\x00-\x7f]+)') |
| 707 |
|
| 708 |
def _generate_unquoted_parts(string, encoding, errors): |
| 709 |
previous_match_end = 0 |
| 710 |
for ascii_match in _asciire.finditer(string): |
| 711 |
start, end = ascii_match.span() |
| 712 |
yield string[previous_match_end:start] # Non-ASCII |
| 713 |
# The ascii_match[1] group == string[start:end]. |
| 714 |
yield _unquote_impl(ascii_match[1]).decode(encoding, errors) |
| 715 |
previous_match_end = end |
| 716 |
yield string[previous_match_end:] # Non-ASCII tail |
| 717 |
|
| 718 |
def unquote(string, encoding='utf-8', errors='replace'): |
| 719 |
"""Replace %xx escapes by their single-character equivalent. The optional |
| 720 |
encoding and errors parameters specify how to decode percent-encoded |
| 721 |
sequences into Unicode characters, as accepted by the bytes.decode() |
| 722 |
method. |
| 723 |
By default, percent-encoded sequences are decoded with UTF-8, and invalid |
| 724 |
sequences are replaced by a placeholder character. |
| 725 |
|
| 726 |
unquote('abc%20def') -> 'abc def'. |
| 727 |
""" |
| 728 |
if isinstance(string, bytes): |
| 729 |
return _unquote_impl(string).decode(encoding, errors) |
| 730 |
if '%' not in string: |
| 731 |
# Is it a string-like object? |
| 732 |
string.split |
| 733 |
return string |
| 734 |
if encoding is None: |
| 735 |
encoding = 'utf-8' |
| 736 |
if errors is None: |
| 737 |
errors = 'replace' |
| 738 |
return ''.join(_generate_unquoted_parts(string, encoding, errors)) |
| 739 |
|
| 740 |
|
| 741 |
def parse_qs(qs, keep_blank_values=False, strict_parsing=False, |
| 742 |
encoding='utf-8', errors='replace', max_num_fields=None, separator='&'): |
| 743 |
"""Parse a query given as a string argument. |
| 744 |
|
| 745 |
Arguments: |
| 746 |
|
| 747 |
qs: percent-encoded query string to be parsed |
| 748 |
|
| 749 |
keep_blank_values: flag indicating whether blank values in |
| 750 |
percent-encoded queries should be treated as blank strings. |
| 751 |
A true value indicates that blanks should be retained as |
| 752 |
blank strings. The default false value indicates that |
| 753 |
blank values are to be ignored and treated as if they were |
| 754 |
not included. |
| 755 |
|
| 756 |
strict_parsing: flag indicating what to do with parsing errors. |
| 757 |
If false (the default), errors are silently ignored. |
| 758 |
If true, errors raise a ValueError exception. |
| 759 |
|
| 760 |
encoding and errors: specify how to decode percent-encoded sequences |
| 761 |
into Unicode characters, as accepted by the bytes.decode() method. |
| 762 |
|
| 763 |
max_num_fields: int. If set, then throws a ValueError if there |
| 764 |
are more than n fields read by parse_qsl(). |
| 765 |
|
| 766 |
separator: str. The symbol to use for separating the query arguments. |
| 767 |
Defaults to &. |
| 768 |
|
| 769 |
Returns a dictionary. |
| 770 |
""" |
| 771 |
parsed_result = {} |
| 772 |
pairs = parse_qsl(qs, keep_blank_values, strict_parsing, |
| 773 |
encoding=encoding, errors=errors, |
| 774 |
max_num_fields=max_num_fields, separator=separator, |
| 775 |
_stacklevel=2) |
| 776 |
for name, value in pairs: |
| 777 |
if name in parsed_result: |
| 778 |
parsed_result[name].append(value) |
| 779 |
else: |
| 780 |
parsed_result[name] = [value] |
| 781 |
return parsed_result |
| 782 |
|
| 783 |
|
| 784 |
def parse_qsl(qs, keep_blank_values=False, strict_parsing=False, |
| 785 |
encoding='utf-8', errors='replace', max_num_fields=None, separator='&', *, _stacklevel=1): |
| 786 |
"""Parse a query given as a string argument. |
| 787 |
|
| 788 |
Arguments: |
| 789 |
|
| 790 |
qs: percent-encoded query string to be parsed |
| 791 |
|
| 792 |
keep_blank_values: flag indicating whether blank values in |
| 793 |
percent-encoded queries should be treated as blank strings. |
| 794 |
A true value indicates that blanks should be retained as blank |
| 795 |
strings. The default false value indicates that blank values |
| 796 |
are to be ignored and treated as if they were not included. |
| 797 |
|
| 798 |
strict_parsing: flag indicating what to do with parsing errors. If |
| 799 |
false (the default), errors are silently ignored. If true, |
| 800 |
errors raise a ValueError exception. |
| 801 |
|
| 802 |
encoding and errors: specify how to decode percent-encoded sequences |
| 803 |
into Unicode characters, as accepted by the bytes.decode() method. |
| 804 |
|
| 805 |
max_num_fields: int. If set, then throws a ValueError |
| 806 |
if there are more than n fields read by parse_qsl(). |
| 807 |
|
| 808 |
separator: str. The symbol to use for separating the query arguments. |
| 809 |
Defaults to &. |
| 810 |
|
| 811 |
Returns a list, as G-d intended. |
| 812 |
""" |
| 813 |
if not separator or not isinstance(separator, (str, bytes)): |
| 814 |
raise ValueError("Separator must be of type string or bytes.") |
| 815 |
if isinstance(qs, str): |
| 816 |
if not isinstance(separator, str): |
| 817 |
separator = str(separator, 'ascii') |
| 818 |
eq = '=' |
| 819 |
def _unquote(s): |
| 820 |
return unquote_plus(s, encoding=encoding, errors=errors) |
| 821 |
elif qs is None: |
| 822 |
return [] |
| 823 |
else: |
| 824 |
try: |
| 825 |
# Use memoryview() to reject integers and iterables, |
| 826 |
# acceptable by the bytes constructor. |
| 827 |
qs = bytes(memoryview(qs)) |
| 828 |
except TypeError: |
| 829 |
if not qs: |
| 830 |
warnings.warn(f"Accepting {type(qs).__name__} objects with " |
| 831 |
f"false value in urllib.parse.parse_qsl() is " |
| 832 |
f"deprecated as of 3.14", |
| 833 |
DeprecationWarning, stacklevel=_stacklevel + 1) |
| 834 |
return [] |
| 835 |
raise |
| 836 |
if isinstance(separator, str): |
| 837 |
separator = bytes(separator, 'ascii') |
| 838 |
eq = b'=' |
| 839 |
def _unquote(s): |
| 840 |
return unquote_to_bytes(s.replace(b'+', b' ')) |
| 841 |
|
| 842 |
if not qs: |
| 843 |
return [] |
| 844 |
|
| 845 |
# If max_num_fields is defined then check that the number of fields |
| 846 |
# is less than max_num_fields. This prevents a memory exhaustion DOS |
| 847 |
# attack via post bodies with many fields. |
| 848 |
if max_num_fields is not None: |
| 849 |
num_fields = 1 + qs.count(separator) |
| 850 |
if max_num_fields < num_fields: |
| 851 |
raise ValueError('Max number of fields exceeded') |
| 852 |
|
| 853 |
r = [] |
| 854 |
for name_value in qs.split(separator): |
| 855 |
if name_value or strict_parsing: |
| 856 |
name, has_eq, value = name_value.partition(eq) |
| 857 |
if not has_eq and strict_parsing: |
| 858 |
raise ValueError("bad query field: %r" % (name_value,)) |
| 859 |
if value or keep_blank_values: |
| 860 |
name = _unquote(name) |
| 861 |
value = _unquote(value) |
| 862 |
r.append((name, value)) |
| 863 |
return r |
| 864 |
|
| 865 |
def unquote_plus(string, encoding='utf-8', errors='replace'): |
| 866 |
"""Like unquote(), but also replace plus signs by spaces, as required for |
| 867 |
unquoting HTML form values. |
| 868 |
|
| 869 |
unquote_plus('%7e/abc+def') -> '~/abc def' |
| 870 |
""" |
| 871 |
string = string.replace('+', ' ') |
| 872 |
return unquote(string, encoding, errors) |
| 873 |
|
| 874 |
_ALWAYS_SAFE = frozenset(b'ABCDEFGHIJKLMNOPQRSTUVWXYZ' |
| 875 |
b'abcdefghijklmnopqrstuvwxyz' |
| 876 |
b'0123456789' |
| 877 |
b'_.-~') |
| 878 |
_ALWAYS_SAFE_BYTES = bytes(_ALWAYS_SAFE) |
| 879 |
|
| 880 |
|
| 881 |
class _Quoter(dict): |
| 882 |
"""A mapping from bytes numbers (in range(0,256)) to strings. |
| 883 |
|
| 884 |
String values are percent-encoded byte values, unless the key < 128, and |
| 885 |
in either of the specified safe set, or the always safe set. |
| 886 |
""" |
| 887 |
# Keeps a cache internally, via __missing__, for efficiency (lookups |
| 888 |
# of cached keys don't call Python code at all). |
| 889 |
def __init__(self, safe): |
| 890 |
"""safe: bytes object.""" |
| 891 |
self.safe = _ALWAYS_SAFE.union(safe) |
| 892 |
|
| 893 |
def __repr__(self): |
| 894 |
return f"<Quoter {dict(self)!r}>" |
| 895 |
|
| 896 |
def __missing__(self, b): |
| 897 |
# Handle a cache miss. Store quoted string in cache and return. |
| 898 |
res = chr(b) if b in self.safe else '%{:02X}'.format(b) |
| 899 |
self[b] = res |
| 900 |
return res |
| 901 |
|
| 902 |
def quote(string, safe='/', encoding=None, errors=None): |
| 903 |
"""quote('abc def') -> 'abc%20def' |
| 904 |
|
| 905 |
Each part of a URL, e.g. the path info, the query, etc., has a |
| 906 |
different set of reserved characters that must be quoted. The |
| 907 |
quote function offers a cautious (not minimal) way to quote a |
| 908 |
string for most of these parts. |
| 909 |
|
| 910 |
RFC 3986 Uniform Resource Identifier (URI): Generic Syntax lists |
| 911 |
the following (un)reserved characters. |
| 912 |
|
| 913 |
unreserved = ALPHA / DIGIT / "-" / "." / "_" / "~" |
| 914 |
reserved = gen-delims / sub-delims |
| 915 |
gen-delims = ":" / "/" / "?" / "#" / "[" / "]" / "@" |
| 916 |
sub-delims = "!" / "$" / "&" / "'" / "(" / ")" |
| 917 |
/ "*" / "+" / "," / ";" / "=" |
| 918 |
|
| 919 |
Each of the reserved characters is reserved in some component of a URL, |
| 920 |
but not necessarily in all of them. |
| 921 |
|
| 922 |
The quote function %-escapes all characters that are neither in the |
| 923 |
unreserved chars ("always safe") nor the additional chars set via the |
| 924 |
safe arg. |
| 925 |
|
| 926 |
The default for the safe arg is '/'. The character is reserved, but in |
| 927 |
typical usage the quote function is being called on a path where the |
| 928 |
existing slash characters are to be preserved. |
| 929 |
|
| 930 |
Python 3.7 updates from using RFC 2396 to RFC 3986 to quote URL strings. |
| 931 |
Now, "~" is included in the set of unreserved characters. |
| 932 |
|
| 933 |
string and safe may be either str or bytes objects. encoding and errors |
| 934 |
must not be specified if string is a bytes object. |
| 935 |
|
| 936 |
The optional encoding and errors parameters specify how to deal with |
| 937 |
non-ASCII characters, as accepted by the str.encode method. |
| 938 |
By default, encoding='utf-8' (characters are encoded with UTF-8), and |
| 939 |
errors='strict' (unsupported characters raise a UnicodeEncodeError). |
| 940 |
""" |
| 941 |
if isinstance(string, str): |
| 942 |
if not string: |
| 943 |
return string |
| 944 |
if encoding is None: |
| 945 |
encoding = 'utf-8' |
| 946 |
if errors is None: |
| 947 |
errors = 'strict' |
| 948 |
string = string.encode(encoding, errors) |
| 949 |
else: |
| 950 |
if encoding is not None: |
| 951 |
raise TypeError("quote() doesn't support 'encoding' for bytes") |
| 952 |
if errors is not None: |
| 953 |
raise TypeError("quote() doesn't support 'errors' for bytes") |
| 954 |
return quote_from_bytes(string, safe) |
| 955 |
|
| 956 |
def quote_plus(string, safe='', encoding=None, errors=None): |
| 957 |
"""Like quote(), but also replace ' ' with '+', as required for quoting |
| 958 |
HTML form values. Plus signs in the original string are escaped unless |
| 959 |
they are included in safe. It also does not have safe default to '/'. |
| 960 |
""" |
| 961 |
# Check if ' ' in string, where string may either be a str or bytes. If |
| 962 |
# there are no spaces, the regular quote will produce the right answer. |
| 963 |
if ((isinstance(string, str) and ' ' not in string) or |
| 964 |
(isinstance(string, bytes) and b' ' not in string)): |
| 965 |
return quote(string, safe, encoding, errors) |
| 966 |
if isinstance(safe, str): |
| 967 |
space = ' ' |
| 968 |
else: |
| 969 |
space = b' ' |
| 970 |
string = quote(string, safe + space, encoding, errors) |
| 971 |
return string.replace(' ', '+') |
| 972 |
|
| 973 |
# Expectation: A typical program is unlikely to create more than 5 of these. |
| 974 |
@functools.lru_cache |
| 975 |
def _byte_quoter_factory(safe): |
| 976 |
return _Quoter(safe).__getitem__ |
| 977 |
|
| 978 |
def quote_from_bytes(bs, safe='/'): |
| 979 |
"""Like quote(), but accepts a bytes object rather than a str, and does |
| 980 |
not perform string-to-bytes encoding. It always returns an ASCII string. |
| 981 |
quote_from_bytes(b'abc def\x3f') -> 'abc%20def%3f' |
| 982 |
""" |
| 983 |
if not isinstance(bs, (bytes, bytearray)): |
| 984 |
raise TypeError("quote_from_bytes() expected bytes") |
| 985 |
if not bs: |
| 986 |
return '' |
| 987 |
if isinstance(safe, str): |
| 988 |
# Normalize 'safe' by converting to bytes and removing non-ASCII chars |
| 989 |
safe = safe.encode('ascii', 'ignore') |
| 990 |
else: |
| 991 |
# List comprehensions are faster than generator expressions. |
| 992 |
safe = bytes([c for c in safe if c < 128]) |
| 993 |
if not bs.rstrip(_ALWAYS_SAFE_BYTES + safe): |
| 994 |
return bs.decode() |
| 995 |
quoter = _byte_quoter_factory(safe) |
| 996 |
if (bs_len := len(bs)) < 200_000: |
| 997 |
return ''.join(map(quoter, bs)) |
| 998 |
else: |
| 999 |
# This saves memory - https://github.com/python/cpython/issues/95865 |
| 1000 |
chunk_size = math.isqrt(bs_len) |
| 1001 |
chunks = [''.join(map(quoter, bs[i:i+chunk_size])) |
| 1002 |
for i in range(0, bs_len, chunk_size)] |
| 1003 |
return ''.join(chunks) |
| 1004 |
|
| 1005 |
def urlencode(query, doseq=False, safe='', encoding=None, errors=None, |
| 1006 |
quote_via=quote_plus): |
| 1007 |
"""Encode a dict or sequence of two-element tuples into a URL query string. |
| 1008 |
|
| 1009 |
If any values in the query arg are sequences and doseq is true, each |
| 1010 |
sequence element is converted to a separate parameter. |
| 1011 |
|
| 1012 |
If the query arg is a sequence of two-element tuples, the order of the |
| 1013 |
parameters in the output will match the order of parameters in the |
| 1014 |
input. |
| 1015 |
|
| 1016 |
The components of a query arg may each be either a string or a bytes type. |
| 1017 |
|
| 1018 |
The safe, encoding, and errors parameters are passed down to the function |
| 1019 |
specified by quote_via (encoding and errors only if a component is a str). |
| 1020 |
""" |
| 1021 |
|
| 1022 |
if hasattr(query, "items"): |
| 1023 |
query = query.items() |
| 1024 |
else: |
| 1025 |
# It's a bother at times that strings and string-like objects are |
| 1026 |
# sequences. |
| 1027 |
try: |
| 1028 |
# non-sequence items should not work with len() |
| 1029 |
# non-empty strings will fail this |
| 1030 |
if len(query) and not isinstance(query[0], tuple): |
| 1031 |
raise TypeError |
| 1032 |
# Zero-length sequences of all types will get here and succeed, |
| 1033 |
# but that's a minor nit. Since the original implementation |
| 1034 |
# allowed empty dicts that type of behavior probably should be |
| 1035 |
# preserved for consistency |
| 1036 |
except TypeError as err: |
| 1037 |
raise TypeError("not a valid non-string sequence " |
| 1038 |
"or mapping object") from err |
| 1039 |
|
| 1040 |
l = [] |
| 1041 |
if not doseq: |
| 1042 |
for k, v in query: |
| 1043 |
if isinstance(k, bytes): |
| 1044 |
k = quote_via(k, safe) |
| 1045 |
else: |
| 1046 |
k = quote_via(str(k), safe, encoding, errors) |
| 1047 |
|
| 1048 |
if isinstance(v, bytes): |
| 1049 |
v = quote_via(v, safe) |
| 1050 |
else: |
| 1051 |
v = quote_via(str(v), safe, encoding, errors) |
| 1052 |
l.append(k + '=' + v) |
| 1053 |
else: |
| 1054 |
for k, v in query: |
| 1055 |
if isinstance(k, bytes): |
| 1056 |
k = quote_via(k, safe) |
| 1057 |
else: |
| 1058 |
k = quote_via(str(k), safe, encoding, errors) |
| 1059 |
|
| 1060 |
if isinstance(v, bytes): |
| 1061 |
v = quote_via(v, safe) |
| 1062 |
l.append(k + '=' + v) |
| 1063 |
elif isinstance(v, str): |
| 1064 |
v = quote_via(v, safe, encoding, errors) |
| 1065 |
l.append(k + '=' + v) |
| 1066 |
else: |
| 1067 |
try: |
| 1068 |
# Is this a sufficient test for sequence-ness? |
| 1069 |
x = len(v) |
| 1070 |
except TypeError: |
| 1071 |
# not a sequence |
| 1072 |
v = quote_via(str(v), safe, encoding, errors) |
| 1073 |
l.append(k + '=' + v) |
| 1074 |
else: |
| 1075 |
# loop over the sequence |
| 1076 |
for elt in v: |
| 1077 |
if isinstance(elt, bytes): |
| 1078 |
elt = quote_via(elt, safe) |
| 1079 |
else: |
| 1080 |
elt = quote_via(str(elt), safe, encoding, errors) |
| 1081 |
l.append(k + '=' + elt) |
| 1082 |
return '&'.join(l) |
| 1083 |
|
| 1084 |
|
| 1085 |
def to_bytes(url): |
| 1086 |
warnings.warn("urllib.parse.to_bytes() is deprecated as of 3.8", |
| 1087 |
DeprecationWarning, stacklevel=2) |
| 1088 |
return _to_bytes(url) |
| 1089 |
|
| 1090 |
|
| 1091 |
def _to_bytes(url): |
| 1092 |
"""to_bytes(u"URL") --> 'URL'.""" |
| 1093 |
# Most URL schemes require ASCII. If that changes, the conversion |
| 1094 |
# can be relaxed. |
| 1095 |
# XXX get rid of to_bytes() |
| 1096 |
if isinstance(url, str): |
| 1097 |
try: |
| 1098 |
url = url.encode("ASCII").decode() |
| 1099 |
except UnicodeError: |
| 1100 |
raise UnicodeError("URL " + repr(url) + |
| 1101 |
" contains non-ASCII characters") |
| 1102 |
return url |
| 1103 |
|
| 1104 |
|
| 1105 |
def unwrap(url): |
| 1106 |
"""Transform a string like '<URL:scheme://host/path>' into 'scheme://host/path'. |
| 1107 |
|
| 1108 |
The string is returned unchanged if it's not a wrapped URL. |
| 1109 |
""" |
| 1110 |
url = str(url).strip() |
| 1111 |
if url[:1] == '<' and url[-1:] == '>': |
| 1112 |
url = url[1:-1].strip() |
| 1113 |
if url[:4] == 'URL:': |
| 1114 |
url = url[4:].strip() |
| 1115 |
return url |
| 1116 |
|
| 1117 |
|
| 1118 |
def splittype(url): |
| 1119 |
warnings.warn("urllib.parse.splittype() is deprecated as of 3.8, " |
| 1120 |
"use urllib.parse.urlparse() instead", |
| 1121 |
DeprecationWarning, stacklevel=2) |
| 1122 |
return _splittype(url) |
| 1123 |
|
| 1124 |
|
| 1125 |
_typeprog = None |
| 1126 |
def _splittype(url): |
| 1127 |
"""splittype('type:opaquestring') --> 'type', 'opaquestring'.""" |
| 1128 |
global _typeprog |
| 1129 |
if _typeprog is None: |
| 1130 |
_typeprog = re.compile('([^/:]+):(.*)', re.DOTALL) |
| 1131 |
|
| 1132 |
match = _typeprog.match(url) |
| 1133 |
if match: |
| 1134 |
scheme, data = match.groups() |
| 1135 |
return scheme.lower(), data |
| 1136 |
return None, url |
| 1137 |
|
| 1138 |
|
| 1139 |
def splithost(url): |
| 1140 |
warnings.warn("urllib.parse.splithost() is deprecated as of 3.8, " |
| 1141 |
"use urllib.parse.urlparse() instead", |
| 1142 |
DeprecationWarning, stacklevel=2) |
| 1143 |
return _splithost(url) |
| 1144 |
|
| 1145 |
|
| 1146 |
_hostprog = None |
| 1147 |
def _splithost(url): |
| 1148 |
"""splithost('//host[:port]/path') --> 'host[:port]', '/path'.""" |
| 1149 |
global _hostprog |
| 1150 |
if _hostprog is None: |
| 1151 |
_hostprog = re.compile('//([^/#?]*)(.*)', re.DOTALL) |
| 1152 |
|
| 1153 |
match = _hostprog.match(url) |
| 1154 |
if match: |
| 1155 |
host_port, path = match.groups() |
| 1156 |
if path and path[0] != '/': |
| 1157 |
path = '/' + path |
| 1158 |
return host_port, path |
| 1159 |
return None, url |
| 1160 |
|
| 1161 |
|
| 1162 |
def splituser(host): |
| 1163 |
warnings.warn("urllib.parse.splituser() is deprecated as of 3.8, " |
| 1164 |
"use urllib.parse.urlparse() instead", |
| 1165 |
DeprecationWarning, stacklevel=2) |
| 1166 |
return _splituser(host) |
| 1167 |
|
| 1168 |
|
| 1169 |
def _splituser(host): |
| 1170 |
"""splituser('user[:passwd]@host[:port]') --> 'user[:passwd]', 'host[:port]'.""" |
| 1171 |
user, delim, host = host.rpartition('@') |
| 1172 |
return (user if delim else None), host |
| 1173 |
|
| 1174 |
|
| 1175 |
def splitpasswd(user): |
| 1176 |
warnings.warn("urllib.parse.splitpasswd() is deprecated as of 3.8, " |
| 1177 |
"use urllib.parse.urlparse() instead", |
| 1178 |
DeprecationWarning, stacklevel=2) |
| 1179 |
return _splitpasswd(user) |
| 1180 |
|
| 1181 |
|
| 1182 |
def _splitpasswd(user): |
| 1183 |
"""splitpasswd('user:passwd') -> 'user', 'passwd'.""" |
| 1184 |
user, delim, passwd = user.partition(':') |
| 1185 |
return user, (passwd if delim else None) |
| 1186 |
|
| 1187 |
|
| 1188 |
def splitport(host): |
| 1189 |
warnings.warn("urllib.parse.splitport() is deprecated as of 3.8, " |
| 1190 |
"use urllib.parse.urlparse() instead", |
| 1191 |
DeprecationWarning, stacklevel=2) |
| 1192 |
return _splitport(host) |
| 1193 |
|
| 1194 |
|
| 1195 |
# splittag('/path#tag') --> '/path', 'tag' |
| 1196 |
_portprog = None |
| 1197 |
def _splitport(host): |
| 1198 |
"""splitport('host:port') --> 'host', 'port'.""" |
| 1199 |
global _portprog |
| 1200 |
if _portprog is None: |
| 1201 |
_portprog = re.compile('(.*):([0-9]*)', re.DOTALL) |
| 1202 |
|
| 1203 |
match = _portprog.fullmatch(host) |
| 1204 |
if match: |
| 1205 |
host, port = match.groups() |
| 1206 |
if port: |
| 1207 |
return host, port |
| 1208 |
return host, None |
| 1209 |
|
| 1210 |
|
| 1211 |
def splitnport(host, defport=-1): |
| 1212 |
warnings.warn("urllib.parse.splitnport() is deprecated as of 3.8, " |
| 1213 |
"use urllib.parse.urlparse() instead", |
| 1214 |
DeprecationWarning, stacklevel=2) |
| 1215 |
return _splitnport(host, defport) |
| 1216 |
|
| 1217 |
|
| 1218 |
def _splitnport(host, defport=-1): |
| 1219 |
"""Split host and port, returning numeric port. |
| 1220 |
Return given default port if no ':' found; defaults to -1. |
| 1221 |
Return numerical port if a valid number is found after ':'. |
| 1222 |
Return None if ':' but not a valid number.""" |
| 1223 |
host, delim, port = host.rpartition(':') |
| 1224 |
if not delim: |
| 1225 |
host = port |
| 1226 |
elif port: |
| 1227 |
if port.isdigit() and port.isascii(): |
| 1228 |
nport = int(port) |
| 1229 |
else: |
| 1230 |
nport = None |
| 1231 |
return host, nport |
| 1232 |
return host, defport |
| 1233 |
|
| 1234 |
|
| 1235 |
def splitquery(url): |
| 1236 |
warnings.warn("urllib.parse.splitquery() is deprecated as of 3.8, " |
| 1237 |
"use urllib.parse.urlparse() instead", |
| 1238 |
DeprecationWarning, stacklevel=2) |
| 1239 |
return _splitquery(url) |
| 1240 |
|
| 1241 |
|
| 1242 |
def _splitquery(url): |
| 1243 |
"""splitquery('/path?query') --> '/path', 'query'.""" |
| 1244 |
path, delim, query = url.rpartition('?') |
| 1245 |
if delim: |
| 1246 |
return path, query |
| 1247 |
return url, None |
| 1248 |
|
| 1249 |
|
| 1250 |
def splittag(url): |
| 1251 |
warnings.warn("urllib.parse.splittag() is deprecated as of 3.8, " |
| 1252 |
"use urllib.parse.urlparse() instead", |
| 1253 |
DeprecationWarning, stacklevel=2) |
| 1254 |
return _splittag(url) |
| 1255 |
|
| 1256 |
|
| 1257 |
def _splittag(url): |
| 1258 |
"""splittag('/path#tag') --> '/path', 'tag'.""" |
| 1259 |
path, delim, tag = url.rpartition('#') |
| 1260 |
if delim: |
| 1261 |
return path, tag |
| 1262 |
return url, None |
| 1263 |
|
| 1264 |
|
| 1265 |
def splitattr(url): |
| 1266 |
warnings.warn("urllib.parse.splitattr() is deprecated as of 3.8, " |
| 1267 |
"use urllib.parse.urlparse() instead", |
| 1268 |
DeprecationWarning, stacklevel=2) |
| 1269 |
return _splitattr(url) |
| 1270 |
|
| 1271 |
|
| 1272 |
def _splitattr(url): |
| 1273 |
"""splitattr('/path;attr1=value1;attr2=value2;...') -> |
| 1274 |
'/path', ['attr1=value1', 'attr2=value2', ...].""" |
| 1275 |
words = url.split(';') |
| 1276 |
return words[0], words[1:] |
| 1277 |
|
| 1278 |
|
| 1279 |
def splitvalue(attr): |
| 1280 |
warnings.warn("urllib.parse.splitvalue() is deprecated as of 3.8, " |
| 1281 |
"use urllib.parse.parse_qsl() instead", |
| 1282 |
DeprecationWarning, stacklevel=2) |
| 1283 |
return _splitvalue(attr) |
| 1284 |
|
| 1285 |
|
| 1286 |
def _splitvalue(attr): |
| 1287 |
"""splitvalue('attr=value') --> 'attr', 'value'.""" |
| 1288 |
attr, delim, value = attr.partition('=') |
| 1289 |
return attr, (value if delim else None) |