Debian Codegraph @

Lib/urllib/request.py

raw
1 """An extensible library for opening URLs using a variety of protocols
2
3 The simplest way to use this module is to call the urlopen function,
4 which accepts a string containing a URL or a Request object (described
5 below). It opens the URL and returns the results as file-like
6 object; the returned object has some extra methods described below.
7
8 The OpenerDirector manages a collection of Handler objects that do
9 all the actual work. Each Handler implements a particular protocol or
10 option. The OpenerDirector is a composite object that invokes the
11 Handlers needed to open the requested URL. For example, the
12 HTTPHandler performs HTTP GET and POST requests and deals with
13 non-error returns. The HTTPRedirectHandler automatically deals with
14 HTTP 301, 302, 303, 307, and 308 redirect errors, and the
15 HTTPDigestAuthHandler deals with digest authentication.
16
17 urlopen(url, data=None) -- Basic usage is the same as original
18 urllib. pass the url and optionally data to post to an HTTP URL, and
19 get a file-like object back. One difference is that you can also pass
20 a Request instance instead of URL. Raises a URLError (subclass of
21 OSError); for HTTP errors, raises an HTTPError, which can also be
22 treated as a valid response.
23
24 build_opener -- Function that creates a new OpenerDirector instance.
25 Will install the default handlers. Accepts one or more Handlers as
26 arguments, either instances or Handler classes that it will
27 instantiate. If one of the argument is a subclass of the default
28 handler, the argument will be installed instead of the default.
29
30 install_opener -- Installs a new opener as the default opener.
31
32 objects of interest:
33
34 OpenerDirector -- Sets up the User Agent as the Python-urllib client and manages
35 the Handler classes, while dealing with requests and responses.
36
37 Request -- An object that encapsulates the state of a request. The
38 state can be as simple as the URL. It can also include extra HTTP
39 headers, e.g. a User-Agent.
40
41 BaseHandler --
42
43 internals:
44 BaseHandler and parent
45 _call_chain conventions
46
47 Example usage:
48
49 import urllib.request
50
51 # set up authentication info
52 authinfo = urllib.request.HTTPBasicAuthHandler()
53 authinfo.add_password(realm='PDQ Application',
54 uri='https://mahler:8092/site-updates.py',
55 user='klem',
56 passwd='geheim$parole')
57
58 proxy_support = urllib.request.ProxyHandler({"http" : "http://ahad-haam:3128"})
59
60 # build a new opener that adds authentication and caching FTP handlers
61 opener = urllib.request.build_opener(proxy_support, authinfo,
62 urllib.request.CacheFTPHandler)
63
64 # install it
65 urllib.request.install_opener(opener)
66
67 f = urllib.request.urlopen('https://www.python.org/')
68 """
69
70 # XXX issues:
71 # If an authentication error handler that tries to perform
72 # authentication for some reason but fails, how should the error be
73 # signalled? The client needs to know the HTTP error code. But if
74 # the handler knows that the problem was, e.g., that it didn't know
75 # that hash algo that requested in the challenge, it would be good to
76 # pass that information along to the client, too.
77 # ftp errors aren't handled cleanly
78 # check digest against correct (i.e. non-apache) implementation
79
80 # Possible extensions:
81 # complex proxies XXX not sure what exactly was meant by this
82 # abstract factory for opener
83
84 import base64
85 import bisect
86 import contextlib
87 import email
88 import hashlib
89 import http.client
90 import io
91 import os
92 import re
93 import socket
94 import string
95 import sys
96 import time
97 import tempfile
98
99
100 from urllib.error import URLError, HTTPError, ContentTooShortError
101 from urllib.parse import (
102 urlparse, urlsplit, urljoin, unwrap, quote, unquote,
103 _splittype, _splithost, _splitport, _splituser, _splitpasswd,
104 _splitattr, _splitvalue, _splittag,
105 unquote_to_bytes, urlunparse)
106 from urllib.response import addinfourl, addclosehook
107
108 # check for SSL
109 try:
110 import ssl # noqa: F401
111 except ImportError:
112 _have_ssl = False
113 else:
114 _have_ssl = True
115
116 __all__ = [
117 # Classes
118 'Request', 'OpenerDirector', 'BaseHandler', 'HTTPDefaultErrorHandler',
119 'HTTPRedirectHandler', 'HTTPCookieProcessor', 'ProxyHandler',
120 'HTTPPasswordMgr', 'HTTPPasswordMgrWithDefaultRealm',
121 'HTTPPasswordMgrWithPriorAuth', 'AbstractBasicAuthHandler',
122 'HTTPBasicAuthHandler', 'ProxyBasicAuthHandler', 'AbstractDigestAuthHandler',
123 'HTTPDigestAuthHandler', 'ProxyDigestAuthHandler', 'HTTPHandler',
124 'FileHandler', 'FTPHandler', 'CacheFTPHandler', 'DataHandler',
125 'UnknownHandler', 'HTTPErrorProcessor',
126 # Functions
127 'urlopen', 'install_opener', 'build_opener',
128 'pathname2url', 'url2pathname', 'getproxies',
129 # Legacy interface
130 'urlretrieve', 'urlcleanup',
131 ]
132
133 # used in User-Agent header sent
134 __version__ = '%d.%d' % sys.version_info[:2]
135
136 _opener = None
137 def urlopen(url, data=None, timeout=socket._GLOBAL_DEFAULT_TIMEOUT,
138 *, context=None):
139 '''Open the URL url, which can be either a string or a Request object.
140
141 *data* must be an object specifying additional data to be sent to
142 the server, or None if no such data is needed. See Request for
143 details.
144
145 urllib.request module uses HTTP/1.1 and includes a "Connection:close"
146 header in its HTTP requests.
147
148 The optional *timeout* parameter specifies a timeout in seconds for
149 blocking operations like the connection attempt (if not specified, the
150 global default timeout setting will be used). This only works for HTTP,
151 HTTPS and FTP connections.
152
153 If *context* is specified, it must be a ssl.SSLContext instance describing
154 the various SSL options. See HTTPSConnection for more details.
155
156
157 This function always returns an object which can work as a
158 context manager and has the properties url, headers, and status.
159 See urllib.response.addinfourl for more detail on these properties.
160
161 For HTTP and HTTPS URLs, this function returns a http.client.HTTPResponse
162 object slightly modified. In addition to the three new methods above, the
163 msg attribute contains the same information as the reason attribute ---
164 the reason phrase returned by the server --- instead of the response
165 headers as it is specified in the documentation for HTTPResponse.
166
167 For FTP, file, and data URLs, this function returns a
168 urllib.response.addinfourl object.
169
170 Note that None may be returned if no handler handles the request (though
171 the default installed global OpenerDirector uses UnknownHandler to ensure
172 this never happens).
173
174 In addition, if proxy settings are detected (for example, when a *_proxy
175 environment variable like http_proxy is set), ProxyHandler is default
176 installed and makes sure the requests are handled through the proxy.
177
178 '''
179 global _opener
180 if context:
181 https_handler = HTTPSHandler(context=context)
182 opener = build_opener(https_handler)
183 elif _opener is None:
184 _opener = opener = build_opener()
185 else:
186 opener = _opener
187 return opener.open(url, data, timeout)
188
189 def install_opener(opener):
190 global _opener
191 _opener = opener
192
193 _url_tempfiles = []
194 def urlretrieve(url, filename=None, reporthook=None, data=None):
195 """
196 Retrieve a URL into a temporary location on disk.
197
198 Requires a URL argument. If a filename is passed, it is used as
199 the temporary file location. The reporthook argument should be
200 a callable that accepts a block number, a read size, and the
201 total file size of the URL target. The data argument should be
202 valid URL encoded data.
203
204 If a filename is passed and the URL points to a local resource,
205 the result is a copy from local file to new file.
206
207 Returns a tuple containing the path to the newly created
208 data file as well as the resulting HTTPMessage object.
209 """
210 url_type, path = _splittype(url)
211
212 with contextlib.closing(urlopen(url, data)) as fp:
213 headers = fp.info()
214
215 # Just return the local path and the "headers" for file://
216 # URLs. No sense in performing a copy unless requested.
217 if url_type == "file" and not filename:
218 return os.path.normpath(path), headers
219
220 # Handle temporary file setup.
221 if filename:
222 tfp = open(filename, 'wb')
223 else:
224 tfp = tempfile.NamedTemporaryFile(delete=False)
225 filename = tfp.name
226 _url_tempfiles.append(filename)
227
228 with tfp:
229 result = filename, headers
230 bs = 1024*8
231 size = -1
232 read = 0
233 blocknum = 0
234 if "content-length" in headers:
235 size = int(headers["Content-Length"])
236
237 if reporthook:
238 reporthook(blocknum, bs, size)
239
240 while block := fp.read(bs):
241 read += len(block)
242 tfp.write(block)
243 blocknum += 1
244 if reporthook:
245 reporthook(blocknum, bs, size)
246
247 if size >= 0 and read < size:
248 raise ContentTooShortError(
249 "retrieval incomplete: got only %i out of %i bytes"
250 % (read, size), result)
251
252 return result
253
254 def urlcleanup():
255 """Clean up temporary files from urlretrieve calls."""
256 for temp_file in _url_tempfiles:
257 try:
258 os.unlink(temp_file)
259 except OSError:
260 pass
261
262 del _url_tempfiles[:]
263 global _opener
264 if _opener:
265 _opener = None
266
267 # copied from cookielib.py
268 _cut_port_re = re.compile(r":\d+$", re.ASCII)
269 def request_host(request):
270 """Return request-host, as defined by RFC 2965.
271
272 Variation from RFC: returned value is lowercased, for convenient
273 comparison.
274
275 """
276 url = request.full_url
277 host = urlparse(url)[1]
278 if host == "":
279 host = request.get_header("Host", "")
280
281 # remove port, if present
282 host = _cut_port_re.sub("", host, 1)
283 return host.lower()
284
285 class Request:
286
287 def __init__(self, url, data=None, headers={},
288 origin_req_host=None, unverifiable=False,
289 method=None):
290 self.full_url = url
291 self.headers = {}
292 self.unredirected_hdrs = {}
293 self._data = None
294 self.data = data
295 self._tunnel_host = None
296 for key, value in headers.items():
297 self.add_header(key, value)
298 if origin_req_host is None:
299 origin_req_host = request_host(self)
300 self.origin_req_host = origin_req_host
301 self.unverifiable = unverifiable
302 if method:
303 self.method = method
304
305 @property
306 def full_url(self):
307 if self.fragment:
308 return '{}#{}'.format(self._full_url, self.fragment)
309 return self._full_url
310
311 @full_url.setter
312 def full_url(self, url):
313 # unwrap('<URL:type://host/path>') --> 'type://host/path'
314 self._full_url = unwrap(url)
315 self._full_url, self.fragment = _splittag(self._full_url)
316 self._parse()
317
318 @full_url.deleter
319 def full_url(self):
320 self._full_url = None
321 self.fragment = None
322 self.selector = ''
323
324 @property
325 def data(self):
326 return self._data
327
328 @data.setter
329 def data(self, data):
330 if data != self._data:
331 self._data = data
332 # issue 16464
333 # if we change data we need to remove content-length header
334 # (cause it's most probably calculated for previous value)
335 if self.has_header("Content-length"):
336 self.remove_header("Content-length")
337
338 @data.deleter
339 def data(self):
340 self.data = None
341
342 def _parse(self):
343 self.type, rest = _splittype(self._full_url)
344 if self.type is None:
345 raise ValueError("unknown url type: %r" % self.full_url)
346 self.host, self.selector = _splithost(rest)
347 if self.host:
348 self.host = unquote(self.host)
349
350 def get_method(self):
351 """Return a string indicating the HTTP request method."""
352 default_method = "POST" if self.data is not None else "GET"
353 return getattr(self, 'method', default_method)
354
355 def get_full_url(self):
356 return self.full_url
357
358 def set_proxy(self, host, type):
359 if self.type == 'https' and not self._tunnel_host:
360 self._tunnel_host = self.host
361 else:
362 self.type= type
363 self.selector = self.full_url
364 self.host = host
365
366 def has_proxy(self):
367 return self.selector == self.full_url
368
369 def add_header(self, key, val):
370 # useful for something like authentication
371 self.headers[key.capitalize()] = val
372
373 def add_unredirected_header(self, key, val):
374 # will not be added to a redirected request
375 self.unredirected_hdrs[key.capitalize()] = val
376
377 def has_header(self, header_name):
378 return (header_name in self.headers or
379 header_name in self.unredirected_hdrs)
380
381 def get_header(self, header_name, default=None):
382 return self.headers.get(
383 header_name,
384 self.unredirected_hdrs.get(header_name, default))
385
386 def remove_header(self, header_name):
387 self.headers.pop(header_name, None)
388 self.unredirected_hdrs.pop(header_name, None)
389
390 def header_items(self):
391 hdrs = {**self.unredirected_hdrs, **self.headers}
392 return list(hdrs.items())
393
394 class OpenerDirector:
395 def __init__(self):
396 client_version = "Python-urllib/%s" % __version__
397 self.addheaders = [('User-agent', client_version)]
398 # self.handlers is retained only for backward compatibility
399 self.handlers = []
400 # manage the individual handlers
401 self.handle_open = {}
402 self.handle_error = {}
403 self.process_response = {}
404 self.process_request = {}
405
406 def add_handler(self, handler):
407 if not hasattr(handler, "add_parent"):
408 raise TypeError("expected BaseHandler instance, got %r" %
409 type(handler))
410
411 added = False
412 for meth in dir(handler):
413 if meth in ["redirect_request", "do_open", "proxy_open"]:
414 # oops, coincidental match
415 continue
416
417 i = meth.find("_")
418 protocol = meth[:i]
419 condition = meth[i+1:]
420
421 if condition.startswith("error"):
422 j = condition.find("_") + i + 1
423 kind = meth[j+1:]
424 try:
425 kind = int(kind)
426 except ValueError:
427 pass
428 lookup = self.handle_error.get(protocol, {})
429 self.handle_error[protocol] = lookup
430 elif condition == "open":
431 kind = protocol
432 lookup = self.handle_open
433 elif condition == "response":
434 kind = protocol
435 lookup = self.process_response
436 elif condition == "request":
437 kind = protocol
438 lookup = self.process_request
439 else:
440 continue
441
442 handlers = lookup.setdefault(kind, [])
443 if handlers:
444 bisect.insort(handlers, handler)
445 else:
446 handlers.append(handler)
447 added = True
448
449 if added:
450 bisect.insort(self.handlers, handler)
451 handler.add_parent(self)
452
453 def close(self):
454 # Only exists for backwards compatibility.
455 pass
456
457 def _call_chain(self, chain, kind, meth_name, *args):
458 # Handlers raise an exception if no one else should try to handle
459 # the request, or return None if they can't but another handler
460 # could. Otherwise, they return the response.
461 handlers = chain.get(kind, ())
462 for handler in handlers:
463 func = getattr(handler, meth_name)
464 result = func(*args)
465 if result is not None:
466 return result
467
468 def open(self, fullurl, data=None, timeout=socket._GLOBAL_DEFAULT_TIMEOUT):
469 # accept a URL or a Request object
470 if isinstance(fullurl, str):
471 req = Request(fullurl, data)
472 else:
473 req = fullurl
474 if data is not None:
475 req.data = data
476
477 req.timeout = timeout
478 protocol = req.type
479
480 # pre-process request
481 meth_name = protocol+"_request"
482 for processor in self.process_request.get(protocol, []):
483 meth = getattr(processor, meth_name)
484 req = meth(req)
485
486 sys.audit('urllib.Request', req.full_url, req.data, req.headers, req.get_method())
487 response = self._open(req, data)
488
489 # post-process response
490 meth_name = protocol+"_response"
491 for processor in self.process_response.get(protocol, []):
492 meth = getattr(processor, meth_name)
493 response = meth(req, response)
494
495 return response
496
497 def _open(self, req, data=None):
498 result = self._call_chain(self.handle_open, 'default',
499 'default_open', req)
500 if result:
501 return result
502
503 protocol = req.type
504 result = self._call_chain(self.handle_open, protocol, protocol +
505 '_open', req)
506 if result:
507 return result
508
509 return self._call_chain(self.handle_open, 'unknown',
510 'unknown_open', req)
511
512 def error(self, proto, *args):
513 if proto in ('http', 'https'):
514 # XXX http[s] protocols are special-cased
515 dict = self.handle_error['http'] # https is not different than http
516 proto = args[2] # YUCK!
517 meth_name = 'http_error_%s' % proto
518 http_err = 1
519 orig_args = args
520 else:
521 dict = self.handle_error
522 meth_name = proto + '_error'
523 http_err = 0
524 args = (dict, proto, meth_name) + args
525 result = self._call_chain(*args)
526 if result:
527 return result
528
529 if http_err:
530 args = (dict, 'default', 'http_error_default') + orig_args
531 return self._call_chain(*args)
532
533 # XXX probably also want an abstract factory that knows when it makes
534 # sense to skip a superclass in favor of a subclass and when it might
535 # make sense to include both
536
537 def build_opener(*handlers):
538 """Create an opener object from a list of handlers.
539
540 The opener will use several default handlers, including support
541 for HTTP, FTP and when applicable HTTPS.
542
543 If any of the handlers passed as arguments are subclasses of the
544 default handlers, the default handlers will not be used.
545 """
546 opener = OpenerDirector()
547 default_classes = [ProxyHandler, UnknownHandler, HTTPHandler,
548 HTTPDefaultErrorHandler, HTTPRedirectHandler,
549 FTPHandler, FileHandler, HTTPErrorProcessor,
550 DataHandler]
551 if hasattr(http.client, "HTTPSConnection"):
552 default_classes.append(HTTPSHandler)
553 skip = set()
554 for klass in default_classes:
555 for check in handlers:
556 if isinstance(check, type):
557 if issubclass(check, klass):
558 skip.add(klass)
559 elif isinstance(check, klass):
560 skip.add(klass)
561 for klass in skip:
562 default_classes.remove(klass)
563
564 for klass in default_classes:
565 opener.add_handler(klass())
566
567 for h in handlers:
568 if isinstance(h, type):
569 h = h()
570 opener.add_handler(h)
571 return opener
572
573 class BaseHandler:
574 handler_order = 500
575
576 def add_parent(self, parent):
577 self.parent = parent
578
579 def close(self):
580 # Only exists for backwards compatibility
581 pass
582
583 def __lt__(self, other):
584 if not hasattr(other, "handler_order"):
585 # Try to preserve the old behavior of having custom classes
586 # inserted after default ones (works only for custom user
587 # classes which are not aware of handler_order).
588 return True
589 return self.handler_order < other.handler_order
590
591
592 class HTTPErrorProcessor(BaseHandler):
593 """Process HTTP error responses."""
594 handler_order = 1000 # after all other processing
595
596 def http_response(self, request, response):
597 code, msg, hdrs = response.code, response.msg, response.info()
598
599 # According to RFC 2616, "2xx" code indicates that the client's
600 # request was successfully received, understood, and accepted.
601 if not (200 <= code < 300):
602 response = self.parent.error(
603 'http', request, response, code, msg, hdrs)
604
605 return response
606
607 https_response = http_response
608
609 class HTTPDefaultErrorHandler(BaseHandler):
610 def http_error_default(self, req, fp, code, msg, hdrs):
611 raise HTTPError(req.full_url, code, msg, hdrs, fp)
612
613 class HTTPRedirectHandler(BaseHandler):
614 # maximum number of redirections to any single URL
615 # this is needed because of the state that cookies introduce
616 max_repeats = 4
617 # maximum total number of redirections (regardless of URL) before
618 # assuming we're in a loop
619 max_redirections = 10
620
621 def redirect_request(self, req, fp, code, msg, headers, newurl):
622 """Return a Request or None in response to a redirect.
623
624 This is called by the http_error_30x methods when a
625 redirection response is received. If a redirection should
626 take place, return a new Request to allow http_error_30x to
627 perform the redirect. Otherwise, raise HTTPError if no-one
628 else should try to handle this url. Return None if you can't
629 but another Handler might.
630 """
631 m = req.get_method()
632 if (not (code in (301, 302, 303, 307, 308) and m in ("GET", "HEAD")
633 or code in (301, 302, 303) and m == "POST")):
634 raise HTTPError(req.full_url, code, msg, headers, fp)
635
636 # Strictly (according to RFC 2616), 301 or 302 in response to
637 # a POST MUST NOT cause a redirection without confirmation
638 # from the user (of urllib.request, in this case). In practice,
639 # essentially all clients do redirect in this case, so we do
640 # the same.
641
642 # Be conciliant with URIs containing a space. This is mainly
643 # redundant with the more complete encoding done in http_error_302(),
644 # but it is kept for compatibility with other callers.
645 newurl = newurl.replace(' ', '%20')
646
647 CONTENT_HEADERS = ("content-length", "content-type")
648 newheaders = {k: v for k, v in req.headers.items()
649 if k.lower() not in CONTENT_HEADERS}
650 return Request(newurl,
651 method="HEAD" if m == "HEAD" else "GET",
652 headers=newheaders,
653 origin_req_host=req.origin_req_host,
654 unverifiable=True)
655
656 # Implementation note: To avoid the server sending us into an
657 # infinite loop, the request object needs to track what URLs we
658 # have already seen. Do this by adding a handler-specific
659 # attribute to the Request object.
660 def http_error_302(self, req, fp, code, msg, headers):
661 # Some servers (incorrectly) return multiple Location headers
662 # (so probably same goes for URI). Use first header.
663 if "location" in headers:
664 newurl = headers["location"]
665 elif "uri" in headers:
666 newurl = headers["uri"]
667 else:
668 return
669
670 # fix a possible malformed URL
671 urlparts = urlparse(newurl)
672
673 # For security reasons we don't allow redirection to anything other
674 # than http, https or ftp.
675
676 if urlparts.scheme not in ('http', 'https', 'ftp', ''):
677 raise HTTPError(
678 newurl, code,
679 "%s - Redirection to url '%s' is not allowed" % (msg, newurl),
680 headers, fp)
681
682 if not urlparts.path and urlparts.netloc:
683 urlparts = list(urlparts)
684 urlparts[2] = "/"
685 newurl = urlunparse(urlparts)
686
687 # http.client.parse_headers() decodes as ISO-8859-1. Recover the
688 # original bytes and percent-encode non-ASCII bytes, and any special
689 # characters such as the space.
690 newurl = quote(
691 newurl, encoding="iso-8859-1", safe=string.punctuation)
692 newurl = urljoin(req.full_url, newurl)
693
694 # XXX Probably want to forget about the state of the current
695 # request, although that might interact poorly with other
696 # handlers that also use handler-specific request attributes
697 new = self.redirect_request(req, fp, code, msg, headers, newurl)
698 if new is None:
699 return
700
701 # loop detection
702 # .redirect_dict has a key url if url was previously visited.
703 if hasattr(req, 'redirect_dict'):
704 visited = new.redirect_dict = req.redirect_dict
705 if (visited.get(newurl, 0) >= self.max_repeats or
706 len(visited) >= self.max_redirections):
707 raise HTTPError(req.full_url, code,
708 self.inf_msg + msg, headers, fp)
709 else:
710 visited = new.redirect_dict = req.redirect_dict = {}
711 visited[newurl] = visited.get(newurl, 0) + 1
712
713 # Don't close the fp until we are sure that we won't use it
714 # with HTTPError.
715 fp.read()
716 fp.close()
717
718 return self.parent.open(new, timeout=req.timeout)
719
720 http_error_301 = http_error_303 = http_error_307 = http_error_308 = http_error_302
721
722 inf_msg = "The HTTP server returned a redirect error that would " \
723 "lead to an infinite loop.\n" \
724 "The last 30x error message was:\n"
725
726
727 def _parse_proxy(proxy):
728 """Return (scheme, user, password, host/port) given a URL or an authority.
729
730 If a URL is supplied, it must have an authority (host:port) component.
731 According to RFC 3986, having an authority component means the URL must
732 have two slashes after the scheme.
733 """
734 scheme, r_scheme = _splittype(proxy)
735 if not r_scheme.startswith("/"):
736 # authority
737 scheme = None
738 authority = proxy
739 else:
740 # URL
741 if not r_scheme.startswith("//"):
742 raise ValueError("proxy URL with no authority: %r" % proxy)
743 # We have an authority, so for RFC 3986-compliant URLs (by ss 3.
744 # and 3.3.), path is empty or starts with '/'
745 if '@' in r_scheme:
746 host_separator = r_scheme.find('@')
747 end = r_scheme.find("/", host_separator)
748 else:
749 end = r_scheme.find("/", 2)
750 if end == -1:
751 end = None
752 authority = r_scheme[2:end]
753 userinfo, hostport = _splituser(authority)
754 if userinfo is not None:
755 user, password = _splitpasswd(userinfo)
756 else:
757 user = password = None
758 return scheme, user, password, hostport
759
760 class ProxyHandler(BaseHandler):
761 # Proxies must be in front
762 handler_order = 100
763
764 def __init__(self, proxies=None):
765 if proxies is None:
766 proxies = getproxies()
767 assert hasattr(proxies, 'keys'), "proxies must be a mapping"
768 self.proxies = proxies
769 for type, url in proxies.items():
770 type = type.lower()
771 setattr(self, '%s_open' % type,
772 lambda r, proxy=url, type=type, meth=self.proxy_open:
773 meth(r, proxy, type))
774
775 def proxy_open(self, req, proxy, type):
776 orig_type = req.type
777 proxy_type, user, password, hostport = _parse_proxy(proxy)
778 if proxy_type is None:
779 proxy_type = orig_type
780
781 if req.host and proxy_bypass(req.host):
782 return None
783
784 if user and password:
785 user_pass = '%s:%s' % (unquote(user),
786 unquote(password))
787 creds = base64.b64encode(user_pass.encode()).decode("ascii")
788 req.add_header('Proxy-authorization', 'Basic ' + creds)
789 hostport = unquote(hostport)
790 req.set_proxy(hostport, proxy_type)
791 if orig_type == proxy_type or orig_type == 'https':
792 # let other handlers take care of it
793 return None
794 else:
795 # need to start over, because the other handlers don't
796 # grok the proxy's URL type
797 # e.g. if we have a constructor arg proxies like so:
798 # {'http': 'ftp://proxy.example.com'}, we may end up turning
799 # a request for http://acme.example.com/a into one for
800 # ftp://proxy.example.com/a
801 return self.parent.open(req, timeout=req.timeout)
802
803 class HTTPPasswordMgr:
804
805 def __init__(self):
806 self.passwd = {}
807
808 def add_password(self, realm, uri, user, passwd):
809 # uri could be a single URI or a sequence
810 if isinstance(uri, str):
811 uri = [uri]
812 if realm not in self.passwd:
813 self.passwd[realm] = {}
814 for default_port in True, False:
815 reduced_uri = tuple(
816 self.reduce_uri(u, default_port) for u in uri)
817 self.passwd[realm][reduced_uri] = (user, passwd)
818
819 def find_user_password(self, realm, authuri):
820 domains = self.passwd.get(realm, {})
821 for default_port in True, False:
822 reduced_authuri = self.reduce_uri(authuri, default_port)
823 for uris, authinfo in domains.items():
824 for uri in uris:
825 if self.is_suburi(uri, reduced_authuri):
826 return authinfo
827 return None, None
828
829 def reduce_uri(self, uri, default_port=True):
830 """Accept authority or URI and extract only the authority and path."""
831 # note HTTP URLs do not have a userinfo component
832 parts = urlsplit(uri)
833 if parts[1]:
834 # URI
835 scheme = parts[0]
836 authority = parts[1]
837 path = parts[2] or '/'
838 else:
839 # host or host:port
840 scheme = None
841 authority = uri
842 path = '/'
843 host, port = _splitport(authority)
844 if default_port and port is None and scheme is not None:
845 dport = {"http": 80,
846 "https": 443,
847 }.get(scheme)
848 if dport is not None:
849 authority = "%s:%d" % (host, dport)
850 return authority, path
851
852 def is_suburi(self, base, test):
853 """Check if test is below base in a URI tree
854
855 Both args must be URIs in reduced form.
856 """
857 if base == test:
858 return True
859 if base[0] != test[0]:
860 return False
861 prefix = base[1]
862 if prefix[-1:] != '/':
863 prefix += '/'
864 return test[1].startswith(prefix)
865
866
867 class HTTPPasswordMgrWithDefaultRealm(HTTPPasswordMgr):
868
869 def find_user_password(self, realm, authuri):
870 user, password = HTTPPasswordMgr.find_user_password(self, realm,
871 authuri)
872 if user is not None:
873 return user, password
874 return HTTPPasswordMgr.find_user_password(self, None, authuri)
875
876
877 class HTTPPasswordMgrWithPriorAuth(HTTPPasswordMgrWithDefaultRealm):
878
879 def __init__(self):
880 self.authenticated = {}
881 super().__init__()
882
883 def add_password(self, realm, uri, user, passwd, is_authenticated=False):
884 self.update_authenticated(uri, is_authenticated)
885 # Add a default for prior auth requests
886 if realm is not None:
887 super().add_password(None, uri, user, passwd)
888 super().add_password(realm, uri, user, passwd)
889
890 def update_authenticated(self, uri, is_authenticated=False):
891 # uri could be a single URI or a sequence
892 if isinstance(uri, str):
893 uri = [uri]
894
895 for default_port in True, False:
896 for u in uri:
897 reduced_uri = self.reduce_uri(u, default_port)
898 self.authenticated[reduced_uri] = is_authenticated
899
900 def is_authenticated(self, authuri):
901 for default_port in True, False:
902 reduced_authuri = self.reduce_uri(authuri, default_port)
903 for uri in self.authenticated:
904 if self.is_suburi(uri, reduced_authuri):
905 return self.authenticated[uri]
906
907
908 class AbstractBasicAuthHandler:
909
910 # XXX this allows for multiple auth-schemes, but will stupidly pick
911 # the last one with a realm specified.
912
913 # allow for double- and single-quoted realm values
914 # (single quotes are a violation of the RFC, but appear in the wild)
915 rx = re.compile('(?:^|,)' # start of the string or ','
916 '[ \t]*' # optional whitespaces
917 '([^ \t,]+)' # scheme like "Basic"
918 '[ \t]+' # mandatory whitespaces
919 # realm=xxx
920 # realm='xxx'
921 # realm="xxx"
922 'realm=(["\']?)([^"\']*)\\2',
923 re.I)
924
925 # XXX could pre-emptively send auth info already accepted (RFC 2617,
926 # end of section 2, and section 1.2 immediately after "credentials"
927 # production).
928
929 def __init__(self, password_mgr=None):
930 if password_mgr is None:
931 password_mgr = HTTPPasswordMgr()
932 self.passwd = password_mgr
933 self.add_password = self.passwd.add_password
934
935 def _parse_realm(self, header):
936 # parse WWW-Authenticate header: accept multiple challenges per header
937 found_challenge = False
938 for mo in AbstractBasicAuthHandler.rx.finditer(header):
939 scheme, quote, realm = mo.groups()
940 if quote not in ['"', "'"]:
941 import warnings
942 warnings.warn("Basic Auth Realm was unquoted",
943 UserWarning, 3)
944
945 yield (scheme, realm)
946
947 found_challenge = True
948
949 if not found_challenge:
950 if header:
951 scheme = header.split()[0]
952 else:
953 scheme = ''
954 yield (scheme, None)
955
956 def http_error_auth_reqed(self, authreq, host, req, headers):
957 # host may be an authority (without userinfo) or a URL with an
958 # authority
959 headers = headers.get_all(authreq)
960 if not headers:
961 # no header found
962 return
963
964 unsupported = None
965 for header in headers:
966 for scheme, realm in self._parse_realm(header):
967 if scheme.lower() != 'basic':
968 unsupported = scheme
969 continue
970
971 if realm is not None:
972 # Use the first matching Basic challenge.
973 # Ignore following challenges even if they use the Basic
974 # scheme.
975 return self.retry_http_basic_auth(host, req, realm)
976
977 if unsupported is not None:
978 raise ValueError("AbstractBasicAuthHandler does not "
979 "support the following scheme: %r"
980 % (scheme,))
981
982 def retry_http_basic_auth(self, host, req, realm):
983 user, pw = self.passwd.find_user_password(realm, host)
984 if pw is not None:
985 raw = "%s:%s" % (user, pw)
986 auth = "Basic " + base64.b64encode(raw.encode()).decode("ascii")
987 if req.get_header(self.auth_header, None) == auth:
988 return None
989 req.add_unredirected_header(self.auth_header, auth)
990 return self.parent.open(req, timeout=req.timeout)
991 else:
992 return None
993
994 def http_request(self, req):
995 if (not hasattr(self.passwd, 'is_authenticated') or
996 not self.passwd.is_authenticated(req.full_url)):
997 return req
998
999 if not req.has_header('Authorization'):
1000 user, passwd = self.passwd.find_user_password(None, req.full_url)
1001 credentials = '{0}:{1}'.format(user, passwd).encode()
1002 auth_str = base64.standard_b64encode(credentials).decode()
1003 req.add_unredirected_header('Authorization',
1004 'Basic {}'.format(auth_str.strip()))
1005 return req
1006
1007 def http_response(self, req, response):
1008 if hasattr(self.passwd, 'is_authenticated'):
1009 if 200 <= response.code < 300:
1010 self.passwd.update_authenticated(req.full_url, True)
1011 else:
1012 self.passwd.update_authenticated(req.full_url, False)
1013 return response
1014
1015 https_request = http_request
1016 https_response = http_response
1017
1018
1019
1020 class HTTPBasicAuthHandler(AbstractBasicAuthHandler, BaseHandler):
1021
1022 auth_header = 'Authorization'
1023
1024 def http_error_401(self, req, fp, code, msg, headers):
1025 url = req.full_url
1026 response = self.http_error_auth_reqed('www-authenticate',
1027 url, req, headers)
1028 return response
1029
1030
1031 class ProxyBasicAuthHandler(AbstractBasicAuthHandler, BaseHandler):
1032
1033 auth_header = 'Proxy-authorization'
1034
1035 def http_error_407(self, req, fp, code, msg, headers):
1036 # http_error_auth_reqed requires that there is no userinfo component in
1037 # authority. Assume there isn't one, since urllib.request does not (and
1038 # should not, RFC 3986 s. 3.2.1) support requests for URLs containing
1039 # userinfo.
1040 authority = req.host
1041 response = self.http_error_auth_reqed('proxy-authenticate',
1042 authority, req, headers)
1043 return response
1044
1045
1046 # Return n random bytes.
1047 _randombytes = os.urandom
1048
1049
1050 class AbstractDigestAuthHandler:
1051 # Digest authentication is specified in RFC 2617/7616.
1052
1053 # XXX The client does not inspect the Authentication-Info header
1054 # in a successful response.
1055
1056 # XXX It should be possible to test this implementation against
1057 # a mock server that just generates a static set of challenges.
1058
1059 # XXX qop="auth-int" supports is shaky
1060
1061 def __init__(self, passwd=None):
1062 if passwd is None:
1063 passwd = HTTPPasswordMgr()
1064 self.passwd = passwd
1065 self.add_password = self.passwd.add_password
1066 self.retried = 0
1067 self.nonce_count = 0
1068 self.last_nonce = None
1069
1070 def reset_retry_count(self):
1071 self.retried = 0
1072
1073 def http_error_auth_reqed(self, auth_header, host, req, headers):
1074 authreq = headers.get(auth_header, None)
1075 if self.retried > 5:
1076 # Don't fail endlessly - if we failed once, we'll probably
1077 # fail a second time. Hm. Unless the Password Manager is
1078 # prompting for the information. Crap. This isn't great
1079 # but it's better than the current 'repeat until recursion
1080 # depth exceeded' approach <wink>
1081 raise HTTPError(req.full_url, 401, "digest auth failed",
1082 headers, None)
1083 else:
1084 self.retried += 1
1085 if authreq:
1086 scheme = authreq.split()[0]
1087 if scheme.lower() == 'digest':
1088 return self.retry_http_digest_auth(req, authreq)
1089 elif scheme.lower() != 'basic':
1090 raise ValueError("AbstractDigestAuthHandler does not support"
1091 " the following scheme: '%s'" % scheme)
1092
1093 def retry_http_digest_auth(self, req, auth):
1094 token, challenge = auth.split(' ', 1)
1095 chal = parse_keqv_list(filter(None, parse_http_list(challenge)))
1096 auth = self.get_authorization(req, chal)
1097 if auth:
1098 auth_val = 'Digest %s' % auth
1099 if req.headers.get(self.auth_header, None) == auth_val:
1100 return None
1101 req.add_unredirected_header(self.auth_header, auth_val)
1102 resp = self.parent.open(req, timeout=req.timeout)
1103 return resp
1104
1105 def get_cnonce(self, nonce):
1106 # The cnonce-value is an opaque
1107 # quoted string value provided by the client and used by both client
1108 # and server to avoid chosen plaintext attacks, to provide mutual
1109 # authentication, and to provide some message integrity protection.
1110 # This isn't a fabulous effort, but it's probably Good Enough.
1111 s = "%s:%s:%s:" % (self.nonce_count, nonce, time.ctime())
1112 b = s.encode("ascii") + _randombytes(8)
1113 dig = hashlib.sha1(b).hexdigest()
1114 return dig[:16]
1115
1116 def get_authorization(self, req, chal):
1117 try:
1118 realm = chal['realm']
1119 nonce = chal['nonce']
1120 qop = chal.get('qop')
1121 algorithm = chal.get('algorithm', 'MD5')
1122 # mod_digest doesn't send an opaque, even though it isn't
1123 # supposed to be optional
1124 opaque = chal.get('opaque', None)
1125 except KeyError:
1126 return None
1127
1128 H, KD = self.get_algorithm_impls(algorithm)
1129 if H is None:
1130 return None
1131
1132 user, pw = self.passwd.find_user_password(realm, req.full_url)
1133 if user is None:
1134 return None
1135
1136 # XXX not implemented yet
1137 if req.data is not None:
1138 entdig = self.get_entity_digest(req.data, chal)
1139 else:
1140 entdig = None
1141
1142 A1 = "%s:%s:%s" % (user, realm, pw)
1143 A2 = "%s:%s" % (req.get_method(),
1144 # XXX selector: what about proxies and full urls
1145 req.selector)
1146 # NOTE: As per RFC 2617, when server sends "auth,auth-int", the client could use either `auth`
1147 # or `auth-int` to the response back. we use `auth` to send the response back.
1148 if qop is None:
1149 respdig = KD(H(A1), "%s:%s" % (nonce, H(A2)))
1150 elif 'auth' in qop.split(','):
1151 if nonce == self.last_nonce:
1152 self.nonce_count += 1
1153 else:
1154 self.nonce_count = 1
1155 self.last_nonce = nonce
1156 ncvalue = '%08x' % self.nonce_count
1157 cnonce = self.get_cnonce(nonce)
1158 noncebit = "%s:%s:%s:%s:%s" % (nonce, ncvalue, cnonce, 'auth', H(A2))
1159 respdig = KD(H(A1), noncebit)
1160 else:
1161 # XXX handle auth-int.
1162 raise URLError("qop '%s' is not supported." % qop)
1163
1164 # XXX should the partial digests be encoded too?
1165
1166 base = 'username="%s", realm="%s", nonce="%s", uri="%s", ' \
1167 'response="%s"' % (user, realm, nonce, req.selector,
1168 respdig)
1169 if opaque:
1170 base += ', opaque="%s"' % opaque
1171 if entdig:
1172 base += ', digest="%s"' % entdig
1173 base += ', algorithm="%s"' % algorithm
1174 if qop:
1175 base += ', qop=auth, nc=%s, cnonce="%s"' % (ncvalue, cnonce)
1176 return base
1177
1178 def get_algorithm_impls(self, algorithm):
1179 # algorithm names taken from RFC 7616 Section 6.1
1180 # lambdas assume digest modules are imported at the top level
1181 if algorithm == 'MD5':
1182 H = lambda x: hashlib.md5(x.encode("ascii")).hexdigest()
1183 elif algorithm == 'SHA': # non-standard, retained for compatibility.
1184 H = lambda x: hashlib.sha1(x.encode("ascii")).hexdigest()
1185 elif algorithm == 'SHA-256':
1186 H = lambda x: hashlib.sha256(x.encode("ascii")).hexdigest()
1187 # XXX MD5-sess
1188 else:
1189 raise ValueError("Unsupported digest authentication "
1190 "algorithm %r" % algorithm)
1191 KD = lambda s, d: H("%s:%s" % (s, d))
1192 return H, KD
1193
1194 def get_entity_digest(self, data, chal):
1195 # XXX not implemented yet
1196 return None
1197
1198
1199 class HTTPDigestAuthHandler(BaseHandler, AbstractDigestAuthHandler):
1200 """An authentication protocol defined by RFC 2069
1201
1202 Digest authentication improves on basic authentication because it
1203 does not transmit passwords in the clear.
1204 """
1205
1206 auth_header = 'Authorization'
1207 handler_order = 490 # before Basic auth
1208
1209 def http_error_401(self, req, fp, code, msg, headers):
1210 host = urlparse(req.full_url)[1]
1211 retry = self.http_error_auth_reqed('www-authenticate',
1212 host, req, headers)
1213 self.reset_retry_count()
1214 return retry
1215
1216
1217 class ProxyDigestAuthHandler(BaseHandler, AbstractDigestAuthHandler):
1218
1219 auth_header = 'Proxy-Authorization'
1220 handler_order = 490 # before Basic auth
1221
1222 def http_error_407(self, req, fp, code, msg, headers):
1223 host = req.host
1224 retry = self.http_error_auth_reqed('proxy-authenticate',
1225 host, req, headers)
1226 self.reset_retry_count()
1227 return retry
1228
1229 class AbstractHTTPHandler(BaseHandler):
1230
1231 def __init__(self, debuglevel=None):
1232 self._debuglevel = debuglevel if debuglevel is not None else http.client.HTTPConnection.debuglevel
1233
1234 def set_http_debuglevel(self, level):
1235 self._debuglevel = level
1236
1237 def _get_content_length(self, request):
1238 return http.client.HTTPConnection._get_content_length(
1239 request.data,
1240 request.get_method())
1241
1242 def do_request_(self, request):
1243 host = request.host
1244 if not host:
1245 raise URLError('no host given')
1246
1247 if request.data is not None: # POST
1248 data = request.data
1249 if isinstance(data, str):
1250 msg = "POST data should be bytes, an iterable of bytes, " \
1251 "or a file object. It cannot be of type str."
1252 raise TypeError(msg)
1253 if not request.has_header('Content-type'):
1254 request.add_unredirected_header(
1255 'Content-type',
1256 'application/x-www-form-urlencoded')
1257 if (not request.has_header('Content-length')
1258 and not request.has_header('Transfer-encoding')):
1259 content_length = self._get_content_length(request)
1260 if content_length is not None:
1261 request.add_unredirected_header(
1262 'Content-length', str(content_length))
1263 else:
1264 request.add_unredirected_header(
1265 'Transfer-encoding', 'chunked')
1266
1267 sel_host = host
1268 if request.has_proxy():
1269 scheme, sel = _splittype(request.selector)
1270 sel_host, sel_path = _splithost(sel)
1271 if not request.has_header('Host'):
1272 request.add_unredirected_header('Host', sel_host)
1273 for name, value in self.parent.addheaders:
1274 name = name.capitalize()
1275 if not request.has_header(name):
1276 request.add_unredirected_header(name, value)
1277
1278 return request
1279
1280 def do_open(self, http_class, req, **http_conn_args):
1281 """Return an HTTPResponse object for the request, using http_class.
1282
1283 http_class must implement the HTTPConnection API from http.client.
1284 """
1285 host = req.host
1286 if not host:
1287 raise URLError('no host given')
1288
1289 # will parse host:port
1290 h = http_class(host, timeout=req.timeout, **http_conn_args)
1291 h.set_debuglevel(self._debuglevel)
1292
1293 headers = dict(req.unredirected_hdrs)
1294 headers.update({k: v for k, v in req.headers.items()
1295 if k not in headers})
1296
1297 # TODO(jhylton): Should this be redesigned to handle
1298 # persistent connections?
1299
1300 # We want to make an HTTP/1.1 request, but the addinfourl
1301 # class isn't prepared to deal with a persistent connection.
1302 # It will try to read all remaining data from the socket,
1303 # which will block while the server waits for the next request.
1304 # So make sure the connection gets closed after the (only)
1305 # request.
1306 headers["Connection"] = "close"
1307 headers = {name.title(): val for name, val in headers.items()}
1308
1309 if req._tunnel_host:
1310 tunnel_headers = {}
1311 proxy_auth_hdr = "Proxy-Authorization"
1312 if proxy_auth_hdr in headers:
1313 tunnel_headers[proxy_auth_hdr] = headers[proxy_auth_hdr]
1314 # Proxy-Authorization should not be sent to origin
1315 # server.
1316 del headers[proxy_auth_hdr]
1317 h.set_tunnel(req._tunnel_host, headers=tunnel_headers)
1318
1319 try:
1320 try:
1321 h.request(req.get_method(), req.selector, req.data, headers,
1322 encode_chunked=req.has_header('Transfer-encoding'))
1323 except OSError as err: # timeout error
1324 raise URLError(err)
1325 r = h.getresponse()
1326 except:
1327 h.close()
1328 raise
1329
1330 # If the server does not send us a 'Connection: close' header,
1331 # HTTPConnection assumes the socket should be left open. Manually
1332 # mark the socket to be closed when this response object goes away.
1333 if h.sock:
1334 h.sock.close()
1335 h.sock = None
1336
1337 r.url = req.get_full_url()
1338 # This line replaces the .msg attribute of the HTTPResponse
1339 # with .headers, because urllib clients expect the response to
1340 # have the reason in .msg. It would be good to mark this
1341 # attribute is deprecated and get then to use info() or
1342 # .headers.
1343 r.msg = r.reason
1344 return r
1345
1346
1347 class HTTPHandler(AbstractHTTPHandler):
1348
1349 def http_open(self, req):
1350 return self.do_open(http.client.HTTPConnection, req)
1351
1352 http_request = AbstractHTTPHandler.do_request_
1353
1354 if hasattr(http.client, 'HTTPSConnection'):
1355
1356 class HTTPSHandler(AbstractHTTPHandler):
1357
1358 def __init__(self, debuglevel=None, context=None, check_hostname=None):
1359 debuglevel = debuglevel if debuglevel is not None else http.client.HTTPSConnection.debuglevel
1360 AbstractHTTPHandler.__init__(self, debuglevel)
1361 if context is None:
1362 http_version = http.client.HTTPSConnection._http_vsn
1363 context = http.client._create_https_context(http_version)
1364 if check_hostname is not None:
1365 context.check_hostname = check_hostname
1366 self._context = context
1367
1368 def https_open(self, req):
1369 return self.do_open(http.client.HTTPSConnection, req,
1370 context=self._context)
1371
1372 https_request = AbstractHTTPHandler.do_request_
1373
1374 __all__.append('HTTPSHandler')
1375
1376 class HTTPCookieProcessor(BaseHandler):
1377 def __init__(self, cookiejar=None):
1378 import http.cookiejar
1379 if cookiejar is None:
1380 cookiejar = http.cookiejar.CookieJar()
1381 self.cookiejar = cookiejar
1382
1383 def http_request(self, request):
1384 self.cookiejar.add_cookie_header(request)
1385 return request
1386
1387 def http_response(self, request, response):
1388 self.cookiejar.extract_cookies(response, request)
1389 return response
1390
1391 https_request = http_request
1392 https_response = http_response
1393
1394 class UnknownHandler(BaseHandler):
1395 def unknown_open(self, req):
1396 type = req.type
1397 raise URLError('unknown url type: %s' % type)
1398
1399 def parse_keqv_list(l):
1400 """Parse list of key=value strings where keys are not duplicated."""
1401 parsed = {}
1402 for elt in l:
1403 k, v = elt.split('=', 1)
1404 if v[0] == '"' and v[-1] == '"':
1405 v = v[1:-1]
1406 parsed[k] = v
1407 return parsed
1408
1409 def parse_http_list(s):
1410 """Parse lists as described by RFC 2068 Section 2.
1411
1412 In particular, parse comma-separated lists where the elements of
1413 the list may include quoted-strings. A quoted-string could
1414 contain a comma. A non-quoted string could have quotes in the
1415 middle. Neither commas nor quotes count if they are escaped.
1416 Only double-quotes count, not single-quotes.
1417 """
1418 res = []
1419 part = ''
1420
1421 escape = quote = False
1422 for cur in s:
1423 if escape:
1424 part += cur
1425 escape = False
1426 continue
1427 if quote:
1428 if cur == '\\':
1429 escape = True
1430 continue
1431 elif cur == '"':
1432 quote = False
1433 part += cur
1434 continue
1435
1436 if cur == ',':
1437 res.append(part)
1438 part = ''
1439 continue
1440
1441 if cur == '"':
1442 quote = True
1443
1444 part += cur
1445
1446 # append last part
1447 if part:
1448 res.append(part)
1449
1450 return [part.strip() for part in res]
1451
1452 class FileHandler(BaseHandler):
1453 # names for the localhost
1454 names = None
1455 def get_names(self):
1456 if FileHandler.names is None:
1457 try:
1458 FileHandler.names = tuple(
1459 socket.gethostbyname_ex('localhost')[2] +
1460 socket.gethostbyname_ex(socket.gethostname())[2])
1461 except socket.gaierror:
1462 FileHandler.names = (socket.gethostbyname('localhost'),)
1463 return FileHandler.names
1464
1465 # not entirely sure what the rules are here
1466 def open_local_file(self, req):
1467 import email.utils
1468 import mimetypes
1469 localfile = url2pathname(req.full_url, require_scheme=True, resolve_host=True)
1470 try:
1471 stats = os.stat(localfile)
1472 size = stats.st_size
1473 modified = email.utils.formatdate(stats.st_mtime, usegmt=True)
1474 mtype = mimetypes.guess_file_type(localfile)[0]
1475 headers = email.message_from_string(
1476 'Content-type: %s\nContent-length: %d\nLast-modified: %s\n' %
1477 (mtype or 'text/plain', size, modified))
1478 origurl = pathname2url(localfile, add_scheme=True)
1479 return addinfourl(open(localfile, 'rb'), headers, origurl)
1480 except OSError as exp:
1481 raise URLError(exp, exp.filename)
1482
1483 file_open = open_local_file
1484
1485 def _is_local_authority(authority, resolve):
1486 # Compare hostnames
1487 if not authority or authority == 'localhost':
1488 return True
1489 try:
1490 hostname = socket.gethostname()
1491 except (socket.gaierror, AttributeError):
1492 pass
1493 else:
1494 if authority == hostname:
1495 return True
1496 # Compare IP addresses
1497 if not resolve:
1498 return False
1499 try:
1500 address = socket.gethostbyname(authority)
1501 except (socket.gaierror, AttributeError, UnicodeEncodeError):
1502 return False
1503 return address in FileHandler().get_names()
1504
1505 class FTPHandler(BaseHandler):
1506 def ftp_open(self, req):
1507 import ftplib
1508 import mimetypes
1509 host = req.host
1510 if not host:
1511 raise URLError('ftp error: no host given')
1512 host, port = _splitport(host)
1513 if port is None:
1514 port = ftplib.FTP_PORT
1515 else:
1516 port = int(port)
1517
1518 # username/password handling
1519 user, host = _splituser(host)
1520 if user:
1521 user, passwd = _splitpasswd(user)
1522 else:
1523 passwd = None
1524 host = unquote(host)
1525 user = user or ''
1526 passwd = passwd or ''
1527
1528 try:
1529 host = socket.gethostbyname(host)
1530 except OSError as msg:
1531 raise URLError(msg)
1532 path, attrs = _splitattr(req.selector)
1533 dirs = path.split('/')
1534 dirs = list(map(unquote, dirs))
1535 dirs, file = dirs[:-1], dirs[-1]
1536 if dirs and not dirs[0]:
1537 dirs = dirs[1:]
1538 fw = None
1539 try:
1540 fw = self.connect_ftp(user, passwd, host, port, dirs, req.timeout)
1541 type = file and 'I' or 'D'
1542 for attr in attrs:
1543 attr, value = _splitvalue(attr)
1544 if attr.lower() == 'type' and \
1545 value in ('a', 'A', 'i', 'I', 'd', 'D'):
1546 type = value.upper()
1547 fp, retrlen = fw.retrfile(file, type)
1548 headers = ""
1549 mtype = mimetypes.guess_type(req.full_url)[0]
1550 if mtype:
1551 headers += "Content-type: %s\n" % mtype
1552 if retrlen is not None and retrlen >= 0:
1553 headers += "Content-length: %d\n" % retrlen
1554 headers = email.message_from_string(headers)
1555 return addinfourl(fp, headers, req.full_url)
1556 except Exception as exp:
1557 if fw is not None and not fw.keepalive:
1558 fw.close()
1559 if isinstance(exp, ftplib.all_errors):
1560 raise URLError(f"ftp error: {exp}") from exp
1561 raise
1562
1563 def connect_ftp(self, user, passwd, host, port, dirs, timeout):
1564 return ftpwrapper(user, passwd, host, port, dirs, timeout,
1565 persistent=False)
1566
1567 class CacheFTPHandler(FTPHandler):
1568 # XXX would be nice to have pluggable cache strategies
1569 # XXX this stuff is definitely not thread safe
1570 def __init__(self):
1571 self.cache = {}
1572 self.timeout = {}
1573 self.soonest = 0
1574 self.delay = 60
1575 self.max_conns = 16
1576
1577 def setTimeout(self, t):
1578 self.delay = t
1579
1580 def setMaxConns(self, m):
1581 self.max_conns = m
1582
1583 def connect_ftp(self, user, passwd, host, port, dirs, timeout):
1584 key = user, host, port, '/'.join(dirs), timeout
1585 conn = self.cache.get(key)
1586 if conn is None or not conn.keepalive:
1587 if conn is not None:
1588 conn.close()
1589 conn = self.cache[key] = ftpwrapper(user, passwd, host, port,
1590 dirs, timeout)
1591 self.timeout[key] = time.time() + self.delay
1592 self.check_cache()
1593 return conn
1594
1595 def check_cache(self):
1596 # first check for old ones
1597 t = time.time()
1598 if self.soonest <= t:
1599 for k, v in list(self.timeout.items()):
1600 if v < t:
1601 self.cache[k].close()
1602 del self.cache[k]
1603 del self.timeout[k]
1604 self.soonest = min(list(self.timeout.values()))
1605
1606 # then check the size
1607 if len(self.cache) == self.max_conns:
1608 for k, v in list(self.timeout.items()):
1609 if v == self.soonest:
1610 del self.cache[k]
1611 del self.timeout[k]
1612 break
1613 self.soonest = min(list(self.timeout.values()))
1614
1615 def clear_cache(self):
1616 for conn in self.cache.values():
1617 conn.close()
1618 self.cache.clear()
1619 self.timeout.clear()
1620
1621 class DataHandler(BaseHandler):
1622 def data_open(self, req):
1623 # data URLs as specified in RFC 2397.
1624 #
1625 # ignores POSTed data
1626 #
1627 # syntax:
1628 # dataurl := "data:" [ mediatype ] [ ";base64" ] "," data
1629 # mediatype := [ type "/" subtype ] *( ";" parameter )
1630 # data := *urlchar
1631 # parameter := attribute "=" value
1632 url = req.full_url
1633
1634 scheme, data = url.split(":",1)
1635 mediatype, data = data.split(",",1)
1636
1637 # Disallow control characters within mediatype.
1638 if re.search(r"[\x00-\x1F\x7F]", mediatype):
1639 raise ValueError(
1640 "Control characters not allowed in data: mediatype")
1641
1642 # even base64 encoded data URLs might be quoted so unquote in any case:
1643 data = unquote_to_bytes(data)
1644 if mediatype.endswith(";base64"):
1645 data = base64.decodebytes(data)
1646 mediatype = mediatype[:-7]
1647
1648 if not mediatype:
1649 mediatype = "text/plain;charset=US-ASCII"
1650
1651 headers = email.message_from_string("Content-type: %s\nContent-length: %d\n" %
1652 (mediatype, len(data)))
1653
1654 return addinfourl(io.BytesIO(data), headers, url)
1655
1656
1657 # Code moved from the old urllib module
1658
1659 def url2pathname(url, *, require_scheme=False, resolve_host=False):
1660 """Convert the given file URL to a local file system path.
1661
1662 The 'file:' scheme prefix must be omitted unless *require_scheme*
1663 is set to true.
1664
1665 The URL authority may be resolved with gethostbyname() if
1666 *resolve_host* is set to true.
1667 """
1668 if not require_scheme:
1669 url = 'file:' + url
1670 scheme, authority, url = urlsplit(url)[:3] # Discard query and fragment.
1671 if scheme != 'file':
1672 raise URLError("URL is missing a 'file:' scheme")
1673 if os.name == 'nt':
1674 if authority[1:2] == ':':
1675 # e.g. file://c:/file.txt
1676 url = authority + url
1677 elif not _is_local_authority(authority, resolve_host):
1678 # e.g. file://server/share/file.txt
1679 url = '//' + authority + url
1680 elif url[:3] == '///':
1681 # e.g. file://///server/share/file.txt
1682 url = url[1:]
1683 else:
1684 if url[:1] == '/' and url[2:3] in (':', '|'):
1685 # Skip past extra slash before DOS drive in URL path.
1686 url = url[1:]
1687 if url[1:2] == '|':
1688 # Older URLs use a pipe after a drive letter
1689 url = url[:1] + ':' + url[2:]
1690 url = url.replace('/', '\\')
1691 elif not _is_local_authority(authority, resolve_host):
1692 raise URLError("file:// scheme is supported only on localhost")
1693 encoding = sys.getfilesystemencoding()
1694 errors = sys.getfilesystemencodeerrors()
1695 return unquote(url, encoding=encoding, errors=errors)
1696
1697
1698 def pathname2url(pathname, *, add_scheme=False):
1699 """Convert the given local file system path to a file URL.
1700
1701 The 'file:' scheme prefix is omitted unless *add_scheme*
1702 is set to true.
1703 """
1704 if os.name == 'nt':
1705 pathname = pathname.replace('\\', '/')
1706 encoding = sys.getfilesystemencoding()
1707 errors = sys.getfilesystemencodeerrors()
1708 scheme = 'file:' if add_scheme else ''
1709 drive, root, tail = os.path.splitroot(pathname)
1710 if drive:
1711 # First, clean up some special forms. We are going to sacrifice the
1712 # additional information anyway
1713 if drive[:4] == '//?/':
1714 drive = drive[4:]
1715 if drive[:4].upper() == 'UNC/':
1716 drive = '//' + drive[4:]
1717 if drive[1:] == ':':
1718 # DOS drive specified. Add three slashes to the start, producing
1719 # an authority section with a zero-length authority, and a path
1720 # section starting with a single slash.
1721 drive = '///' + drive
1722 drive = quote(drive, encoding=encoding, errors=errors, safe='/:')
1723 elif root:
1724 # Add explicitly empty authority to absolute path. If the path
1725 # starts with exactly one slash then this change is mostly
1726 # cosmetic, but if it begins with two or more slashes then this
1727 # avoids interpreting the path as a URL authority.
1728 root = '//' + root
1729 tail = quote(tail, encoding=encoding, errors=errors)
1730 return scheme + drive + root + tail
1731
1732
1733 # Utility functions
1734
1735 _localhost = None
1736 def localhost():
1737 """Return the IP address of the magic hostname 'localhost'."""
1738 global _localhost
1739 if _localhost is None:
1740 _localhost = socket.gethostbyname('localhost')
1741 return _localhost
1742
1743 _thishost = None
1744 def thishost():
1745 """Return the IP addresses of the current host."""
1746 global _thishost
1747 if _thishost is None:
1748 try:
1749 _thishost = tuple(socket.gethostbyname_ex(socket.gethostname())[2])
1750 except socket.gaierror:
1751 _thishost = tuple(socket.gethostbyname_ex('localhost')[2])
1752 return _thishost
1753
1754 _ftperrors = None
1755 def ftperrors():
1756 """Return the set of errors raised by the FTP class."""
1757 global _ftperrors
1758 if _ftperrors is None:
1759 import ftplib
1760 _ftperrors = ftplib.all_errors
1761 return _ftperrors
1762
1763 _noheaders = None
1764 def noheaders():
1765 """Return an empty email Message object."""
1766 global _noheaders
1767 if _noheaders is None:
1768 _noheaders = email.message_from_string("")
1769 return _noheaders
1770
1771
1772 # Utility classes
1773
1774 class ftpwrapper:
1775 """Class used by open_ftp() for cache of open FTP connections."""
1776
1777 def __init__(self, user, passwd, host, port, dirs, timeout=None,
1778 persistent=True):
1779 self.user = user
1780 self.passwd = passwd
1781 self.host = host
1782 self.port = port
1783 self.dirs = dirs
1784 self.timeout = timeout
1785 self.refcount = 0
1786 self.keepalive = persistent
1787 try:
1788 self.init()
1789 except:
1790 self.close()
1791 raise
1792
1793 def init(self):
1794 import ftplib
1795 self.busy = 0
1796 self.ftp = ftplib.FTP()
1797 self.ftp.connect(self.host, self.port, self.timeout)
1798 self.ftp.login(self.user, self.passwd)
1799 _target = '/'.join(self.dirs)
1800 self.ftp.cwd(_target)
1801
1802 def retrfile(self, file, type):
1803 import ftplib
1804 self.endtransfer()
1805 if type in ('d', 'D'): cmd = 'TYPE A'; isdir = 1
1806 else: cmd = 'TYPE ' + type; isdir = 0
1807 try:
1808 self.ftp.voidcmd(cmd)
1809 except ftplib.all_errors:
1810 self.init()
1811 self.ftp.voidcmd(cmd)
1812 conn = None
1813 if file and not isdir:
1814 # Try to retrieve as a file
1815 try:
1816 cmd = 'RETR ' + file
1817 conn, retrlen = self.ftp.ntransfercmd(cmd)
1818 except ftplib.error_perm as reason:
1819 if str(reason)[:3] != '550':
1820 raise URLError(f'ftp error: {reason}') from reason
1821 if not conn:
1822 # Set transfer mode to ASCII!
1823 self.ftp.voidcmd('TYPE A')
1824 # Try a directory listing. Verify that directory exists.
1825 if file:
1826 pwd = self.ftp.pwd()
1827 try:
1828 try:
1829 self.ftp.cwd(file)
1830 except ftplib.error_perm as reason:
1831 raise URLError('ftp error: %r' % reason) from reason
1832 finally:
1833 self.ftp.cwd(pwd)
1834 cmd = 'LIST ' + file
1835 else:
1836 cmd = 'LIST'
1837 conn, retrlen = self.ftp.ntransfercmd(cmd)
1838 self.busy = 1
1839
1840 ftpobj = addclosehook(conn.makefile('rb'), self.file_close)
1841 self.refcount += 1
1842 conn.close()
1843 # Pass back both a suitably decorated object and a retrieval length
1844 return (ftpobj, retrlen)
1845
1846 def endtransfer(self):
1847 if not self.busy:
1848 return
1849 self.busy = 0
1850 try:
1851 self.ftp.voidresp()
1852 except ftperrors():
1853 pass
1854
1855 def close(self):
1856 self.keepalive = False
1857 if self.refcount <= 0:
1858 self.real_close()
1859
1860 def file_close(self):
1861 self.endtransfer()
1862 self.refcount -= 1
1863 if self.refcount <= 0 and not self.keepalive:
1864 self.real_close()
1865
1866 def real_close(self):
1867 self.endtransfer()
1868 try:
1869 self.ftp.close()
1870 except ftperrors():
1871 pass
1872
1873 # Proxy handling
1874 def getproxies_environment():
1875 """Return a dictionary of scheme -> proxy server URL mappings.
1876
1877 Scan the environment for variables named <scheme>_proxy;
1878 this seems to be the standard convention.
1879 """
1880 # in order to prefer lowercase variables, process environment in
1881 # two passes: first matches any, second pass matches lowercase only
1882
1883 # select only environment variables which end in (after making lowercase) _proxy
1884 proxies = {}
1885 environment = []
1886 for name in os.environ:
1887 # fast screen underscore position before more expensive case-folding
1888 if len(name) > 5 and name[-6] == "_" and name[-5:].lower() == "proxy":
1889 value = os.environ[name]
1890 proxy_name = name[:-6].lower()
1891 environment.append((name, value, proxy_name))
1892 if value:
1893 proxies[proxy_name] = value
1894 # CVE-2016-1000110 - If we are running as CGI script, forget HTTP_PROXY
1895 # (non-all-lowercase) as it may be set from the web server by a "Proxy:"
1896 # header from the client
1897 # If "proxy" is lowercase, it will still be used thanks to the next block
1898 if 'REQUEST_METHOD' in os.environ:
1899 proxies.pop('http', None)
1900 for name, value, proxy_name in environment:
1901 # not case-folded, checking here for lower-case env vars only
1902 if name[-6:] == '_proxy':
1903 if value:
1904 proxies[proxy_name] = value
1905 else:
1906 proxies.pop(proxy_name, None)
1907 return proxies
1908
1909 def proxy_bypass_environment(host, proxies=None):
1910 """Test if proxies should not be used for a particular host.
1911
1912 Checks the proxy dict for the value of no_proxy, which should
1913 be a list of comma separated DNS suffixes, or '*' for all hosts.
1914
1915 """
1916 if proxies is None:
1917 proxies = getproxies_environment()
1918 # don't bypass, if no_proxy isn't specified
1919 try:
1920 no_proxy = proxies['no']
1921 except KeyError:
1922 return False
1923 # '*' is special case for always bypass
1924 if no_proxy == '*':
1925 return True
1926 host = host.lower()
1927 # strip port off host
1928 hostonly, port = _splitport(host)
1929 # check if the host ends with any of the DNS suffixes
1930 for name in no_proxy.split(','):
1931 name = name.strip()
1932 if name:
1933 name = name.lstrip('.') # ignore leading dots
1934 name = name.lower()
1935 if hostonly == name or host == name:
1936 return True
1937 name = '.' + name
1938 if hostonly.endswith(name) or host.endswith(name):
1939 return True
1940 # otherwise, don't bypass
1941 return False
1942
1943
1944 # This code tests an OSX specific data structure but is testable on all
1945 # platforms
1946 def _proxy_bypass_macosx_sysconf(host, proxy_settings):
1947 """
1948 Return True iff this host shouldn't be accessed using a proxy
1949
1950 This function uses the MacOSX framework SystemConfiguration
1951 to fetch the proxy information.
1952
1953 proxy_settings come from _scproxy._get_proxy_settings or get mocked ie:
1954 { 'exclude_simple': bool,
1955 'exceptions': ['foo.bar', '*.bar.com', '127.0.0.1', '10.1', '10.0/16']
1956 }
1957 """
1958 from fnmatch import fnmatch
1959 from ipaddress import AddressValueError, IPv4Address
1960
1961 hostonly, port = _splitport(host)
1962
1963 def ip2num(ipAddr):
1964 parts = ipAddr.split('.')
1965 parts = list(map(int, parts))
1966 if len(parts) != 4:
1967 parts = (parts + [0, 0, 0, 0])[:4]
1968 return (parts[0] << 24) | (parts[1] << 16) | (parts[2] << 8) | parts[3]
1969
1970 # Check for simple host names:
1971 if '.' not in host:
1972 if proxy_settings['exclude_simple']:
1973 return True
1974
1975 hostIP = None
1976 try:
1977 hostIP = int(IPv4Address(hostonly))
1978 except AddressValueError:
1979 pass
1980
1981 for value in proxy_settings.get('exceptions', ()):
1982 # Items in the list are strings like these: *.local, 169.254/16
1983 if not value: continue
1984
1985 m = re.match(r"(\d+(?:\.\d+)*)(/\d+)?", value)
1986 if m is not None and hostIP is not None:
1987 base = ip2num(m.group(1))
1988 mask = m.group(2)
1989 if mask is None:
1990 mask = 8 * (m.group(1).count('.') + 1)
1991 else:
1992 mask = int(mask[1:])
1993
1994 if mask < 0 or mask > 32:
1995 # System libraries ignore invalid prefix lengths
1996 continue
1997
1998 mask = 32 - mask
1999
2000 if (hostIP >> mask) == (base >> mask):
2001 return True
2002
2003 elif fnmatch(host, value):
2004 return True
2005
2006 return False
2007
2008
2009 # Same as _proxy_bypass_macosx_sysconf, testable on all platforms
2010 def _proxy_bypass_winreg_override(host, override):
2011 """Return True if the host should bypass the proxy server.
2012
2013 The proxy override list is obtained from the Windows
2014 Internet settings proxy override registry value.
2015
2016 An example of a proxy override value is:
2017 "www.example.com;*.example.net; 192.168.0.1"
2018 """
2019 from fnmatch import fnmatch
2020
2021 host, _ = _splitport(host)
2022 proxy_override = override.split(';')
2023 for test in proxy_override:
2024 test = test.strip()
2025 # "<local>" should bypass the proxy server for all intranet addresses
2026 if test == '<local>':
2027 if '.' not in host:
2028 return True
2029 elif fnmatch(host, test):
2030 return True
2031 return False
2032
2033
2034 if sys.platform == 'darwin':
2035 from _scproxy import _get_proxy_settings, _get_proxies
2036
2037 def proxy_bypass_macosx_sysconf(host):
2038 proxy_settings = _get_proxy_settings()
2039 return _proxy_bypass_macosx_sysconf(host, proxy_settings)
2040
2041 def getproxies_macosx_sysconf():
2042 """Return a dictionary of scheme -> proxy server URL mappings.
2043
2044 This function uses the MacOSX framework SystemConfiguration
2045 to fetch the proxy information.
2046 """
2047 return _get_proxies()
2048
2049
2050
2051 def proxy_bypass(host):
2052 """Return True, if host should be bypassed.
2053
2054 Checks proxy settings gathered from the environment, if specified,
2055 or from the MacOSX framework SystemConfiguration.
2056
2057 """
2058 proxies = getproxies_environment()
2059 if proxies:
2060 return proxy_bypass_environment(host, proxies)
2061 else:
2062 return proxy_bypass_macosx_sysconf(host)
2063
2064 def getproxies():
2065 return getproxies_environment() or getproxies_macosx_sysconf()
2066
2067
2068 elif os.name == 'nt':
2069 def getproxies_registry():
2070 """Return a dictionary of scheme -> proxy server URL mappings.
2071
2072 Win32 uses the registry to store proxies.
2073
2074 """
2075 proxies = {}
2076 try:
2077 import winreg
2078 except ImportError:
2079 # Std module, so should be around - but you never know!
2080 return proxies
2081 try:
2082 internetSettings = winreg.OpenKey(winreg.HKEY_CURRENT_USER,
2083 r'Software\Microsoft\Windows\CurrentVersion\Internet Settings')
2084 proxyEnable = winreg.QueryValueEx(internetSettings,
2085 'ProxyEnable')[0]
2086 if proxyEnable:
2087 # Returned as Unicode but problems if not converted to ASCII
2088 proxyServer = str(winreg.QueryValueEx(internetSettings,
2089 'ProxyServer')[0])
2090 if '=' not in proxyServer and ';' not in proxyServer:
2091 # Use one setting for all protocols.
2092 proxyServer = 'http={0};https={0};ftp={0}'.format(proxyServer)
2093 for p in proxyServer.split(';'):
2094 protocol, address = p.split('=', 1)
2095 # See if address has a type:// prefix
2096 if not re.match('(?:[^/:]+)://', address):
2097 # Add type:// prefix to address without specifying type
2098 if protocol in ('http', 'https', 'ftp'):
2099 # The default proxy type of Windows is HTTP
2100 address = 'http://' + address
2101 elif protocol == 'socks':
2102 address = 'socks://' + address
2103 proxies[protocol] = address
2104 # Use SOCKS proxy for HTTP(S) protocols
2105 if proxies.get('socks'):
2106 # The default SOCKS proxy type of Windows is SOCKS4
2107 address = re.sub(r'^socks://', 'socks4://', proxies['socks'])
2108 proxies['http'] = proxies.get('http') or address
2109 proxies['https'] = proxies.get('https') or address
2110 internetSettings.Close()
2111 except (OSError, ValueError, TypeError):
2112 # Either registry key not found etc, or the value in an
2113 # unexpected format.
2114 # proxies already set up to be empty so nothing to do
2115 pass
2116 return proxies
2117
2118 def getproxies():
2119 """Return a dictionary of scheme -> proxy server URL mappings.
2120
2121 Returns settings gathered from the environment, if specified,
2122 or the registry.
2123
2124 """
2125 return getproxies_environment() or getproxies_registry()
2126
2127 def proxy_bypass_registry(host):
2128 try:
2129 import winreg
2130 except ImportError:
2131 # Std modules, so should be around - but you never know!
2132 return False
2133 try:
2134 internetSettings = winreg.OpenKey(winreg.HKEY_CURRENT_USER,
2135 r'Software\Microsoft\Windows\CurrentVersion\Internet Settings')
2136 proxyEnable = winreg.QueryValueEx(internetSettings,
2137 'ProxyEnable')[0]
2138 proxyOverride = str(winreg.QueryValueEx(internetSettings,
2139 'ProxyOverride')[0])
2140 # ^^^^ Returned as Unicode but problems if not converted to ASCII
2141 except OSError:
2142 return False
2143 if not proxyEnable or not proxyOverride:
2144 return False
2145 return _proxy_bypass_winreg_override(host, proxyOverride)
2146
2147 def proxy_bypass(host):
2148 """Return True, if host should be bypassed.
2149
2150 Checks proxy settings gathered from the environment, if specified,
2151 or the registry.
2152
2153 """
2154 proxies = getproxies_environment()
2155 if proxies:
2156 return proxy_bypass_environment(host, proxies)
2157 else:
2158 return proxy_bypass_registry(host)
2159
2160 else:
2161 # By default use environment variables
2162 getproxies = getproxies_environment
2163 proxy_bypass = proxy_bypass_environment