Debian Codegraph @

Lib/test/test_urllib2_localnet.py

raw
1 import base64
2 import os
3 import email
4 import urllib.parse
5 import urllib.request
6 import http.server
7 import threading
8 import unittest
9 import hashlib
10
11 from test import support
12 from test.support import hashlib_helper
13 from test.support import threading_helper
14
15 try:
16 import ssl
17 except ImportError:
18 ssl = None
19
20 support.requires_working_socket(module=True)
21
22 here = os.path.dirname(__file__)
23 # Self-signed cert file for 'localhost'
24 CERT_localhost = os.path.join(here, 'certdata', 'keycert.pem')
25 # Self-signed cert file for 'fakehostname'
26 CERT_fakehostname = os.path.join(here, 'certdata', 'keycert2.pem')
27
28
29 # Loopback http server infrastructure
30
31 class LoopbackHttpServer(http.server.HTTPServer):
32 """HTTP server w/ a few modifications that make it useful for
33 loopback testing purposes.
34 """
35
36 def __init__(self, server_address, RequestHandlerClass):
37 http.server.HTTPServer.__init__(self,
38 server_address,
39 RequestHandlerClass)
40
41 # Set the timeout of our listening socket really low so
42 # that we can stop the server easily.
43 self.socket.settimeout(0.1)
44
45 def get_request(self):
46 """HTTPServer method, overridden."""
47
48 request, client_address = self.socket.accept()
49
50 # It's a loopback connection, so setting the timeout
51 # really low shouldn't affect anything, but should make
52 # deadlocks less likely to occur.
53 request.settimeout(10.0)
54
55 return (request, client_address)
56
57 class LoopbackHttpServerThread(threading.Thread):
58 """Stoppable thread that runs a loopback http server."""
59
60 def __init__(self, request_handler):
61 threading.Thread.__init__(self)
62 self._stop_server = False
63 self.ready = threading.Event()
64 request_handler.protocol_version = "HTTP/1.0"
65 self.httpd = LoopbackHttpServer(("127.0.0.1", 0),
66 request_handler)
67 self.port = self.httpd.server_port
68
69 def stop(self):
70 """Stops the webserver if it's currently running."""
71
72 self._stop_server = True
73
74 self.join()
75 self.httpd.server_close()
76
77 def run(self):
78 self.ready.set()
79 while not self._stop_server:
80 self.httpd.handle_request()
81
82 # Authentication infrastructure
83
84 class DigestAuthHandler:
85 """Handler for performing digest authentication."""
86
87 def __init__(self):
88 self._request_num = 0
89 self._nonces = []
90 self._users = {}
91 self._realm_name = "Test Realm"
92 self._qop = "auth"
93
94 def set_qop(self, qop):
95 self._qop = qop
96
97 def set_users(self, users):
98 assert isinstance(users, dict)
99 self._users = users
100
101 def set_realm(self, realm):
102 self._realm_name = realm
103
104 def _generate_nonce(self):
105 self._request_num += 1
106 nonce = hashlib.md5(str(self._request_num).encode("ascii")).hexdigest()
107 self._nonces.append(nonce)
108 return nonce
109
110 def _create_auth_dict(self, auth_str):
111 first_space_index = auth_str.find(" ")
112 auth_str = auth_str[first_space_index+1:]
113
114 parts = auth_str.split(",")
115
116 auth_dict = {}
117 for part in parts:
118 name, value = part.split("=")
119 name = name.strip()
120 if value[0] == '"' and value[-1] == '"':
121 value = value[1:-1]
122 else:
123 value = value.strip()
124 auth_dict[name] = value
125 return auth_dict
126
127 def _validate_auth(self, auth_dict, password, method, uri):
128 final_dict = {}
129 final_dict.update(auth_dict)
130 final_dict["password"] = password
131 final_dict["method"] = method
132 final_dict["uri"] = uri
133 HA1_str = "%(username)s:%(realm)s:%(password)s" % final_dict
134 HA1 = hashlib.md5(HA1_str.encode("ascii")).hexdigest()
135 HA2_str = "%(method)s:%(uri)s" % final_dict
136 HA2 = hashlib.md5(HA2_str.encode("ascii")).hexdigest()
137 final_dict["HA1"] = HA1
138 final_dict["HA2"] = HA2
139 response_str = "%(HA1)s:%(nonce)s:%(nc)s:" \
140 "%(cnonce)s:%(qop)s:%(HA2)s" % final_dict
141 response = hashlib.md5(response_str.encode("ascii")).hexdigest()
142
143 return response == auth_dict["response"]
144
145 def _return_auth_challenge(self, request_handler):
146 request_handler.send_response(407, "Proxy Authentication Required")
147 request_handler.send_header("Content-Type", "text/html")
148 request_handler.send_header(
149 'Proxy-Authenticate', 'Digest realm="%s", '
150 'qop="%s",'
151 'nonce="%s", ' % \
152 (self._realm_name, self._qop, self._generate_nonce()))
153 # XXX: Not sure if we're supposed to add this next header or
154 # not.
155 #request_handler.send_header('Connection', 'close')
156 request_handler.end_headers()
157 request_handler.wfile.write(b"Proxy Authentication Required.")
158 return False
159
160 def handle_request(self, request_handler):
161 """Performs digest authentication on the given HTTP request
162 handler. Returns True if authentication was successful, False
163 otherwise.
164
165 If no users have been set, then digest auth is effectively
166 disabled and this method will always return True.
167 """
168
169 if len(self._users) == 0:
170 return True
171
172 if "Proxy-Authorization" not in request_handler.headers:
173 return self._return_auth_challenge(request_handler)
174 else:
175 auth_dict = self._create_auth_dict(
176 request_handler.headers["Proxy-Authorization"]
177 )
178 if auth_dict["username"] in self._users:
179 password = self._users[ auth_dict["username"] ]
180 else:
181 return self._return_auth_challenge(request_handler)
182 if not auth_dict.get("nonce") in self._nonces:
183 return self._return_auth_challenge(request_handler)
184 else:
185 self._nonces.remove(auth_dict["nonce"])
186
187 auth_validated = False
188
189 # MSIE uses short_path in its validation, but Python's
190 # urllib.request uses the full path, so we're going to see if
191 # either of them works here.
192
193 for path in [request_handler.path, request_handler.short_path]:
194 if self._validate_auth(auth_dict,
195 password,
196 request_handler.command,
197 path):
198 auth_validated = True
199
200 if not auth_validated:
201 return self._return_auth_challenge(request_handler)
202 return True
203
204
205 class BasicAuthHandler(http.server.BaseHTTPRequestHandler):
206 """Handler for performing basic authentication."""
207 # Server side values
208 USER = 'testUser'
209 PASSWD = 'testPass'
210 REALM = 'Test'
211 USER_PASSWD = "%s:%s" % (USER, PASSWD)
212 ENCODED_AUTH = base64.b64encode(USER_PASSWD.encode('ascii')).decode('ascii')
213
214 def __init__(self, *args, **kwargs):
215 http.server.BaseHTTPRequestHandler.__init__(self, *args, **kwargs)
216
217 def log_message(self, format, *args):
218 # Suppress console log message
219 pass
220
221 def do_HEAD(self):
222 self.send_response(200)
223 self.send_header("Content-type", "text/html")
224 self.end_headers()
225
226 def do_AUTHHEAD(self):
227 self.send_response(401)
228 self.send_header("WWW-Authenticate", "Basic realm=\"%s\"" % self.REALM)
229 self.send_header("Content-type", "text/html")
230 self.end_headers()
231
232 def do_GET(self):
233 if not self.headers.get("Authorization", ""):
234 self.do_AUTHHEAD()
235 self.wfile.write(b"No Auth header received")
236 elif self.headers.get(
237 "Authorization", "") == "Basic " + self.ENCODED_AUTH:
238 self.send_response(200)
239 self.end_headers()
240 self.wfile.write(b"It works")
241 else:
242 # Request Unauthorized
243 self.do_AUTHHEAD()
244
245
246
247 # Proxy test infrastructure
248
249 class FakeProxyHandler(http.server.BaseHTTPRequestHandler):
250 """This is a 'fake proxy' that makes it look like the entire
251 internet has gone down due to a sudden zombie invasion. It main
252 utility is in providing us with authentication support for
253 testing.
254 """
255
256 def __init__(self, digest_auth_handler, *args, **kwargs):
257 # This has to be set before calling our parent's __init__(), which will
258 # try to call do_GET().
259 self.digest_auth_handler = digest_auth_handler
260 http.server.BaseHTTPRequestHandler.__init__(self, *args, **kwargs)
261
262 def log_message(self, format, *args):
263 # Uncomment the next line for debugging.
264 # sys.stderr.write(format % args)
265 pass
266
267 def do_GET(self):
268 (scm, netloc, path, params, query, fragment) = urllib.parse.urlparse(
269 self.path, "http")
270 self.short_path = path
271 if self.digest_auth_handler.handle_request(self):
272 self.send_response(200, "OK")
273 self.send_header("Content-Type", "text/html")
274 self.end_headers()
275 self.wfile.write(bytes("You've reached %s!<BR>" % self.path,
276 "ascii"))
277 self.wfile.write(b"Our apologies, but our server is down due to "
278 b"a sudden zombie invasion.")
279
280 # Test cases
281
282 class BasicAuthTests(unittest.TestCase):
283 USER = "testUser"
284 PASSWD = "testPass"
285 INCORRECT_PASSWD = "Incorrect"
286 REALM = "Test"
287
288 def setUp(self):
289 super(BasicAuthTests, self).setUp()
290 # With Basic Authentication
291 def http_server_with_basic_auth_handler(*args, **kwargs):
292 return BasicAuthHandler(*args, **kwargs)
293 self.server = LoopbackHttpServerThread(http_server_with_basic_auth_handler)
294 self.addCleanup(self.stop_server)
295 self.server_url = 'http://127.0.0.1:%s' % self.server.port
296 self.server.start()
297 self.server.ready.wait()
298
299 def stop_server(self):
300 self.server.stop()
301 self.server = None
302
303 def tearDown(self):
304 super(BasicAuthTests, self).tearDown()
305
306 def test_basic_auth_success(self):
307 ah = urllib.request.HTTPBasicAuthHandler()
308 ah.add_password(self.REALM, self.server_url, self.USER, self.PASSWD)
309 urllib.request.install_opener(urllib.request.build_opener(ah))
310 try:
311 self.assertTrue(urllib.request.urlopen(self.server_url))
312 except urllib.error.HTTPError:
313 self.fail("Basic auth failed for the url: %s" % self.server_url)
314
315 def test_basic_auth_httperror(self):
316 ah = urllib.request.HTTPBasicAuthHandler()
317 ah.add_password(self.REALM, self.server_url, self.USER, self.INCORRECT_PASSWD)
318 urllib.request.install_opener(urllib.request.build_opener(ah))
319 with self.assertRaises(urllib.error.HTTPError) as cm:
320 urllib.request.urlopen(self.server_url)
321 cm.exception.close()
322
323
324 @hashlib_helper.requires_hashdigest("md5", openssl=True)
325 class ProxyAuthTests(unittest.TestCase):
326 URL = "http://localhost"
327
328 USER = "tester"
329 PASSWD = "test123"
330 REALM = "TestRealm"
331
332 def setUp(self):
333 super(ProxyAuthTests, self).setUp()
334 # Ignore proxy bypass settings in the environment.
335 def restore_environ(old_environ):
336 os.environ.clear()
337 os.environ.update(old_environ)
338 self.addCleanup(restore_environ, os.environ.copy())
339 os.environ['NO_PROXY'] = ''
340 os.environ['no_proxy'] = ''
341
342 self.digest_auth_handler = DigestAuthHandler()
343 self.digest_auth_handler.set_users({self.USER: self.PASSWD})
344 self.digest_auth_handler.set_realm(self.REALM)
345 # With Digest Authentication.
346 def create_fake_proxy_handler(*args, **kwargs):
347 return FakeProxyHandler(self.digest_auth_handler, *args, **kwargs)
348
349 self.server = LoopbackHttpServerThread(create_fake_proxy_handler)
350 self.addCleanup(self.stop_server)
351 self.server.start()
352 self.server.ready.wait()
353 proxy_url = "http://127.0.0.1:%d" % self.server.port
354 handler = urllib.request.ProxyHandler({"http" : proxy_url})
355 self.proxy_digest_handler = urllib.request.ProxyDigestAuthHandler()
356 self.opener = urllib.request.build_opener(
357 handler, self.proxy_digest_handler)
358
359 def stop_server(self):
360 self.server.stop()
361 self.server = None
362
363 def test_proxy_with_bad_password_raises_httperror(self):
364 self.proxy_digest_handler.add_password(self.REALM, self.URL,
365 self.USER, self.PASSWD+"bad")
366 self.digest_auth_handler.set_qop("auth")
367 with self.assertRaises(urllib.error.HTTPError) as cm:
368 self.opener.open(self.URL)
369 cm.exception.close()
370
371 def test_proxy_with_no_password_raises_httperror(self):
372 self.digest_auth_handler.set_qop("auth")
373 with self.assertRaises(urllib.error.HTTPError) as cm:
374 self.opener.open(self.URL)
375 cm.exception.close()
376
377 def test_proxy_qop_auth_works(self):
378 self.proxy_digest_handler.add_password(self.REALM, self.URL,
379 self.USER, self.PASSWD)
380 self.digest_auth_handler.set_qop("auth")
381 with self.opener.open(self.URL) as result:
382 while result.read():
383 pass
384
385 def test_proxy_qop_auth_int_works_or_throws_urlerror(self):
386 self.proxy_digest_handler.add_password(self.REALM, self.URL,
387 self.USER, self.PASSWD)
388 self.digest_auth_handler.set_qop("auth-int")
389 try:
390 result = self.opener.open(self.URL)
391 except urllib.error.URLError:
392 # It's okay if we don't support auth-int, but we certainly
393 # shouldn't receive any kind of exception here other than
394 # a URLError.
395 pass
396 else:
397 with result:
398 while result.read():
399 pass
400
401
402 def GetRequestHandler(responses):
403
404 class FakeHTTPRequestHandler(http.server.BaseHTTPRequestHandler):
405
406 server_version = "TestHTTP/"
407 requests = []
408 headers_received = []
409 port = 80
410
411 def do_GET(self):
412 body = self.send_head()
413 while body:
414 done = self.wfile.write(body)
415 body = body[done:]
416
417 def do_POST(self):
418 content_length = self.headers["Content-Length"]
419 post_data = self.rfile.read(int(content_length))
420 self.do_GET()
421 self.requests.append(post_data)
422
423 def send_head(self):
424 FakeHTTPRequestHandler.headers_received = self.headers
425 self.requests.append(self.path)
426 response_code, headers, body = responses.pop(0)
427
428 self.send_response(response_code)
429
430 for (header, value) in headers:
431 self.send_header(header, value % {'port':self.port})
432 if body:
433 self.send_header("Content-type", "text/plain")
434 self.end_headers()
435 return body
436 self.end_headers()
437
438 def log_message(self, *args):
439 pass
440
441
442 return FakeHTTPRequestHandler
443
444
445 class TestUrlopen(unittest.TestCase):
446 """Tests urllib.request.urlopen using the network.
447
448 These tests are not exhaustive. Assuming that testing using files does a
449 good job overall of some of the basic interface features. There are no
450 tests exercising the optional 'data' and 'proxies' arguments. No tests
451 for transparent redirection have been written.
452 """
453
454 def setUp(self):
455 super(TestUrlopen, self).setUp()
456
457 # clear _opener global variable
458 self.addCleanup(urllib.request.urlcleanup)
459
460 # Ignore proxies for localhost tests.
461 def restore_environ(old_environ):
462 os.environ.clear()
463 os.environ.update(old_environ)
464 self.addCleanup(restore_environ, os.environ.copy())
465 os.environ['NO_PROXY'] = '*'
466 os.environ['no_proxy'] = '*'
467
468 def urlopen(self, url, data=None, **kwargs):
469 l = []
470 f = urllib.request.urlopen(url, data, **kwargs)
471 try:
472 # Exercise various methods
473 l.extend(f.readlines(200))
474 l.append(f.readline())
475 l.append(f.read(1024))
476 l.append(f.read())
477 finally:
478 f.close()
479 return b"".join(l)
480
481 def stop_server(self):
482 self.server.stop()
483 self.server = None
484
485 def start_server(self, responses=None):
486 if responses is None:
487 responses = [(200, [], b"we don't care")]
488 handler = GetRequestHandler(responses)
489
490 self.server = LoopbackHttpServerThread(handler)
491 self.addCleanup(self.stop_server)
492 self.server.start()
493 self.server.ready.wait()
494 port = self.server.port
495 handler.port = port
496 return handler
497
498 def start_https_server(self, responses=None, **kwargs):
499 if not hasattr(urllib.request, 'HTTPSHandler'):
500 self.skipTest('ssl support required')
501 from test.ssl_servers import make_https_server
502 if responses is None:
503 responses = [(200, [], b"we care a bit")]
504 handler = GetRequestHandler(responses)
505 server = make_https_server(self, handler_class=handler, **kwargs)
506 handler.port = server.port
507 return handler
508
509 def test_redirection(self):
510 expected_response = b"We got here..."
511 responses = [
512 (302, [("Location", "http://localhost:%(port)s/somewhere_else")],
513 ""),
514 (200, [], expected_response)
515 ]
516
517 handler = self.start_server(responses)
518 data = self.urlopen("http://localhost:%s/" % handler.port)
519 self.assertEqual(data, expected_response)
520 self.assertEqual(handler.requests, ["/", "/somewhere_else"])
521
522 def test_chunked(self):
523 expected_response = b"hello world"
524 chunked_start = (
525 b'a\r\n'
526 b'hello worl\r\n'
527 b'1\r\n'
528 b'd\r\n'
529 b'0\r\n'
530 )
531 response = [(200, [("Transfer-Encoding", "chunked")], chunked_start)]
532 handler = self.start_server(response)
533 data = self.urlopen("http://localhost:%s/" % handler.port)
534 self.assertEqual(data, expected_response)
535
536 def test_404(self):
537 expected_response = b"Bad bad bad..."
538 handler = self.start_server([(404, [], expected_response)])
539
540 try:
541 self.urlopen("http://localhost:%s/weeble" % handler.port)
542 except urllib.error.URLError as f:
543 data = f.read()
544 f.close()
545 else:
546 self.fail("404 should raise URLError")
547
548 self.assertEqual(data, expected_response)
549 self.assertEqual(handler.requests, ["/weeble"])
550
551 def test_200(self):
552 expected_response = b"pycon 2008..."
553 handler = self.start_server([(200, [], expected_response)])
554 data = self.urlopen("http://localhost:%s/bizarre" % handler.port)
555 self.assertEqual(data, expected_response)
556 self.assertEqual(handler.requests, ["/bizarre"])
557
558 def test_200_with_parameters(self):
559 expected_response = b"pycon 2008..."
560 handler = self.start_server([(200, [], expected_response)])
561 data = self.urlopen("http://localhost:%s/bizarre" % handler.port,
562 b"get=with_feeling")
563 self.assertEqual(data, expected_response)
564 self.assertEqual(handler.requests, ["/bizarre", b"get=with_feeling"])
565
566 def test_https(self):
567 handler = self.start_https_server()
568 context = ssl.create_default_context(cafile=CERT_localhost)
569 data = self.urlopen("https://localhost:%s/bizarre" % handler.port, context=context)
570 self.assertEqual(data, b"we care a bit")
571
572 def test_https_sni(self):
573 if ssl is None:
574 self.skipTest("ssl module required")
575 if not ssl.HAS_SNI:
576 self.skipTest("SNI support required in OpenSSL")
577 sni_name = None
578 def cb_sni(ssl_sock, server_name, initial_context):
579 nonlocal sni_name
580 sni_name = server_name
581 context = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER)
582 context.set_servername_callback(cb_sni)
583 handler = self.start_https_server(context=context, certfile=CERT_localhost)
584 context = ssl.create_default_context(cafile=CERT_localhost)
585 self.urlopen("https://localhost:%s" % handler.port, context=context)
586 self.assertEqual(sni_name, "localhost")
587
588 def test_sending_headers(self):
589 handler = self.start_server()
590 req = urllib.request.Request("http://localhost:%s/" % handler.port,
591 headers={"Range": "bytes=20-39"})
592 with urllib.request.urlopen(req):
593 pass
594 self.assertEqual(handler.headers_received["Range"], "bytes=20-39")
595
596 def test_sending_headers_camel(self):
597 handler = self.start_server()
598 req = urllib.request.Request("http://localhost:%s/" % handler.port,
599 headers={"X-SoMe-hEader": "foobar"})
600 with urllib.request.urlopen(req):
601 pass
602 self.assertIn("X-Some-Header", handler.headers_received.keys())
603 self.assertNotIn("X-SoMe-hEader", handler.headers_received.keys())
604
605 def test_basic(self):
606 handler = self.start_server()
607 with urllib.request.urlopen("http://localhost:%s" % handler.port) as open_url:
608 for attr in ("read", "close", "info", "geturl"):
609 self.assertHasAttr(open_url, attr)
610 self.assertTrue(open_url.read(), "calling 'read' failed")
611
612 def test_info(self):
613 handler = self.start_server()
614 open_url = urllib.request.urlopen(
615 "http://localhost:%s" % handler.port)
616 with open_url:
617 info_obj = open_url.info()
618 self.assertIsInstance(info_obj, email.message.Message,
619 "object returned by 'info' is not an "
620 "instance of email.message.Message")
621 self.assertEqual(info_obj.get_content_subtype(), "plain")
622
623 def test_geturl(self):
624 # Make sure same URL as opened is returned by geturl.
625 handler = self.start_server()
626 open_url = urllib.request.urlopen("http://localhost:%s" % handler.port)
627 with open_url:
628 url = open_url.geturl()
629 self.assertEqual(url, "http://localhost:%s" % handler.port)
630
631 def test_iteration(self):
632 expected_response = b"pycon 2008..."
633 handler = self.start_server([(200, [], expected_response)])
634 data = urllib.request.urlopen("http://localhost:%s" % handler.port)
635 for line in data:
636 self.assertEqual(line, expected_response)
637
638 def test_line_iteration(self):
639 lines = [b"We\n", b"got\n", b"here\n", b"verylong " * 8192 + b"\n"]
640 expected_response = b"".join(lines)
641 handler = self.start_server([(200, [], expected_response)])
642 data = urllib.request.urlopen("http://localhost:%s" % handler.port)
643 for index, line in enumerate(data):
644 self.assertEqual(line, lines[index],
645 "Fetched line number %s doesn't match expected:\n"
646 " Expected length was %s, got %s" %
647 (index, len(lines[index]), len(line)))
648 self.assertEqual(index + 1, len(lines))
649
650 def test_issue16464(self):
651 # See https://bugs.python.org/issue16464
652 # and https://bugs.python.org/issue46648
653 handler = self.start_server([
654 (200, [], b'any'),
655 (200, [], b'any'),
656 ])
657 opener = urllib.request.build_opener()
658 request = urllib.request.Request("http://localhost:%s" % handler.port)
659 self.assertEqual(None, request.data)
660
661 opener.open(request, "1".encode("us-ascii"))
662 self.assertEqual(b"1", request.data)
663 self.assertEqual("1", request.get_header("Content-length"))
664
665 opener.open(request, "1234567890".encode("us-ascii"))
666 self.assertEqual(b"1234567890", request.data)
667 self.assertEqual("10", request.get_header("Content-length"))
668
669 def setUpModule():
670 thread_info = threading_helper.threading_setup()
671 unittest.addModuleCleanup(threading_helper.threading_cleanup, *thread_info)
672
673
674 if __name__ == "__main__":
675 unittest.main()