Debian Codegraph @

Lib/test/test_urllib2.py

raw
1 import unittest
2 from test import support
3 from test.support import os_helper
4 from test.support import requires_subprocess
5 from test.support import warnings_helper
6 from test import test_urllib
7 from unittest import mock
8
9 import os
10 import io
11 import ftplib
12 import socket
13 import array
14 import sys
15 import tempfile
16 import subprocess
17
18 import urllib.request
19 # The proxy bypass method imported below has logic specific to the
20 # corresponding system but is testable on all platforms.
21 from urllib.request import (Request, OpenerDirector, HTTPBasicAuthHandler,
22 HTTPPasswordMgrWithPriorAuth, _parse_proxy,
23 _proxy_bypass_winreg_override,
24 _proxy_bypass_macosx_sysconf,
25 AbstractDigestAuthHandler)
26 from urllib.parse import urlsplit
27 import urllib.error
28 import http.client
29
30
31 support.requires_working_socket(module=True)
32
33 # XXX
34 # Request
35 # CacheFTPHandler (hard to write)
36 # parse_keqv_list, parse_http_list, HTTPDigestAuthHandler
37
38
39 class TrivialTests(unittest.TestCase):
40
41 def test___all__(self):
42 # Verify which names are exposed
43 for module in 'request', 'response', 'parse', 'error', 'robotparser':
44 context = {}
45 exec('from urllib.%s import *' % module, context)
46 del context['__builtins__']
47 for k, v in context.items():
48 self.assertEqual(v.__module__, 'urllib.%s' % module,
49 "%r is exposed in 'urllib.%s' but defined in %r" %
50 (k, module, v.__module__))
51
52 def test_trivial(self):
53 # A couple trivial tests
54
55 # clear _opener global variable
56 self.addCleanup(urllib.request.urlcleanup)
57
58 self.assertRaises(ValueError, urllib.request.urlopen, 'bogus url')
59
60 # XXX Name hacking to get this to work on Windows.
61 fname = os.path.abspath(urllib.request.__file__).replace(os.sep, '/')
62
63 if os.name == 'nt':
64 file_url = "file:///%s" % fname
65 else:
66 file_url = "file://%s" % fname
67
68 with urllib.request.urlopen(file_url) as f:
69 f.read()
70
71 def test_parse_http_list(self):
72 tests = [
73 ('a,b,c', ['a', 'b', 'c']),
74 ('path"o,l"og"i"cal, example', ['path"o,l"og"i"cal', 'example']),
75 ('a, b, "c", "d", "e,f", g, h',
76 ['a', 'b', '"c"', '"d"', '"e,f"', 'g', 'h']),
77 ('a="b\\"c", d="e\\,f", g="h\\\\i"',
78 ['a="b"c"', 'd="e,f"', 'g="h\\i"'])]
79 for string, list in tests:
80 self.assertEqual(urllib.request.parse_http_list(string), list)
81
82 def test_URLError_reasonstr(self):
83 err = urllib.error.URLError('reason')
84 self.assertIn(err.reason, str(err))
85
86
87 class RequestHdrsTests(unittest.TestCase):
88
89 def test_request_headers_dict(self):
90 """
91 The Request.headers dictionary is not a documented interface. It
92 should stay that way, because the complete set of headers are only
93 accessible through the .get_header(), .has_header(), .header_items()
94 interface. However, .headers pre-dates those methods, and so real code
95 will be using the dictionary.
96
97 The introduction in 2.4 of those methods was a mistake for the same
98 reason: code that previously saw all (urllib2 user)-provided headers in
99 .headers now sees only a subset.
100
101 """
102 url = "http://example.com"
103 self.assertEqual(Request(url,
104 headers={"Spam-eggs": "blah"}
105 ).headers["Spam-eggs"], "blah")
106 self.assertEqual(Request(url,
107 headers={"spam-EggS": "blah"}
108 ).headers["Spam-eggs"], "blah")
109
110 def test_request_headers_methods(self):
111 """
112 Note the case normalization of header names here, to
113 .capitalize()-case. This should be preserved for
114 backwards-compatibility. (In the HTTP case, normalization to
115 .title()-case is done by urllib2 before sending headers to
116 http.client).
117
118 Note that e.g. r.has_header("spam-EggS") is currently False, and
119 r.get_header("spam-EggS") returns None, but that could be changed in
120 future.
121
122 Method r.remove_header should remove items both from r.headers and
123 r.unredirected_hdrs dictionaries
124 """
125 url = "http://example.com"
126 req = Request(url, headers={"Spam-eggs": "blah"})
127 self.assertTrue(req.has_header("Spam-eggs"))
128 self.assertEqual(req.header_items(), [('Spam-eggs', 'blah')])
129
130 req.add_header("Foo-Bar", "baz")
131 self.assertEqual(sorted(req.header_items()),
132 [('Foo-bar', 'baz'), ('Spam-eggs', 'blah')])
133 self.assertFalse(req.has_header("Not-there"))
134 self.assertIsNone(req.get_header("Not-there"))
135 self.assertEqual(req.get_header("Not-there", "default"), "default")
136
137 req.remove_header("Spam-eggs")
138 self.assertFalse(req.has_header("Spam-eggs"))
139
140 req.add_unredirected_header("Unredirected-spam", "Eggs")
141 self.assertTrue(req.has_header("Unredirected-spam"))
142
143 req.remove_header("Unredirected-spam")
144 self.assertFalse(req.has_header("Unredirected-spam"))
145
146 def test_password_manager(self):
147 mgr = urllib.request.HTTPPasswordMgr()
148 add = mgr.add_password
149 find_user_pass = mgr.find_user_password
150
151 add("Some Realm", "http://example.com/", "joe", "password")
152 add("Some Realm", "http://example.com/ni", "ni", "ni")
153 add("Some Realm", "http://c.example.com:3128", "3", "c")
154 add("Some Realm", "d.example.com", "4", "d")
155 add("Some Realm", "e.example.com:3128", "5", "e")
156
157 # For the same realm, password set the highest path is the winner.
158 self.assertEqual(find_user_pass("Some Realm", "example.com"),
159 ('joe', 'password'))
160 self.assertEqual(find_user_pass("Some Realm", "http://example.com/ni"),
161 ('joe', 'password'))
162 self.assertEqual(find_user_pass("Some Realm", "http://example.com"),
163 ('joe', 'password'))
164 self.assertEqual(find_user_pass("Some Realm", "http://example.com/"),
165 ('joe', 'password'))
166 self.assertEqual(find_user_pass("Some Realm",
167 "http://example.com/spam"),
168 ('joe', 'password'))
169 self.assertEqual(find_user_pass("Some Realm",
170 "http://example.com/spam/spam"),
171 ('joe', 'password'))
172
173 # You can have different passwords for different paths.
174
175 add("c", "http://example.com/foo", "foo", "ni")
176 add("c", "http://example.com/bar", "bar", "nini")
177 add("c", "http://example.com/foo/bar", "foobar", "nibar")
178
179 self.assertEqual(find_user_pass("c", "http://example.com/foo"),
180 ('foo', 'ni'))
181 self.assertEqual(find_user_pass("c", "http://example.com/bar"),
182 ('bar', 'nini'))
183 self.assertEqual(find_user_pass("c", "http://example.com/foo/"),
184 ('foo', 'ni'))
185 self.assertEqual(find_user_pass("c", "http://example.com/foo/bar"),
186 ('foo', 'ni'))
187 self.assertEqual(find_user_pass("c", "http://example.com/foo/baz"),
188 ('foo', 'ni'))
189 self.assertEqual(find_user_pass("c", "http://example.com/foobar"),
190 (None, None))
191
192 add("c", "http://example.com/baz/", "baz", "ninini")
193
194 self.assertEqual(find_user_pass("c", "http://example.com/baz"),
195 (None, None))
196 self.assertEqual(find_user_pass("c", "http://example.com/baz/"),
197 ('baz', 'ninini'))
198 self.assertEqual(find_user_pass("c", "http://example.com/baz/bar"),
199 ('baz', 'ninini'))
200
201 # For the same path, newer password should be considered.
202
203 add("b", "http://example.com/", "first", "blah")
204 add("b", "http://example.com/", "second", "spam")
205
206 self.assertEqual(find_user_pass("b", "http://example.com/"),
207 ('second', 'spam'))
208
209 # No special relationship between a.example.com and example.com:
210
211 add("a", "http://example.com", "1", "a")
212 self.assertEqual(find_user_pass("a", "http://example.com/"),
213 ('1', 'a'))
214
215 self.assertEqual(find_user_pass("a", "http://a.example.com/"),
216 (None, None))
217
218 # Ports:
219
220 self.assertEqual(find_user_pass("Some Realm", "c.example.com"),
221 (None, None))
222 self.assertEqual(find_user_pass("Some Realm", "c.example.com:3128"),
223 ('3', 'c'))
224 self.assertEqual(
225 find_user_pass("Some Realm", "http://c.example.com:3128"),
226 ('3', 'c'))
227 self.assertEqual(find_user_pass("Some Realm", "d.example.com"),
228 ('4', 'd'))
229 self.assertEqual(find_user_pass("Some Realm", "e.example.com:3128"),
230 ('5', 'e'))
231
232 def test_password_manager_default_port(self):
233 """
234 The point to note here is that we can't guess the default port if
235 there's no scheme. This applies to both add_password and
236 find_user_password.
237 """
238 mgr = urllib.request.HTTPPasswordMgr()
239 add = mgr.add_password
240 find_user_pass = mgr.find_user_password
241 add("f", "http://g.example.com:80", "10", "j")
242 add("g", "http://h.example.com", "11", "k")
243 add("h", "i.example.com:80", "12", "l")
244 add("i", "j.example.com", "13", "m")
245 self.assertEqual(find_user_pass("f", "g.example.com:100"),
246 (None, None))
247 self.assertEqual(find_user_pass("f", "g.example.com:80"),
248 ('10', 'j'))
249 self.assertEqual(find_user_pass("f", "g.example.com"),
250 (None, None))
251 self.assertEqual(find_user_pass("f", "http://g.example.com:100"),
252 (None, None))
253 self.assertEqual(find_user_pass("f", "http://g.example.com:80"),
254 ('10', 'j'))
255 self.assertEqual(find_user_pass("f", "http://g.example.com"),
256 ('10', 'j'))
257 self.assertEqual(find_user_pass("g", "h.example.com"), ('11', 'k'))
258 self.assertEqual(find_user_pass("g", "h.example.com:80"), ('11', 'k'))
259 self.assertEqual(find_user_pass("g", "http://h.example.com:80"),
260 ('11', 'k'))
261 self.assertEqual(find_user_pass("h", "i.example.com"), (None, None))
262 self.assertEqual(find_user_pass("h", "i.example.com:80"), ('12', 'l'))
263 self.assertEqual(find_user_pass("h", "http://i.example.com:80"),
264 ('12', 'l'))
265 self.assertEqual(find_user_pass("i", "j.example.com"), ('13', 'm'))
266 self.assertEqual(find_user_pass("i", "j.example.com:80"),
267 (None, None))
268 self.assertEqual(find_user_pass("i", "http://j.example.com"),
269 ('13', 'm'))
270 self.assertEqual(find_user_pass("i", "http://j.example.com:80"),
271 (None, None))
272
273
274 class MockOpener:
275 addheaders = []
276
277 def open(self, req, data=None, timeout=socket._GLOBAL_DEFAULT_TIMEOUT):
278 self.req, self.data, self.timeout = req, data, timeout
279
280 def error(self, proto, *args):
281 self.proto, self.args = proto, args
282
283
284 class MockFile:
285 def read(self, count=None):
286 pass
287
288 def readline(self, count=None):
289 pass
290
291 def close(self):
292 pass
293
294
295 class MockHeaders(dict):
296 def getheaders(self, name):
297 return list(self.values())
298
299
300 class MockResponse(io.StringIO):
301 def __init__(self, code, msg, headers, data, url=None):
302 io.StringIO.__init__(self, data)
303 self.code, self.msg, self.headers, self.url = code, msg, headers, url
304
305 def info(self):
306 return self.headers
307
308 def geturl(self):
309 return self.url
310
311
312 class MockCookieJar:
313 def add_cookie_header(self, request):
314 self.ach_req = request
315
316 def extract_cookies(self, response, request):
317 self.ec_req, self.ec_r = request, response
318
319
320 class FakeMethod:
321 def __init__(self, meth_name, action, handle):
322 self.meth_name = meth_name
323 self.handle = handle
324 self.action = action
325
326 def __call__(self, *args):
327 return self.handle(self.meth_name, self.action, *args)
328
329
330 class MockHTTPResponse(io.IOBase):
331 def __init__(self, fp, msg, status, reason):
332 self.fp = fp
333 self.msg = msg
334 self.status = status
335 self.reason = reason
336 self.code = 200
337
338 def read(self):
339 return ''
340
341 def info(self):
342 return {}
343
344 def geturl(self):
345 return self.url
346
347
348 class MockHTTPClass:
349 def __init__(self):
350 self.level = 0
351 self.req_headers = []
352 self.data = None
353 self.raise_on_endheaders = False
354 self.sock = None
355 self._tunnel_headers = {}
356
357 def __call__(self, host, timeout=socket._GLOBAL_DEFAULT_TIMEOUT):
358 self.host = host
359 self.timeout = timeout
360 return self
361
362 def set_debuglevel(self, level):
363 self.level = level
364
365 def set_tunnel(self, host, port=None, headers=None):
366 self._tunnel_host = host
367 self._tunnel_port = port
368 if headers:
369 self._tunnel_headers = headers
370 else:
371 self._tunnel_headers.clear()
372
373 def request(self, method, url, body=None, headers=None, *,
374 encode_chunked=False):
375 self.method = method
376 self.selector = url
377 if headers is not None:
378 self.req_headers += headers.items()
379 self.req_headers.sort()
380 if body:
381 self.data = body
382 self.encode_chunked = encode_chunked
383 if self.raise_on_endheaders:
384 raise OSError()
385
386 def getresponse(self):
387 return MockHTTPResponse(MockFile(), {}, 200, "OK")
388
389 def close(self):
390 pass
391
392
393 class MockHandler:
394 # useful for testing handler machinery
395 # see add_ordered_mock_handlers() docstring
396 handler_order = 500
397
398 def __init__(self, methods):
399 self._define_methods(methods)
400
401 def _define_methods(self, methods):
402 for spec in methods:
403 if len(spec) == 2:
404 name, action = spec
405 else:
406 name, action = spec, None
407 meth = FakeMethod(name, action, self.handle)
408 setattr(self.__class__, name, meth)
409
410 def handle(self, fn_name, action, *args, **kwds):
411 self.parent.calls.append((self, fn_name, args, kwds))
412 if action is None:
413 return None
414 elif action == "return self":
415 return self
416 elif action == "return response":
417 res = MockResponse(200, "OK", {}, "")
418 return res
419 elif action == "return request":
420 return Request("http://blah/")
421 elif action.startswith("error"):
422 code = action[action.rfind(" ")+1:]
423 try:
424 code = int(code)
425 except ValueError:
426 pass
427 res = MockResponse(200, "OK", {}, "")
428 return self.parent.error("http", args[0], res, code, "", {})
429 elif action == "raise":
430 raise urllib.error.URLError("blah")
431 assert False
432
433 def close(self):
434 pass
435
436 def add_parent(self, parent):
437 self.parent = parent
438 self.parent.calls = []
439
440 def __lt__(self, other):
441 if not hasattr(other, "handler_order"):
442 # No handler_order, leave in original order. Yuck.
443 return True
444 return self.handler_order < other.handler_order
445
446
447 def add_ordered_mock_handlers(opener, meth_spec):
448 """Create MockHandlers and add them to an OpenerDirector.
449
450 meth_spec: list of lists of tuples and strings defining methods to define
451 on handlers. eg:
452
453 [["http_error", "ftp_open"], ["http_open"]]
454
455 defines methods .http_error() and .ftp_open() on one handler, and
456 .http_open() on another. These methods just record their arguments and
457 return None. Using a tuple instead of a string causes the method to
458 perform some action (see MockHandler.handle()), eg:
459
460 [["http_error"], [("http_open", "return request")]]
461
462 defines .http_error() on one handler (which simply returns None), and
463 .http_open() on another handler, which returns a Request object.
464
465 """
466 handlers = []
467 count = 0
468 for meths in meth_spec:
469 class MockHandlerSubclass(MockHandler):
470 pass
471
472 h = MockHandlerSubclass(meths)
473 h.handler_order += count
474 h.add_parent(opener)
475 count = count + 1
476 handlers.append(h)
477 opener.add_handler(h)
478 return handlers
479
480
481 def build_test_opener(*handler_instances):
482 opener = OpenerDirector()
483 for h in handler_instances:
484 opener.add_handler(h)
485 return opener
486
487
488 class MockHTTPHandler(urllib.request.HTTPHandler):
489 # Very simple mock HTTP handler with no special behavior other than using a mock HTTP connection
490
491 def __init__(self, debuglevel=None):
492 super(MockHTTPHandler, self).__init__(debuglevel=debuglevel)
493 self.httpconn = MockHTTPClass()
494
495 def http_open(self, req):
496 return self.do_open(self.httpconn, req)
497
498
499 class MockHTTPHandlerRedirect(urllib.request.BaseHandler):
500 # useful for testing redirections and auth
501 # sends supplied headers and code as first response
502 # sends 200 OK as second response
503 def __init__(self, code, headers):
504 self.code = code
505 self.headers = headers
506 self.reset()
507
508 def reset(self):
509 self._count = 0
510 self.requests = []
511
512 def http_open(self, req):
513 import email, copy
514 self.requests.append(copy.deepcopy(req))
515 if self._count == 0:
516 self._count = self._count + 1
517 name = http.client.responses[self.code]
518 msg = email.message_from_string(self.headers)
519 return self.parent.error(
520 "http", req, MockFile(), self.code, name, msg)
521 else:
522 self.req = req
523 msg = email.message_from_string("\r\n\r\n")
524 return MockResponse(200, "OK", msg, "", req.get_full_url())
525
526
527 if hasattr(http.client, 'HTTPSConnection'):
528 class MockHTTPSHandler(urllib.request.HTTPSHandler):
529 # Useful for testing the Proxy-Authorization request by verifying the
530 # properties of httpcon
531
532 def __init__(self, debuglevel=None, context=None, check_hostname=None):
533 super(MockHTTPSHandler, self).__init__(debuglevel, context, check_hostname)
534 self.httpconn = MockHTTPClass()
535
536 def https_open(self, req):
537 return self.do_open(self.httpconn, req)
538
539
540 class MockHTTPHandlerCheckAuth(urllib.request.BaseHandler):
541 # useful for testing auth
542 # sends supplied code response
543 # checks if auth header is specified in request
544 def __init__(self, code):
545 self.code = code
546 self.has_auth_header = False
547
548 def reset(self):
549 self.has_auth_header = False
550
551 def http_open(self, req):
552 if req.has_header('Authorization'):
553 self.has_auth_header = True
554 name = http.client.responses[self.code]
555 return MockResponse(self.code, name, MockFile(), "", req.get_full_url())
556
557
558
559 class MockPasswordManager:
560 def add_password(self, realm, uri, user, password):
561 self.realm = realm
562 self.url = uri
563 self.user = user
564 self.password = password
565
566 def find_user_password(self, realm, authuri):
567 self.target_realm = realm
568 self.target_url = authuri
569 return self.user, self.password
570
571
572 class OpenerDirectorTests(unittest.TestCase):
573
574 def test_add_non_handler(self):
575 class NonHandler(object):
576 pass
577 self.assertRaises(TypeError,
578 OpenerDirector().add_handler, NonHandler())
579
580 def test_badly_named_methods(self):
581 # test work-around for three methods that accidentally follow the
582 # naming conventions for handler methods
583 # (*_open() / *_request() / *_response())
584
585 # These used to call the accidentally-named methods, causing a
586 # TypeError in real code; here, returning self from these mock
587 # methods would either cause no exception, or AttributeError.
588
589 from urllib.error import URLError
590
591 o = OpenerDirector()
592 meth_spec = [
593 [("do_open", "return self"), ("proxy_open", "return self")],
594 [("redirect_request", "return self")],
595 ]
596 add_ordered_mock_handlers(o, meth_spec)
597 o.add_handler(urllib.request.UnknownHandler())
598 for scheme in "do", "proxy", "redirect":
599 self.assertRaises(URLError, o.open, scheme+"://example.com/")
600
601 def test_handled(self):
602 # handler returning non-None means no more handlers will be called
603 o = OpenerDirector()
604 meth_spec = [
605 ["http_open", "ftp_open", "http_error_302"],
606 ["ftp_open"],
607 [("http_open", "return self")],
608 [("http_open", "return self")],
609 ]
610 handlers = add_ordered_mock_handlers(o, meth_spec)
611
612 req = Request("http://example.com/")
613 r = o.open(req)
614 # Second .http_open() gets called, third doesn't, since second returned
615 # non-None. Handlers without .http_open() never get any methods called
616 # on them.
617 # In fact, second mock handler defining .http_open() returns self
618 # (instead of response), which becomes the OpenerDirector's return
619 # value.
620 self.assertEqual(r, handlers[2])
621 calls = [(handlers[0], "http_open"), (handlers[2], "http_open")]
622 for expected, got in zip(calls, o.calls):
623 handler, name, args, kwds = got
624 self.assertEqual((handler, name), expected)
625 self.assertEqual(args, (req,))
626
627 def test_handler_order(self):
628 o = OpenerDirector()
629 handlers = []
630 for meths, handler_order in [([("http_open", "return self")], 500),
631 (["http_open"], 0)]:
632 class MockHandlerSubclass(MockHandler):
633 pass
634
635 h = MockHandlerSubclass(meths)
636 h.handler_order = handler_order
637 handlers.append(h)
638 o.add_handler(h)
639
640 o.open("http://example.com/")
641 # handlers called in reverse order, thanks to their sort order
642 self.assertEqual(o.calls[0][0], handlers[1])
643 self.assertEqual(o.calls[1][0], handlers[0])
644
645 def test_raise(self):
646 # raising URLError stops processing of request
647 o = OpenerDirector()
648 meth_spec = [
649 [("http_open", "raise")],
650 [("http_open", "return self")],
651 ]
652 handlers = add_ordered_mock_handlers(o, meth_spec)
653
654 req = Request("http://example.com/")
655 self.assertRaises(urllib.error.URLError, o.open, req)
656 self.assertEqual(o.calls, [(handlers[0], "http_open", (req,), {})])
657
658 def test_http_error(self):
659 # XXX http_error_default
660 # http errors are a special case
661 o = OpenerDirector()
662 meth_spec = [
663 [("http_open", "error 302")],
664 [("http_error_400", "raise"), "http_open"],
665 [("http_error_302", "return response"), "http_error_303",
666 "http_error"],
667 [("http_error_302")],
668 ]
669 handlers = add_ordered_mock_handlers(o, meth_spec)
670 req = Request("http://example.com/")
671 o.open(req)
672 assert len(o.calls) == 2
673 calls = [(handlers[0], "http_open", (req,)),
674 (handlers[2], "http_error_302",
675 (req, support.ALWAYS_EQ, 302, "", {}))]
676 for expected, got in zip(calls, o.calls):
677 handler, method_name, args = expected
678 self.assertEqual((handler, method_name), got[:2])
679 self.assertEqual(args, got[2])
680
681 def test_processors(self):
682 # *_request / *_response methods get called appropriately
683 o = OpenerDirector()
684 meth_spec = [
685 [("http_request", "return request"),
686 ("http_response", "return response")],
687 [("http_request", "return request"),
688 ("http_response", "return response")],
689 ]
690 handlers = add_ordered_mock_handlers(o, meth_spec)
691
692 req = Request("http://example.com/")
693 o.open(req)
694 # processor methods are called on *all* handlers that define them,
695 # not just the first handler that handles the request
696 calls = [
697 (handlers[0], "http_request"), (handlers[1], "http_request"),
698 (handlers[0], "http_response"), (handlers[1], "http_response")]
699
700 for i, (handler, name, args, kwds) in enumerate(o.calls):
701 if i < 2:
702 # *_request
703 self.assertEqual((handler, name), calls[i])
704 self.assertEqual(len(args), 1)
705 self.assertIsInstance(args[0], Request)
706 else:
707 # *_response
708 self.assertEqual((handler, name), calls[i])
709 self.assertEqual(len(args), 2)
710 self.assertIsInstance(args[0], Request)
711 # response from opener.open is None, because there's no
712 # handler that defines http_open to handle it
713 if args[1] is not None:
714 self.assertIsInstance(args[1], MockResponse)
715
716
717 class HandlerTests(unittest.TestCase):
718
719 def test_ftp(self):
720 class MockFTPWrapper:
721 def __init__(self, data):
722 self.data = data
723
724 def retrfile(self, filename, filetype):
725 self.filename, self.filetype = filename, filetype
726 return io.StringIO(self.data), len(self.data)
727
728 def close(self):
729 pass
730
731 class NullFTPHandler(urllib.request.FTPHandler):
732 def __init__(self, data):
733 self.data = data
734
735 def connect_ftp(self, user, passwd, host, port, dirs,
736 timeout=socket._GLOBAL_DEFAULT_TIMEOUT):
737 self.user, self.passwd = user, passwd
738 self.host, self.port = host, port
739 self.dirs = dirs
740 self.ftpwrapper = MockFTPWrapper(self.data)
741 return self.ftpwrapper
742
743 data = "rheum rhaponicum"
744 h = NullFTPHandler(data)
745 h.parent = MockOpener()
746
747 for url, host, port, user, passwd, type_, dirs, filename, mimetype in [
748 ("ftp://localhost/foo/bar/baz.html",
749 "localhost", ftplib.FTP_PORT, "", "", "I",
750 ["foo", "bar"], "baz.html", "text/html"),
751 ("ftp://parrot@localhost/foo/bar/baz.html",
752 "localhost", ftplib.FTP_PORT, "parrot", "", "I",
753 ["foo", "bar"], "baz.html", "text/html"),
754 ("ftp://%25parrot@localhost/foo/bar/baz.html",
755 "localhost", ftplib.FTP_PORT, "%parrot", "", "I",
756 ["foo", "bar"], "baz.html", "text/html"),
757 ("ftp://%2542parrot@localhost/foo/bar/baz.html",
758 "localhost", ftplib.FTP_PORT, "%42parrot", "", "I",
759 ["foo", "bar"], "baz.html", "text/html"),
760 ("ftp://localhost:80/foo/bar/",
761 "localhost", 80, "", "", "D",
762 ["foo", "bar"], "", None),
763 ("ftp://localhost/baz.gif;type=a",
764 "localhost", ftplib.FTP_PORT, "", "", "A",
765 [], "baz.gif", "image/gif"),
766 ]:
767 req = Request(url)
768 req.timeout = None
769 r = h.ftp_open(req)
770 # ftp authentication not yet implemented by FTPHandler
771 self.assertEqual(h.user, user)
772 self.assertEqual(h.passwd, passwd)
773 self.assertEqual(h.host, socket.gethostbyname(host))
774 self.assertEqual(h.port, port)
775 self.assertEqual(h.dirs, dirs)
776 self.assertEqual(h.ftpwrapper.filename, filename)
777 self.assertEqual(h.ftpwrapper.filetype, type_)
778 headers = r.info()
779 self.assertEqual(headers.get("Content-type"), mimetype)
780 self.assertEqual(int(headers["Content-length"]), len(data))
781 r.close()
782
783 @support.requires_resource("network")
784 def test_ftp_error(self):
785 class ErrorFTPHandler(urllib.request.FTPHandler):
786 def __init__(self, exception):
787 self._exception = exception
788
789 def connect_ftp(self, user, passwd, host, port, dirs,
790 timeout=socket._GLOBAL_DEFAULT_TIMEOUT):
791 raise self._exception
792
793 exception = ftplib.error_perm(
794 "500 OOPS: cannot change directory:/nonexistent")
795 h = ErrorFTPHandler(exception)
796 urlopen = urllib.request.build_opener(h).open
797 try:
798 urlopen("ftp://www.pythontest.net/")
799 except urllib.error.URLError as raised:
800 self.assertEqual(raised.reason,
801 f"ftp error: {exception.args[0]}")
802 else:
803 self.fail("Did not raise ftplib exception")
804
805 def test_file(self):
806 import email.utils
807 h = urllib.request.FileHandler()
808 o = h.parent = MockOpener()
809
810 TESTFN = os_helper.TESTFN
811 towrite = b"hello, world\n"
812 canonurl = urllib.request.pathname2url(os.path.abspath(TESTFN), add_scheme=True)
813 parsed = urlsplit(canonurl)
814 if parsed.netloc:
815 raise unittest.SkipTest("non-local working directory")
816 urls = [
817 canonurl,
818 parsed._replace(netloc='localhost').geturl(),
819 parsed._replace(netloc=socket.gethostbyname('localhost')).geturl(),
820 ]
821 try:
822 localaddr = socket.gethostbyname(socket.gethostname())
823 except socket.gaierror:
824 localaddr = ''
825 if localaddr:
826 urls.append(parsed._replace(netloc=localaddr).geturl())
827
828 for url in urls:
829 f = open(TESTFN, "wb")
830 try:
831 try:
832 f.write(towrite)
833 finally:
834 f.close()
835
836 r = h.file_open(Request(url))
837 try:
838 data = r.read()
839 headers = r.info()
840 respurl = r.geturl()
841 finally:
842 r.close()
843 stats = os.stat(TESTFN)
844 modified = email.utils.formatdate(stats.st_mtime, usegmt=True)
845 finally:
846 os.remove(TESTFN)
847 self.assertEqual(data, towrite)
848 self.assertEqual(headers["Content-type"], "text/plain")
849 self.assertEqual(headers["Content-length"], "13")
850 self.assertEqual(headers["Last-modified"], modified)
851 self.assertEqual(respurl, canonurl)
852
853 for url in [
854 parsed._replace(netloc='localhost:80').geturl(),
855 "file:///file_does_not_exist.txt",
856 "file://not-a-local-host.com//dir/file.txt",
857 "file://%s:80%s/%s" % (socket.gethostbyname('localhost'),
858 os.getcwd(), TESTFN),
859 "file://somerandomhost.ontheinternet.com%s/%s" %
860 (os.getcwd(), TESTFN),
861 ]:
862 try:
863 f = open(TESTFN, "wb")
864 try:
865 f.write(towrite)
866 finally:
867 f.close()
868
869 self.assertRaises(urllib.error.URLError,
870 h.file_open, Request(url))
871 finally:
872 os.remove(TESTFN)
873
874 h = urllib.request.FileHandler()
875 o = h.parent = MockOpener()
876 # XXXX why does // mean ftp (and /// mean not ftp!), and where
877 # is file: scheme specified? I think this is really a bug, and
878 # what was intended was to distinguish between URLs like:
879 # file:/blah.txt (a file)
880 # file://localhost/blah.txt (a file)
881 # file:///blah.txt (a file)
882 # file://ftp.example.com/blah.txt (an ftp URL)
883 for url, ftp in [
884 ("file://ftp.example.com//foo.txt", False),
885 ("file://ftp.example.com///foo.txt", False),
886 ("file://ftp.example.com/foo.txt", False),
887 ("file://somehost//foo/something.txt", False),
888 ("file://localhost//foo/something.txt", False),
889 ]:
890 req = Request(url)
891 try:
892 h.file_open(req)
893 except urllib.error.URLError:
894 self.assertFalse(ftp)
895 else:
896 self.assertIs(o.req, req)
897 self.assertEqual(req.type, "ftp")
898 self.assertEqual(req.type == "ftp", ftp)
899
900 def test_http(self):
901
902 h = urllib.request.AbstractHTTPHandler()
903 o = h.parent = MockOpener()
904
905 url = "http://example.com/"
906 for method, data in [("GET", None), ("POST", b"blah")]:
907 req = Request(url, data, {"Foo": "bar"})
908 req.timeout = None
909 req.add_unredirected_header("Spam", "eggs")
910 http = MockHTTPClass()
911 r = h.do_open(http, req)
912
913 # result attributes
914 r.read; r.readline # wrapped MockFile methods
915 r.info; r.geturl # addinfourl methods
916 r.code, r.msg == 200, "OK" # added from MockHTTPClass.getreply()
917 hdrs = r.info()
918 hdrs.get; hdrs.__contains__ # r.info() gives dict from .getreply()
919 self.assertEqual(r.geturl(), url)
920
921 self.assertEqual(http.host, "example.com")
922 self.assertEqual(http.level, 0)
923 self.assertEqual(http.method, method)
924 self.assertEqual(http.selector, "/")
925 self.assertEqual(http.req_headers,
926 [("Connection", "close"),
927 ("Foo", "bar"), ("Spam", "eggs")])
928 self.assertEqual(http.data, data)
929
930 # check OSError converted to URLError
931 http.raise_on_endheaders = True
932 self.assertRaises(urllib.error.URLError, h.do_open, http, req)
933
934 # Check for TypeError on POST data which is str.
935 req = Request("http://example.com/","badpost")
936 self.assertRaises(TypeError, h.do_request_, req)
937
938 # check adding of standard headers
939 o.addheaders = [("Spam", "eggs")]
940 for data in b"", None: # POST, GET
941 req = Request("http://example.com/", data)
942 r = MockResponse(200, "OK", {}, "")
943 newreq = h.do_request_(req)
944 if data is None: # GET
945 self.assertNotIn("Content-length", req.unredirected_hdrs)
946 self.assertNotIn("Content-type", req.unredirected_hdrs)
947 else: # POST
948 self.assertEqual(req.unredirected_hdrs["Content-length"], "0")
949 self.assertEqual(req.unredirected_hdrs["Content-type"],
950 "application/x-www-form-urlencoded")
951 # XXX the details of Host could be better tested
952 self.assertEqual(req.unredirected_hdrs["Host"], "example.com")
953 self.assertEqual(req.unredirected_hdrs["Spam"], "eggs")
954
955 # don't clobber existing headers
956 req.add_unredirected_header("Content-length", "foo")
957 req.add_unredirected_header("Content-type", "bar")
958 req.add_unredirected_header("Host", "baz")
959 req.add_unredirected_header("Spam", "foo")
960 newreq = h.do_request_(req)
961 self.assertEqual(req.unredirected_hdrs["Content-length"], "foo")
962 self.assertEqual(req.unredirected_hdrs["Content-type"], "bar")
963 self.assertEqual(req.unredirected_hdrs["Host"], "baz")
964 self.assertEqual(req.unredirected_hdrs["Spam"], "foo")
965
966 def test_http_body_file(self):
967 # A regular file - chunked encoding is used unless Content Length is
968 # already set.
969
970 h = urllib.request.AbstractHTTPHandler()
971 o = h.parent = MockOpener()
972
973 file_obj = tempfile.NamedTemporaryFile(mode='w+b', delete=False)
974 file_path = file_obj.name
975 file_obj.close()
976 self.addCleanup(os.unlink, file_path)
977
978 with open(file_path, "rb") as f:
979 req = Request("http://example.com/", f, {})
980 newreq = h.do_request_(req)
981 te = newreq.get_header('Transfer-encoding')
982 self.assertEqual(te, "chunked")
983 self.assertFalse(newreq.has_header('Content-length'))
984
985 with open(file_path, "rb") as f:
986 req = Request("http://example.com/", f, {"Content-Length": 30})
987 newreq = h.do_request_(req)
988 self.assertEqual(int(newreq.get_header('Content-length')), 30)
989 self.assertFalse(newreq.has_header("Transfer-encoding"))
990
991 def test_http_body_fileobj(self):
992 # A file object - chunked encoding is used
993 # unless Content Length is already set.
994 # (Note that there are some subtle differences to a regular
995 # file, that is why we are testing both cases.)
996
997 h = urllib.request.AbstractHTTPHandler()
998 o = h.parent = MockOpener()
999 file_obj = io.BytesIO()
1000
1001 req = Request("http://example.com/", file_obj, {})
1002 newreq = h.do_request_(req)
1003 self.assertEqual(newreq.get_header('Transfer-encoding'), 'chunked')
1004 self.assertFalse(newreq.has_header('Content-length'))
1005
1006 headers = {"Content-Length": 30}
1007 req = Request("http://example.com/", file_obj, headers)
1008 newreq = h.do_request_(req)
1009 self.assertEqual(int(newreq.get_header('Content-length')), 30)
1010 self.assertFalse(newreq.has_header("Transfer-encoding"))
1011
1012 file_obj.close()
1013
1014 @requires_subprocess()
1015 def test_http_body_pipe(self):
1016 # A file reading from a pipe.
1017 # A pipe cannot be seek'ed. There is no way to determine the
1018 # content length up front. Thus, do_request_() should fall
1019 # back to Transfer-encoding chunked.
1020
1021 h = urllib.request.AbstractHTTPHandler()
1022 o = h.parent = MockOpener()
1023
1024 cmd = [sys.executable, "-c", r"pass"]
1025 for headers in {}, {"Content-Length": 30}:
1026 with subprocess.Popen(cmd, stdout=subprocess.PIPE) as proc:
1027 req = Request("http://example.com/", proc.stdout, headers)
1028 newreq = h.do_request_(req)
1029 if not headers:
1030 self.assertEqual(newreq.get_header('Content-length'), None)
1031 self.assertEqual(newreq.get_header('Transfer-encoding'),
1032 'chunked')
1033 else:
1034 self.assertEqual(int(newreq.get_header('Content-length')),
1035 30)
1036
1037 def test_http_body_iterable(self):
1038 # Generic iterable. There is no way to determine the content
1039 # length up front. Fall back to Transfer-encoding chunked.
1040
1041 h = urllib.request.AbstractHTTPHandler()
1042 o = h.parent = MockOpener()
1043
1044 def iterable_body():
1045 yield b"one"
1046
1047 for headers in {}, {"Content-Length": 11}:
1048 req = Request("http://example.com/", iterable_body(), headers)
1049 newreq = h.do_request_(req)
1050 if not headers:
1051 self.assertEqual(newreq.get_header('Content-length'), None)
1052 self.assertEqual(newreq.get_header('Transfer-encoding'),
1053 'chunked')
1054 else:
1055 self.assertEqual(int(newreq.get_header('Content-length')), 11)
1056
1057 def test_http_body_empty_seq(self):
1058 # Zero-length iterable body should be treated like any other iterable
1059 h = urllib.request.AbstractHTTPHandler()
1060 h.parent = MockOpener()
1061 req = h.do_request_(Request("http://example.com/", ()))
1062 self.assertEqual(req.get_header("Transfer-encoding"), "chunked")
1063 self.assertFalse(req.has_header("Content-length"))
1064
1065 def test_http_body_array(self):
1066 # array.array Iterable - Content Length is calculated
1067
1068 h = urllib.request.AbstractHTTPHandler()
1069 o = h.parent = MockOpener()
1070
1071 iterable_array = array.array("I",[1,2,3,4])
1072
1073 for headers in {}, {"Content-Length": 16}:
1074 req = Request("http://example.com/", iterable_array, headers)
1075 newreq = h.do_request_(req)
1076 self.assertEqual(int(newreq.get_header('Content-length')),16)
1077
1078 def test_http_handler_global_debuglevel(self):
1079 with mock.patch.object(http.client.HTTPConnection, 'debuglevel', 6):
1080 o = OpenerDirector()
1081 h = MockHTTPHandler()
1082 o.add_handler(h)
1083 o.open("http://www.example.com")
1084 self.assertEqual(h._debuglevel, 6)
1085
1086 def test_http_handler_local_debuglevel(self):
1087 o = OpenerDirector()
1088 h = MockHTTPHandler(debuglevel=5)
1089 o.add_handler(h)
1090 o.open("http://www.example.com")
1091 self.assertEqual(h._debuglevel, 5)
1092
1093 @unittest.skipUnless(hasattr(http.client, 'HTTPSConnection'), 'HTTPSConnection required for HTTPS tests.')
1094 def test_https_handler_global_debuglevel(self):
1095 with mock.patch.object(http.client.HTTPSConnection, 'debuglevel', 7):
1096 o = OpenerDirector()
1097 h = MockHTTPSHandler()
1098 o.add_handler(h)
1099 o.open("https://www.example.com")
1100 self.assertEqual(h._debuglevel, 7)
1101
1102 @unittest.skipUnless(hasattr(http.client, 'HTTPSConnection'), 'HTTPSConnection required for HTTPS tests.')
1103 def test_https_handler_local_debuglevel(self):
1104 o = OpenerDirector()
1105 h = MockHTTPSHandler(debuglevel=4)
1106 o.add_handler(h)
1107 o.open("https://www.example.com")
1108 self.assertEqual(h._debuglevel, 4)
1109
1110 def test_http_doubleslash(self):
1111 # Checks the presence of any unnecessary double slash in url does not
1112 # break anything. Previously, a double slash directly after the host
1113 # could cause incorrect parsing.
1114 h = urllib.request.AbstractHTTPHandler()
1115 h.parent = MockOpener()
1116
1117 data = b""
1118 ds_urls = [
1119 "http://example.com/foo/bar/baz.html",
1120 "http://example.com//foo/bar/baz.html",
1121 "http://example.com/foo//bar/baz.html",
1122 "http://example.com/foo/bar//baz.html"
1123 ]
1124
1125 for ds_url in ds_urls:
1126 ds_req = Request(ds_url, data)
1127
1128 # Check whether host is determined correctly if there is no proxy
1129 np_ds_req = h.do_request_(ds_req)
1130 self.assertEqual(np_ds_req.unredirected_hdrs["Host"], "example.com")
1131
1132 # Check whether host is determined correctly if there is a proxy
1133 ds_req.set_proxy("someproxy:3128", None)
1134 p_ds_req = h.do_request_(ds_req)
1135 self.assertEqual(p_ds_req.unredirected_hdrs["Host"], "example.com")
1136
1137 def test_full_url_setter(self):
1138 # Checks to ensure that components are set correctly after setting the
1139 # full_url of a Request object
1140
1141 urls = [
1142 'http://example.com?foo=bar#baz',
1143 'http://example.com?foo=bar&spam=eggs#bash',
1144 'http://example.com',
1145 ]
1146
1147 # testing a reusable request instance, but the url parameter is
1148 # required, so just use a dummy one to instantiate
1149 r = Request('http://example.com')
1150 for url in urls:
1151 r.full_url = url
1152 parsed = urlsplit(url)
1153
1154 self.assertEqual(r.get_full_url(), url)
1155 # full_url setter uses splittag to split into components.
1156 # splittag sets the fragment as None while urlparse sets it to ''
1157 self.assertEqual(r.fragment or '', parsed.fragment)
1158 self.assertEqual(urlsplit(r.get_full_url()).query, parsed.query)
1159
1160 def test_full_url_deleter(self):
1161 r = Request('http://www.example.com')
1162 del r.full_url
1163 self.assertIsNone(r.full_url)
1164 self.assertIsNone(r.fragment)
1165 self.assertEqual(r.selector, '')
1166
1167 def test_fixpath_in_weirdurls(self):
1168 # Issue4493: urllib2 to supply '/' when to urls where path does not
1169 # start with'/'
1170
1171 h = urllib.request.AbstractHTTPHandler()
1172 h.parent = MockOpener()
1173
1174 weird_url = 'http://www.python.org?getspam'
1175 req = Request(weird_url)
1176 newreq = h.do_request_(req)
1177 self.assertEqual(newreq.host, 'www.python.org')
1178 self.assertEqual(newreq.selector, '/?getspam')
1179
1180 url_without_path = 'http://www.python.org'
1181 req = Request(url_without_path)
1182 newreq = h.do_request_(req)
1183 self.assertEqual(newreq.host, 'www.python.org')
1184 self.assertEqual(newreq.selector, '')
1185
1186 def test_errors(self):
1187 h = urllib.request.HTTPErrorProcessor()
1188 o = h.parent = MockOpener()
1189
1190 url = "http://example.com/"
1191 req = Request(url)
1192 # all 2xx are passed through
1193 r = MockResponse(200, "OK", {}, "", url)
1194 newr = h.http_response(req, r)
1195 self.assertIs(r, newr)
1196 self.assertNotHasAttr(o, "proto") # o.error not called
1197 r = MockResponse(202, "Accepted", {}, "", url)
1198 newr = h.http_response(req, r)
1199 self.assertIs(r, newr)
1200 self.assertNotHasAttr(o, "proto") # o.error not called
1201 r = MockResponse(206, "Partial content", {}, "", url)
1202 newr = h.http_response(req, r)
1203 self.assertIs(r, newr)
1204 self.assertNotHasAttr(o, "proto") # o.error not called
1205 # anything else calls o.error (and MockOpener returns None, here)
1206 r = MockResponse(502, "Bad gateway", {}, "", url)
1207 self.assertIsNone(h.http_response(req, r))
1208 self.assertEqual(o.proto, "http") # o.error called
1209 self.assertEqual(o.args, (req, r, 502, "Bad gateway", {}))
1210
1211 def test_cookies(self):
1212 cj = MockCookieJar()
1213 h = urllib.request.HTTPCookieProcessor(cj)
1214 h.parent = MockOpener()
1215
1216 req = Request("http://example.com/")
1217 r = MockResponse(200, "OK", {}, "")
1218 newreq = h.http_request(req)
1219 self.assertIs(cj.ach_req, req)
1220 self.assertIs(cj.ach_req, newreq)
1221 self.assertEqual(req.origin_req_host, "example.com")
1222 self.assertFalse(req.unverifiable)
1223 newr = h.http_response(req, r)
1224 self.assertIs(cj.ec_req, req)
1225 self.assertIs(cj.ec_r, r)
1226 self.assertIs(r, newr)
1227
1228 def test_redirect(self):
1229 from_url = "http://example.com/a.html"
1230 to_url = "http://example.com/b.html"
1231 h = urllib.request.HTTPRedirectHandler()
1232 o = h.parent = MockOpener()
1233
1234 # ordinary redirect behaviour
1235 for code in 301, 302, 303, 307, 308:
1236 for data in None, "blah\nblah\n":
1237 method = getattr(h, "http_error_%s" % code)
1238 req = Request(from_url, data)
1239 req.timeout = socket._GLOBAL_DEFAULT_TIMEOUT
1240 req.add_header("Nonsense", "viking=withhold")
1241 if data is not None:
1242 req.add_header("Content-Length", str(len(data)))
1243 req.add_unredirected_header("Spam", "spam")
1244 try:
1245 method(req, MockFile(), code, "Blah",
1246 MockHeaders({"location": to_url}))
1247 except urllib.error.HTTPError as err:
1248 # 307 and 308 in response to POST require user OK
1249 self.assertIn(code, (307, 308))
1250 self.assertIsNotNone(data)
1251 err.close()
1252 self.assertEqual(o.req.get_full_url(), to_url)
1253 try:
1254 self.assertEqual(o.req.get_method(), "GET")
1255 except AttributeError:
1256 self.assertFalse(o.req.data)
1257
1258 # now it's a GET, there should not be headers regarding content
1259 # (possibly dragged from before being a POST)
1260 headers = [x.lower() for x in o.req.headers]
1261 self.assertNotIn("content-length", headers)
1262 self.assertNotIn("content-type", headers)
1263
1264 self.assertEqual(o.req.headers["Nonsense"],
1265 "viking=withhold")
1266 self.assertNotIn("Spam", o.req.headers)
1267 self.assertNotIn("Spam", o.req.unredirected_hdrs)
1268
1269 # loop detection
1270 req = Request(from_url)
1271 req.timeout = socket._GLOBAL_DEFAULT_TIMEOUT
1272
1273 def redirect(h, req, url=to_url):
1274 h.http_error_302(req, MockFile(), 302, "Blah",
1275 MockHeaders({"location": url}))
1276 # Note that the *original* request shares the same record of
1277 # redirections with the sub-requests caused by the redirections.
1278
1279 # detect infinite loop redirect of a URL to itself
1280 req = Request(from_url, origin_req_host="example.com")
1281 count = 0
1282 req.timeout = socket._GLOBAL_DEFAULT_TIMEOUT
1283 try:
1284 while 1:
1285 redirect(h, req, "http://example.com/")
1286 count = count + 1
1287 except urllib.error.HTTPError as err:
1288 # don't stop until max_repeats, because cookies may introduce state
1289 self.assertEqual(count, urllib.request.HTTPRedirectHandler.max_repeats)
1290 err.close()
1291
1292 # detect endless non-repeating chain of redirects
1293 req = Request(from_url, origin_req_host="example.com")
1294 count = 0
1295 req.timeout = socket._GLOBAL_DEFAULT_TIMEOUT
1296 try:
1297 while 1:
1298 redirect(h, req, "http://example.com/%d" % count)
1299 count = count + 1
1300 except urllib.error.HTTPError as err:
1301 self.assertEqual(count,
1302 urllib.request.HTTPRedirectHandler.max_redirections)
1303 err.close()
1304
1305 def test_invalid_redirect(self):
1306 from_url = "http://example.com/a.html"
1307 valid_schemes = ['http','https','ftp']
1308 invalid_schemes = ['file','imap','ldap']
1309 schemeless_url = "example.com/b.html"
1310 h = urllib.request.HTTPRedirectHandler()
1311 o = h.parent = MockOpener()
1312 req = Request(from_url)
1313 req.timeout = socket._GLOBAL_DEFAULT_TIMEOUT
1314
1315 for scheme in invalid_schemes:
1316 invalid_url = scheme + '://' + schemeless_url
1317 with self.assertRaises(urllib.error.HTTPError) as cm:
1318 h.http_error_302(
1319 req, MockFile(), 302, "Security Loophole",
1320 MockHeaders({"location": invalid_url}))
1321 cm.exception.close()
1322
1323 for scheme in valid_schemes:
1324 valid_url = scheme + '://' + schemeless_url
1325 h.http_error_302(req, MockFile(), 302, "That's fine",
1326 MockHeaders({"location": valid_url}))
1327 self.assertEqual(o.req.get_full_url(), valid_url)
1328
1329 def test_relative_redirect(self):
1330 from_url = "http://example.com/a.html"
1331 relative_url = "/b.html"
1332 h = urllib.request.HTTPRedirectHandler()
1333 o = h.parent = MockOpener()
1334 req = Request(from_url)
1335 req.timeout = socket._GLOBAL_DEFAULT_TIMEOUT
1336
1337 valid_url = urllib.parse.urljoin(from_url,relative_url)
1338 h.http_error_302(req, MockFile(), 302, "That's fine",
1339 MockHeaders({"location": valid_url}))
1340 self.assertEqual(o.req.get_full_url(), valid_url)
1341
1342 def test_cookie_redirect(self):
1343 # cookies shouldn't leak into redirected requests
1344 from http.cookiejar import CookieJar
1345 from test.test_http_cookiejar import interact_netscape
1346
1347 cj = CookieJar()
1348 interact_netscape(cj, "http://www.example.com/", "spam=eggs")
1349 hh = MockHTTPHandlerRedirect(302, "Location: http://www.cracker.com/\r\n\r\n")
1350 hdeh = urllib.request.HTTPDefaultErrorHandler()
1351 hrh = urllib.request.HTTPRedirectHandler()
1352 cp = urllib.request.HTTPCookieProcessor(cj)
1353 o = build_test_opener(hh, hdeh, hrh, cp)
1354 o.open("http://www.example.com/")
1355 self.assertFalse(hh.req.has_header("Cookie"))
1356
1357 def test_redirect_fragment(self):
1358 redirected_url = 'http://www.example.com/index.html#OK\r\n\r\n'
1359 hh = MockHTTPHandlerRedirect(302, 'Location: ' + redirected_url)
1360 hdeh = urllib.request.HTTPDefaultErrorHandler()
1361 hrh = urllib.request.HTTPRedirectHandler()
1362 o = build_test_opener(hh, hdeh, hrh)
1363 fp = o.open('http://www.example.com')
1364 self.assertEqual(fp.geturl(), redirected_url.strip())
1365
1366 def test_redirect_no_path(self):
1367 # Issue 14132: Relative redirect strips original path
1368
1369 # clear _opener global variable
1370 self.addCleanup(urllib.request.urlcleanup)
1371
1372 real_class = http.client.HTTPConnection
1373 response1 = b"HTTP/1.1 302 Found\r\nLocation: ?query\r\n\r\n"
1374 http.client.HTTPConnection = test_urllib.fakehttp(response1)
1375 self.addCleanup(setattr, http.client, "HTTPConnection", real_class)
1376 urls = iter(("/path", "/path?query"))
1377 def request(conn, method, url, *pos, **kw):
1378 self.assertEqual(url, next(urls))
1379 real_class.request(conn, method, url, *pos, **kw)
1380 # Change response for subsequent connection
1381 conn.__class__.fakedata = b"HTTP/1.1 200 OK\r\n\r\nHello!"
1382 http.client.HTTPConnection.request = request
1383 fp = urllib.request.urlopen("http://python.org/path")
1384 self.assertEqual(fp.geturl(), "http://python.org/path?query")
1385
1386 def test_redirect_encoding(self):
1387 # Some characters in the redirect target may need special handling,
1388 # but most ASCII characters should be treated as already encoded
1389 class Handler(urllib.request.HTTPHandler):
1390 def http_open(self, req):
1391 result = self.do_open(self.connection, req)
1392 self.last_buf = self.connection.buf
1393 # Set up a normal response for the next request
1394 self.connection = test_urllib.fakehttp(
1395 b'HTTP/1.1 200 OK\r\n'
1396 b'Content-Length: 3\r\n'
1397 b'\r\n'
1398 b'123'
1399 )
1400 return result
1401 handler = Handler()
1402 opener = urllib.request.build_opener(handler)
1403 tests = (
1404 (b'/p\xC3\xA5-dansk/', b'/p%C3%A5-dansk/'),
1405 (b'/spaced%20path/', b'/spaced%20path/'),
1406 (b'/spaced path/', b'/spaced%20path/'),
1407 (b'/?p\xC3\xA5-dansk', b'/?p%C3%A5-dansk'),
1408 )
1409 for [location, result] in tests:
1410 with self.subTest(repr(location)):
1411 handler.connection = test_urllib.fakehttp(
1412 b'HTTP/1.1 302 Redirect\r\n'
1413 b'Location: ' + location + b'\r\n'
1414 b'\r\n'
1415 )
1416 response = opener.open('http://example.com/')
1417 expected = b'GET ' + result + b' '
1418 request = handler.last_buf
1419 self.assertStartsWith(request, expected)
1420
1421 def test_redirect_head_request(self):
1422 from_url = "http://example.com/a.html"
1423 to_url = "http://example.com/b.html"
1424 h = urllib.request.HTTPRedirectHandler()
1425 req = Request(from_url, method="HEAD")
1426 fp = MockFile()
1427 new_req = h.redirect_request(req, fp, 302, "Found", {}, to_url)
1428 self.assertEqual(new_req.get_method(), "HEAD")
1429
1430 def test_proxy(self):
1431 u = "proxy.example.com:3128"
1432 for d in dict(http=u), dict(HTTP=u):
1433 o = OpenerDirector()
1434 ph = urllib.request.ProxyHandler(d)
1435 o.add_handler(ph)
1436 meth_spec = [
1437 [("http_open", "return response")]
1438 ]
1439 handlers = add_ordered_mock_handlers(o, meth_spec)
1440
1441 req = Request("http://acme.example.com/")
1442 self.assertEqual(req.host, "acme.example.com")
1443 o.open(req)
1444 self.assertEqual(req.host, u)
1445 self.assertEqual([(handlers[0], "http_open")],
1446 [tup[0:2] for tup in o.calls])
1447
1448 def test_proxy_no_proxy(self):
1449 env = self.enterContext(os_helper.EnvironmentVarGuard())
1450 env['no_proxy'] = 'python.org'
1451 o = OpenerDirector()
1452 ph = urllib.request.ProxyHandler(dict(http="proxy.example.com"))
1453 o.add_handler(ph)
1454 req = Request("http://www.perl.org/")
1455 self.assertEqual(req.host, "www.perl.org")
1456 o.open(req)
1457 self.assertEqual(req.host, "proxy.example.com")
1458 req = Request("http://www.python.org")
1459 self.assertEqual(req.host, "www.python.org")
1460 o.open(req)
1461 self.assertEqual(req.host, "www.python.org")
1462
1463 def test_proxy_no_proxy_all(self):
1464 env = self.enterContext(os_helper.EnvironmentVarGuard())
1465 env['no_proxy'] = '*'
1466 o = OpenerDirector()
1467 ph = urllib.request.ProxyHandler(dict(http="proxy.example.com"))
1468 o.add_handler(ph)
1469 req = Request("http://www.python.org")
1470 self.assertEqual(req.host, "www.python.org")
1471 o.open(req)
1472 self.assertEqual(req.host, "www.python.org")
1473
1474 def test_proxy_https(self):
1475 o = OpenerDirector()
1476 ph = urllib.request.ProxyHandler(dict(https="proxy.example.com:3128"))
1477 o.add_handler(ph)
1478 meth_spec = [
1479 [("https_open", "return response")]
1480 ]
1481 handlers = add_ordered_mock_handlers(o, meth_spec)
1482
1483 req = Request("https://www.example.com/")
1484 self.assertEqual(req.host, "www.example.com")
1485 o.open(req)
1486 self.assertEqual(req.host, "proxy.example.com:3128")
1487 self.assertEqual([(handlers[0], "https_open")],
1488 [tup[0:2] for tup in o.calls])
1489
1490 @unittest.skipUnless(hasattr(http.client, 'HTTPSConnection'), 'HTTPSConnection required for HTTPS tests.')
1491 def test_proxy_https_proxy_authorization(self):
1492 o = OpenerDirector()
1493 ph = urllib.request.ProxyHandler(dict(https='proxy.example.com:3128'))
1494 o.add_handler(ph)
1495 https_handler = MockHTTPSHandler()
1496 o.add_handler(https_handler)
1497 req = Request("https://www.example.com/")
1498 req.add_header("Proxy-Authorization", "FooBar")
1499 req.add_header("User-Agent", "Grail")
1500 self.assertEqual(req.host, "www.example.com")
1501 self.assertIsNone(req._tunnel_host)
1502 o.open(req)
1503 # Verify Proxy-Authorization gets tunneled to request.
1504 # httpsconn req_headers do not have the Proxy-Authorization header but
1505 # the req will have.
1506 self.assertNotIn(("Proxy-Authorization", "FooBar"),
1507 https_handler.httpconn.req_headers)
1508 self.assertIn(("User-Agent", "Grail"),
1509 https_handler.httpconn.req_headers)
1510 self.assertIsNotNone(req._tunnel_host)
1511 self.assertEqual(req.host, "proxy.example.com:3128")
1512 self.assertEqual(req.get_header("Proxy-authorization"), "FooBar")
1513
1514 @unittest.skipUnless(os.name == "nt", "only relevant for Windows")
1515 def test_winreg_proxy_bypass(self):
1516 proxy_override = "www.example.com;*.example.net; 192.168.0.1"
1517 proxy_bypass = _proxy_bypass_winreg_override
1518 for host in ("www.example.com", "www.example.net", "192.168.0.1"):
1519 self.assertTrue(proxy_bypass(host, proxy_override),
1520 "expected bypass of %s to be true" % host)
1521
1522 for host in ("example.com", "www.example.org", "example.net",
1523 "192.168.0.2"):
1524 self.assertFalse(proxy_bypass(host, proxy_override),
1525 "expected bypass of %s to be False" % host)
1526
1527 # check intranet address bypass
1528 proxy_override = "example.com; <local>"
1529 self.assertTrue(proxy_bypass("example.com", proxy_override),
1530 "expected bypass of %s to be true" % host)
1531 self.assertFalse(proxy_bypass("example.net", proxy_override),
1532 "expected bypass of %s to be False" % host)
1533 for host in ("test", "localhost"):
1534 self.assertTrue(proxy_bypass(host, proxy_override),
1535 "expect <local> to bypass intranet address '%s'"
1536 % host)
1537
1538 @unittest.skipUnless(sys.platform == 'darwin', "only relevant for OSX")
1539 def test_osx_proxy_bypass(self):
1540 bypass = {
1541 'exclude_simple': False,
1542 'exceptions': ['foo.bar', '*.bar.com', '127.0.0.1', '10.10',
1543 '10.0/16']
1544 }
1545 # Check hosts that should trigger the proxy bypass
1546 for host in ('foo.bar', 'www.bar.com', '127.0.0.1', '10.10.0.1',
1547 '10.0.0.1'):
1548 self.assertTrue(_proxy_bypass_macosx_sysconf(host, bypass),
1549 'expected bypass of %s to be True' % host)
1550 # Check hosts that should not trigger the proxy bypass
1551 for host in ('abc.foo.bar', 'bar.com', '127.0.0.2', '10.11.0.1',
1552 'notinbypass'):
1553 self.assertFalse(_proxy_bypass_macosx_sysconf(host, bypass),
1554 'expected bypass of %s to be False' % host)
1555
1556 # Check the exclude_simple flag
1557 bypass = {'exclude_simple': True, 'exceptions': []}
1558 self.assertTrue(_proxy_bypass_macosx_sysconf('test', bypass))
1559
1560 # Check that invalid prefix lengths are ignored
1561 bypass = {
1562 'exclude_simple': False,
1563 'exceptions': [ '10.0.0.0/40', '172.19.10.0/24' ]
1564 }
1565 host = '172.19.10.5'
1566 self.assertTrue(_proxy_bypass_macosx_sysconf(host, bypass),
1567 'expected bypass of %s to be True' % host)
1568 host = '10.0.1.5'
1569 self.assertFalse(_proxy_bypass_macosx_sysconf(host, bypass),
1570 'expected bypass of %s to be False' % host)
1571
1572 def check_basic_auth(self, headers, realm):
1573 with self.subTest(realm=realm, headers=headers):
1574 opener = OpenerDirector()
1575 password_manager = MockPasswordManager()
1576 auth_handler = urllib.request.HTTPBasicAuthHandler(password_manager)
1577 body = '\r\n'.join(headers) + '\r\n\r\n'
1578 http_handler = MockHTTPHandlerRedirect(401, body)
1579 opener.add_handler(auth_handler)
1580 opener.add_handler(http_handler)
1581 self._test_basic_auth(opener, auth_handler, "Authorization",
1582 realm, http_handler, password_manager,
1583 "http://acme.example.com/protected",
1584 "http://acme.example.com/protected")
1585
1586 def test_basic_auth(self):
1587 realm = "[email protected]"
1588 realm2 = "[email protected]"
1589 basic = f'Basic realm="{realm}"'
1590 basic2 = f'Basic realm="{realm2}"'
1591 other_no_realm = 'Otherscheme xxx'
1592 digest = (f'Digest realm="{realm2}", '
1593 f'qop="auth, auth-int", '
1594 f'nonce="dcd98b7102dd2f0e8b11d0f600bfb0c093", '
1595 f'opaque="5ccc069c403ebaf9f0171e9517f40e41"')
1596 for realm_str in (
1597 # test "quote" and 'quote'
1598 f'Basic realm="{realm}"',
1599 f"Basic realm='{realm}'",
1600
1601 # charset is ignored
1602 f'Basic realm="{realm}", charset="UTF-8"',
1603
1604 # Multiple challenges per header
1605 f'{basic}, {basic2}',
1606 f'{basic}, {other_no_realm}',
1607 f'{other_no_realm}, {basic}',
1608 f'{basic}, {digest}',
1609 f'{digest}, {basic}',
1610 ):
1611 headers = [f'WWW-Authenticate: {realm_str}']
1612 self.check_basic_auth(headers, realm)
1613
1614 # no quote: expect a warning
1615 with warnings_helper.check_warnings(("Basic Auth Realm was unquoted",
1616 UserWarning)):
1617 headers = [f'WWW-Authenticate: Basic realm={realm}']
1618 self.check_basic_auth(headers, realm)
1619
1620 # Multiple headers: one challenge per header.
1621 # Use the first Basic realm.
1622 for challenges in (
1623 [basic, basic2],
1624 [basic, digest],
1625 [digest, basic],
1626 ):
1627 headers = [f'WWW-Authenticate: {challenge}'
1628 for challenge in challenges]
1629 self.check_basic_auth(headers, realm)
1630
1631 def test_proxy_basic_auth(self):
1632 opener = OpenerDirector()
1633 ph = urllib.request.ProxyHandler(dict(http="proxy.example.com:3128"))
1634 opener.add_handler(ph)
1635 password_manager = MockPasswordManager()
1636 auth_handler = urllib.request.ProxyBasicAuthHandler(password_manager)
1637 realm = "ACME Networks"
1638 http_handler = MockHTTPHandlerRedirect(
1639 407, 'Proxy-Authenticate: Basic realm="%s"\r\n\r\n' % realm)
1640 opener.add_handler(auth_handler)
1641 opener.add_handler(http_handler)
1642 self._test_basic_auth(opener, auth_handler, "Proxy-authorization",
1643 realm, http_handler, password_manager,
1644 "http://acme.example.com:3128/protected",
1645 "proxy.example.com:3128",
1646 )
1647
1648 def test_basic_and_digest_auth_handlers(self):
1649 # HTTPDigestAuthHandler raised an exception if it couldn't handle a 40*
1650 # response (https://bugs.python.org/issue1479302), where it should instead
1651 # return None to allow another handler (especially
1652 # HTTPBasicAuthHandler) to handle the response.
1653
1654 # Also (https://bugs.python.org/issue14797027, RFC 2617 section 1.2), we must
1655 # try digest first (since it's the strongest auth scheme), so we record
1656 # order of calls here to check digest comes first:
1657 class RecordingOpenerDirector(OpenerDirector):
1658 def __init__(self):
1659 OpenerDirector.__init__(self)
1660 self.recorded = []
1661
1662 def record(self, info):
1663 self.recorded.append(info)
1664
1665 class TestDigestAuthHandler(urllib.request.HTTPDigestAuthHandler):
1666 def http_error_401(self, *args, **kwds):
1667 self.parent.record("digest")
1668 urllib.request.HTTPDigestAuthHandler.http_error_401(self,
1669 *args, **kwds)
1670
1671 class TestBasicAuthHandler(urllib.request.HTTPBasicAuthHandler):
1672 def http_error_401(self, *args, **kwds):
1673 self.parent.record("basic")
1674 urllib.request.HTTPBasicAuthHandler.http_error_401(self,
1675 *args, **kwds)
1676
1677 opener = RecordingOpenerDirector()
1678 password_manager = MockPasswordManager()
1679 digest_handler = TestDigestAuthHandler(password_manager)
1680 basic_handler = TestBasicAuthHandler(password_manager)
1681 realm = "ACME Networks"
1682 http_handler = MockHTTPHandlerRedirect(
1683 401, 'WWW-Authenticate: Basic realm="%s"\r\n\r\n' % realm)
1684 opener.add_handler(basic_handler)
1685 opener.add_handler(digest_handler)
1686 opener.add_handler(http_handler)
1687
1688 # check basic auth isn't blocked by digest handler failing
1689 self._test_basic_auth(opener, basic_handler, "Authorization",
1690 realm, http_handler, password_manager,
1691 "http://acme.example.com/protected",
1692 "http://acme.example.com/protected",
1693 )
1694 # check digest was tried before basic (twice, because
1695 # _test_basic_auth called .open() twice)
1696 self.assertEqual(opener.recorded, ["digest", "basic"]*2)
1697
1698 def test_unsupported_auth_digest_handler(self):
1699 opener = OpenerDirector()
1700 # While using DigestAuthHandler
1701 digest_auth_handler = urllib.request.HTTPDigestAuthHandler(None)
1702 http_handler = MockHTTPHandlerRedirect(
1703 401, 'WWW-Authenticate: Kerberos\r\n\r\n')
1704 opener.add_handler(digest_auth_handler)
1705 opener.add_handler(http_handler)
1706 self.assertRaises(ValueError, opener.open, "http://www.example.com")
1707
1708 def test_unsupported_auth_basic_handler(self):
1709 # While using BasicAuthHandler
1710 opener = OpenerDirector()
1711 basic_auth_handler = urllib.request.HTTPBasicAuthHandler(None)
1712 http_handler = MockHTTPHandlerRedirect(
1713 401, 'WWW-Authenticate: NTLM\r\n\r\n')
1714 opener.add_handler(basic_auth_handler)
1715 opener.add_handler(http_handler)
1716 self.assertRaises(ValueError, opener.open, "http://www.example.com")
1717
1718 def _test_basic_auth(self, opener, auth_handler, auth_header,
1719 realm, http_handler, password_manager,
1720 request_url, protected_url):
1721 import base64
1722 user, password = "wile", "coyote"
1723
1724 # .add_password() fed through to password manager
1725 auth_handler.add_password(realm, request_url, user, password)
1726 self.assertEqual(realm, password_manager.realm)
1727 self.assertEqual(request_url, password_manager.url)
1728 self.assertEqual(user, password_manager.user)
1729 self.assertEqual(password, password_manager.password)
1730
1731 opener.open(request_url)
1732
1733 # should have asked the password manager for the username/password
1734 self.assertEqual(password_manager.target_realm, realm)
1735 self.assertEqual(password_manager.target_url, protected_url)
1736
1737 # expect one request without authorization, then one with
1738 self.assertEqual(len(http_handler.requests), 2)
1739 self.assertFalse(http_handler.requests[0].has_header(auth_header))
1740 userpass = bytes('%s:%s' % (user, password), "ascii")
1741 auth_hdr_value = ('Basic ' +
1742 base64.encodebytes(userpass).strip().decode())
1743 self.assertEqual(http_handler.requests[1].get_header(auth_header),
1744 auth_hdr_value)
1745 self.assertEqual(http_handler.requests[1].unredirected_hdrs[auth_header],
1746 auth_hdr_value)
1747 # if the password manager can't find a password, the handler won't
1748 # handle the HTTP auth error
1749 password_manager.user = password_manager.password = None
1750 http_handler.reset()
1751 opener.open(request_url)
1752 self.assertEqual(len(http_handler.requests), 1)
1753 self.assertFalse(http_handler.requests[0].has_header(auth_header))
1754
1755 def test_basic_prior_auth_auto_send(self):
1756 # Assume already authenticated if is_authenticated=True
1757 # for APIs like Github that don't return 401
1758
1759 user, password = "wile", "coyote"
1760 request_url = "http://acme.example.com/protected"
1761
1762 http_handler = MockHTTPHandlerCheckAuth(200)
1763
1764 pwd_manager = HTTPPasswordMgrWithPriorAuth()
1765 auth_prior_handler = HTTPBasicAuthHandler(pwd_manager)
1766 auth_prior_handler.add_password(
1767 None, request_url, user, password, is_authenticated=True)
1768
1769 self.assertTrue(pwd_manager.is_authenticated(request_url))
1770 self.assertTrue(pwd_manager.is_authenticated(request_url + '/nested'))
1771 self.assertFalse(pwd_manager.is_authenticated(request_url + 'plain'))
1772
1773 opener = OpenerDirector()
1774 opener.add_handler(auth_prior_handler)
1775 opener.add_handler(http_handler)
1776
1777 opener.open(request_url)
1778
1779 # expect request to be sent with auth header
1780 self.assertTrue(http_handler.has_auth_header)
1781
1782 def test_basic_prior_auth_send_after_first_success(self):
1783 # Auto send auth header after authentication is successful once
1784
1785 user, password = 'wile', 'coyote'
1786 request_url = 'http://acme.example.com/protected'
1787 realm = 'ACME'
1788
1789 pwd_manager = HTTPPasswordMgrWithPriorAuth()
1790 auth_prior_handler = HTTPBasicAuthHandler(pwd_manager)
1791 auth_prior_handler.add_password(realm, request_url, user, password)
1792
1793 is_auth = pwd_manager.is_authenticated(request_url)
1794 self.assertFalse(is_auth)
1795
1796 opener = OpenerDirector()
1797 opener.add_handler(auth_prior_handler)
1798
1799 http_handler = MockHTTPHandlerRedirect(
1800 401, 'WWW-Authenticate: Basic realm="%s"\r\n\r\n' % None)
1801 opener.add_handler(http_handler)
1802
1803 opener.open(request_url)
1804
1805 is_auth = pwd_manager.is_authenticated(request_url)
1806 self.assertTrue(is_auth)
1807
1808 http_handler = MockHTTPHandlerCheckAuth(200)
1809 self.assertFalse(http_handler.has_auth_header)
1810
1811 opener = OpenerDirector()
1812 opener.add_handler(auth_prior_handler)
1813 opener.add_handler(http_handler)
1814
1815 # After getting 200 from MockHTTPHandler
1816 # Next request sends header in the first request
1817 opener.open(request_url)
1818
1819 # expect request to be sent with auth header
1820 self.assertTrue(http_handler.has_auth_header)
1821
1822 def test_http_closed(self):
1823 """Test the connection is cleaned up when the response is closed"""
1824 for (transfer, data) in (
1825 ("Connection: close", b"data"),
1826 ("Transfer-Encoding: chunked", b"4\r\ndata\r\n0\r\n\r\n"),
1827 ("Content-Length: 4", b"data"),
1828 ):
1829 header = "HTTP/1.1 200 OK\r\n{}\r\n\r\n".format(transfer)
1830 conn = test_urllib.fakehttp(header.encode() + data)
1831 handler = urllib.request.AbstractHTTPHandler()
1832 req = Request("http://dummy/")
1833 req.timeout = None
1834 with handler.do_open(conn, req) as resp:
1835 resp.read()
1836 self.assertTrue(conn.fakesock.closed,
1837 "Connection not closed with {!r}".format(transfer))
1838
1839 def test_invalid_closed(self):
1840 """Test the connection is cleaned up after an invalid response"""
1841 conn = test_urllib.fakehttp(b"")
1842 handler = urllib.request.AbstractHTTPHandler()
1843 req = Request("http://dummy/")
1844 req.timeout = None
1845 with self.assertRaises(http.client.BadStatusLine):
1846 handler.do_open(conn, req)
1847 self.assertTrue(conn.fakesock.closed, "Connection not closed")
1848
1849
1850 class MiscTests(unittest.TestCase):
1851
1852 def opener_has_handler(self, opener, handler_class):
1853 self.assertTrue(any(h.__class__ == handler_class
1854 for h in opener.handlers))
1855
1856 def test_build_opener(self):
1857 class MyHTTPHandler(urllib.request.HTTPHandler):
1858 pass
1859
1860 class FooHandler(urllib.request.BaseHandler):
1861 def foo_open(self):
1862 pass
1863
1864 class BarHandler(urllib.request.BaseHandler):
1865 def bar_open(self):
1866 pass
1867
1868 build_opener = urllib.request.build_opener
1869
1870 o = build_opener(FooHandler, BarHandler)
1871 self.opener_has_handler(o, FooHandler)
1872 self.opener_has_handler(o, BarHandler)
1873
1874 # can take a mix of classes and instances
1875 o = build_opener(FooHandler, BarHandler())
1876 self.opener_has_handler(o, FooHandler)
1877 self.opener_has_handler(o, BarHandler)
1878
1879 # subclasses of default handlers override default handlers
1880 o = build_opener(MyHTTPHandler)
1881 self.opener_has_handler(o, MyHTTPHandler)
1882
1883 # a particular case of overriding: default handlers can be passed
1884 # in explicitly
1885 o = build_opener()
1886 self.opener_has_handler(o, urllib.request.HTTPHandler)
1887 o = build_opener(urllib.request.HTTPHandler)
1888 self.opener_has_handler(o, urllib.request.HTTPHandler)
1889 o = build_opener(urllib.request.HTTPHandler())
1890 self.opener_has_handler(o, urllib.request.HTTPHandler)
1891
1892 # Issue2670: multiple handlers sharing the same base class
1893 class MyOtherHTTPHandler(urllib.request.HTTPHandler):
1894 pass
1895
1896 o = build_opener(MyHTTPHandler, MyOtherHTTPHandler)
1897 self.opener_has_handler(o, MyHTTPHandler)
1898 self.opener_has_handler(o, MyOtherHTTPHandler)
1899
1900 def test_HTTPError_interface(self):
1901 """
1902 Issue 13211 reveals that HTTPError didn't implement the URLError
1903 interface even though HTTPError is a subclass of URLError.
1904 """
1905 msg = 'something bad happened'
1906 url = code = fp = None
1907 hdrs = 'Content-Length: 42'
1908 err = urllib.error.HTTPError(url, code, msg, hdrs, fp)
1909 self.assertHasAttr(err, 'reason')
1910 self.assertEqual(err.reason, 'something bad happened')
1911 self.assertHasAttr(err, 'headers')
1912 self.assertEqual(err.headers, 'Content-Length: 42')
1913 expected_errmsg = 'HTTP Error %s: %s' % (err.code, err.msg)
1914 self.assertEqual(str(err), expected_errmsg)
1915 expected_errmsg = '<HTTPError %s: %r>' % (err.code, err.msg)
1916 self.assertEqual(repr(err), expected_errmsg)
1917 err.close()
1918
1919 def test_gh_98778(self):
1920 x = urllib.error.HTTPError("url", 405, "METHOD NOT ALLOWED", None, None)
1921 self.assertEqual(getattr(x, "__notes__", ()), ())
1922 self.assertIsInstance(x.fp.read(), bytes)
1923 x.close()
1924
1925 def test_parse_proxy(self):
1926 parse_proxy_test_cases = [
1927 ('proxy.example.com',
1928 (None, None, None, 'proxy.example.com')),
1929 ('proxy.example.com:3128',
1930 (None, None, None, 'proxy.example.com:3128')),
1931 ('proxy.example.com', (None, None, None, 'proxy.example.com')),
1932 ('proxy.example.com:3128',
1933 (None, None, None, 'proxy.example.com:3128')),
1934 # The authority component may optionally include userinfo
1935 # (assumed to be # username:password):
1936 ('joe:[email protected]',
1937 (None, 'joe', 'password', 'proxy.example.com')),
1938 ('joe:[email protected]:3128',
1939 (None, 'joe', 'password', 'proxy.example.com:3128')),
1940 #Examples with URLS
1941 ('http://proxy.example.com/',
1942 ('http', None, None, 'proxy.example.com')),
1943 ('http://proxy.example.com:3128/',
1944 ('http', None, None, 'proxy.example.com:3128')),
1945 ('http://joe:[email protected]/',
1946 ('http', 'joe', 'password', 'proxy.example.com')),
1947 ('http://joe:[email protected]:3128',
1948 ('http', 'joe', 'password', 'proxy.example.com:3128')),
1949 # Everything after the authority is ignored
1950 ('ftp://joe:[email protected]/rubbish:3128',
1951 ('ftp', 'joe', 'password', 'proxy.example.com')),
1952 # Test for no trailing '/' case
1953 ('http://joe:[email protected]',
1954 ('http', 'joe', 'password', 'proxy.example.com')),
1955 # Testcases with '/' character in username, password
1956 ('http://user/name:password@localhost:22',
1957 ('http', 'user/name', 'password', 'localhost:22')),
1958 ('http://username:pass/word@localhost:22',
1959 ('http', 'username', 'pass/word', 'localhost:22')),
1960 ('http://user/name:pass/word@localhost:22',
1961 ('http', 'user/name', 'pass/word', 'localhost:22')),
1962 ]
1963
1964
1965 for tc, expected in parse_proxy_test_cases:
1966 self.assertEqual(_parse_proxy(tc), expected)
1967
1968 self.assertRaises(ValueError, _parse_proxy, 'file:/ftp.example.com'),
1969
1970
1971 skip_libssl_fips_mode = unittest.skipIf(
1972 support.is_libssl_fips_mode(),
1973 "conservative skip due to OpenSSL FIPS mode possible algorithm nerfing",
1974 )
1975
1976
1977 class TestDigestAuthAlgorithms(unittest.TestCase):
1978 def setUp(self):
1979 self.handler = AbstractDigestAuthHandler()
1980
1981 @skip_libssl_fips_mode
1982 def test_md5_algorithm(self):
1983 H, KD = self.handler.get_algorithm_impls('MD5')
1984 self.assertEqual(H("foo"), "acbd18db4cc2f85cedef654fccc4a4d8")
1985 self.assertEqual(KD("foo", "bar"), "4e99e8c12de7e01535248d2bac85e732")
1986
1987 @skip_libssl_fips_mode
1988 def test_sha_algorithm(self):
1989 H, KD = self.handler.get_algorithm_impls('SHA')
1990 self.assertEqual(H("foo"), "0beec7b5ea3f0fdbc95d0dd47f3c5bc275da8a33")
1991 self.assertEqual(KD("foo", "bar"), "54dcbe67d21d5eb39493d46d89ae1f412d3bd6de")
1992
1993 @skip_libssl_fips_mode
1994 def test_sha256_algorithm(self):
1995 H, KD = self.handler.get_algorithm_impls('SHA-256')
1996 self.assertEqual(H("foo"), "2c26b46b68ffc68ff99b453c1d30413413422d706483bfa0f98a5e886266e7ae")
1997 self.assertEqual(KD("foo", "bar"), "a765a8beaa9d561d4c5cbed29d8f4e30870297fdfa9cb7d6e9848a95fec9f937")
1998
1999 def test_invalid_algorithm(self):
2000 with self.assertRaises(ValueError) as exc:
2001 self.handler.get_algorithm_impls('invalid')
2002 self.assertEqual(
2003 str(exc.exception),
2004 "Unsupported digest authentication algorithm 'invalid'"
2005 )
2006
2007
2008 class RequestTests(unittest.TestCase):
2009 class PutRequest(Request):
2010 method = 'PUT'
2011
2012 def setUp(self):
2013 self.get = Request("http://www.python.org/~jeremy/")
2014 self.post = Request("http://www.python.org/~jeremy/",
2015 "data",
2016 headers={"X-Test": "test"})
2017 self.head = Request("http://www.python.org/~jeremy/", method='HEAD')
2018 self.put = self.PutRequest("http://www.python.org/~jeremy/")
2019 self.force_post = self.PutRequest("http://www.python.org/~jeremy/",
2020 method="POST")
2021
2022 def test_method(self):
2023 self.assertEqual("POST", self.post.get_method())
2024 self.assertEqual("GET", self.get.get_method())
2025 self.assertEqual("HEAD", self.head.get_method())
2026 self.assertEqual("PUT", self.put.get_method())
2027 self.assertEqual("POST", self.force_post.get_method())
2028
2029 def test_data(self):
2030 self.assertFalse(self.get.data)
2031 self.assertEqual("GET", self.get.get_method())
2032 self.get.data = "spam"
2033 self.assertTrue(self.get.data)
2034 self.assertEqual("POST", self.get.get_method())
2035
2036 # issue 16464
2037 # if we change data we need to remove content-length header
2038 # (cause it's most probably calculated for previous value)
2039 def test_setting_data_should_remove_content_length(self):
2040 self.assertNotIn("Content-length", self.get.unredirected_hdrs)
2041 self.get.add_unredirected_header("Content-length", 42)
2042 self.assertEqual(42, self.get.unredirected_hdrs["Content-length"])
2043 self.get.data = "spam"
2044 self.assertNotIn("Content-length", self.get.unredirected_hdrs)
2045
2046 # issue 17485 same for deleting data.
2047 def test_deleting_data_should_remove_content_length(self):
2048 self.assertNotIn("Content-length", self.get.unredirected_hdrs)
2049 self.get.data = 'foo'
2050 self.get.add_unredirected_header("Content-length", 3)
2051 self.assertEqual(3, self.get.unredirected_hdrs["Content-length"])
2052 del self.get.data
2053 self.assertNotIn("Content-length", self.get.unredirected_hdrs)
2054
2055 def test_get_full_url(self):
2056 self.assertEqual("http://www.python.org/~jeremy/",
2057 self.get.get_full_url())
2058
2059 def test_selector(self):
2060 self.assertEqual("/~jeremy/", self.get.selector)
2061 req = Request("http://www.python.org/")
2062 self.assertEqual("/", req.selector)
2063
2064 def test_get_type(self):
2065 self.assertEqual("http", self.get.type)
2066
2067 def test_get_host(self):
2068 self.assertEqual("www.python.org", self.get.host)
2069
2070 def test_get_host_unquote(self):
2071 req = Request("http://www.%70ython.org/")
2072 self.assertEqual("www.python.org", req.host)
2073
2074 def test_proxy(self):
2075 self.assertFalse(self.get.has_proxy())
2076 self.get.set_proxy("www.perl.org", "http")
2077 self.assertTrue(self.get.has_proxy())
2078 self.assertEqual("www.python.org", self.get.origin_req_host)
2079 self.assertEqual("www.perl.org", self.get.host)
2080
2081 def test_wrapped_url(self):
2082 req = Request("<URL:http://www.python.org>")
2083 self.assertEqual("www.python.org", req.host)
2084
2085 def test_url_fragment(self):
2086 req = Request("http://www.python.org/?qs=query#fragment=true")
2087 self.assertEqual("/?qs=query", req.selector)
2088 req = Request("http://www.python.org/#fun=true")
2089 self.assertEqual("/", req.selector)
2090
2091 # Issue 11703: geturl() omits fragment in the original URL.
2092 url = 'http://docs.python.org/library/urllib2.html#OK'
2093 req = Request(url)
2094 self.assertEqual(req.get_full_url(), url)
2095
2096 def test_url_fullurl_get_full_url(self):
2097 urls = ['http://docs.python.org',
2098 'http://docs.python.org/library/urllib2.html#OK',
2099 'http://www.python.org/?qs=query#fragment=true']
2100 for url in urls:
2101 req = Request(url)
2102 self.assertEqual(req.get_full_url(), req.full_url)
2103
2104
2105 if __name__ == "__main__":
2106 unittest.main()