| 1 |
import unittest |
| 2 |
from test import support |
| 3 |
from test.support import os_helper |
| 4 |
from test.support import requires_subprocess |
| 5 |
from test.support import warnings_helper |
| 6 |
from test import test_urllib |
| 7 |
from unittest import mock |
| 8 |
|
| 9 |
import os |
| 10 |
import io |
| 11 |
import ftplib |
| 12 |
import socket |
| 13 |
import array |
| 14 |
import sys |
| 15 |
import tempfile |
| 16 |
import subprocess |
| 17 |
|
| 18 |
import urllib.request |
| 19 |
# The proxy bypass method imported below has logic specific to the |
| 20 |
# corresponding system but is testable on all platforms. |
| 21 |
from urllib.request import (Request, OpenerDirector, HTTPBasicAuthHandler, |
| 22 |
HTTPPasswordMgrWithPriorAuth, _parse_proxy, |
| 23 |
_proxy_bypass_winreg_override, |
| 24 |
_proxy_bypass_macosx_sysconf, |
| 25 |
AbstractDigestAuthHandler) |
| 26 |
from urllib.parse import urlsplit |
| 27 |
import urllib.error |
| 28 |
import http.client |
| 29 |
|
| 30 |
|
| 31 |
support.requires_working_socket(module=True) |
| 32 |
|
| 33 |
# XXX |
| 34 |
# Request |
| 35 |
# CacheFTPHandler (hard to write) |
| 36 |
# parse_keqv_list, parse_http_list, HTTPDigestAuthHandler |
| 37 |
|
| 38 |
|
| 39 |
class TrivialTests(unittest.TestCase): |
| 40 |
|
| 41 |
def test___all__(self): |
| 42 |
# Verify which names are exposed |
| 43 |
for module in 'request', 'response', 'parse', 'error', 'robotparser': |
| 44 |
context = {} |
| 45 |
exec('from urllib.%s import *' % module, context) |
| 46 |
del context['__builtins__'] |
| 47 |
for k, v in context.items(): |
| 48 |
self.assertEqual(v.__module__, 'urllib.%s' % module, |
| 49 |
"%r is exposed in 'urllib.%s' but defined in %r" % |
| 50 |
(k, module, v.__module__)) |
| 51 |
|
| 52 |
def test_trivial(self): |
| 53 |
# A couple trivial tests |
| 54 |
|
| 55 |
# clear _opener global variable |
| 56 |
self.addCleanup(urllib.request.urlcleanup) |
| 57 |
|
| 58 |
self.assertRaises(ValueError, urllib.request.urlopen, 'bogus url') |
| 59 |
|
| 60 |
# XXX Name hacking to get this to work on Windows. |
| 61 |
fname = os.path.abspath(urllib.request.__file__).replace(os.sep, '/') |
| 62 |
|
| 63 |
if os.name == 'nt': |
| 64 |
file_url = "file:///%s" % fname |
| 65 |
else: |
| 66 |
file_url = "file://%s" % fname |
| 67 |
|
| 68 |
with urllib.request.urlopen(file_url) as f: |
| 69 |
f.read() |
| 70 |
|
| 71 |
def test_parse_http_list(self): |
| 72 |
tests = [ |
| 73 |
('a,b,c', ['a', 'b', 'c']), |
| 74 |
('path"o,l"og"i"cal, example', ['path"o,l"og"i"cal', 'example']), |
| 75 |
('a, b, "c", "d", "e,f", g, h', |
| 76 |
['a', 'b', '"c"', '"d"', '"e,f"', 'g', 'h']), |
| 77 |
('a="b\\"c", d="e\\,f", g="h\\\\i"', |
| 78 |
['a="b"c"', 'd="e,f"', 'g="h\\i"'])] |
| 79 |
for string, list in tests: |
| 80 |
self.assertEqual(urllib.request.parse_http_list(string), list) |
| 81 |
|
| 82 |
def test_URLError_reasonstr(self): |
| 83 |
err = urllib.error.URLError('reason') |
| 84 |
self.assertIn(err.reason, str(err)) |
| 85 |
|
| 86 |
|
| 87 |
class RequestHdrsTests(unittest.TestCase): |
| 88 |
|
| 89 |
def test_request_headers_dict(self): |
| 90 |
""" |
| 91 |
The Request.headers dictionary is not a documented interface. It |
| 92 |
should stay that way, because the complete set of headers are only |
| 93 |
accessible through the .get_header(), .has_header(), .header_items() |
| 94 |
interface. However, .headers pre-dates those methods, and so real code |
| 95 |
will be using the dictionary. |
| 96 |
|
| 97 |
The introduction in 2.4 of those methods was a mistake for the same |
| 98 |
reason: code that previously saw all (urllib2 user)-provided headers in |
| 99 |
.headers now sees only a subset. |
| 100 |
|
| 101 |
""" |
| 102 |
url = "http://example.com" |
| 103 |
self.assertEqual(Request(url, |
| 104 |
headers={"Spam-eggs": "blah"} |
| 105 |
).headers["Spam-eggs"], "blah") |
| 106 |
self.assertEqual(Request(url, |
| 107 |
headers={"spam-EggS": "blah"} |
| 108 |
).headers["Spam-eggs"], "blah") |
| 109 |
|
| 110 |
def test_request_headers_methods(self): |
| 111 |
""" |
| 112 |
Note the case normalization of header names here, to |
| 113 |
.capitalize()-case. This should be preserved for |
| 114 |
backwards-compatibility. (In the HTTP case, normalization to |
| 115 |
.title()-case is done by urllib2 before sending headers to |
| 116 |
http.client). |
| 117 |
|
| 118 |
Note that e.g. r.has_header("spam-EggS") is currently False, and |
| 119 |
r.get_header("spam-EggS") returns None, but that could be changed in |
| 120 |
future. |
| 121 |
|
| 122 |
Method r.remove_header should remove items both from r.headers and |
| 123 |
r.unredirected_hdrs dictionaries |
| 124 |
""" |
| 125 |
url = "http://example.com" |
| 126 |
req = Request(url, headers={"Spam-eggs": "blah"}) |
| 127 |
self.assertTrue(req.has_header("Spam-eggs")) |
| 128 |
self.assertEqual(req.header_items(), [('Spam-eggs', 'blah')]) |
| 129 |
|
| 130 |
req.add_header("Foo-Bar", "baz") |
| 131 |
self.assertEqual(sorted(req.header_items()), |
| 132 |
[('Foo-bar', 'baz'), ('Spam-eggs', 'blah')]) |
| 133 |
self.assertFalse(req.has_header("Not-there")) |
| 134 |
self.assertIsNone(req.get_header("Not-there")) |
| 135 |
self.assertEqual(req.get_header("Not-there", "default"), "default") |
| 136 |
|
| 137 |
req.remove_header("Spam-eggs") |
| 138 |
self.assertFalse(req.has_header("Spam-eggs")) |
| 139 |
|
| 140 |
req.add_unredirected_header("Unredirected-spam", "Eggs") |
| 141 |
self.assertTrue(req.has_header("Unredirected-spam")) |
| 142 |
|
| 143 |
req.remove_header("Unredirected-spam") |
| 144 |
self.assertFalse(req.has_header("Unredirected-spam")) |
| 145 |
|
| 146 |
def test_password_manager(self): |
| 147 |
mgr = urllib.request.HTTPPasswordMgr() |
| 148 |
add = mgr.add_password |
| 149 |
find_user_pass = mgr.find_user_password |
| 150 |
|
| 151 |
add("Some Realm", "http://example.com/", "joe", "password") |
| 152 |
add("Some Realm", "http://example.com/ni", "ni", "ni") |
| 153 |
add("Some Realm", "http://c.example.com:3128", "3", "c") |
| 154 |
add("Some Realm", "d.example.com", "4", "d") |
| 155 |
add("Some Realm", "e.example.com:3128", "5", "e") |
| 156 |
|
| 157 |
# For the same realm, password set the highest path is the winner. |
| 158 |
self.assertEqual(find_user_pass("Some Realm", "example.com"), |
| 159 |
('joe', 'password')) |
| 160 |
self.assertEqual(find_user_pass("Some Realm", "http://example.com/ni"), |
| 161 |
('joe', 'password')) |
| 162 |
self.assertEqual(find_user_pass("Some Realm", "http://example.com"), |
| 163 |
('joe', 'password')) |
| 164 |
self.assertEqual(find_user_pass("Some Realm", "http://example.com/"), |
| 165 |
('joe', 'password')) |
| 166 |
self.assertEqual(find_user_pass("Some Realm", |
| 167 |
"http://example.com/spam"), |
| 168 |
('joe', 'password')) |
| 169 |
self.assertEqual(find_user_pass("Some Realm", |
| 170 |
"http://example.com/spam/spam"), |
| 171 |
('joe', 'password')) |
| 172 |
|
| 173 |
# You can have different passwords for different paths. |
| 174 |
|
| 175 |
add("c", "http://example.com/foo", "foo", "ni") |
| 176 |
add("c", "http://example.com/bar", "bar", "nini") |
| 177 |
add("c", "http://example.com/foo/bar", "foobar", "nibar") |
| 178 |
|
| 179 |
self.assertEqual(find_user_pass("c", "http://example.com/foo"), |
| 180 |
('foo', 'ni')) |
| 181 |
self.assertEqual(find_user_pass("c", "http://example.com/bar"), |
| 182 |
('bar', 'nini')) |
| 183 |
self.assertEqual(find_user_pass("c", "http://example.com/foo/"), |
| 184 |
('foo', 'ni')) |
| 185 |
self.assertEqual(find_user_pass("c", "http://example.com/foo/bar"), |
| 186 |
('foo', 'ni')) |
| 187 |
self.assertEqual(find_user_pass("c", "http://example.com/foo/baz"), |
| 188 |
('foo', 'ni')) |
| 189 |
self.assertEqual(find_user_pass("c", "http://example.com/foobar"), |
| 190 |
(None, None)) |
| 191 |
|
| 192 |
add("c", "http://example.com/baz/", "baz", "ninini") |
| 193 |
|
| 194 |
self.assertEqual(find_user_pass("c", "http://example.com/baz"), |
| 195 |
(None, None)) |
| 196 |
self.assertEqual(find_user_pass("c", "http://example.com/baz/"), |
| 197 |
('baz', 'ninini')) |
| 198 |
self.assertEqual(find_user_pass("c", "http://example.com/baz/bar"), |
| 199 |
('baz', 'ninini')) |
| 200 |
|
| 201 |
# For the same path, newer password should be considered. |
| 202 |
|
| 203 |
add("b", "http://example.com/", "first", "blah") |
| 204 |
add("b", "http://example.com/", "second", "spam") |
| 205 |
|
| 206 |
self.assertEqual(find_user_pass("b", "http://example.com/"), |
| 207 |
('second', 'spam')) |
| 208 |
|
| 209 |
# No special relationship between a.example.com and example.com: |
| 210 |
|
| 211 |
add("a", "http://example.com", "1", "a") |
| 212 |
self.assertEqual(find_user_pass("a", "http://example.com/"), |
| 213 |
('1', 'a')) |
| 214 |
|
| 215 |
self.assertEqual(find_user_pass("a", "http://a.example.com/"), |
| 216 |
(None, None)) |
| 217 |
|
| 218 |
# Ports: |
| 219 |
|
| 220 |
self.assertEqual(find_user_pass("Some Realm", "c.example.com"), |
| 221 |
(None, None)) |
| 222 |
self.assertEqual(find_user_pass("Some Realm", "c.example.com:3128"), |
| 223 |
('3', 'c')) |
| 224 |
self.assertEqual( |
| 225 |
find_user_pass("Some Realm", "http://c.example.com:3128"), |
| 226 |
('3', 'c')) |
| 227 |
self.assertEqual(find_user_pass("Some Realm", "d.example.com"), |
| 228 |
('4', 'd')) |
| 229 |
self.assertEqual(find_user_pass("Some Realm", "e.example.com:3128"), |
| 230 |
('5', 'e')) |
| 231 |
|
| 232 |
def test_password_manager_default_port(self): |
| 233 |
""" |
| 234 |
The point to note here is that we can't guess the default port if |
| 235 |
there's no scheme. This applies to both add_password and |
| 236 |
find_user_password. |
| 237 |
""" |
| 238 |
mgr = urllib.request.HTTPPasswordMgr() |
| 239 |
add = mgr.add_password |
| 240 |
find_user_pass = mgr.find_user_password |
| 241 |
add("f", "http://g.example.com:80", "10", "j") |
| 242 |
add("g", "http://h.example.com", "11", "k") |
| 243 |
add("h", "i.example.com:80", "12", "l") |
| 244 |
add("i", "j.example.com", "13", "m") |
| 245 |
self.assertEqual(find_user_pass("f", "g.example.com:100"), |
| 246 |
(None, None)) |
| 247 |
self.assertEqual(find_user_pass("f", "g.example.com:80"), |
| 248 |
('10', 'j')) |
| 249 |
self.assertEqual(find_user_pass("f", "g.example.com"), |
| 250 |
(None, None)) |
| 251 |
self.assertEqual(find_user_pass("f", "http://g.example.com:100"), |
| 252 |
(None, None)) |
| 253 |
self.assertEqual(find_user_pass("f", "http://g.example.com:80"), |
| 254 |
('10', 'j')) |
| 255 |
self.assertEqual(find_user_pass("f", "http://g.example.com"), |
| 256 |
('10', 'j')) |
| 257 |
self.assertEqual(find_user_pass("g", "h.example.com"), ('11', 'k')) |
| 258 |
self.assertEqual(find_user_pass("g", "h.example.com:80"), ('11', 'k')) |
| 259 |
self.assertEqual(find_user_pass("g", "http://h.example.com:80"), |
| 260 |
('11', 'k')) |
| 261 |
self.assertEqual(find_user_pass("h", "i.example.com"), (None, None)) |
| 262 |
self.assertEqual(find_user_pass("h", "i.example.com:80"), ('12', 'l')) |
| 263 |
self.assertEqual(find_user_pass("h", "http://i.example.com:80"), |
| 264 |
('12', 'l')) |
| 265 |
self.assertEqual(find_user_pass("i", "j.example.com"), ('13', 'm')) |
| 266 |
self.assertEqual(find_user_pass("i", "j.example.com:80"), |
| 267 |
(None, None)) |
| 268 |
self.assertEqual(find_user_pass("i", "http://j.example.com"), |
| 269 |
('13', 'm')) |
| 270 |
self.assertEqual(find_user_pass("i", "http://j.example.com:80"), |
| 271 |
(None, None)) |
| 272 |
|
| 273 |
|
| 274 |
class MockOpener: |
| 275 |
addheaders = [] |
| 276 |
|
| 277 |
def open(self, req, data=None, timeout=socket._GLOBAL_DEFAULT_TIMEOUT): |
| 278 |
self.req, self.data, self.timeout = req, data, timeout |
| 279 |
|
| 280 |
def error(self, proto, *args): |
| 281 |
self.proto, self.args = proto, args |
| 282 |
|
| 283 |
|
| 284 |
class MockFile: |
| 285 |
def read(self, count=None): |
| 286 |
pass |
| 287 |
|
| 288 |
def readline(self, count=None): |
| 289 |
pass |
| 290 |
|
| 291 |
def close(self): |
| 292 |
pass |
| 293 |
|
| 294 |
|
| 295 |
class MockHeaders(dict): |
| 296 |
def getheaders(self, name): |
| 297 |
return list(self.values()) |
| 298 |
|
| 299 |
|
| 300 |
class MockResponse(io.StringIO): |
| 301 |
def __init__(self, code, msg, headers, data, url=None): |
| 302 |
io.StringIO.__init__(self, data) |
| 303 |
self.code, self.msg, self.headers, self.url = code, msg, headers, url |
| 304 |
|
| 305 |
def info(self): |
| 306 |
return self.headers |
| 307 |
|
| 308 |
def geturl(self): |
| 309 |
return self.url |
| 310 |
|
| 311 |
|
| 312 |
class MockCookieJar: |
| 313 |
def add_cookie_header(self, request): |
| 314 |
self.ach_req = request |
| 315 |
|
| 316 |
def extract_cookies(self, response, request): |
| 317 |
self.ec_req, self.ec_r = request, response |
| 318 |
|
| 319 |
|
| 320 |
class FakeMethod: |
| 321 |
def __init__(self, meth_name, action, handle): |
| 322 |
self.meth_name = meth_name |
| 323 |
self.handle = handle |
| 324 |
self.action = action |
| 325 |
|
| 326 |
def __call__(self, *args): |
| 327 |
return self.handle(self.meth_name, self.action, *args) |
| 328 |
|
| 329 |
|
| 330 |
class MockHTTPResponse(io.IOBase): |
| 331 |
def __init__(self, fp, msg, status, reason): |
| 332 |
self.fp = fp |
| 333 |
self.msg = msg |
| 334 |
self.status = status |
| 335 |
self.reason = reason |
| 336 |
self.code = 200 |
| 337 |
|
| 338 |
def read(self): |
| 339 |
return '' |
| 340 |
|
| 341 |
def info(self): |
| 342 |
return {} |
| 343 |
|
| 344 |
def geturl(self): |
| 345 |
return self.url |
| 346 |
|
| 347 |
|
| 348 |
class MockHTTPClass: |
| 349 |
def __init__(self): |
| 350 |
self.level = 0 |
| 351 |
self.req_headers = [] |
| 352 |
self.data = None |
| 353 |
self.raise_on_endheaders = False |
| 354 |
self.sock = None |
| 355 |
self._tunnel_headers = {} |
| 356 |
|
| 357 |
def __call__(self, host, timeout=socket._GLOBAL_DEFAULT_TIMEOUT): |
| 358 |
self.host = host |
| 359 |
self.timeout = timeout |
| 360 |
return self |
| 361 |
|
| 362 |
def set_debuglevel(self, level): |
| 363 |
self.level = level |
| 364 |
|
| 365 |
def set_tunnel(self, host, port=None, headers=None): |
| 366 |
self._tunnel_host = host |
| 367 |
self._tunnel_port = port |
| 368 |
if headers: |
| 369 |
self._tunnel_headers = headers |
| 370 |
else: |
| 371 |
self._tunnel_headers.clear() |
| 372 |
|
| 373 |
def request(self, method, url, body=None, headers=None, *, |
| 374 |
encode_chunked=False): |
| 375 |
self.method = method |
| 376 |
self.selector = url |
| 377 |
if headers is not None: |
| 378 |
self.req_headers += headers.items() |
| 379 |
self.req_headers.sort() |
| 380 |
if body: |
| 381 |
self.data = body |
| 382 |
self.encode_chunked = encode_chunked |
| 383 |
if self.raise_on_endheaders: |
| 384 |
raise OSError() |
| 385 |
|
| 386 |
def getresponse(self): |
| 387 |
return MockHTTPResponse(MockFile(), {}, 200, "OK") |
| 388 |
|
| 389 |
def close(self): |
| 390 |
pass |
| 391 |
|
| 392 |
|
| 393 |
class MockHandler: |
| 394 |
# useful for testing handler machinery |
| 395 |
# see add_ordered_mock_handlers() docstring |
| 396 |
handler_order = 500 |
| 397 |
|
| 398 |
def __init__(self, methods): |
| 399 |
self._define_methods(methods) |
| 400 |
|
| 401 |
def _define_methods(self, methods): |
| 402 |
for spec in methods: |
| 403 |
if len(spec) == 2: |
| 404 |
name, action = spec |
| 405 |
else: |
| 406 |
name, action = spec, None |
| 407 |
meth = FakeMethod(name, action, self.handle) |
| 408 |
setattr(self.__class__, name, meth) |
| 409 |
|
| 410 |
def handle(self, fn_name, action, *args, **kwds): |
| 411 |
self.parent.calls.append((self, fn_name, args, kwds)) |
| 412 |
if action is None: |
| 413 |
return None |
| 414 |
elif action == "return self": |
| 415 |
return self |
| 416 |
elif action == "return response": |
| 417 |
res = MockResponse(200, "OK", {}, "") |
| 418 |
return res |
| 419 |
elif action == "return request": |
| 420 |
return Request("http://blah/") |
| 421 |
elif action.startswith("error"): |
| 422 |
code = action[action.rfind(" ")+1:] |
| 423 |
try: |
| 424 |
code = int(code) |
| 425 |
except ValueError: |
| 426 |
pass |
| 427 |
res = MockResponse(200, "OK", {}, "") |
| 428 |
return self.parent.error("http", args[0], res, code, "", {}) |
| 429 |
elif action == "raise": |
| 430 |
raise urllib.error.URLError("blah") |
| 431 |
assert False |
| 432 |
|
| 433 |
def close(self): |
| 434 |
pass |
| 435 |
|
| 436 |
def add_parent(self, parent): |
| 437 |
self.parent = parent |
| 438 |
self.parent.calls = [] |
| 439 |
|
| 440 |
def __lt__(self, other): |
| 441 |
if not hasattr(other, "handler_order"): |
| 442 |
# No handler_order, leave in original order. Yuck. |
| 443 |
return True |
| 444 |
return self.handler_order < other.handler_order |
| 445 |
|
| 446 |
|
| 447 |
def add_ordered_mock_handlers(opener, meth_spec): |
| 448 |
"""Create MockHandlers and add them to an OpenerDirector. |
| 449 |
|
| 450 |
meth_spec: list of lists of tuples and strings defining methods to define |
| 451 |
on handlers. eg: |
| 452 |
|
| 453 |
[["http_error", "ftp_open"], ["http_open"]] |
| 454 |
|
| 455 |
defines methods .http_error() and .ftp_open() on one handler, and |
| 456 |
.http_open() on another. These methods just record their arguments and |
| 457 |
return None. Using a tuple instead of a string causes the method to |
| 458 |
perform some action (see MockHandler.handle()), eg: |
| 459 |
|
| 460 |
[["http_error"], [("http_open", "return request")]] |
| 461 |
|
| 462 |
defines .http_error() on one handler (which simply returns None), and |
| 463 |
.http_open() on another handler, which returns a Request object. |
| 464 |
|
| 465 |
""" |
| 466 |
handlers = [] |
| 467 |
count = 0 |
| 468 |
for meths in meth_spec: |
| 469 |
class MockHandlerSubclass(MockHandler): |
| 470 |
pass |
| 471 |
|
| 472 |
h = MockHandlerSubclass(meths) |
| 473 |
h.handler_order += count |
| 474 |
h.add_parent(opener) |
| 475 |
count = count + 1 |
| 476 |
handlers.append(h) |
| 477 |
opener.add_handler(h) |
| 478 |
return handlers |
| 479 |
|
| 480 |
|
| 481 |
def build_test_opener(*handler_instances): |
| 482 |
opener = OpenerDirector() |
| 483 |
for h in handler_instances: |
| 484 |
opener.add_handler(h) |
| 485 |
return opener |
| 486 |
|
| 487 |
|
| 488 |
class MockHTTPHandler(urllib.request.HTTPHandler): |
| 489 |
# Very simple mock HTTP handler with no special behavior other than using a mock HTTP connection |
| 490 |
|
| 491 |
def __init__(self, debuglevel=None): |
| 492 |
super(MockHTTPHandler, self).__init__(debuglevel=debuglevel) |
| 493 |
self.httpconn = MockHTTPClass() |
| 494 |
|
| 495 |
def http_open(self, req): |
| 496 |
return self.do_open(self.httpconn, req) |
| 497 |
|
| 498 |
|
| 499 |
class MockHTTPHandlerRedirect(urllib.request.BaseHandler): |
| 500 |
# useful for testing redirections and auth |
| 501 |
# sends supplied headers and code as first response |
| 502 |
# sends 200 OK as second response |
| 503 |
def __init__(self, code, headers): |
| 504 |
self.code = code |
| 505 |
self.headers = headers |
| 506 |
self.reset() |
| 507 |
|
| 508 |
def reset(self): |
| 509 |
self._count = 0 |
| 510 |
self.requests = [] |
| 511 |
|
| 512 |
def http_open(self, req): |
| 513 |
import email, copy |
| 514 |
self.requests.append(copy.deepcopy(req)) |
| 515 |
if self._count == 0: |
| 516 |
self._count = self._count + 1 |
| 517 |
name = http.client.responses[self.code] |
| 518 |
msg = email.message_from_string(self.headers) |
| 519 |
return self.parent.error( |
| 520 |
"http", req, MockFile(), self.code, name, msg) |
| 521 |
else: |
| 522 |
self.req = req |
| 523 |
msg = email.message_from_string("\r\n\r\n") |
| 524 |
return MockResponse(200, "OK", msg, "", req.get_full_url()) |
| 525 |
|
| 526 |
|
| 527 |
if hasattr(http.client, 'HTTPSConnection'): |
| 528 |
class MockHTTPSHandler(urllib.request.HTTPSHandler): |
| 529 |
# Useful for testing the Proxy-Authorization request by verifying the |
| 530 |
# properties of httpcon |
| 531 |
|
| 532 |
def __init__(self, debuglevel=None, context=None, check_hostname=None): |
| 533 |
super(MockHTTPSHandler, self).__init__(debuglevel, context, check_hostname) |
| 534 |
self.httpconn = MockHTTPClass() |
| 535 |
|
| 536 |
def https_open(self, req): |
| 537 |
return self.do_open(self.httpconn, req) |
| 538 |
|
| 539 |
|
| 540 |
class MockHTTPHandlerCheckAuth(urllib.request.BaseHandler): |
| 541 |
# useful for testing auth |
| 542 |
# sends supplied code response |
| 543 |
# checks if auth header is specified in request |
| 544 |
def __init__(self, code): |
| 545 |
self.code = code |
| 546 |
self.has_auth_header = False |
| 547 |
|
| 548 |
def reset(self): |
| 549 |
self.has_auth_header = False |
| 550 |
|
| 551 |
def http_open(self, req): |
| 552 |
if req.has_header('Authorization'): |
| 553 |
self.has_auth_header = True |
| 554 |
name = http.client.responses[self.code] |
| 555 |
return MockResponse(self.code, name, MockFile(), "", req.get_full_url()) |
| 556 |
|
| 557 |
|
| 558 |
|
| 559 |
class MockPasswordManager: |
| 560 |
def add_password(self, realm, uri, user, password): |
| 561 |
self.realm = realm |
| 562 |
self.url = uri |
| 563 |
self.user = user |
| 564 |
self.password = password |
| 565 |
|
| 566 |
def find_user_password(self, realm, authuri): |
| 567 |
self.target_realm = realm |
| 568 |
self.target_url = authuri |
| 569 |
return self.user, self.password |
| 570 |
|
| 571 |
|
| 572 |
class OpenerDirectorTests(unittest.TestCase): |
| 573 |
|
| 574 |
def test_add_non_handler(self): |
| 575 |
class NonHandler(object): |
| 576 |
pass |
| 577 |
self.assertRaises(TypeError, |
| 578 |
OpenerDirector().add_handler, NonHandler()) |
| 579 |
|
| 580 |
def test_badly_named_methods(self): |
| 581 |
# test work-around for three methods that accidentally follow the |
| 582 |
# naming conventions for handler methods |
| 583 |
# (*_open() / *_request() / *_response()) |
| 584 |
|
| 585 |
# These used to call the accidentally-named methods, causing a |
| 586 |
# TypeError in real code; here, returning self from these mock |
| 587 |
# methods would either cause no exception, or AttributeError. |
| 588 |
|
| 589 |
from urllib.error import URLError |
| 590 |
|
| 591 |
o = OpenerDirector() |
| 592 |
meth_spec = [ |
| 593 |
[("do_open", "return self"), ("proxy_open", "return self")], |
| 594 |
[("redirect_request", "return self")], |
| 595 |
] |
| 596 |
add_ordered_mock_handlers(o, meth_spec) |
| 597 |
o.add_handler(urllib.request.UnknownHandler()) |
| 598 |
for scheme in "do", "proxy", "redirect": |
| 599 |
self.assertRaises(URLError, o.open, scheme+"://example.com/") |
| 600 |
|
| 601 |
def test_handled(self): |
| 602 |
# handler returning non-None means no more handlers will be called |
| 603 |
o = OpenerDirector() |
| 604 |
meth_spec = [ |
| 605 |
["http_open", "ftp_open", "http_error_302"], |
| 606 |
["ftp_open"], |
| 607 |
[("http_open", "return self")], |
| 608 |
[("http_open", "return self")], |
| 609 |
] |
| 610 |
handlers = add_ordered_mock_handlers(o, meth_spec) |
| 611 |
|
| 612 |
req = Request("http://example.com/") |
| 613 |
r = o.open(req) |
| 614 |
# Second .http_open() gets called, third doesn't, since second returned |
| 615 |
# non-None. Handlers without .http_open() never get any methods called |
| 616 |
# on them. |
| 617 |
# In fact, second mock handler defining .http_open() returns self |
| 618 |
# (instead of response), which becomes the OpenerDirector's return |
| 619 |
# value. |
| 620 |
self.assertEqual(r, handlers[2]) |
| 621 |
calls = [(handlers[0], "http_open"), (handlers[2], "http_open")] |
| 622 |
for expected, got in zip(calls, o.calls): |
| 623 |
handler, name, args, kwds = got |
| 624 |
self.assertEqual((handler, name), expected) |
| 625 |
self.assertEqual(args, (req,)) |
| 626 |
|
| 627 |
def test_handler_order(self): |
| 628 |
o = OpenerDirector() |
| 629 |
handlers = [] |
| 630 |
for meths, handler_order in [([("http_open", "return self")], 500), |
| 631 |
(["http_open"], 0)]: |
| 632 |
class MockHandlerSubclass(MockHandler): |
| 633 |
pass |
| 634 |
|
| 635 |
h = MockHandlerSubclass(meths) |
| 636 |
h.handler_order = handler_order |
| 637 |
handlers.append(h) |
| 638 |
o.add_handler(h) |
| 639 |
|
| 640 |
o.open("http://example.com/") |
| 641 |
# handlers called in reverse order, thanks to their sort order |
| 642 |
self.assertEqual(o.calls[0][0], handlers[1]) |
| 643 |
self.assertEqual(o.calls[1][0], handlers[0]) |
| 644 |
|
| 645 |
def test_raise(self): |
| 646 |
# raising URLError stops processing of request |
| 647 |
o = OpenerDirector() |
| 648 |
meth_spec = [ |
| 649 |
[("http_open", "raise")], |
| 650 |
[("http_open", "return self")], |
| 651 |
] |
| 652 |
handlers = add_ordered_mock_handlers(o, meth_spec) |
| 653 |
|
| 654 |
req = Request("http://example.com/") |
| 655 |
self.assertRaises(urllib.error.URLError, o.open, req) |
| 656 |
self.assertEqual(o.calls, [(handlers[0], "http_open", (req,), {})]) |
| 657 |
|
| 658 |
def test_http_error(self): |
| 659 |
# XXX http_error_default |
| 660 |
# http errors are a special case |
| 661 |
o = OpenerDirector() |
| 662 |
meth_spec = [ |
| 663 |
[("http_open", "error 302")], |
| 664 |
[("http_error_400", "raise"), "http_open"], |
| 665 |
[("http_error_302", "return response"), "http_error_303", |
| 666 |
"http_error"], |
| 667 |
[("http_error_302")], |
| 668 |
] |
| 669 |
handlers = add_ordered_mock_handlers(o, meth_spec) |
| 670 |
req = Request("http://example.com/") |
| 671 |
o.open(req) |
| 672 |
assert len(o.calls) == 2 |
| 673 |
calls = [(handlers[0], "http_open", (req,)), |
| 674 |
(handlers[2], "http_error_302", |
| 675 |
(req, support.ALWAYS_EQ, 302, "", {}))] |
| 676 |
for expected, got in zip(calls, o.calls): |
| 677 |
handler, method_name, args = expected |
| 678 |
self.assertEqual((handler, method_name), got[:2]) |
| 679 |
self.assertEqual(args, got[2]) |
| 680 |
|
| 681 |
def test_processors(self): |
| 682 |
# *_request / *_response methods get called appropriately |
| 683 |
o = OpenerDirector() |
| 684 |
meth_spec = [ |
| 685 |
[("http_request", "return request"), |
| 686 |
("http_response", "return response")], |
| 687 |
[("http_request", "return request"), |
| 688 |
("http_response", "return response")], |
| 689 |
] |
| 690 |
handlers = add_ordered_mock_handlers(o, meth_spec) |
| 691 |
|
| 692 |
req = Request("http://example.com/") |
| 693 |
o.open(req) |
| 694 |
# processor methods are called on *all* handlers that define them, |
| 695 |
# not just the first handler that handles the request |
| 696 |
calls = [ |
| 697 |
(handlers[0], "http_request"), (handlers[1], "http_request"), |
| 698 |
(handlers[0], "http_response"), (handlers[1], "http_response")] |
| 699 |
|
| 700 |
for i, (handler, name, args, kwds) in enumerate(o.calls): |
| 701 |
if i < 2: |
| 702 |
# *_request |
| 703 |
self.assertEqual((handler, name), calls[i]) |
| 704 |
self.assertEqual(len(args), 1) |
| 705 |
self.assertIsInstance(args[0], Request) |
| 706 |
else: |
| 707 |
# *_response |
| 708 |
self.assertEqual((handler, name), calls[i]) |
| 709 |
self.assertEqual(len(args), 2) |
| 710 |
self.assertIsInstance(args[0], Request) |
| 711 |
# response from opener.open is None, because there's no |
| 712 |
# handler that defines http_open to handle it |
| 713 |
if args[1] is not None: |
| 714 |
self.assertIsInstance(args[1], MockResponse) |
| 715 |
|
| 716 |
|
| 717 |
class HandlerTests(unittest.TestCase): |
| 718 |
|
| 719 |
def test_ftp(self): |
| 720 |
class MockFTPWrapper: |
| 721 |
def __init__(self, data): |
| 722 |
self.data = data |
| 723 |
|
| 724 |
def retrfile(self, filename, filetype): |
| 725 |
self.filename, self.filetype = filename, filetype |
| 726 |
return io.StringIO(self.data), len(self.data) |
| 727 |
|
| 728 |
def close(self): |
| 729 |
pass |
| 730 |
|
| 731 |
class NullFTPHandler(urllib.request.FTPHandler): |
| 732 |
def __init__(self, data): |
| 733 |
self.data = data |
| 734 |
|
| 735 |
def connect_ftp(self, user, passwd, host, port, dirs, |
| 736 |
timeout=socket._GLOBAL_DEFAULT_TIMEOUT): |
| 737 |
self.user, self.passwd = user, passwd |
| 738 |
self.host, self.port = host, port |
| 739 |
self.dirs = dirs |
| 740 |
self.ftpwrapper = MockFTPWrapper(self.data) |
| 741 |
return self.ftpwrapper |
| 742 |
|
| 743 |
data = "rheum rhaponicum" |
| 744 |
h = NullFTPHandler(data) |
| 745 |
h.parent = MockOpener() |
| 746 |
|
| 747 |
for url, host, port, user, passwd, type_, dirs, filename, mimetype in [ |
| 748 |
("ftp://localhost/foo/bar/baz.html", |
| 749 |
"localhost", ftplib.FTP_PORT, "", "", "I", |
| 750 |
["foo", "bar"], "baz.html", "text/html"), |
| 751 |
("ftp://parrot@localhost/foo/bar/baz.html", |
| 752 |
"localhost", ftplib.FTP_PORT, "parrot", "", "I", |
| 753 |
["foo", "bar"], "baz.html", "text/html"), |
| 754 |
("ftp://%25parrot@localhost/foo/bar/baz.html", |
| 755 |
"localhost", ftplib.FTP_PORT, "%parrot", "", "I", |
| 756 |
["foo", "bar"], "baz.html", "text/html"), |
| 757 |
("ftp://%2542parrot@localhost/foo/bar/baz.html", |
| 758 |
"localhost", ftplib.FTP_PORT, "%42parrot", "", "I", |
| 759 |
["foo", "bar"], "baz.html", "text/html"), |
| 760 |
("ftp://localhost:80/foo/bar/", |
| 761 |
"localhost", 80, "", "", "D", |
| 762 |
["foo", "bar"], "", None), |
| 763 |
("ftp://localhost/baz.gif;type=a", |
| 764 |
"localhost", ftplib.FTP_PORT, "", "", "A", |
| 765 |
[], "baz.gif", "image/gif"), |
| 766 |
]: |
| 767 |
req = Request(url) |
| 768 |
req.timeout = None |
| 769 |
r = h.ftp_open(req) |
| 770 |
# ftp authentication not yet implemented by FTPHandler |
| 771 |
self.assertEqual(h.user, user) |
| 772 |
self.assertEqual(h.passwd, passwd) |
| 773 |
self.assertEqual(h.host, socket.gethostbyname(host)) |
| 774 |
self.assertEqual(h.port, port) |
| 775 |
self.assertEqual(h.dirs, dirs) |
| 776 |
self.assertEqual(h.ftpwrapper.filename, filename) |
| 777 |
self.assertEqual(h.ftpwrapper.filetype, type_) |
| 778 |
headers = r.info() |
| 779 |
self.assertEqual(headers.get("Content-type"), mimetype) |
| 780 |
self.assertEqual(int(headers["Content-length"]), len(data)) |
| 781 |
r.close() |
| 782 |
|
| 783 |
@support.requires_resource("network") |
| 784 |
def test_ftp_error(self): |
| 785 |
class ErrorFTPHandler(urllib.request.FTPHandler): |
| 786 |
def __init__(self, exception): |
| 787 |
self._exception = exception |
| 788 |
|
| 789 |
def connect_ftp(self, user, passwd, host, port, dirs, |
| 790 |
timeout=socket._GLOBAL_DEFAULT_TIMEOUT): |
| 791 |
raise self._exception |
| 792 |
|
| 793 |
exception = ftplib.error_perm( |
| 794 |
"500 OOPS: cannot change directory:/nonexistent") |
| 795 |
h = ErrorFTPHandler(exception) |
| 796 |
urlopen = urllib.request.build_opener(h).open |
| 797 |
try: |
| 798 |
urlopen("ftp://www.pythontest.net/") |
| 799 |
except urllib.error.URLError as raised: |
| 800 |
self.assertEqual(raised.reason, |
| 801 |
f"ftp error: {exception.args[0]}") |
| 802 |
else: |
| 803 |
self.fail("Did not raise ftplib exception") |
| 804 |
|
| 805 |
def test_file(self): |
| 806 |
import email.utils |
| 807 |
h = urllib.request.FileHandler() |
| 808 |
o = h.parent = MockOpener() |
| 809 |
|
| 810 |
TESTFN = os_helper.TESTFN |
| 811 |
towrite = b"hello, world\n" |
| 812 |
canonurl = urllib.request.pathname2url(os.path.abspath(TESTFN), add_scheme=True) |
| 813 |
parsed = urlsplit(canonurl) |
| 814 |
if parsed.netloc: |
| 815 |
raise unittest.SkipTest("non-local working directory") |
| 816 |
urls = [ |
| 817 |
canonurl, |
| 818 |
parsed._replace(netloc='localhost').geturl(), |
| 819 |
parsed._replace(netloc=socket.gethostbyname('localhost')).geturl(), |
| 820 |
] |
| 821 |
try: |
| 822 |
localaddr = socket.gethostbyname(socket.gethostname()) |
| 823 |
except socket.gaierror: |
| 824 |
localaddr = '' |
| 825 |
if localaddr: |
| 826 |
urls.append(parsed._replace(netloc=localaddr).geturl()) |
| 827 |
|
| 828 |
for url in urls: |
| 829 |
f = open(TESTFN, "wb") |
| 830 |
try: |
| 831 |
try: |
| 832 |
f.write(towrite) |
| 833 |
finally: |
| 834 |
f.close() |
| 835 |
|
| 836 |
r = h.file_open(Request(url)) |
| 837 |
try: |
| 838 |
data = r.read() |
| 839 |
headers = r.info() |
| 840 |
respurl = r.geturl() |
| 841 |
finally: |
| 842 |
r.close() |
| 843 |
stats = os.stat(TESTFN) |
| 844 |
modified = email.utils.formatdate(stats.st_mtime, usegmt=True) |
| 845 |
finally: |
| 846 |
os.remove(TESTFN) |
| 847 |
self.assertEqual(data, towrite) |
| 848 |
self.assertEqual(headers["Content-type"], "text/plain") |
| 849 |
self.assertEqual(headers["Content-length"], "13") |
| 850 |
self.assertEqual(headers["Last-modified"], modified) |
| 851 |
self.assertEqual(respurl, canonurl) |
| 852 |
|
| 853 |
for url in [ |
| 854 |
parsed._replace(netloc='localhost:80').geturl(), |
| 855 |
"file:///file_does_not_exist.txt", |
| 856 |
"file://not-a-local-host.com//dir/file.txt", |
| 857 |
"file://%s:80%s/%s" % (socket.gethostbyname('localhost'), |
| 858 |
os.getcwd(), TESTFN), |
| 859 |
"file://somerandomhost.ontheinternet.com%s/%s" % |
| 860 |
(os.getcwd(), TESTFN), |
| 861 |
]: |
| 862 |
try: |
| 863 |
f = open(TESTFN, "wb") |
| 864 |
try: |
| 865 |
f.write(towrite) |
| 866 |
finally: |
| 867 |
f.close() |
| 868 |
|
| 869 |
self.assertRaises(urllib.error.URLError, |
| 870 |
h.file_open, Request(url)) |
| 871 |
finally: |
| 872 |
os.remove(TESTFN) |
| 873 |
|
| 874 |
h = urllib.request.FileHandler() |
| 875 |
o = h.parent = MockOpener() |
| 876 |
# XXXX why does // mean ftp (and /// mean not ftp!), and where |
| 877 |
# is file: scheme specified? I think this is really a bug, and |
| 878 |
# what was intended was to distinguish between URLs like: |
| 879 |
# file:/blah.txt (a file) |
| 880 |
# file://localhost/blah.txt (a file) |
| 881 |
# file:///blah.txt (a file) |
| 882 |
# file://ftp.example.com/blah.txt (an ftp URL) |
| 883 |
for url, ftp in [ |
| 884 |
("file://ftp.example.com//foo.txt", False), |
| 885 |
("file://ftp.example.com///foo.txt", False), |
| 886 |
("file://ftp.example.com/foo.txt", False), |
| 887 |
("file://somehost//foo/something.txt", False), |
| 888 |
("file://localhost//foo/something.txt", False), |
| 889 |
]: |
| 890 |
req = Request(url) |
| 891 |
try: |
| 892 |
h.file_open(req) |
| 893 |
except urllib.error.URLError: |
| 894 |
self.assertFalse(ftp) |
| 895 |
else: |
| 896 |
self.assertIs(o.req, req) |
| 897 |
self.assertEqual(req.type, "ftp") |
| 898 |
self.assertEqual(req.type == "ftp", ftp) |
| 899 |
|
| 900 |
def test_http(self): |
| 901 |
|
| 902 |
h = urllib.request.AbstractHTTPHandler() |
| 903 |
o = h.parent = MockOpener() |
| 904 |
|
| 905 |
url = "http://example.com/" |
| 906 |
for method, data in [("GET", None), ("POST", b"blah")]: |
| 907 |
req = Request(url, data, {"Foo": "bar"}) |
| 908 |
req.timeout = None |
| 909 |
req.add_unredirected_header("Spam", "eggs") |
| 910 |
http = MockHTTPClass() |
| 911 |
r = h.do_open(http, req) |
| 912 |
|
| 913 |
# result attributes |
| 914 |
r.read; r.readline # wrapped MockFile methods |
| 915 |
r.info; r.geturl # addinfourl methods |
| 916 |
r.code, r.msg == 200, "OK" # added from MockHTTPClass.getreply() |
| 917 |
hdrs = r.info() |
| 918 |
hdrs.get; hdrs.__contains__ # r.info() gives dict from .getreply() |
| 919 |
self.assertEqual(r.geturl(), url) |
| 920 |
|
| 921 |
self.assertEqual(http.host, "example.com") |
| 922 |
self.assertEqual(http.level, 0) |
| 923 |
self.assertEqual(http.method, method) |
| 924 |
self.assertEqual(http.selector, "/") |
| 925 |
self.assertEqual(http.req_headers, |
| 926 |
[("Connection", "close"), |
| 927 |
("Foo", "bar"), ("Spam", "eggs")]) |
| 928 |
self.assertEqual(http.data, data) |
| 929 |
|
| 930 |
# check OSError converted to URLError |
| 931 |
http.raise_on_endheaders = True |
| 932 |
self.assertRaises(urllib.error.URLError, h.do_open, http, req) |
| 933 |
|
| 934 |
# Check for TypeError on POST data which is str. |
| 935 |
req = Request("http://example.com/","badpost") |
| 936 |
self.assertRaises(TypeError, h.do_request_, req) |
| 937 |
|
| 938 |
# check adding of standard headers |
| 939 |
o.addheaders = [("Spam", "eggs")] |
| 940 |
for data in b"", None: # POST, GET |
| 941 |
req = Request("http://example.com/", data) |
| 942 |
r = MockResponse(200, "OK", {}, "") |
| 943 |
newreq = h.do_request_(req) |
| 944 |
if data is None: # GET |
| 945 |
self.assertNotIn("Content-length", req.unredirected_hdrs) |
| 946 |
self.assertNotIn("Content-type", req.unredirected_hdrs) |
| 947 |
else: # POST |
| 948 |
self.assertEqual(req.unredirected_hdrs["Content-length"], "0") |
| 949 |
self.assertEqual(req.unredirected_hdrs["Content-type"], |
| 950 |
"application/x-www-form-urlencoded") |
| 951 |
# XXX the details of Host could be better tested |
| 952 |
self.assertEqual(req.unredirected_hdrs["Host"], "example.com") |
| 953 |
self.assertEqual(req.unredirected_hdrs["Spam"], "eggs") |
| 954 |
|
| 955 |
# don't clobber existing headers |
| 956 |
req.add_unredirected_header("Content-length", "foo") |
| 957 |
req.add_unredirected_header("Content-type", "bar") |
| 958 |
req.add_unredirected_header("Host", "baz") |
| 959 |
req.add_unredirected_header("Spam", "foo") |
| 960 |
newreq = h.do_request_(req) |
| 961 |
self.assertEqual(req.unredirected_hdrs["Content-length"], "foo") |
| 962 |
self.assertEqual(req.unredirected_hdrs["Content-type"], "bar") |
| 963 |
self.assertEqual(req.unredirected_hdrs["Host"], "baz") |
| 964 |
self.assertEqual(req.unredirected_hdrs["Spam"], "foo") |
| 965 |
|
| 966 |
def test_http_body_file(self): |
| 967 |
# A regular file - chunked encoding is used unless Content Length is |
| 968 |
# already set. |
| 969 |
|
| 970 |
h = urllib.request.AbstractHTTPHandler() |
| 971 |
o = h.parent = MockOpener() |
| 972 |
|
| 973 |
file_obj = tempfile.NamedTemporaryFile(mode='w+b', delete=False) |
| 974 |
file_path = file_obj.name |
| 975 |
file_obj.close() |
| 976 |
self.addCleanup(os.unlink, file_path) |
| 977 |
|
| 978 |
with open(file_path, "rb") as f: |
| 979 |
req = Request("http://example.com/", f, {}) |
| 980 |
newreq = h.do_request_(req) |
| 981 |
te = newreq.get_header('Transfer-encoding') |
| 982 |
self.assertEqual(te, "chunked") |
| 983 |
self.assertFalse(newreq.has_header('Content-length')) |
| 984 |
|
| 985 |
with open(file_path, "rb") as f: |
| 986 |
req = Request("http://example.com/", f, {"Content-Length": 30}) |
| 987 |
newreq = h.do_request_(req) |
| 988 |
self.assertEqual(int(newreq.get_header('Content-length')), 30) |
| 989 |
self.assertFalse(newreq.has_header("Transfer-encoding")) |
| 990 |
|
| 991 |
def test_http_body_fileobj(self): |
| 992 |
# A file object - chunked encoding is used |
| 993 |
# unless Content Length is already set. |
| 994 |
# (Note that there are some subtle differences to a regular |
| 995 |
# file, that is why we are testing both cases.) |
| 996 |
|
| 997 |
h = urllib.request.AbstractHTTPHandler() |
| 998 |
o = h.parent = MockOpener() |
| 999 |
file_obj = io.BytesIO() |
| 1000 |
|
| 1001 |
req = Request("http://example.com/", file_obj, {}) |
| 1002 |
newreq = h.do_request_(req) |
| 1003 |
self.assertEqual(newreq.get_header('Transfer-encoding'), 'chunked') |
| 1004 |
self.assertFalse(newreq.has_header('Content-length')) |
| 1005 |
|
| 1006 |
headers = {"Content-Length": 30} |
| 1007 |
req = Request("http://example.com/", file_obj, headers) |
| 1008 |
newreq = h.do_request_(req) |
| 1009 |
self.assertEqual(int(newreq.get_header('Content-length')), 30) |
| 1010 |
self.assertFalse(newreq.has_header("Transfer-encoding")) |
| 1011 |
|
| 1012 |
file_obj.close() |
| 1013 |
|
| 1014 |
@requires_subprocess() |
| 1015 |
def test_http_body_pipe(self): |
| 1016 |
# A file reading from a pipe. |
| 1017 |
# A pipe cannot be seek'ed. There is no way to determine the |
| 1018 |
# content length up front. Thus, do_request_() should fall |
| 1019 |
# back to Transfer-encoding chunked. |
| 1020 |
|
| 1021 |
h = urllib.request.AbstractHTTPHandler() |
| 1022 |
o = h.parent = MockOpener() |
| 1023 |
|
| 1024 |
cmd = [sys.executable, "-c", r"pass"] |
| 1025 |
for headers in {}, {"Content-Length": 30}: |
| 1026 |
with subprocess.Popen(cmd, stdout=subprocess.PIPE) as proc: |
| 1027 |
req = Request("http://example.com/", proc.stdout, headers) |
| 1028 |
newreq = h.do_request_(req) |
| 1029 |
if not headers: |
| 1030 |
self.assertEqual(newreq.get_header('Content-length'), None) |
| 1031 |
self.assertEqual(newreq.get_header('Transfer-encoding'), |
| 1032 |
'chunked') |
| 1033 |
else: |
| 1034 |
self.assertEqual(int(newreq.get_header('Content-length')), |
| 1035 |
30) |
| 1036 |
|
| 1037 |
def test_http_body_iterable(self): |
| 1038 |
# Generic iterable. There is no way to determine the content |
| 1039 |
# length up front. Fall back to Transfer-encoding chunked. |
| 1040 |
|
| 1041 |
h = urllib.request.AbstractHTTPHandler() |
| 1042 |
o = h.parent = MockOpener() |
| 1043 |
|
| 1044 |
def iterable_body(): |
| 1045 |
yield b"one" |
| 1046 |
|
| 1047 |
for headers in {}, {"Content-Length": 11}: |
| 1048 |
req = Request("http://example.com/", iterable_body(), headers) |
| 1049 |
newreq = h.do_request_(req) |
| 1050 |
if not headers: |
| 1051 |
self.assertEqual(newreq.get_header('Content-length'), None) |
| 1052 |
self.assertEqual(newreq.get_header('Transfer-encoding'), |
| 1053 |
'chunked') |
| 1054 |
else: |
| 1055 |
self.assertEqual(int(newreq.get_header('Content-length')), 11) |
| 1056 |
|
| 1057 |
def test_http_body_empty_seq(self): |
| 1058 |
# Zero-length iterable body should be treated like any other iterable |
| 1059 |
h = urllib.request.AbstractHTTPHandler() |
| 1060 |
h.parent = MockOpener() |
| 1061 |
req = h.do_request_(Request("http://example.com/", ())) |
| 1062 |
self.assertEqual(req.get_header("Transfer-encoding"), "chunked") |
| 1063 |
self.assertFalse(req.has_header("Content-length")) |
| 1064 |
|
| 1065 |
def test_http_body_array(self): |
| 1066 |
# array.array Iterable - Content Length is calculated |
| 1067 |
|
| 1068 |
h = urllib.request.AbstractHTTPHandler() |
| 1069 |
o = h.parent = MockOpener() |
| 1070 |
|
| 1071 |
iterable_array = array.array("I",[1,2,3,4]) |
| 1072 |
|
| 1073 |
for headers in {}, {"Content-Length": 16}: |
| 1074 |
req = Request("http://example.com/", iterable_array, headers) |
| 1075 |
newreq = h.do_request_(req) |
| 1076 |
self.assertEqual(int(newreq.get_header('Content-length')),16) |
| 1077 |
|
| 1078 |
def test_http_handler_global_debuglevel(self): |
| 1079 |
with mock.patch.object(http.client.HTTPConnection, 'debuglevel', 6): |
| 1080 |
o = OpenerDirector() |
| 1081 |
h = MockHTTPHandler() |
| 1082 |
o.add_handler(h) |
| 1083 |
o.open("http://www.example.com") |
| 1084 |
self.assertEqual(h._debuglevel, 6) |
| 1085 |
|
| 1086 |
def test_http_handler_local_debuglevel(self): |
| 1087 |
o = OpenerDirector() |
| 1088 |
h = MockHTTPHandler(debuglevel=5) |
| 1089 |
o.add_handler(h) |
| 1090 |
o.open("http://www.example.com") |
| 1091 |
self.assertEqual(h._debuglevel, 5) |
| 1092 |
|
| 1093 |
@unittest.skipUnless(hasattr(http.client, 'HTTPSConnection'), 'HTTPSConnection required for HTTPS tests.') |
| 1094 |
def test_https_handler_global_debuglevel(self): |
| 1095 |
with mock.patch.object(http.client.HTTPSConnection, 'debuglevel', 7): |
| 1096 |
o = OpenerDirector() |
| 1097 |
h = MockHTTPSHandler() |
| 1098 |
o.add_handler(h) |
| 1099 |
o.open("https://www.example.com") |
| 1100 |
self.assertEqual(h._debuglevel, 7) |
| 1101 |
|
| 1102 |
@unittest.skipUnless(hasattr(http.client, 'HTTPSConnection'), 'HTTPSConnection required for HTTPS tests.') |
| 1103 |
def test_https_handler_local_debuglevel(self): |
| 1104 |
o = OpenerDirector() |
| 1105 |
h = MockHTTPSHandler(debuglevel=4) |
| 1106 |
o.add_handler(h) |
| 1107 |
o.open("https://www.example.com") |
| 1108 |
self.assertEqual(h._debuglevel, 4) |
| 1109 |
|
| 1110 |
def test_http_doubleslash(self): |
| 1111 |
# Checks the presence of any unnecessary double slash in url does not |
| 1112 |
# break anything. Previously, a double slash directly after the host |
| 1113 |
# could cause incorrect parsing. |
| 1114 |
h = urllib.request.AbstractHTTPHandler() |
| 1115 |
h.parent = MockOpener() |
| 1116 |
|
| 1117 |
data = b"" |
| 1118 |
ds_urls = [ |
| 1119 |
"http://example.com/foo/bar/baz.html", |
| 1120 |
"http://example.com//foo/bar/baz.html", |
| 1121 |
"http://example.com/foo//bar/baz.html", |
| 1122 |
"http://example.com/foo/bar//baz.html" |
| 1123 |
] |
| 1124 |
|
| 1125 |
for ds_url in ds_urls: |
| 1126 |
ds_req = Request(ds_url, data) |
| 1127 |
|
| 1128 |
# Check whether host is determined correctly if there is no proxy |
| 1129 |
np_ds_req = h.do_request_(ds_req) |
| 1130 |
self.assertEqual(np_ds_req.unredirected_hdrs["Host"], "example.com") |
| 1131 |
|
| 1132 |
# Check whether host is determined correctly if there is a proxy |
| 1133 |
ds_req.set_proxy("someproxy:3128", None) |
| 1134 |
p_ds_req = h.do_request_(ds_req) |
| 1135 |
self.assertEqual(p_ds_req.unredirected_hdrs["Host"], "example.com") |
| 1136 |
|
| 1137 |
def test_full_url_setter(self): |
| 1138 |
# Checks to ensure that components are set correctly after setting the |
| 1139 |
# full_url of a Request object |
| 1140 |
|
| 1141 |
urls = [ |
| 1142 |
'http://example.com?foo=bar#baz', |
| 1143 |
'http://example.com?foo=bar&spam=eggs#bash', |
| 1144 |
'http://example.com', |
| 1145 |
] |
| 1146 |
|
| 1147 |
# testing a reusable request instance, but the url parameter is |
| 1148 |
# required, so just use a dummy one to instantiate |
| 1149 |
r = Request('http://example.com') |
| 1150 |
for url in urls: |
| 1151 |
r.full_url = url |
| 1152 |
parsed = urlsplit(url) |
| 1153 |
|
| 1154 |
self.assertEqual(r.get_full_url(), url) |
| 1155 |
# full_url setter uses splittag to split into components. |
| 1156 |
# splittag sets the fragment as None while urlparse sets it to '' |
| 1157 |
self.assertEqual(r.fragment or '', parsed.fragment) |
| 1158 |
self.assertEqual(urlsplit(r.get_full_url()).query, parsed.query) |
| 1159 |
|
| 1160 |
def test_full_url_deleter(self): |
| 1161 |
r = Request('http://www.example.com') |
| 1162 |
del r.full_url |
| 1163 |
self.assertIsNone(r.full_url) |
| 1164 |
self.assertIsNone(r.fragment) |
| 1165 |
self.assertEqual(r.selector, '') |
| 1166 |
|
| 1167 |
def test_fixpath_in_weirdurls(self): |
| 1168 |
# Issue4493: urllib2 to supply '/' when to urls where path does not |
| 1169 |
# start with'/' |
| 1170 |
|
| 1171 |
h = urllib.request.AbstractHTTPHandler() |
| 1172 |
h.parent = MockOpener() |
| 1173 |
|
| 1174 |
weird_url = 'http://www.python.org?getspam' |
| 1175 |
req = Request(weird_url) |
| 1176 |
newreq = h.do_request_(req) |
| 1177 |
self.assertEqual(newreq.host, 'www.python.org') |
| 1178 |
self.assertEqual(newreq.selector, '/?getspam') |
| 1179 |
|
| 1180 |
url_without_path = 'http://www.python.org' |
| 1181 |
req = Request(url_without_path) |
| 1182 |
newreq = h.do_request_(req) |
| 1183 |
self.assertEqual(newreq.host, 'www.python.org') |
| 1184 |
self.assertEqual(newreq.selector, '') |
| 1185 |
|
| 1186 |
def test_errors(self): |
| 1187 |
h = urllib.request.HTTPErrorProcessor() |
| 1188 |
o = h.parent = MockOpener() |
| 1189 |
|
| 1190 |
url = "http://example.com/" |
| 1191 |
req = Request(url) |
| 1192 |
# all 2xx are passed through |
| 1193 |
r = MockResponse(200, "OK", {}, "", url) |
| 1194 |
newr = h.http_response(req, r) |
| 1195 |
self.assertIs(r, newr) |
| 1196 |
self.assertNotHasAttr(o, "proto") # o.error not called |
| 1197 |
r = MockResponse(202, "Accepted", {}, "", url) |
| 1198 |
newr = h.http_response(req, r) |
| 1199 |
self.assertIs(r, newr) |
| 1200 |
self.assertNotHasAttr(o, "proto") # o.error not called |
| 1201 |
r = MockResponse(206, "Partial content", {}, "", url) |
| 1202 |
newr = h.http_response(req, r) |
| 1203 |
self.assertIs(r, newr) |
| 1204 |
self.assertNotHasAttr(o, "proto") # o.error not called |
| 1205 |
# anything else calls o.error (and MockOpener returns None, here) |
| 1206 |
r = MockResponse(502, "Bad gateway", {}, "", url) |
| 1207 |
self.assertIsNone(h.http_response(req, r)) |
| 1208 |
self.assertEqual(o.proto, "http") # o.error called |
| 1209 |
self.assertEqual(o.args, (req, r, 502, "Bad gateway", {})) |
| 1210 |
|
| 1211 |
def test_cookies(self): |
| 1212 |
cj = MockCookieJar() |
| 1213 |
h = urllib.request.HTTPCookieProcessor(cj) |
| 1214 |
h.parent = MockOpener() |
| 1215 |
|
| 1216 |
req = Request("http://example.com/") |
| 1217 |
r = MockResponse(200, "OK", {}, "") |
| 1218 |
newreq = h.http_request(req) |
| 1219 |
self.assertIs(cj.ach_req, req) |
| 1220 |
self.assertIs(cj.ach_req, newreq) |
| 1221 |
self.assertEqual(req.origin_req_host, "example.com") |
| 1222 |
self.assertFalse(req.unverifiable) |
| 1223 |
newr = h.http_response(req, r) |
| 1224 |
self.assertIs(cj.ec_req, req) |
| 1225 |
self.assertIs(cj.ec_r, r) |
| 1226 |
self.assertIs(r, newr) |
| 1227 |
|
| 1228 |
def test_redirect(self): |
| 1229 |
from_url = "http://example.com/a.html" |
| 1230 |
to_url = "http://example.com/b.html" |
| 1231 |
h = urllib.request.HTTPRedirectHandler() |
| 1232 |
o = h.parent = MockOpener() |
| 1233 |
|
| 1234 |
# ordinary redirect behaviour |
| 1235 |
for code in 301, 302, 303, 307, 308: |
| 1236 |
for data in None, "blah\nblah\n": |
| 1237 |
method = getattr(h, "http_error_%s" % code) |
| 1238 |
req = Request(from_url, data) |
| 1239 |
req.timeout = socket._GLOBAL_DEFAULT_TIMEOUT |
| 1240 |
req.add_header("Nonsense", "viking=withhold") |
| 1241 |
if data is not None: |
| 1242 |
req.add_header("Content-Length", str(len(data))) |
| 1243 |
req.add_unredirected_header("Spam", "spam") |
| 1244 |
try: |
| 1245 |
method(req, MockFile(), code, "Blah", |
| 1246 |
MockHeaders({"location": to_url})) |
| 1247 |
except urllib.error.HTTPError as err: |
| 1248 |
# 307 and 308 in response to POST require user OK |
| 1249 |
self.assertIn(code, (307, 308)) |
| 1250 |
self.assertIsNotNone(data) |
| 1251 |
err.close() |
| 1252 |
self.assertEqual(o.req.get_full_url(), to_url) |
| 1253 |
try: |
| 1254 |
self.assertEqual(o.req.get_method(), "GET") |
| 1255 |
except AttributeError: |
| 1256 |
self.assertFalse(o.req.data) |
| 1257 |
|
| 1258 |
# now it's a GET, there should not be headers regarding content |
| 1259 |
# (possibly dragged from before being a POST) |
| 1260 |
headers = [x.lower() for x in o.req.headers] |
| 1261 |
self.assertNotIn("content-length", headers) |
| 1262 |
self.assertNotIn("content-type", headers) |
| 1263 |
|
| 1264 |
self.assertEqual(o.req.headers["Nonsense"], |
| 1265 |
"viking=withhold") |
| 1266 |
self.assertNotIn("Spam", o.req.headers) |
| 1267 |
self.assertNotIn("Spam", o.req.unredirected_hdrs) |
| 1268 |
|
| 1269 |
# loop detection |
| 1270 |
req = Request(from_url) |
| 1271 |
req.timeout = socket._GLOBAL_DEFAULT_TIMEOUT |
| 1272 |
|
| 1273 |
def redirect(h, req, url=to_url): |
| 1274 |
h.http_error_302(req, MockFile(), 302, "Blah", |
| 1275 |
MockHeaders({"location": url})) |
| 1276 |
# Note that the *original* request shares the same record of |
| 1277 |
# redirections with the sub-requests caused by the redirections. |
| 1278 |
|
| 1279 |
# detect infinite loop redirect of a URL to itself |
| 1280 |
req = Request(from_url, origin_req_host="example.com") |
| 1281 |
count = 0 |
| 1282 |
req.timeout = socket._GLOBAL_DEFAULT_TIMEOUT |
| 1283 |
try: |
| 1284 |
while 1: |
| 1285 |
redirect(h, req, "http://example.com/") |
| 1286 |
count = count + 1 |
| 1287 |
except urllib.error.HTTPError as err: |
| 1288 |
# don't stop until max_repeats, because cookies may introduce state |
| 1289 |
self.assertEqual(count, urllib.request.HTTPRedirectHandler.max_repeats) |
| 1290 |
err.close() |
| 1291 |
|
| 1292 |
# detect endless non-repeating chain of redirects |
| 1293 |
req = Request(from_url, origin_req_host="example.com") |
| 1294 |
count = 0 |
| 1295 |
req.timeout = socket._GLOBAL_DEFAULT_TIMEOUT |
| 1296 |
try: |
| 1297 |
while 1: |
| 1298 |
redirect(h, req, "http://example.com/%d" % count) |
| 1299 |
count = count + 1 |
| 1300 |
except urllib.error.HTTPError as err: |
| 1301 |
self.assertEqual(count, |
| 1302 |
urllib.request.HTTPRedirectHandler.max_redirections) |
| 1303 |
err.close() |
| 1304 |
|
| 1305 |
def test_invalid_redirect(self): |
| 1306 |
from_url = "http://example.com/a.html" |
| 1307 |
valid_schemes = ['http','https','ftp'] |
| 1308 |
invalid_schemes = ['file','imap','ldap'] |
| 1309 |
schemeless_url = "example.com/b.html" |
| 1310 |
h = urllib.request.HTTPRedirectHandler() |
| 1311 |
o = h.parent = MockOpener() |
| 1312 |
req = Request(from_url) |
| 1313 |
req.timeout = socket._GLOBAL_DEFAULT_TIMEOUT |
| 1314 |
|
| 1315 |
for scheme in invalid_schemes: |
| 1316 |
invalid_url = scheme + '://' + schemeless_url |
| 1317 |
with self.assertRaises(urllib.error.HTTPError) as cm: |
| 1318 |
h.http_error_302( |
| 1319 |
req, MockFile(), 302, "Security Loophole", |
| 1320 |
MockHeaders({"location": invalid_url})) |
| 1321 |
cm.exception.close() |
| 1322 |
|
| 1323 |
for scheme in valid_schemes: |
| 1324 |
valid_url = scheme + '://' + schemeless_url |
| 1325 |
h.http_error_302(req, MockFile(), 302, "That's fine", |
| 1326 |
MockHeaders({"location": valid_url})) |
| 1327 |
self.assertEqual(o.req.get_full_url(), valid_url) |
| 1328 |
|
| 1329 |
def test_relative_redirect(self): |
| 1330 |
from_url = "http://example.com/a.html" |
| 1331 |
relative_url = "/b.html" |
| 1332 |
h = urllib.request.HTTPRedirectHandler() |
| 1333 |
o = h.parent = MockOpener() |
| 1334 |
req = Request(from_url) |
| 1335 |
req.timeout = socket._GLOBAL_DEFAULT_TIMEOUT |
| 1336 |
|
| 1337 |
valid_url = urllib.parse.urljoin(from_url,relative_url) |
| 1338 |
h.http_error_302(req, MockFile(), 302, "That's fine", |
| 1339 |
MockHeaders({"location": valid_url})) |
| 1340 |
self.assertEqual(o.req.get_full_url(), valid_url) |
| 1341 |
|
| 1342 |
def test_cookie_redirect(self): |
| 1343 |
# cookies shouldn't leak into redirected requests |
| 1344 |
from http.cookiejar import CookieJar |
| 1345 |
from test.test_http_cookiejar import interact_netscape |
| 1346 |
|
| 1347 |
cj = CookieJar() |
| 1348 |
interact_netscape(cj, "http://www.example.com/", "spam=eggs") |
| 1349 |
hh = MockHTTPHandlerRedirect(302, "Location: http://www.cracker.com/\r\n\r\n") |
| 1350 |
hdeh = urllib.request.HTTPDefaultErrorHandler() |
| 1351 |
hrh = urllib.request.HTTPRedirectHandler() |
| 1352 |
cp = urllib.request.HTTPCookieProcessor(cj) |
| 1353 |
o = build_test_opener(hh, hdeh, hrh, cp) |
| 1354 |
o.open("http://www.example.com/") |
| 1355 |
self.assertFalse(hh.req.has_header("Cookie")) |
| 1356 |
|
| 1357 |
def test_redirect_fragment(self): |
| 1358 |
redirected_url = 'http://www.example.com/index.html#OK\r\n\r\n' |
| 1359 |
hh = MockHTTPHandlerRedirect(302, 'Location: ' + redirected_url) |
| 1360 |
hdeh = urllib.request.HTTPDefaultErrorHandler() |
| 1361 |
hrh = urllib.request.HTTPRedirectHandler() |
| 1362 |
o = build_test_opener(hh, hdeh, hrh) |
| 1363 |
fp = o.open('http://www.example.com') |
| 1364 |
self.assertEqual(fp.geturl(), redirected_url.strip()) |
| 1365 |
|
| 1366 |
def test_redirect_no_path(self): |
| 1367 |
# Issue 14132: Relative redirect strips original path |
| 1368 |
|
| 1369 |
# clear _opener global variable |
| 1370 |
self.addCleanup(urllib.request.urlcleanup) |
| 1371 |
|
| 1372 |
real_class = http.client.HTTPConnection |
| 1373 |
response1 = b"HTTP/1.1 302 Found\r\nLocation: ?query\r\n\r\n" |
| 1374 |
http.client.HTTPConnection = test_urllib.fakehttp(response1) |
| 1375 |
self.addCleanup(setattr, http.client, "HTTPConnection", real_class) |
| 1376 |
urls = iter(("/path", "/path?query")) |
| 1377 |
def request(conn, method, url, *pos, **kw): |
| 1378 |
self.assertEqual(url, next(urls)) |
| 1379 |
real_class.request(conn, method, url, *pos, **kw) |
| 1380 |
# Change response for subsequent connection |
| 1381 |
conn.__class__.fakedata = b"HTTP/1.1 200 OK\r\n\r\nHello!" |
| 1382 |
http.client.HTTPConnection.request = request |
| 1383 |
fp = urllib.request.urlopen("http://python.org/path") |
| 1384 |
self.assertEqual(fp.geturl(), "http://python.org/path?query") |
| 1385 |
|
| 1386 |
def test_redirect_encoding(self): |
| 1387 |
# Some characters in the redirect target may need special handling, |
| 1388 |
# but most ASCII characters should be treated as already encoded |
| 1389 |
class Handler(urllib.request.HTTPHandler): |
| 1390 |
def http_open(self, req): |
| 1391 |
result = self.do_open(self.connection, req) |
| 1392 |
self.last_buf = self.connection.buf |
| 1393 |
# Set up a normal response for the next request |
| 1394 |
self.connection = test_urllib.fakehttp( |
| 1395 |
b'HTTP/1.1 200 OK\r\n' |
| 1396 |
b'Content-Length: 3\r\n' |
| 1397 |
b'\r\n' |
| 1398 |
b'123' |
| 1399 |
) |
| 1400 |
return result |
| 1401 |
handler = Handler() |
| 1402 |
opener = urllib.request.build_opener(handler) |
| 1403 |
tests = ( |
| 1404 |
(b'/p\xC3\xA5-dansk/', b'/p%C3%A5-dansk/'), |
| 1405 |
(b'/spaced%20path/', b'/spaced%20path/'), |
| 1406 |
(b'/spaced path/', b'/spaced%20path/'), |
| 1407 |
(b'/?p\xC3\xA5-dansk', b'/?p%C3%A5-dansk'), |
| 1408 |
) |
| 1409 |
for [location, result] in tests: |
| 1410 |
with self.subTest(repr(location)): |
| 1411 |
handler.connection = test_urllib.fakehttp( |
| 1412 |
b'HTTP/1.1 302 Redirect\r\n' |
| 1413 |
b'Location: ' + location + b'\r\n' |
| 1414 |
b'\r\n' |
| 1415 |
) |
| 1416 |
response = opener.open('http://example.com/') |
| 1417 |
expected = b'GET ' + result + b' ' |
| 1418 |
request = handler.last_buf |
| 1419 |
self.assertStartsWith(request, expected) |
| 1420 |
|
| 1421 |
def test_redirect_head_request(self): |
| 1422 |
from_url = "http://example.com/a.html" |
| 1423 |
to_url = "http://example.com/b.html" |
| 1424 |
h = urllib.request.HTTPRedirectHandler() |
| 1425 |
req = Request(from_url, method="HEAD") |
| 1426 |
fp = MockFile() |
| 1427 |
new_req = h.redirect_request(req, fp, 302, "Found", {}, to_url) |
| 1428 |
self.assertEqual(new_req.get_method(), "HEAD") |
| 1429 |
|
| 1430 |
def test_proxy(self): |
| 1431 |
u = "proxy.example.com:3128" |
| 1432 |
for d in dict(http=u), dict(HTTP=u): |
| 1433 |
o = OpenerDirector() |
| 1434 |
ph = urllib.request.ProxyHandler(d) |
| 1435 |
o.add_handler(ph) |
| 1436 |
meth_spec = [ |
| 1437 |
[("http_open", "return response")] |
| 1438 |
] |
| 1439 |
handlers = add_ordered_mock_handlers(o, meth_spec) |
| 1440 |
|
| 1441 |
req = Request("http://acme.example.com/") |
| 1442 |
self.assertEqual(req.host, "acme.example.com") |
| 1443 |
o.open(req) |
| 1444 |
self.assertEqual(req.host, u) |
| 1445 |
self.assertEqual([(handlers[0], "http_open")], |
| 1446 |
[tup[0:2] for tup in o.calls]) |
| 1447 |
|
| 1448 |
def test_proxy_no_proxy(self): |
| 1449 |
env = self.enterContext(os_helper.EnvironmentVarGuard()) |
| 1450 |
env['no_proxy'] = 'python.org' |
| 1451 |
o = OpenerDirector() |
| 1452 |
ph = urllib.request.ProxyHandler(dict(http="proxy.example.com")) |
| 1453 |
o.add_handler(ph) |
| 1454 |
req = Request("http://www.perl.org/") |
| 1455 |
self.assertEqual(req.host, "www.perl.org") |
| 1456 |
o.open(req) |
| 1457 |
self.assertEqual(req.host, "proxy.example.com") |
| 1458 |
req = Request("http://www.python.org") |
| 1459 |
self.assertEqual(req.host, "www.python.org") |
| 1460 |
o.open(req) |
| 1461 |
self.assertEqual(req.host, "www.python.org") |
| 1462 |
|
| 1463 |
def test_proxy_no_proxy_all(self): |
| 1464 |
env = self.enterContext(os_helper.EnvironmentVarGuard()) |
| 1465 |
env['no_proxy'] = '*' |
| 1466 |
o = OpenerDirector() |
| 1467 |
ph = urllib.request.ProxyHandler(dict(http="proxy.example.com")) |
| 1468 |
o.add_handler(ph) |
| 1469 |
req = Request("http://www.python.org") |
| 1470 |
self.assertEqual(req.host, "www.python.org") |
| 1471 |
o.open(req) |
| 1472 |
self.assertEqual(req.host, "www.python.org") |
| 1473 |
|
| 1474 |
def test_proxy_https(self): |
| 1475 |
o = OpenerDirector() |
| 1476 |
ph = urllib.request.ProxyHandler(dict(https="proxy.example.com:3128")) |
| 1477 |
o.add_handler(ph) |
| 1478 |
meth_spec = [ |
| 1479 |
[("https_open", "return response")] |
| 1480 |
] |
| 1481 |
handlers = add_ordered_mock_handlers(o, meth_spec) |
| 1482 |
|
| 1483 |
req = Request("https://www.example.com/") |
| 1484 |
self.assertEqual(req.host, "www.example.com") |
| 1485 |
o.open(req) |
| 1486 |
self.assertEqual(req.host, "proxy.example.com:3128") |
| 1487 |
self.assertEqual([(handlers[0], "https_open")], |
| 1488 |
[tup[0:2] for tup in o.calls]) |
| 1489 |
|
| 1490 |
@unittest.skipUnless(hasattr(http.client, 'HTTPSConnection'), 'HTTPSConnection required for HTTPS tests.') |
| 1491 |
def test_proxy_https_proxy_authorization(self): |
| 1492 |
o = OpenerDirector() |
| 1493 |
ph = urllib.request.ProxyHandler(dict(https='proxy.example.com:3128')) |
| 1494 |
o.add_handler(ph) |
| 1495 |
https_handler = MockHTTPSHandler() |
| 1496 |
o.add_handler(https_handler) |
| 1497 |
req = Request("https://www.example.com/") |
| 1498 |
req.add_header("Proxy-Authorization", "FooBar") |
| 1499 |
req.add_header("User-Agent", "Grail") |
| 1500 |
self.assertEqual(req.host, "www.example.com") |
| 1501 |
self.assertIsNone(req._tunnel_host) |
| 1502 |
o.open(req) |
| 1503 |
# Verify Proxy-Authorization gets tunneled to request. |
| 1504 |
# httpsconn req_headers do not have the Proxy-Authorization header but |
| 1505 |
# the req will have. |
| 1506 |
self.assertNotIn(("Proxy-Authorization", "FooBar"), |
| 1507 |
https_handler.httpconn.req_headers) |
| 1508 |
self.assertIn(("User-Agent", "Grail"), |
| 1509 |
https_handler.httpconn.req_headers) |
| 1510 |
self.assertIsNotNone(req._tunnel_host) |
| 1511 |
self.assertEqual(req.host, "proxy.example.com:3128") |
| 1512 |
self.assertEqual(req.get_header("Proxy-authorization"), "FooBar") |
| 1513 |
|
| 1514 |
@unittest.skipUnless(os.name == "nt", "only relevant for Windows") |
| 1515 |
def test_winreg_proxy_bypass(self): |
| 1516 |
proxy_override = "www.example.com;*.example.net; 192.168.0.1" |
| 1517 |
proxy_bypass = _proxy_bypass_winreg_override |
| 1518 |
for host in ("www.example.com", "www.example.net", "192.168.0.1"): |
| 1519 |
self.assertTrue(proxy_bypass(host, proxy_override), |
| 1520 |
"expected bypass of %s to be true" % host) |
| 1521 |
|
| 1522 |
for host in ("example.com", "www.example.org", "example.net", |
| 1523 |
"192.168.0.2"): |
| 1524 |
self.assertFalse(proxy_bypass(host, proxy_override), |
| 1525 |
"expected bypass of %s to be False" % host) |
| 1526 |
|
| 1527 |
# check intranet address bypass |
| 1528 |
proxy_override = "example.com; <local>" |
| 1529 |
self.assertTrue(proxy_bypass("example.com", proxy_override), |
| 1530 |
"expected bypass of %s to be true" % host) |
| 1531 |
self.assertFalse(proxy_bypass("example.net", proxy_override), |
| 1532 |
"expected bypass of %s to be False" % host) |
| 1533 |
for host in ("test", "localhost"): |
| 1534 |
self.assertTrue(proxy_bypass(host, proxy_override), |
| 1535 |
"expect <local> to bypass intranet address '%s'" |
| 1536 |
% host) |
| 1537 |
|
| 1538 |
@unittest.skipUnless(sys.platform == 'darwin', "only relevant for OSX") |
| 1539 |
def test_osx_proxy_bypass(self): |
| 1540 |
bypass = { |
| 1541 |
'exclude_simple': False, |
| 1542 |
'exceptions': ['foo.bar', '*.bar.com', '127.0.0.1', '10.10', |
| 1543 |
'10.0/16'] |
| 1544 |
} |
| 1545 |
# Check hosts that should trigger the proxy bypass |
| 1546 |
for host in ('foo.bar', 'www.bar.com', '127.0.0.1', '10.10.0.1', |
| 1547 |
'10.0.0.1'): |
| 1548 |
self.assertTrue(_proxy_bypass_macosx_sysconf(host, bypass), |
| 1549 |
'expected bypass of %s to be True' % host) |
| 1550 |
# Check hosts that should not trigger the proxy bypass |
| 1551 |
for host in ('abc.foo.bar', 'bar.com', '127.0.0.2', '10.11.0.1', |
| 1552 |
'notinbypass'): |
| 1553 |
self.assertFalse(_proxy_bypass_macosx_sysconf(host, bypass), |
| 1554 |
'expected bypass of %s to be False' % host) |
| 1555 |
|
| 1556 |
# Check the exclude_simple flag |
| 1557 |
bypass = {'exclude_simple': True, 'exceptions': []} |
| 1558 |
self.assertTrue(_proxy_bypass_macosx_sysconf('test', bypass)) |
| 1559 |
|
| 1560 |
# Check that invalid prefix lengths are ignored |
| 1561 |
bypass = { |
| 1562 |
'exclude_simple': False, |
| 1563 |
'exceptions': [ '10.0.0.0/40', '172.19.10.0/24' ] |
| 1564 |
} |
| 1565 |
host = '172.19.10.5' |
| 1566 |
self.assertTrue(_proxy_bypass_macosx_sysconf(host, bypass), |
| 1567 |
'expected bypass of %s to be True' % host) |
| 1568 |
host = '10.0.1.5' |
| 1569 |
self.assertFalse(_proxy_bypass_macosx_sysconf(host, bypass), |
| 1570 |
'expected bypass of %s to be False' % host) |
| 1571 |
|
| 1572 |
def check_basic_auth(self, headers, realm): |
| 1573 |
with self.subTest(realm=realm, headers=headers): |
| 1574 |
opener = OpenerDirector() |
| 1575 |
password_manager = MockPasswordManager() |
| 1576 |
auth_handler = urllib.request.HTTPBasicAuthHandler(password_manager) |
| 1577 |
body = '\r\n'.join(headers) + '\r\n\r\n' |
| 1578 |
http_handler = MockHTTPHandlerRedirect(401, body) |
| 1579 |
opener.add_handler(auth_handler) |
| 1580 |
opener.add_handler(http_handler) |
| 1581 |
self._test_basic_auth(opener, auth_handler, "Authorization", |
| 1582 |
realm, http_handler, password_manager, |
| 1583 |
"http://acme.example.com/protected", |
| 1584 |
"http://acme.example.com/protected") |
| 1585 |
|
| 1586 |
def test_basic_auth(self): |
| 1587 |
realm = "[email protected]" |
| 1588 |
realm2 = "[email protected]" |
| 1589 |
basic = f'Basic realm="{realm}"' |
| 1590 |
basic2 = f'Basic realm="{realm2}"' |
| 1591 |
other_no_realm = 'Otherscheme xxx' |
| 1592 |
digest = (f'Digest realm="{realm2}", ' |
| 1593 |
f'qop="auth, auth-int", ' |
| 1594 |
f'nonce="dcd98b7102dd2f0e8b11d0f600bfb0c093", ' |
| 1595 |
f'opaque="5ccc069c403ebaf9f0171e9517f40e41"') |
| 1596 |
for realm_str in ( |
| 1597 |
# test "quote" and 'quote' |
| 1598 |
f'Basic realm="{realm}"', |
| 1599 |
f"Basic realm='{realm}'", |
| 1600 |
|
| 1601 |
# charset is ignored |
| 1602 |
f'Basic realm="{realm}", charset="UTF-8"', |
| 1603 |
|
| 1604 |
# Multiple challenges per header |
| 1605 |
f'{basic}, {basic2}', |
| 1606 |
f'{basic}, {other_no_realm}', |
| 1607 |
f'{other_no_realm}, {basic}', |
| 1608 |
f'{basic}, {digest}', |
| 1609 |
f'{digest}, {basic}', |
| 1610 |
): |
| 1611 |
headers = [f'WWW-Authenticate: {realm_str}'] |
| 1612 |
self.check_basic_auth(headers, realm) |
| 1613 |
|
| 1614 |
# no quote: expect a warning |
| 1615 |
with warnings_helper.check_warnings(("Basic Auth Realm was unquoted", |
| 1616 |
UserWarning)): |
| 1617 |
headers = [f'WWW-Authenticate: Basic realm={realm}'] |
| 1618 |
self.check_basic_auth(headers, realm) |
| 1619 |
|
| 1620 |
# Multiple headers: one challenge per header. |
| 1621 |
# Use the first Basic realm. |
| 1622 |
for challenges in ( |
| 1623 |
[basic, basic2], |
| 1624 |
[basic, digest], |
| 1625 |
[digest, basic], |
| 1626 |
): |
| 1627 |
headers = [f'WWW-Authenticate: {challenge}' |
| 1628 |
for challenge in challenges] |
| 1629 |
self.check_basic_auth(headers, realm) |
| 1630 |
|
| 1631 |
def test_proxy_basic_auth(self): |
| 1632 |
opener = OpenerDirector() |
| 1633 |
ph = urllib.request.ProxyHandler(dict(http="proxy.example.com:3128")) |
| 1634 |
opener.add_handler(ph) |
| 1635 |
password_manager = MockPasswordManager() |
| 1636 |
auth_handler = urllib.request.ProxyBasicAuthHandler(password_manager) |
| 1637 |
realm = "ACME Networks" |
| 1638 |
http_handler = MockHTTPHandlerRedirect( |
| 1639 |
407, 'Proxy-Authenticate: Basic realm="%s"\r\n\r\n' % realm) |
| 1640 |
opener.add_handler(auth_handler) |
| 1641 |
opener.add_handler(http_handler) |
| 1642 |
self._test_basic_auth(opener, auth_handler, "Proxy-authorization", |
| 1643 |
realm, http_handler, password_manager, |
| 1644 |
"http://acme.example.com:3128/protected", |
| 1645 |
"proxy.example.com:3128", |
| 1646 |
) |
| 1647 |
|
| 1648 |
def test_basic_and_digest_auth_handlers(self): |
| 1649 |
# HTTPDigestAuthHandler raised an exception if it couldn't handle a 40* |
| 1650 |
# response (https://bugs.python.org/issue1479302), where it should instead |
| 1651 |
# return None to allow another handler (especially |
| 1652 |
# HTTPBasicAuthHandler) to handle the response. |
| 1653 |
|
| 1654 |
# Also (https://bugs.python.org/issue14797027, RFC 2617 section 1.2), we must |
| 1655 |
# try digest first (since it's the strongest auth scheme), so we record |
| 1656 |
# order of calls here to check digest comes first: |
| 1657 |
class RecordingOpenerDirector(OpenerDirector): |
| 1658 |
def __init__(self): |
| 1659 |
OpenerDirector.__init__(self) |
| 1660 |
self.recorded = [] |
| 1661 |
|
| 1662 |
def record(self, info): |
| 1663 |
self.recorded.append(info) |
| 1664 |
|
| 1665 |
class TestDigestAuthHandler(urllib.request.HTTPDigestAuthHandler): |
| 1666 |
def http_error_401(self, *args, **kwds): |
| 1667 |
self.parent.record("digest") |
| 1668 |
urllib.request.HTTPDigestAuthHandler.http_error_401(self, |
| 1669 |
*args, **kwds) |
| 1670 |
|
| 1671 |
class TestBasicAuthHandler(urllib.request.HTTPBasicAuthHandler): |
| 1672 |
def http_error_401(self, *args, **kwds): |
| 1673 |
self.parent.record("basic") |
| 1674 |
urllib.request.HTTPBasicAuthHandler.http_error_401(self, |
| 1675 |
*args, **kwds) |
| 1676 |
|
| 1677 |
opener = RecordingOpenerDirector() |
| 1678 |
password_manager = MockPasswordManager() |
| 1679 |
digest_handler = TestDigestAuthHandler(password_manager) |
| 1680 |
basic_handler = TestBasicAuthHandler(password_manager) |
| 1681 |
realm = "ACME Networks" |
| 1682 |
http_handler = MockHTTPHandlerRedirect( |
| 1683 |
401, 'WWW-Authenticate: Basic realm="%s"\r\n\r\n' % realm) |
| 1684 |
opener.add_handler(basic_handler) |
| 1685 |
opener.add_handler(digest_handler) |
| 1686 |
opener.add_handler(http_handler) |
| 1687 |
|
| 1688 |
# check basic auth isn't blocked by digest handler failing |
| 1689 |
self._test_basic_auth(opener, basic_handler, "Authorization", |
| 1690 |
realm, http_handler, password_manager, |
| 1691 |
"http://acme.example.com/protected", |
| 1692 |
"http://acme.example.com/protected", |
| 1693 |
) |
| 1694 |
# check digest was tried before basic (twice, because |
| 1695 |
# _test_basic_auth called .open() twice) |
| 1696 |
self.assertEqual(opener.recorded, ["digest", "basic"]*2) |
| 1697 |
|
| 1698 |
def test_unsupported_auth_digest_handler(self): |
| 1699 |
opener = OpenerDirector() |
| 1700 |
# While using DigestAuthHandler |
| 1701 |
digest_auth_handler = urllib.request.HTTPDigestAuthHandler(None) |
| 1702 |
http_handler = MockHTTPHandlerRedirect( |
| 1703 |
401, 'WWW-Authenticate: Kerberos\r\n\r\n') |
| 1704 |
opener.add_handler(digest_auth_handler) |
| 1705 |
opener.add_handler(http_handler) |
| 1706 |
self.assertRaises(ValueError, opener.open, "http://www.example.com") |
| 1707 |
|
| 1708 |
def test_unsupported_auth_basic_handler(self): |
| 1709 |
# While using BasicAuthHandler |
| 1710 |
opener = OpenerDirector() |
| 1711 |
basic_auth_handler = urllib.request.HTTPBasicAuthHandler(None) |
| 1712 |
http_handler = MockHTTPHandlerRedirect( |
| 1713 |
401, 'WWW-Authenticate: NTLM\r\n\r\n') |
| 1714 |
opener.add_handler(basic_auth_handler) |
| 1715 |
opener.add_handler(http_handler) |
| 1716 |
self.assertRaises(ValueError, opener.open, "http://www.example.com") |
| 1717 |
|
| 1718 |
def _test_basic_auth(self, opener, auth_handler, auth_header, |
| 1719 |
realm, http_handler, password_manager, |
| 1720 |
request_url, protected_url): |
| 1721 |
import base64 |
| 1722 |
user, password = "wile", "coyote" |
| 1723 |
|
| 1724 |
# .add_password() fed through to password manager |
| 1725 |
auth_handler.add_password(realm, request_url, user, password) |
| 1726 |
self.assertEqual(realm, password_manager.realm) |
| 1727 |
self.assertEqual(request_url, password_manager.url) |
| 1728 |
self.assertEqual(user, password_manager.user) |
| 1729 |
self.assertEqual(password, password_manager.password) |
| 1730 |
|
| 1731 |
opener.open(request_url) |
| 1732 |
|
| 1733 |
# should have asked the password manager for the username/password |
| 1734 |
self.assertEqual(password_manager.target_realm, realm) |
| 1735 |
self.assertEqual(password_manager.target_url, protected_url) |
| 1736 |
|
| 1737 |
# expect one request without authorization, then one with |
| 1738 |
self.assertEqual(len(http_handler.requests), 2) |
| 1739 |
self.assertFalse(http_handler.requests[0].has_header(auth_header)) |
| 1740 |
userpass = bytes('%s:%s' % (user, password), "ascii") |
| 1741 |
auth_hdr_value = ('Basic ' + |
| 1742 |
base64.encodebytes(userpass).strip().decode()) |
| 1743 |
self.assertEqual(http_handler.requests[1].get_header(auth_header), |
| 1744 |
auth_hdr_value) |
| 1745 |
self.assertEqual(http_handler.requests[1].unredirected_hdrs[auth_header], |
| 1746 |
auth_hdr_value) |
| 1747 |
# if the password manager can't find a password, the handler won't |
| 1748 |
# handle the HTTP auth error |
| 1749 |
password_manager.user = password_manager.password = None |
| 1750 |
http_handler.reset() |
| 1751 |
opener.open(request_url) |
| 1752 |
self.assertEqual(len(http_handler.requests), 1) |
| 1753 |
self.assertFalse(http_handler.requests[0].has_header(auth_header)) |
| 1754 |
|
| 1755 |
def test_basic_prior_auth_auto_send(self): |
| 1756 |
# Assume already authenticated if is_authenticated=True |
| 1757 |
# for APIs like Github that don't return 401 |
| 1758 |
|
| 1759 |
user, password = "wile", "coyote" |
| 1760 |
request_url = "http://acme.example.com/protected" |
| 1761 |
|
| 1762 |
http_handler = MockHTTPHandlerCheckAuth(200) |
| 1763 |
|
| 1764 |
pwd_manager = HTTPPasswordMgrWithPriorAuth() |
| 1765 |
auth_prior_handler = HTTPBasicAuthHandler(pwd_manager) |
| 1766 |
auth_prior_handler.add_password( |
| 1767 |
None, request_url, user, password, is_authenticated=True) |
| 1768 |
|
| 1769 |
self.assertTrue(pwd_manager.is_authenticated(request_url)) |
| 1770 |
self.assertTrue(pwd_manager.is_authenticated(request_url + '/nested')) |
| 1771 |
self.assertFalse(pwd_manager.is_authenticated(request_url + 'plain')) |
| 1772 |
|
| 1773 |
opener = OpenerDirector() |
| 1774 |
opener.add_handler(auth_prior_handler) |
| 1775 |
opener.add_handler(http_handler) |
| 1776 |
|
| 1777 |
opener.open(request_url) |
| 1778 |
|
| 1779 |
# expect request to be sent with auth header |
| 1780 |
self.assertTrue(http_handler.has_auth_header) |
| 1781 |
|
| 1782 |
def test_basic_prior_auth_send_after_first_success(self): |
| 1783 |
# Auto send auth header after authentication is successful once |
| 1784 |
|
| 1785 |
user, password = 'wile', 'coyote' |
| 1786 |
request_url = 'http://acme.example.com/protected' |
| 1787 |
realm = 'ACME' |
| 1788 |
|
| 1789 |
pwd_manager = HTTPPasswordMgrWithPriorAuth() |
| 1790 |
auth_prior_handler = HTTPBasicAuthHandler(pwd_manager) |
| 1791 |
auth_prior_handler.add_password(realm, request_url, user, password) |
| 1792 |
|
| 1793 |
is_auth = pwd_manager.is_authenticated(request_url) |
| 1794 |
self.assertFalse(is_auth) |
| 1795 |
|
| 1796 |
opener = OpenerDirector() |
| 1797 |
opener.add_handler(auth_prior_handler) |
| 1798 |
|
| 1799 |
http_handler = MockHTTPHandlerRedirect( |
| 1800 |
401, 'WWW-Authenticate: Basic realm="%s"\r\n\r\n' % None) |
| 1801 |
opener.add_handler(http_handler) |
| 1802 |
|
| 1803 |
opener.open(request_url) |
| 1804 |
|
| 1805 |
is_auth = pwd_manager.is_authenticated(request_url) |
| 1806 |
self.assertTrue(is_auth) |
| 1807 |
|
| 1808 |
http_handler = MockHTTPHandlerCheckAuth(200) |
| 1809 |
self.assertFalse(http_handler.has_auth_header) |
| 1810 |
|
| 1811 |
opener = OpenerDirector() |
| 1812 |
opener.add_handler(auth_prior_handler) |
| 1813 |
opener.add_handler(http_handler) |
| 1814 |
|
| 1815 |
# After getting 200 from MockHTTPHandler |
| 1816 |
# Next request sends header in the first request |
| 1817 |
opener.open(request_url) |
| 1818 |
|
| 1819 |
# expect request to be sent with auth header |
| 1820 |
self.assertTrue(http_handler.has_auth_header) |
| 1821 |
|
| 1822 |
def test_http_closed(self): |
| 1823 |
"""Test the connection is cleaned up when the response is closed""" |
| 1824 |
for (transfer, data) in ( |
| 1825 |
("Connection: close", b"data"), |
| 1826 |
("Transfer-Encoding: chunked", b"4\r\ndata\r\n0\r\n\r\n"), |
| 1827 |
("Content-Length: 4", b"data"), |
| 1828 |
): |
| 1829 |
header = "HTTP/1.1 200 OK\r\n{}\r\n\r\n".format(transfer) |
| 1830 |
conn = test_urllib.fakehttp(header.encode() + data) |
| 1831 |
handler = urllib.request.AbstractHTTPHandler() |
| 1832 |
req = Request("http://dummy/") |
| 1833 |
req.timeout = None |
| 1834 |
with handler.do_open(conn, req) as resp: |
| 1835 |
resp.read() |
| 1836 |
self.assertTrue(conn.fakesock.closed, |
| 1837 |
"Connection not closed with {!r}".format(transfer)) |
| 1838 |
|
| 1839 |
def test_invalid_closed(self): |
| 1840 |
"""Test the connection is cleaned up after an invalid response""" |
| 1841 |
conn = test_urllib.fakehttp(b"") |
| 1842 |
handler = urllib.request.AbstractHTTPHandler() |
| 1843 |
req = Request("http://dummy/") |
| 1844 |
req.timeout = None |
| 1845 |
with self.assertRaises(http.client.BadStatusLine): |
| 1846 |
handler.do_open(conn, req) |
| 1847 |
self.assertTrue(conn.fakesock.closed, "Connection not closed") |
| 1848 |
|
| 1849 |
|
| 1850 |
class MiscTests(unittest.TestCase): |
| 1851 |
|
| 1852 |
def opener_has_handler(self, opener, handler_class): |
| 1853 |
self.assertTrue(any(h.__class__ == handler_class |
| 1854 |
for h in opener.handlers)) |
| 1855 |
|
| 1856 |
def test_build_opener(self): |
| 1857 |
class MyHTTPHandler(urllib.request.HTTPHandler): |
| 1858 |
pass |
| 1859 |
|
| 1860 |
class FooHandler(urllib.request.BaseHandler): |
| 1861 |
def foo_open(self): |
| 1862 |
pass |
| 1863 |
|
| 1864 |
class BarHandler(urllib.request.BaseHandler): |
| 1865 |
def bar_open(self): |
| 1866 |
pass |
| 1867 |
|
| 1868 |
build_opener = urllib.request.build_opener |
| 1869 |
|
| 1870 |
o = build_opener(FooHandler, BarHandler) |
| 1871 |
self.opener_has_handler(o, FooHandler) |
| 1872 |
self.opener_has_handler(o, BarHandler) |
| 1873 |
|
| 1874 |
# can take a mix of classes and instances |
| 1875 |
o = build_opener(FooHandler, BarHandler()) |
| 1876 |
self.opener_has_handler(o, FooHandler) |
| 1877 |
self.opener_has_handler(o, BarHandler) |
| 1878 |
|
| 1879 |
# subclasses of default handlers override default handlers |
| 1880 |
o = build_opener(MyHTTPHandler) |
| 1881 |
self.opener_has_handler(o, MyHTTPHandler) |
| 1882 |
|
| 1883 |
# a particular case of overriding: default handlers can be passed |
| 1884 |
# in explicitly |
| 1885 |
o = build_opener() |
| 1886 |
self.opener_has_handler(o, urllib.request.HTTPHandler) |
| 1887 |
o = build_opener(urllib.request.HTTPHandler) |
| 1888 |
self.opener_has_handler(o, urllib.request.HTTPHandler) |
| 1889 |
o = build_opener(urllib.request.HTTPHandler()) |
| 1890 |
self.opener_has_handler(o, urllib.request.HTTPHandler) |
| 1891 |
|
| 1892 |
# Issue2670: multiple handlers sharing the same base class |
| 1893 |
class MyOtherHTTPHandler(urllib.request.HTTPHandler): |
| 1894 |
pass |
| 1895 |
|
| 1896 |
o = build_opener(MyHTTPHandler, MyOtherHTTPHandler) |
| 1897 |
self.opener_has_handler(o, MyHTTPHandler) |
| 1898 |
self.opener_has_handler(o, MyOtherHTTPHandler) |
| 1899 |
|
| 1900 |
def test_HTTPError_interface(self): |
| 1901 |
""" |
| 1902 |
Issue 13211 reveals that HTTPError didn't implement the URLError |
| 1903 |
interface even though HTTPError is a subclass of URLError. |
| 1904 |
""" |
| 1905 |
msg = 'something bad happened' |
| 1906 |
url = code = fp = None |
| 1907 |
hdrs = 'Content-Length: 42' |
| 1908 |
err = urllib.error.HTTPError(url, code, msg, hdrs, fp) |
| 1909 |
self.assertHasAttr(err, 'reason') |
| 1910 |
self.assertEqual(err.reason, 'something bad happened') |
| 1911 |
self.assertHasAttr(err, 'headers') |
| 1912 |
self.assertEqual(err.headers, 'Content-Length: 42') |
| 1913 |
expected_errmsg = 'HTTP Error %s: %s' % (err.code, err.msg) |
| 1914 |
self.assertEqual(str(err), expected_errmsg) |
| 1915 |
expected_errmsg = '<HTTPError %s: %r>' % (err.code, err.msg) |
| 1916 |
self.assertEqual(repr(err), expected_errmsg) |
| 1917 |
err.close() |
| 1918 |
|
| 1919 |
def test_gh_98778(self): |
| 1920 |
x = urllib.error.HTTPError("url", 405, "METHOD NOT ALLOWED", None, None) |
| 1921 |
self.assertEqual(getattr(x, "__notes__", ()), ()) |
| 1922 |
self.assertIsInstance(x.fp.read(), bytes) |
| 1923 |
x.close() |
| 1924 |
|
| 1925 |
def test_parse_proxy(self): |
| 1926 |
parse_proxy_test_cases = [ |
| 1927 |
('proxy.example.com', |
| 1928 |
(None, None, None, 'proxy.example.com')), |
| 1929 |
('proxy.example.com:3128', |
| 1930 |
(None, None, None, 'proxy.example.com:3128')), |
| 1931 |
('proxy.example.com', (None, None, None, 'proxy.example.com')), |
| 1932 |
('proxy.example.com:3128', |
| 1933 |
(None, None, None, 'proxy.example.com:3128')), |
| 1934 |
# The authority component may optionally include userinfo |
| 1935 |
# (assumed to be # username:password): |
| 1936 |
('joe:[email protected]', |
| 1937 |
(None, 'joe', 'password', 'proxy.example.com')), |
| 1938 |
('joe:[email protected]:3128', |
| 1939 |
(None, 'joe', 'password', 'proxy.example.com:3128')), |
| 1940 |
#Examples with URLS |
| 1941 |
('http://proxy.example.com/', |
| 1942 |
('http', None, None, 'proxy.example.com')), |
| 1943 |
('http://proxy.example.com:3128/', |
| 1944 |
('http', None, None, 'proxy.example.com:3128')), |
| 1945 |
('http://joe:[email protected]/', |
| 1946 |
('http', 'joe', 'password', 'proxy.example.com')), |
| 1947 |
('http://joe:[email protected]:3128', |
| 1948 |
('http', 'joe', 'password', 'proxy.example.com:3128')), |
| 1949 |
# Everything after the authority is ignored |
| 1950 |
('ftp://joe:[email protected]/rubbish:3128', |
| 1951 |
('ftp', 'joe', 'password', 'proxy.example.com')), |
| 1952 |
# Test for no trailing '/' case |
| 1953 |
('http://joe:[email protected]', |
| 1954 |
('http', 'joe', 'password', 'proxy.example.com')), |
| 1955 |
# Testcases with '/' character in username, password |
| 1956 |
('http://user/name:password@localhost:22', |
| 1957 |
('http', 'user/name', 'password', 'localhost:22')), |
| 1958 |
('http://username:pass/word@localhost:22', |
| 1959 |
('http', 'username', 'pass/word', 'localhost:22')), |
| 1960 |
('http://user/name:pass/word@localhost:22', |
| 1961 |
('http', 'user/name', 'pass/word', 'localhost:22')), |
| 1962 |
] |
| 1963 |
|
| 1964 |
|
| 1965 |
for tc, expected in parse_proxy_test_cases: |
| 1966 |
self.assertEqual(_parse_proxy(tc), expected) |
| 1967 |
|
| 1968 |
self.assertRaises(ValueError, _parse_proxy, 'file:/ftp.example.com'), |
| 1969 |
|
| 1970 |
|
| 1971 |
skip_libssl_fips_mode = unittest.skipIf( |
| 1972 |
support.is_libssl_fips_mode(), |
| 1973 |
"conservative skip due to OpenSSL FIPS mode possible algorithm nerfing", |
| 1974 |
) |
| 1975 |
|
| 1976 |
|
| 1977 |
class TestDigestAuthAlgorithms(unittest.TestCase): |
| 1978 |
def setUp(self): |
| 1979 |
self.handler = AbstractDigestAuthHandler() |
| 1980 |
|
| 1981 |
@skip_libssl_fips_mode |
| 1982 |
def test_md5_algorithm(self): |
| 1983 |
H, KD = self.handler.get_algorithm_impls('MD5') |
| 1984 |
self.assertEqual(H("foo"), "acbd18db4cc2f85cedef654fccc4a4d8") |
| 1985 |
self.assertEqual(KD("foo", "bar"), "4e99e8c12de7e01535248d2bac85e732") |
| 1986 |
|
| 1987 |
@skip_libssl_fips_mode |
| 1988 |
def test_sha_algorithm(self): |
| 1989 |
H, KD = self.handler.get_algorithm_impls('SHA') |
| 1990 |
self.assertEqual(H("foo"), "0beec7b5ea3f0fdbc95d0dd47f3c5bc275da8a33") |
| 1991 |
self.assertEqual(KD("foo", "bar"), "54dcbe67d21d5eb39493d46d89ae1f412d3bd6de") |
| 1992 |
|
| 1993 |
@skip_libssl_fips_mode |
| 1994 |
def test_sha256_algorithm(self): |
| 1995 |
H, KD = self.handler.get_algorithm_impls('SHA-256') |
| 1996 |
self.assertEqual(H("foo"), "2c26b46b68ffc68ff99b453c1d30413413422d706483bfa0f98a5e886266e7ae") |
| 1997 |
self.assertEqual(KD("foo", "bar"), "a765a8beaa9d561d4c5cbed29d8f4e30870297fdfa9cb7d6e9848a95fec9f937") |
| 1998 |
|
| 1999 |
def test_invalid_algorithm(self): |
| 2000 |
with self.assertRaises(ValueError) as exc: |
| 2001 |
self.handler.get_algorithm_impls('invalid') |
| 2002 |
self.assertEqual( |
| 2003 |
str(exc.exception), |
| 2004 |
"Unsupported digest authentication algorithm 'invalid'" |
| 2005 |
) |
| 2006 |
|
| 2007 |
|
| 2008 |
class RequestTests(unittest.TestCase): |
| 2009 |
class PutRequest(Request): |
| 2010 |
method = 'PUT' |
| 2011 |
|
| 2012 |
def setUp(self): |
| 2013 |
self.get = Request("http://www.python.org/~jeremy/") |
| 2014 |
self.post = Request("http://www.python.org/~jeremy/", |
| 2015 |
"data", |
| 2016 |
headers={"X-Test": "test"}) |
| 2017 |
self.head = Request("http://www.python.org/~jeremy/", method='HEAD') |
| 2018 |
self.put = self.PutRequest("http://www.python.org/~jeremy/") |
| 2019 |
self.force_post = self.PutRequest("http://www.python.org/~jeremy/", |
| 2020 |
method="POST") |
| 2021 |
|
| 2022 |
def test_method(self): |
| 2023 |
self.assertEqual("POST", self.post.get_method()) |
| 2024 |
self.assertEqual("GET", self.get.get_method()) |
| 2025 |
self.assertEqual("HEAD", self.head.get_method()) |
| 2026 |
self.assertEqual("PUT", self.put.get_method()) |
| 2027 |
self.assertEqual("POST", self.force_post.get_method()) |
| 2028 |
|
| 2029 |
def test_data(self): |
| 2030 |
self.assertFalse(self.get.data) |
| 2031 |
self.assertEqual("GET", self.get.get_method()) |
| 2032 |
self.get.data = "spam" |
| 2033 |
self.assertTrue(self.get.data) |
| 2034 |
self.assertEqual("POST", self.get.get_method()) |
| 2035 |
|
| 2036 |
# issue 16464 |
| 2037 |
# if we change data we need to remove content-length header |
| 2038 |
# (cause it's most probably calculated for previous value) |
| 2039 |
def test_setting_data_should_remove_content_length(self): |
| 2040 |
self.assertNotIn("Content-length", self.get.unredirected_hdrs) |
| 2041 |
self.get.add_unredirected_header("Content-length", 42) |
| 2042 |
self.assertEqual(42, self.get.unredirected_hdrs["Content-length"]) |
| 2043 |
self.get.data = "spam" |
| 2044 |
self.assertNotIn("Content-length", self.get.unredirected_hdrs) |
| 2045 |
|
| 2046 |
# issue 17485 same for deleting data. |
| 2047 |
def test_deleting_data_should_remove_content_length(self): |
| 2048 |
self.assertNotIn("Content-length", self.get.unredirected_hdrs) |
| 2049 |
self.get.data = 'foo' |
| 2050 |
self.get.add_unredirected_header("Content-length", 3) |
| 2051 |
self.assertEqual(3, self.get.unredirected_hdrs["Content-length"]) |
| 2052 |
del self.get.data |
| 2053 |
self.assertNotIn("Content-length", self.get.unredirected_hdrs) |
| 2054 |
|
| 2055 |
def test_get_full_url(self): |
| 2056 |
self.assertEqual("http://www.python.org/~jeremy/", |
| 2057 |
self.get.get_full_url()) |
| 2058 |
|
| 2059 |
def test_selector(self): |
| 2060 |
self.assertEqual("/~jeremy/", self.get.selector) |
| 2061 |
req = Request("http://www.python.org/") |
| 2062 |
self.assertEqual("/", req.selector) |
| 2063 |
|
| 2064 |
def test_get_type(self): |
| 2065 |
self.assertEqual("http", self.get.type) |
| 2066 |
|
| 2067 |
def test_get_host(self): |
| 2068 |
self.assertEqual("www.python.org", self.get.host) |
| 2069 |
|
| 2070 |
def test_get_host_unquote(self): |
| 2071 |
req = Request("http://www.%70ython.org/") |
| 2072 |
self.assertEqual("www.python.org", req.host) |
| 2073 |
|
| 2074 |
def test_proxy(self): |
| 2075 |
self.assertFalse(self.get.has_proxy()) |
| 2076 |
self.get.set_proxy("www.perl.org", "http") |
| 2077 |
self.assertTrue(self.get.has_proxy()) |
| 2078 |
self.assertEqual("www.python.org", self.get.origin_req_host) |
| 2079 |
self.assertEqual("www.perl.org", self.get.host) |
| 2080 |
|
| 2081 |
def test_wrapped_url(self): |
| 2082 |
req = Request("<URL:http://www.python.org>") |
| 2083 |
self.assertEqual("www.python.org", req.host) |
| 2084 |
|
| 2085 |
def test_url_fragment(self): |
| 2086 |
req = Request("http://www.python.org/?qs=query#fragment=true") |
| 2087 |
self.assertEqual("/?qs=query", req.selector) |
| 2088 |
req = Request("http://www.python.org/#fun=true") |
| 2089 |
self.assertEqual("/", req.selector) |
| 2090 |
|
| 2091 |
# Issue 11703: geturl() omits fragment in the original URL. |
| 2092 |
url = 'http://docs.python.org/library/urllib2.html#OK' |
| 2093 |
req = Request(url) |
| 2094 |
self.assertEqual(req.get_full_url(), url) |
| 2095 |
|
| 2096 |
def test_url_fullurl_get_full_url(self): |
| 2097 |
urls = ['http://docs.python.org', |
| 2098 |
'http://docs.python.org/library/urllib2.html#OK', |
| 2099 |
'http://www.python.org/?qs=query#fragment=true'] |
| 2100 |
for url in urls: |
| 2101 |
req = Request(url) |
| 2102 |
self.assertEqual(req.get_full_url(), req.full_url) |
| 2103 |
|
| 2104 |
|
| 2105 |
if __name__ == "__main__": |
| 2106 |
unittest.main() |