Debian Codegraph @

Lib/test/test_urllib2net.py

raw
1 import contextlib
2 import errno
3 import sysconfig
4 import unittest
5 from unittest import mock
6 from test import support
7 from test.support import os_helper
8 from test.support import socket_helper
9 from test.support import ResourceDenied
10 from test.support.warnings_helper import check_no_resource_warning
11
12 import os
13 import socket
14 import urllib.error
15 import urllib.request
16 import sys
17
18 support.requires("network")
19
20
21 def _retry_thrice(func, exc, *args, **kwargs):
22 for i in range(3):
23 try:
24 return func(*args, **kwargs)
25 except exc as e:
26 last_exc = e
27 continue
28 raise last_exc
29
30 def _wrap_with_retry_thrice(func, exc):
31 def wrapped(*args, **kwargs):
32 return _retry_thrice(func, exc, *args, **kwargs)
33 return wrapped
34
35 # Connecting to remote hosts is flaky. Make it more robust by retrying
36 # the connection several times.
37 _urlopen_with_retry = _wrap_with_retry_thrice(urllib.request.urlopen,
38 urllib.error.URLError)
39
40
41 class TransientResource(object):
42
43 """Raise ResourceDenied if an exception is raised while the context manager
44 is in effect that matches the specified exception and attributes."""
45
46 def __init__(self, exc, **kwargs):
47 self.exc = exc
48 self.attrs = kwargs
49
50 def __enter__(self):
51 return self
52
53 def __exit__(self, type_=None, value=None, traceback=None):
54 """If type_ is a subclass of self.exc and value has attributes matching
55 self.attrs, raise ResourceDenied. Otherwise let the exception
56 propagate (if any)."""
57 if type_ is not None and issubclass(self.exc, type_):
58 for attr, attr_value in self.attrs.items():
59 if not hasattr(value, attr):
60 break
61 if getattr(value, attr) != attr_value:
62 break
63 else:
64 raise ResourceDenied("an optional resource is not available")
65
66 # Context managers that raise ResourceDenied when various issues
67 # with the internet connection manifest themselves as exceptions.
68 # XXX deprecate these and use transient_internet() instead
69 time_out = TransientResource(OSError, errno=errno.ETIMEDOUT)
70 socket_peer_reset = TransientResource(OSError, errno=errno.ECONNRESET)
71 ioerror_peer_reset = TransientResource(OSError, errno=errno.ECONNRESET)
72
73
74 class AuthTests(unittest.TestCase):
75 """Tests urllib2 authentication features."""
76
77 ## Disabled at the moment since there is no page under python.org which
78 ## could be used to HTTP authentication.
79 #
80 # def test_basic_auth(self):
81 # import http.client
82 #
83 # test_url = "http://www.python.org/test/test_urllib2/basic_auth"
84 # test_hostport = "www.python.org"
85 # test_realm = 'Test Realm'
86 # test_user = 'test.test_urllib2net'
87 # test_password = 'blah'
88 #
89 # # failure
90 # try:
91 # _urlopen_with_retry(test_url)
92 # except urllib2.HTTPError, exc:
93 # self.assertEqual(exc.code, 401)
94 # else:
95 # self.fail("urlopen() should have failed with 401")
96 #
97 # # success
98 # auth_handler = urllib2.HTTPBasicAuthHandler()
99 # auth_handler.add_password(test_realm, test_hostport,
100 # test_user, test_password)
101 # opener = urllib2.build_opener(auth_handler)
102 # f = opener.open('http://localhost/')
103 # response = _urlopen_with_retry("http://www.python.org/")
104 #
105 # # The 'userinfo' URL component is deprecated by RFC 3986 for security
106 # # reasons, let's not implement it! (it's already implemented for proxy
107 # # specification strings (that is, URLs or authorities specifying a
108 # # proxy), so we must keep that)
109 # self.assertRaises(http.client.InvalidURL,
110 # urllib2.urlopen, "http://evil:[email protected]")
111
112
113 class CloseSocketTest(unittest.TestCase):
114
115 def test_close(self):
116 # clear _opener global variable
117 self.addCleanup(urllib.request.urlcleanup)
118
119 # calling .close() on urllib2's response objects should close the
120 # underlying socket
121 url = support.TEST_HTTP_URL
122 with socket_helper.transient_internet(url):
123 response = _urlopen_with_retry(url)
124 sock = response.fp
125 self.assertFalse(sock.closed)
126 response.close()
127 self.assertTrue(sock.closed)
128
129 class OtherNetworkTests(unittest.TestCase):
130 def setUp(self):
131 if 0: # for debugging
132 import logging
133 logger = logging.getLogger("test_urllib2net")
134 logger.addHandler(logging.StreamHandler())
135
136 # XXX The rest of these tests aren't very good -- they don't check much.
137 # They do sometimes catch some major disasters, though.
138
139 @support.requires_resource('walltime')
140 def test_ftp(self):
141 # Testing the same URL twice exercises the caching in CacheFTPHandler
142 urls = [
143 'ftp://www.pythontest.net/README',
144 'ftp://www.pythontest.net/README',
145 ('ftp://www.pythontest.net/non-existent-file',
146 None, urllib.error.URLError),
147 ]
148 self._test_urls(urls, self._extra_handlers())
149
150 @support.requires_resource('walltime')
151 @unittest.skipIf(sysconfig.get_platform() == 'linux-ppc64le',
152 'leaks on PPC64LE (gh-140691)')
153 def test_ftp_no_leak(self):
154 # gh-140691: When the data connection (but not control connection)
155 # cannot be made established, we shouldn't leave an open socket object.
156
157 class MockError(OSError):
158 pass
159
160 orig_create_connection = socket.create_connection
161 def patched_create_connection(address, *args, **kwargs):
162 """Simulate REJECTing connections to ports other than 21"""
163 host, port = address
164 if port != 21:
165 raise MockError()
166 return orig_create_connection(address, *args, **kwargs)
167
168 url = 'ftp://www.pythontest.net/README'
169 entry = url, None, urllib.error.URLError
170 no_cache_handlers = [urllib.request.FTPHandler()]
171 cache_handlers = self._extra_handlers()
172 with mock.patch('socket.create_connection', patched_create_connection):
173 with check_no_resource_warning(self):
174 # Try without CacheFTPHandler
175 self._test_urls([entry], handlers=no_cache_handlers,
176 retry=False)
177 with check_no_resource_warning(self):
178 # Try with CacheFTPHandler (uncached)
179 self._test_urls([entry], cache_handlers, retry=False)
180 with check_no_resource_warning(self):
181 # Try with CacheFTPHandler (cached)
182 self._test_urls([entry], cache_handlers, retry=False)
183 # Try without the mock: the handler should not use a closed connection
184 with check_no_resource_warning(self):
185 self._test_urls([url], cache_handlers, retry=False)
186
187 def test_file(self):
188 TESTFN = os_helper.TESTFN
189 f = open(TESTFN, 'w')
190 try:
191 f.write('hi there\n')
192 f.close()
193 urls = [
194 urllib.request.pathname2url(os.path.abspath(TESTFN), add_scheme=True),
195 ('file:///nonsensename/etc/passwd', None,
196 urllib.error.URLError),
197 ]
198 self._test_urls(urls, self._extra_handlers(), retry=True)
199 finally:
200 os.remove(TESTFN)
201
202 self.assertRaises(ValueError, urllib.request.urlopen,'./relative_path/to/file')
203
204 # XXX Following test depends on machine configurations that are internal
205 # to CNRI. Need to set up a public server with the right authentication
206 # configuration for test purposes.
207
208 ## def test_cnri(self):
209 ## if socket.gethostname() == 'bitdiddle':
210 ## localhost = 'bitdiddle.cnri.reston.va.us'
211 ## elif socket.gethostname() == 'bitdiddle.concentric.net':
212 ## localhost = 'localhost'
213 ## else:
214 ## localhost = None
215 ## if localhost is not None:
216 ## urls = [
217 ## 'file://%s/etc/passwd' % localhost,
218 ## 'http://%s/simple/' % localhost,
219 ## 'http://%s/digest/' % localhost,
220 ## 'http://%s/not/found.h' % localhost,
221 ## ]
222
223 ## bauth = HTTPBasicAuthHandler()
224 ## bauth.add_password('basic_test_realm', localhost, 'jhylton',
225 ## 'password')
226 ## dauth = HTTPDigestAuthHandler()
227 ## dauth.add_password('digest_test_realm', localhost, 'jhylton',
228 ## 'password')
229
230 ## self._test_urls(urls, self._extra_handlers()+[bauth, dauth])
231
232 def test_urlwithfrag(self):
233 urlwith_frag = "http://www.pythontest.net/index.html#frag"
234 with socket_helper.transient_internet(urlwith_frag):
235 req = urllib.request.Request(urlwith_frag)
236 res = urllib.request.urlopen(req)
237 self.assertEqual(res.geturl(),
238 "http://www.pythontest.net/index.html#frag")
239
240 @support.requires_resource('walltime')
241 def test_redirect_url_withfrag(self):
242 redirect_url_with_frag = "http://www.pythontest.net/redir/with_frag/"
243 with socket_helper.transient_internet(redirect_url_with_frag):
244 req = urllib.request.Request(redirect_url_with_frag)
245 res = urllib.request.urlopen(req)
246 self.assertEqual(res.geturl(),
247 "http://www.pythontest.net/elsewhere/#frag")
248
249 def test_custom_headers(self):
250 url = support.TEST_HTTP_URL
251 with socket_helper.transient_internet(url):
252 opener = urllib.request.build_opener()
253 request = urllib.request.Request(url)
254 self.assertFalse(request.header_items())
255 opener.open(request)
256 self.assertTrue(request.header_items())
257 self.assertTrue(request.has_header('User-agent'))
258 request.add_header('User-Agent','Test-Agent')
259 opener.open(request)
260 self.assertEqual(request.get_header('User-agent'),'Test-Agent')
261
262 @unittest.skip('XXX: http://www.imdb.com is gone')
263 def test_sites_no_connection_close(self):
264 # Some sites do not send Connection: close header.
265 # Verify that those work properly. (#issue12576)
266
267 URL = 'http://www.imdb.com' # mangles Connection:close
268
269 with socket_helper.transient_internet(URL):
270 try:
271 with urllib.request.urlopen(URL) as res:
272 pass
273 except ValueError:
274 self.fail("urlopen failed for site not sending \
275 Connection:close")
276 else:
277 self.assertTrue(res)
278
279 req = urllib.request.urlopen(URL)
280 res = req.read()
281 self.assertTrue(res)
282
283 def _test_urls(self, urls, handlers, retry=True):
284 import time
285 import logging
286 debug = logging.getLogger("test_urllib2").debug
287
288 urlopen = urllib.request.build_opener(*handlers).open
289 if retry:
290 urlopen = _wrap_with_retry_thrice(urlopen, urllib.error.URLError)
291
292 for url in urls:
293 with self.subTest(url=url):
294 if isinstance(url, tuple):
295 url, req, expected_err = url
296 else:
297 req = expected_err = None
298
299 if expected_err:
300 context = self.assertRaises(expected_err)
301 else:
302 context = contextlib.nullcontext()
303
304 with socket_helper.transient_internet(url):
305 f = None
306 with context:
307 f = urlopen(url, req, support.INTERNET_TIMEOUT)
308 if f is not None:
309 try:
310 with time_out, \
311 socket_peer_reset, \
312 ioerror_peer_reset:
313 buf = f.read()
314 debug("read %d bytes" % len(buf))
315 except TimeoutError:
316 print("<timeout: %s>" % url, file=sys.stderr)
317 f.close()
318 time.sleep(0.1)
319
320 def _extra_handlers(self):
321 handlers = []
322
323 cfh = urllib.request.CacheFTPHandler()
324 self.addCleanup(cfh.clear_cache)
325 cfh.setTimeout(1)
326 handlers.append(cfh)
327
328 return handlers
329
330
331 class TimeoutTest(unittest.TestCase):
332 def setUp(self):
333 # clear _opener global variable
334 self.addCleanup(urllib.request.urlcleanup)
335
336 def test_http_basic(self):
337 self.assertIsNone(socket.getdefaulttimeout())
338 url = support.TEST_HTTP_URL
339 with socket_helper.transient_internet(url, timeout=None):
340 u = _urlopen_with_retry(url)
341 self.addCleanup(u.close)
342 self.assertIsNone(u.fp.raw._sock.gettimeout())
343
344 def test_http_default_timeout(self):
345 self.assertIsNone(socket.getdefaulttimeout())
346 url = support.TEST_HTTP_URL
347 with socket_helper.transient_internet(url):
348 socket.setdefaulttimeout(60)
349 try:
350 u = _urlopen_with_retry(url)
351 self.addCleanup(u.close)
352 finally:
353 socket.setdefaulttimeout(None)
354 self.assertEqual(u.fp.raw._sock.gettimeout(), 60)
355
356 def test_http_no_timeout(self):
357 self.assertIsNone(socket.getdefaulttimeout())
358 url = support.TEST_HTTP_URL
359 with socket_helper.transient_internet(url):
360 socket.setdefaulttimeout(60)
361 try:
362 u = _urlopen_with_retry(url, timeout=None)
363 self.addCleanup(u.close)
364 finally:
365 socket.setdefaulttimeout(None)
366 self.assertIsNone(u.fp.raw._sock.gettimeout())
367
368 def test_http_timeout(self):
369 url = support.TEST_HTTP_URL
370 with socket_helper.transient_internet(url):
371 u = _urlopen_with_retry(url, timeout=120)
372 self.addCleanup(u.close)
373 self.assertEqual(u.fp.raw._sock.gettimeout(), 120)
374
375 FTP_HOST = 'ftp://www.pythontest.net/'
376
377 @support.requires_resource('walltime')
378 def test_ftp_basic(self):
379 self.assertIsNone(socket.getdefaulttimeout())
380 with socket_helper.transient_internet(self.FTP_HOST, timeout=None):
381 u = _urlopen_with_retry(self.FTP_HOST)
382 self.addCleanup(u.close)
383 self.assertIsNone(u.fp.fp.raw._sock.gettimeout())
384
385 def test_ftp_default_timeout(self):
386 self.assertIsNone(socket.getdefaulttimeout())
387 with socket_helper.transient_internet(self.FTP_HOST):
388 socket.setdefaulttimeout(60)
389 try:
390 u = _urlopen_with_retry(self.FTP_HOST)
391 self.addCleanup(u.close)
392 finally:
393 socket.setdefaulttimeout(None)
394 self.assertEqual(u.fp.fp.raw._sock.gettimeout(), 60)
395
396 @support.requires_resource('walltime')
397 def test_ftp_no_timeout(self):
398 self.assertIsNone(socket.getdefaulttimeout())
399 with socket_helper.transient_internet(self.FTP_HOST):
400 socket.setdefaulttimeout(60)
401 try:
402 u = _urlopen_with_retry(self.FTP_HOST, timeout=None)
403 self.addCleanup(u.close)
404 finally:
405 socket.setdefaulttimeout(None)
406 self.assertIsNone(u.fp.fp.raw._sock.gettimeout())
407
408 @support.requires_resource('walltime')
409 def test_ftp_timeout(self):
410 with socket_helper.transient_internet(self.FTP_HOST):
411 u = _urlopen_with_retry(self.FTP_HOST, timeout=60)
412 self.addCleanup(u.close)
413 self.assertEqual(u.fp.fp.raw._sock.gettimeout(), 60)
414
415
416 if __name__ == "__main__":
417 unittest.main()