Debian Codegraph @

Lib/test/test_httplib.py

raw
1 import enum
2 import errno
3 from http import client, HTTPStatus
4 import io
5 import itertools
6 import os
7 import array
8 import re
9 import socket
10 import threading
11
12 import unittest
13 from unittest import mock
14 TestCase = unittest.TestCase
15
16 from test import support
17 from test.support import os_helper
18 from test.support import socket_helper
19
20 support.requires_working_socket(module=True)
21
22 here = os.path.dirname(__file__)
23 # Self-signed cert file for 'localhost'
24 CERT_localhost = os.path.join(here, 'certdata', 'keycert.pem')
25 # Self-signed cert file for 'fakehostname'
26 CERT_fakehostname = os.path.join(here, 'certdata', 'keycert2.pem')
27 # Self-signed cert file for self-signed.pythontest.net
28 CERT_selfsigned_pythontestdotnet = os.path.join(
29 here, 'certdata', 'selfsigned_pythontestdotnet.pem',
30 )
31
32 # constants for testing chunked encoding
33 chunked_start = (
34 'HTTP/1.1 200 OK\r\n'
35 'Transfer-Encoding: chunked\r\n\r\n'
36 'a\r\n'
37 'hello worl\r\n'
38 '3\r\n'
39 'd! \r\n'
40 '8\r\n'
41 'and now \r\n'
42 '22\r\n'
43 'for something completely different\r\n'
44 )
45 chunked_expected = b'hello world! and now for something completely different'
46 chunk_extension = ";foo=bar"
47 last_chunk = "0\r\n"
48 last_chunk_extended = "0" + chunk_extension + "\r\n"
49 trailers = "X-Dummy: foo\r\nX-Dumm2: bar\r\n"
50 chunked_end = "\r\n"
51
52 HOST = socket_helper.HOST
53
54 class FakeSocket:
55 def __init__(self, text, fileclass=io.BytesIO, host=None, port=None):
56 if isinstance(text, str):
57 text = text.encode("ascii")
58 self.text = text
59 self.fileclass = fileclass
60 self.data = b''
61 self.sendall_calls = 0
62 self.file_closed = False
63 self.host = host
64 self.port = port
65
66 def sendall(self, data):
67 self.sendall_calls += 1
68 self.data += data
69
70 def makefile(self, mode, bufsize=None):
71 if mode != 'r' and mode != 'rb':
72 raise client.UnimplementedFileMode()
73 # keep the file around so we can check how much was read from it
74 self.file = self.fileclass(self.text)
75 self.file.close = self.file_close #nerf close ()
76 return self.file
77
78 def file_close(self):
79 self.file_closed = True
80
81 def close(self):
82 pass
83
84 def setsockopt(self, level, optname, value):
85 pass
86
87 class EPipeSocket(FakeSocket):
88
89 def __init__(self, text, pipe_trigger):
90 # When sendall() is called with pipe_trigger, raise EPIPE.
91 FakeSocket.__init__(self, text)
92 self.pipe_trigger = pipe_trigger
93
94 def sendall(self, data):
95 if self.pipe_trigger in data:
96 raise OSError(errno.EPIPE, "gotcha")
97 self.data += data
98
99 def close(self):
100 pass
101
102 class NoEOFBytesIO(io.BytesIO):
103 """Like BytesIO, but raises AssertionError on EOF.
104
105 This is used below to test that http.client doesn't try to read
106 more from the underlying file than it should.
107 """
108 def read(self, n=-1):
109 data = io.BytesIO.read(self, n)
110 if data == b'':
111 raise AssertionError('caller tried to read past EOF')
112 return data
113
114 def readline(self, length=None):
115 data = io.BytesIO.readline(self, length)
116 if data == b'':
117 raise AssertionError('caller tried to read past EOF')
118 return data
119
120 class FakeSocketHTTPConnection(client.HTTPConnection):
121 """HTTPConnection subclass using FakeSocket; counts connect() calls"""
122
123 def __init__(self, *args):
124 self.connections = 0
125 super().__init__('example.com')
126 self.fake_socket_args = args
127 self._create_connection = self.create_connection
128
129 def connect(self):
130 """Count the number of times connect() is invoked"""
131 self.connections += 1
132 return super().connect()
133
134 def create_connection(self, *pos, **kw):
135 return FakeSocket(*self.fake_socket_args)
136
137 class HeaderTests(TestCase):
138 def test_auto_headers(self):
139 # Some headers are added automatically, but should not be added by
140 # .request() if they are explicitly set.
141
142 class HeaderCountingBuffer(list):
143 def __init__(self):
144 self.count = {}
145 def append(self, item):
146 kv = item.split(b':')
147 if len(kv) > 1:
148 # item is a 'Key: Value' header string
149 lcKey = kv[0].decode('ascii').lower()
150 self.count.setdefault(lcKey, 0)
151 self.count[lcKey] += 1
152 list.append(self, item)
153
154 for explicit_header in True, False:
155 for header in 'Content-length', 'Host', 'Accept-encoding':
156 conn = client.HTTPConnection('example.com')
157 conn.sock = FakeSocket('blahblahblah')
158 conn._buffer = HeaderCountingBuffer()
159
160 body = 'spamspamspam'
161 headers = {}
162 if explicit_header:
163 headers[header] = str(len(body))
164 conn.request('POST', '/', body, headers)
165 self.assertEqual(conn._buffer.count[header.lower()], 1)
166
167 def test_content_length_0(self):
168
169 class ContentLengthChecker(list):
170 def __init__(self):
171 list.__init__(self)
172 self.content_length = None
173 def append(self, item):
174 kv = item.split(b':', 1)
175 if len(kv) > 1 and kv[0].lower() == b'content-length':
176 self.content_length = kv[1].strip()
177 list.append(self, item)
178
179 # Here, we're testing that methods expecting a body get a
180 # content-length set to zero if the body is empty (either None or '')
181 bodies = (None, '')
182 methods_with_body = ('PUT', 'POST', 'PATCH')
183 for method, body in itertools.product(methods_with_body, bodies):
184 conn = client.HTTPConnection('example.com')
185 conn.sock = FakeSocket(None)
186 conn._buffer = ContentLengthChecker()
187 conn.request(method, '/', body)
188 self.assertEqual(
189 conn._buffer.content_length, b'0',
190 'Header Content-Length incorrect on {}'.format(method)
191 )
192
193 # For these methods, we make sure that content-length is not set when
194 # the body is None because it might cause unexpected behaviour on the
195 # server.
196 methods_without_body = (
197 'GET', 'CONNECT', 'DELETE', 'HEAD', 'OPTIONS', 'TRACE',
198 )
199 for method in methods_without_body:
200 conn = client.HTTPConnection('example.com')
201 conn.sock = FakeSocket(None)
202 conn._buffer = ContentLengthChecker()
203 conn.request(method, '/', None)
204 self.assertEqual(
205 conn._buffer.content_length, None,
206 'Header Content-Length set for empty body on {}'.format(method)
207 )
208
209 # If the body is set to '', that's considered to be "present but
210 # empty" rather than "missing", so content length would be set, even
211 # for methods that don't expect a body.
212 for method in methods_without_body:
213 conn = client.HTTPConnection('example.com')
214 conn.sock = FakeSocket(None)
215 conn._buffer = ContentLengthChecker()
216 conn.request(method, '/', '')
217 self.assertEqual(
218 conn._buffer.content_length, b'0',
219 'Header Content-Length incorrect on {}'.format(method)
220 )
221
222 # If the body is set, make sure Content-Length is set.
223 for method in itertools.chain(methods_without_body, methods_with_body):
224 conn = client.HTTPConnection('example.com')
225 conn.sock = FakeSocket(None)
226 conn._buffer = ContentLengthChecker()
227 conn.request(method, '/', ' ')
228 self.assertEqual(
229 conn._buffer.content_length, b'1',
230 'Header Content-Length incorrect on {}'.format(method)
231 )
232
233 def test_putheader(self):
234 conn = client.HTTPConnection('example.com')
235 conn.sock = FakeSocket(None)
236 conn.putrequest('GET','/')
237 conn.putheader('Content-length', 42)
238 self.assertIn(b'Content-length: 42', conn._buffer)
239
240 conn.putheader('Foo', ' bar ')
241 self.assertIn(b'Foo: bar ', conn._buffer)
242 conn.putheader('Bar', '\tbaz\t')
243 self.assertIn(b'Bar: \tbaz\t', conn._buffer)
244 conn.putheader('Authorization', 'Bearer mytoken')
245 self.assertIn(b'Authorization: Bearer mytoken', conn._buffer)
246 conn.putheader('IterHeader', 'IterA', 'IterB')
247 self.assertIn(b'IterHeader: IterA\r\n\tIterB', conn._buffer)
248 conn.putheader('LatinHeader', b'\xFF')
249 self.assertIn(b'LatinHeader: \xFF', conn._buffer)
250 conn.putheader('Utf8Header', b'\xc3\x80')
251 self.assertIn(b'Utf8Header: \xc3\x80', conn._buffer)
252 conn.putheader('C1-Control', b'next\x85line')
253 self.assertIn(b'C1-Control: next\x85line', conn._buffer)
254 conn.putheader('Embedded-Fold-Space', 'is\r\n allowed')
255 self.assertIn(b'Embedded-Fold-Space: is\r\n allowed', conn._buffer)
256 conn.putheader('Embedded-Fold-Tab', 'is\r\n\tallowed')
257 self.assertIn(b'Embedded-Fold-Tab: is\r\n\tallowed', conn._buffer)
258 conn.putheader('Key Space', 'value')
259 self.assertIn(b'Key Space: value', conn._buffer)
260 conn.putheader('KeySpace ', 'value')
261 self.assertIn(b'KeySpace : value', conn._buffer)
262 conn.putheader(b'Nonbreak\xa0Space', 'value')
263 self.assertIn(b'Nonbreak\xa0Space: value', conn._buffer)
264 conn.putheader(b'\xa0NonbreakSpace', 'value')
265 self.assertIn(b'\xa0NonbreakSpace: value', conn._buffer)
266
267 def test_ipv6host_header(self):
268 # Default host header on IPv6 transaction should be wrapped by [] if
269 # it is an IPv6 address
270 expected = b'GET /foo HTTP/1.1\r\nHost: [2001::]:81\r\n' \
271 b'Accept-Encoding: identity\r\n\r\n'
272 conn = client.HTTPConnection('[2001::]:81')
273 sock = FakeSocket('')
274 conn.sock = sock
275 conn.request('GET', '/foo')
276 self.assertStartsWith(sock.data, expected)
277
278 expected = b'GET /foo HTTP/1.1\r\nHost: [2001:102A::]\r\n' \
279 b'Accept-Encoding: identity\r\n\r\n'
280 conn = client.HTTPConnection('[2001:102A::]')
281 sock = FakeSocket('')
282 conn.sock = sock
283 conn.request('GET', '/foo')
284 self.assertStartsWith(sock.data, expected)
285
286 expected = b'GET /foo HTTP/1.1\r\nHost: [fe80::]\r\n' \
287 b'Accept-Encoding: identity\r\n\r\n'
288 conn = client.HTTPConnection('[fe80::%2]')
289 sock = FakeSocket('')
290 conn.sock = sock
291 conn.request('GET', '/foo')
292 self.assertStartsWith(sock.data, expected)
293
294 expected = b'GET /foo HTTP/1.1\r\nHost: [fe80::]:81\r\n' \
295 b'Accept-Encoding: identity\r\n\r\n'
296 conn = client.HTTPConnection('[fe80::%2]:81')
297 sock = FakeSocket('')
298 conn.sock = sock
299 conn.request('GET', '/foo')
300 self.assertStartsWith(sock.data, expected)
301
302 def test_malformed_headers_coped_with(self):
303 # Issue 19996
304 body = "HTTP/1.1 200 OK\r\nFirst: val\r\n: nval\r\nSecond: val\r\n\r\n"
305 sock = FakeSocket(body)
306 resp = client.HTTPResponse(sock)
307 resp.begin()
308
309 self.assertEqual(resp.getheader('First'), 'val')
310 self.assertEqual(resp.getheader('Second'), 'val')
311
312 def test_parse_all_octets(self):
313 # Ensure no valid header field octet breaks the parser
314 body = (
315 b'HTTP/1.1 200 OK\r\n'
316 b"!#$%&'*+-.^_`|~: value\r\n" # Special token characters
317 b'VCHAR: ' + bytes(range(0x21, 0x7E + 1)) + b'\r\n'
318 b'obs-text: ' + bytes(range(0x80, 0xFF + 1)) + b'\r\n'
319 b'obs-fold: text\r\n'
320 b' folded with space\r\n'
321 b'\tfolded with tab\r\n'
322 b'Content-Length: 0\r\n'
323 b'\r\n'
324 )
325 sock = FakeSocket(body)
326 resp = client.HTTPResponse(sock)
327 resp.begin()
328 self.assertEqual(resp.getheader('Content-Length'), '0')
329 self.assertEqual(resp.msg['Content-Length'], '0')
330 self.assertEqual(resp.getheader("!#$%&'*+-.^_`|~"), 'value')
331 self.assertEqual(resp.msg["!#$%&'*+-.^_`|~"], 'value')
332 vchar = ''.join(map(chr, range(0x21, 0x7E + 1)))
333 self.assertEqual(resp.getheader('VCHAR'), vchar)
334 self.assertEqual(resp.msg['VCHAR'], vchar)
335 self.assertIsNotNone(resp.getheader('obs-text'))
336 self.assertIn('obs-text', resp.msg)
337 for folded in (resp.getheader('obs-fold'), resp.msg['obs-fold']):
338 self.assertStartsWith(folded, 'text')
339 self.assertIn(' folded with space', folded)
340 self.assertEndsWith(folded, 'folded with tab')
341
342 def test_invalid_headers(self):
343 conn = client.HTTPConnection('example.com')
344 conn.sock = FakeSocket('')
345 conn.putrequest('GET', '/')
346
347 # http://tools.ietf.org/html/rfc7230#section-3.2.4, whitespace is no
348 # longer allowed in header names
349 cases = (
350 (b'Invalid\r\nName', b'ValidValue'),
351 (b'Invalid\rName', b'ValidValue'),
352 (b'Invalid\nName', b'ValidValue'),
353 (b'\r\nInvalidName', b'ValidValue'),
354 (b'\rInvalidName', b'ValidValue'),
355 (b'\nInvalidName', b'ValidValue'),
356 (b' InvalidName', b'ValidValue'),
357 (b'\tInvalidName', b'ValidValue'),
358 (b'Invalid:Name', b'ValidValue'),
359 (b':InvalidName', b'ValidValue'),
360 (b'ValidName', b'Invalid\r\nValue'),
361 (b'ValidName', b'Invalid\rValue'),
362 (b'ValidName', b'Invalid\nValue'),
363 (b'ValidName', b'InvalidValue\r\n'),
364 (b'ValidName', b'InvalidValue\r'),
365 (b'ValidName', b'InvalidValue\n'),
366 )
367 for name, value in cases:
368 with self.subTest((name, value)):
369 with self.assertRaisesRegex(ValueError, 'Invalid header'):
370 conn.putheader(name, value)
371
372 def test_invalid_tunnel_headers(self):
373 cases = (
374 ('Invalid\r\nName', 'ValidValue'),
375 ('Invalid\rName', 'ValidValue'),
376 ('Invalid\nName', 'ValidValue'),
377 ('\r\nInvalidName', 'ValidValue'),
378 ('\rInvalidName', 'ValidValue'),
379 ('\nInvalidName', 'ValidValue'),
380 (' InvalidName', 'ValidValue'),
381 ('\tInvalidName', 'ValidValue'),
382 ('Invalid:Name', 'ValidValue'),
383 (':InvalidName', 'ValidValue'),
384 ('ValidName', 'Invalid\r\nValue'),
385 ('ValidName', 'Invalid\rValue'),
386 ('ValidName', 'Invalid\nValue'),
387 ('ValidName', 'InvalidValue\r\n'),
388 ('ValidName', 'InvalidValue\r'),
389 ('ValidName', 'InvalidValue\n'),
390 )
391 for name, value in cases:
392 with self.subTest((name, value)):
393 conn = client.HTTPConnection('example.com')
394 conn.set_tunnel('tunnel', headers={
395 name: value
396 })
397 conn.sock = FakeSocket('')
398 with self.assertRaisesRegex(ValueError, 'Invalid header'):
399 conn._tunnel() # Called in .connect()
400
401 def test_invalid_tunnel_host(self):
402 cases = (
403 'invalid\r.host',
404 '\ninvalid.host',
405 'invalid.host\r\n',
406 'invalid.host\x00',
407 'invalid host',
408 )
409 for tunnel_host in cases:
410 with self.subTest(tunnel_host):
411 conn = client.HTTPConnection('example.com')
412 conn.set_tunnel(tunnel_host)
413 conn.sock = FakeSocket('')
414 with self.assertRaisesRegex(ValueError, 'Tunnel host can\'t contain control characters'):
415 conn._tunnel() # Called in .connect()
416
417 def test_headers_debuglevel(self):
418 body = (
419 b'HTTP/1.1 200 OK\r\n'
420 b'First: val\r\n'
421 b'Second: val1\r\n'
422 b'Second: val2\r\n'
423 )
424 sock = FakeSocket(body)
425 resp = client.HTTPResponse(sock, debuglevel=1)
426 with support.captured_stdout() as output:
427 resp.begin()
428 lines = output.getvalue().splitlines()
429 self.assertEqual(lines[0], "reply: 'HTTP/1.1 200 OK\\r\\n'")
430 self.assertEqual(lines[1], "header: First: val")
431 self.assertEqual(lines[2], "header: Second: val1")
432 self.assertEqual(lines[3], "header: Second: val2")
433
434
435 class HttpMethodTests(TestCase):
436 def test_invalid_method_names(self):
437 methods = (
438 'GET\r',
439 'POST\n',
440 'PUT\n\r',
441 'POST\nValue',
442 'POST\nHOST:abc',
443 'GET\nrHost:abc\n',
444 'POST\rRemainder:\r',
445 'GET\rHOST:\n',
446 '\nPUT'
447 )
448
449 for method in methods:
450 with self.assertRaisesRegex(
451 ValueError, "method can't contain control characters"):
452 conn = client.HTTPConnection('example.com')
453 conn.sock = FakeSocket(None)
454 conn.request(method=method, url="/")
455
456
457 class TransferEncodingTest(TestCase):
458 expected_body = b"It's just a flesh wound"
459
460 def test_endheaders_chunked(self):
461 conn = client.HTTPConnection('example.com')
462 conn.sock = FakeSocket(b'')
463 conn.putrequest('POST', '/')
464 conn.endheaders(self._make_body(), encode_chunked=True)
465
466 _, _, body = self._parse_request(conn.sock.data)
467 body = self._parse_chunked(body)
468 self.assertEqual(body, self.expected_body)
469
470 def test_explicit_headers(self):
471 # explicit chunked
472 conn = client.HTTPConnection('example.com')
473 conn.sock = FakeSocket(b'')
474 # this shouldn't actually be automatically chunk-encoded because the
475 # calling code has explicitly stated that it's taking care of it
476 conn.request(
477 'POST', '/', self._make_body(), {'Transfer-Encoding': 'chunked'})
478
479 _, headers, body = self._parse_request(conn.sock.data)
480 self.assertNotIn('content-length', [k.lower() for k in headers.keys()])
481 self.assertEqual(headers['Transfer-Encoding'], 'chunked')
482 self.assertEqual(body, self.expected_body)
483
484 # explicit chunked, string body
485 conn = client.HTTPConnection('example.com')
486 conn.sock = FakeSocket(b'')
487 conn.request(
488 'POST', '/', self.expected_body.decode('latin-1'),
489 {'Transfer-Encoding': 'chunked'})
490
491 _, headers, body = self._parse_request(conn.sock.data)
492 self.assertNotIn('content-length', [k.lower() for k in headers.keys()])
493 self.assertEqual(headers['Transfer-Encoding'], 'chunked')
494 self.assertEqual(body, self.expected_body)
495
496 # User-specified TE, but request() does the chunk encoding
497 conn = client.HTTPConnection('example.com')
498 conn.sock = FakeSocket(b'')
499 conn.request('POST', '/',
500 headers={'Transfer-Encoding': 'gzip, chunked'},
501 encode_chunked=True,
502 body=self._make_body())
503 _, headers, body = self._parse_request(conn.sock.data)
504 self.assertNotIn('content-length', [k.lower() for k in headers])
505 self.assertEqual(headers['Transfer-Encoding'], 'gzip, chunked')
506 self.assertEqual(self._parse_chunked(body), self.expected_body)
507
508 def test_request(self):
509 for empty_lines in (False, True,):
510 conn = client.HTTPConnection('example.com')
511 conn.sock = FakeSocket(b'')
512 conn.request(
513 'POST', '/', self._make_body(empty_lines=empty_lines))
514
515 _, headers, body = self._parse_request(conn.sock.data)
516 body = self._parse_chunked(body)
517 self.assertEqual(body, self.expected_body)
518 self.assertEqual(headers['Transfer-Encoding'], 'chunked')
519
520 # Content-Length and Transfer-Encoding SHOULD not be sent in the
521 # same request
522 self.assertNotIn('content-length', [k.lower() for k in headers])
523
524 def test_empty_body(self):
525 # Zero-length iterable should be treated like any other iterable
526 conn = client.HTTPConnection('example.com')
527 conn.sock = FakeSocket(b'')
528 conn.request('POST', '/', ())
529 _, headers, body = self._parse_request(conn.sock.data)
530 self.assertEqual(headers['Transfer-Encoding'], 'chunked')
531 self.assertNotIn('content-length', [k.lower() for k in headers])
532 self.assertEqual(body, b"0\r\n\r\n")
533
534 def _make_body(self, empty_lines=False):
535 lines = self.expected_body.split(b' ')
536 for idx, line in enumerate(lines):
537 # for testing handling empty lines
538 if empty_lines and idx % 2:
539 yield b''
540 if idx < len(lines) - 1:
541 yield line + b' '
542 else:
543 yield line
544
545 def _parse_request(self, data):
546 lines = data.split(b'\r\n')
547 request = lines[0]
548 headers = {}
549 n = 1
550 while n < len(lines) and len(lines[n]) > 0:
551 key, val = lines[n].split(b':')
552 key = key.decode('latin-1').strip()
553 headers[key] = val.decode('latin-1').strip()
554 n += 1
555
556 return request, headers, b'\r\n'.join(lines[n + 1:])
557
558 def _parse_chunked(self, data):
559 body = []
560 trailers = {}
561 n = 0
562 lines = data.split(b'\r\n')
563 # parse body
564 while True:
565 size, chunk = lines[n:n+2]
566 size = int(size, 16)
567
568 if size == 0:
569 n += 1
570 break
571
572 self.assertEqual(size, len(chunk))
573 body.append(chunk)
574
575 n += 2
576 # we /should/ hit the end chunk, but check against the size of
577 # lines so we're not stuck in an infinite loop should we get
578 # malformed data
579 if n > len(lines):
580 break
581
582 return b''.join(body)
583
584
585 class BasicTest(TestCase):
586 def test_dir_with_added_behavior_on_status(self):
587 # see issue40084
588 self.assertTrue({'description', 'name', 'phrase', 'value'} <= set(dir(HTTPStatus(404))))
589
590 def test_simple_httpstatus(self):
591 class CheckedHTTPStatus(enum.IntEnum):
592 """HTTP status codes and reason phrases
593
594 Status codes from the following RFCs are all observed:
595
596 * RFC 7231: Hypertext Transfer Protocol (HTTP/1.1), obsoletes 2616
597 * RFC 6585: Additional HTTP Status Codes
598 * RFC 3229: Delta encoding in HTTP
599 * RFC 4918: HTTP Extensions for WebDAV, obsoletes 2518
600 * RFC 5842: Binding Extensions to WebDAV
601 * RFC 7238: Permanent Redirect
602 * RFC 2295: Transparent Content Negotiation in HTTP
603 * RFC 2774: An HTTP Extension Framework
604 * RFC 7725: An HTTP Status Code to Report Legal Obstacles
605 * RFC 7540: Hypertext Transfer Protocol Version 2 (HTTP/2)
606 * RFC 2324: Hyper Text Coffee Pot Control Protocol (HTCPCP/1.0)
607 * RFC 8297: An HTTP Status Code for Indicating Hints
608 * RFC 8470: Using Early Data in HTTP
609 """
610 def __new__(cls, value, phrase, description=''):
611 obj = int.__new__(cls, value)
612 obj._value_ = value
613
614 obj.phrase = phrase
615 obj.description = description
616 return obj
617
618 @property
619 def is_informational(self):
620 return 100 <= self <= 199
621
622 @property
623 def is_success(self):
624 return 200 <= self <= 299
625
626 @property
627 def is_redirection(self):
628 return 300 <= self <= 399
629
630 @property
631 def is_client_error(self):
632 return 400 <= self <= 499
633
634 @property
635 def is_server_error(self):
636 return 500 <= self <= 599
637
638 # informational
639 CONTINUE = 100, 'Continue', 'Request received, please continue'
640 SWITCHING_PROTOCOLS = (101, 'Switching Protocols',
641 'Switching to new protocol; obey Upgrade header')
642 PROCESSING = 102, 'Processing', 'Server is processing the request'
643 EARLY_HINTS = (103, 'Early Hints',
644 'Headers sent to prepare for the response')
645 # success
646 OK = 200, 'OK', 'Request fulfilled, document follows'
647 CREATED = 201, 'Created', 'Document created, URL follows'
648 ACCEPTED = (202, 'Accepted',
649 'Request accepted, processing continues off-line')
650 NON_AUTHORITATIVE_INFORMATION = (203,
651 'Non-Authoritative Information', 'Request fulfilled from cache')
652 NO_CONTENT = 204, 'No Content', 'Request fulfilled, nothing follows'
653 RESET_CONTENT = 205, 'Reset Content', 'Clear input form for further input'
654 PARTIAL_CONTENT = 206, 'Partial Content', 'Partial content follows'
655 MULTI_STATUS = (207, 'Multi-Status',
656 'Response contains multiple statuses in the body')
657 ALREADY_REPORTED = (208, 'Already Reported',
658 'Operation has already been reported')
659 IM_USED = 226, 'IM Used', 'Request completed using instance manipulations'
660 # redirection
661 MULTIPLE_CHOICES = (300, 'Multiple Choices',
662 'Object has several resources -- see URI list')
663 MOVED_PERMANENTLY = (301, 'Moved Permanently',
664 'Object moved permanently -- see URI list')
665 FOUND = 302, 'Found', 'Object moved temporarily -- see URI list'
666 SEE_OTHER = 303, 'See Other', 'Object moved -- see Method and URL list'
667 NOT_MODIFIED = (304, 'Not Modified',
668 'Document has not changed since given time')
669 USE_PROXY = (305, 'Use Proxy',
670 'You must use proxy specified in Location to access this resource')
671 TEMPORARY_REDIRECT = (307, 'Temporary Redirect',
672 'Object moved temporarily -- see URI list')
673 PERMANENT_REDIRECT = (308, 'Permanent Redirect',
674 'Object moved permanently -- see URI list')
675 # client error
676 BAD_REQUEST = (400, 'Bad Request',
677 'Bad request syntax or unsupported method')
678 UNAUTHORIZED = (401, 'Unauthorized',
679 'No permission -- see authorization schemes')
680 PAYMENT_REQUIRED = (402, 'Payment Required',
681 'No payment -- see charging schemes')
682 FORBIDDEN = (403, 'Forbidden',
683 'Request forbidden -- authorization will not help')
684 NOT_FOUND = (404, 'Not Found',
685 'Nothing matches the given URI')
686 METHOD_NOT_ALLOWED = (405, 'Method Not Allowed',
687 'Specified method is invalid for this resource')
688 NOT_ACCEPTABLE = (406, 'Not Acceptable',
689 'URI not available in preferred format')
690 PROXY_AUTHENTICATION_REQUIRED = (407,
691 'Proxy Authentication Required',
692 'You must authenticate with this proxy before proceeding')
693 REQUEST_TIMEOUT = (408, 'Request Timeout',
694 'Request timed out; try again later')
695 CONFLICT = 409, 'Conflict', 'Request conflict'
696 GONE = (410, 'Gone',
697 'URI no longer exists and has been permanently removed')
698 LENGTH_REQUIRED = (411, 'Length Required',
699 'Client must specify Content-Length')
700 PRECONDITION_FAILED = (412, 'Precondition Failed',
701 'Precondition in headers is false')
702 CONTENT_TOO_LARGE = (413, 'Content Too Large',
703 'Content is too large')
704 REQUEST_ENTITY_TOO_LARGE = CONTENT_TOO_LARGE
705 URI_TOO_LONG = (414, 'URI Too Long', 'URI is too long')
706 REQUEST_URI_TOO_LONG = URI_TOO_LONG
707 UNSUPPORTED_MEDIA_TYPE = (415, 'Unsupported Media Type',
708 'Entity body in unsupported format')
709 RANGE_NOT_SATISFIABLE = (416,
710 'Range Not Satisfiable',
711 'Cannot satisfy request range')
712 REQUESTED_RANGE_NOT_SATISFIABLE = RANGE_NOT_SATISFIABLE
713 EXPECTATION_FAILED = (417, 'Expectation Failed',
714 'Expect condition could not be satisfied')
715 IM_A_TEAPOT = (418, 'I\'m a Teapot',
716 'Server refuses to brew coffee because it is a teapot')
717 MISDIRECTED_REQUEST = (421, 'Misdirected Request',
718 'Server is not able to produce a response')
719 UNPROCESSABLE_CONTENT = (422, 'Unprocessable Content',
720 'Server is not able to process the contained instructions')
721 UNPROCESSABLE_ENTITY = UNPROCESSABLE_CONTENT
722 LOCKED = 423, 'Locked', 'Resource of a method is locked'
723 FAILED_DEPENDENCY = (424, 'Failed Dependency',
724 'Dependent action of the request failed')
725 TOO_EARLY = (425, 'Too Early',
726 'Server refuses to process a request that might be replayed')
727 UPGRADE_REQUIRED = (426, 'Upgrade Required',
728 'Server refuses to perform the request using the current protocol')
729 PRECONDITION_REQUIRED = (428, 'Precondition Required',
730 'The origin server requires the request to be conditional')
731 TOO_MANY_REQUESTS = (429, 'Too Many Requests',
732 'The user has sent too many requests in '
733 'a given amount of time ("rate limiting")')
734 REQUEST_HEADER_FIELDS_TOO_LARGE = (431,
735 'Request Header Fields Too Large',
736 'The server is unwilling to process the request because its header '
737 'fields are too large')
738 UNAVAILABLE_FOR_LEGAL_REASONS = (451,
739 'Unavailable For Legal Reasons',
740 'The server is denying access to the '
741 'resource as a consequence of a legal demand')
742 # server errors
743 INTERNAL_SERVER_ERROR = (500, 'Internal Server Error',
744 'Server got itself in trouble')
745 NOT_IMPLEMENTED = (501, 'Not Implemented',
746 'Server does not support this operation')
747 BAD_GATEWAY = (502, 'Bad Gateway',
748 'Invalid responses from another server/proxy')
749 SERVICE_UNAVAILABLE = (503, 'Service Unavailable',
750 'The server cannot process the request due to a high load')
751 GATEWAY_TIMEOUT = (504, 'Gateway Timeout',
752 'The gateway server did not receive a timely response')
753 HTTP_VERSION_NOT_SUPPORTED = (505, 'HTTP Version Not Supported',
754 'Cannot fulfill request')
755 VARIANT_ALSO_NEGOTIATES = (506, 'Variant Also Negotiates',
756 'Server has an internal configuration error')
757 INSUFFICIENT_STORAGE = (507, 'Insufficient Storage',
758 'Server is not able to store the representation')
759 LOOP_DETECTED = (508, 'Loop Detected',
760 'Server encountered an infinite loop while processing a request')
761 NOT_EXTENDED = (510, 'Not Extended',
762 'Request does not meet the resource access policy')
763 NETWORK_AUTHENTICATION_REQUIRED = (511,
764 'Network Authentication Required',
765 'The client needs to authenticate to gain network access')
766 enum._test_simple_enum(CheckedHTTPStatus, HTTPStatus)
767
768 def test_httpstatus_range(self):
769 """Checks that the statuses are in the 100-599 range"""
770
771 for member in HTTPStatus.__members__.values():
772 self.assertGreaterEqual(member, 100)
773 self.assertLessEqual(member, 599)
774
775 def test_httpstatus_category(self):
776 """Checks that the statuses belong to the standard categories"""
777
778 categories = (
779 ((100, 199), "is_informational"),
780 ((200, 299), "is_success"),
781 ((300, 399), "is_redirection"),
782 ((400, 499), "is_client_error"),
783 ((500, 599), "is_server_error"),
784 )
785 for member in HTTPStatus.__members__.values():
786 for (lower, upper), category in categories:
787 category_indicator = getattr(member, category)
788 if lower <= member <= upper:
789 self.assertTrue(category_indicator)
790 else:
791 self.assertFalse(category_indicator)
792
793 def test_status_lines(self):
794 # Test HTTP status lines
795
796 body = "HTTP/1.1 200 Ok\r\n\r\nText"
797 sock = FakeSocket(body)
798 resp = client.HTTPResponse(sock)
799 resp.begin()
800 self.assertEqual(resp.read(0), b'') # Issue #20007
801 self.assertFalse(resp.isclosed())
802 self.assertFalse(resp.closed)
803 self.assertEqual(resp.read(), b"Text")
804 self.assertTrue(resp.isclosed())
805 self.assertFalse(resp.closed)
806 resp.close()
807 self.assertTrue(resp.closed)
808
809 body = "HTTP/1.1 400.100 Not Ok\r\n\r\nText"
810 sock = FakeSocket(body)
811 resp = client.HTTPResponse(sock)
812 self.assertRaises(client.BadStatusLine, resp.begin)
813
814 def test_bad_status_repr(self):
815 exc = client.BadStatusLine('')
816 self.assertEqual(repr(exc), '''BadStatusLine("''")''')
817
818 def test_partial_reads(self):
819 # if we have Content-Length, HTTPResponse knows when to close itself,
820 # the same behaviour as when we read the whole thing with read()
821 body = "HTTP/1.1 200 Ok\r\nContent-Length: 4\r\n\r\nText"
822 sock = FakeSocket(body)
823 resp = client.HTTPResponse(sock)
824 resp.begin()
825 self.assertEqual(resp.read(2), b'Te')
826 self.assertFalse(resp.isclosed())
827 self.assertEqual(resp.read(2), b'xt')
828 self.assertTrue(resp.isclosed())
829 self.assertFalse(resp.closed)
830 resp.close()
831 self.assertTrue(resp.closed)
832
833 def test_mixed_reads(self):
834 # readline() should update the remaining length, so that read() knows
835 # how much data is left and does not raise IncompleteRead
836 body = "HTTP/1.1 200 Ok\r\nContent-Length: 13\r\n\r\nText\r\nAnother"
837 sock = FakeSocket(body)
838 resp = client.HTTPResponse(sock)
839 resp.begin()
840 self.assertEqual(resp.readline(), b'Text\r\n')
841 self.assertFalse(resp.isclosed())
842 self.assertEqual(resp.read(), b'Another')
843 self.assertTrue(resp.isclosed())
844 self.assertFalse(resp.closed)
845 resp.close()
846 self.assertTrue(resp.closed)
847
848 def test_partial_readintos(self):
849 # if we have Content-Length, HTTPResponse knows when to close itself,
850 # the same behaviour as when we read the whole thing with read()
851 body = "HTTP/1.1 200 Ok\r\nContent-Length: 4\r\n\r\nText"
852 sock = FakeSocket(body)
853 resp = client.HTTPResponse(sock)
854 resp.begin()
855 b = bytearray(2)
856 n = resp.readinto(b)
857 self.assertEqual(n, 2)
858 self.assertEqual(bytes(b), b'Te')
859 self.assertFalse(resp.isclosed())
860 n = resp.readinto(b)
861 self.assertEqual(n, 2)
862 self.assertEqual(bytes(b), b'xt')
863 self.assertTrue(resp.isclosed())
864 self.assertFalse(resp.closed)
865 resp.close()
866 self.assertTrue(resp.closed)
867
868 def test_partial_reads_past_end(self):
869 # if we have Content-Length, clip reads to the end
870 body = "HTTP/1.1 200 Ok\r\nContent-Length: 4\r\n\r\nText"
871 sock = FakeSocket(body)
872 resp = client.HTTPResponse(sock)
873 resp.begin()
874 self.assertEqual(resp.read(10), b'Text')
875 self.assertTrue(resp.isclosed())
876 self.assertFalse(resp.closed)
877 resp.close()
878 self.assertTrue(resp.closed)
879
880 def test_partial_readintos_past_end(self):
881 # if we have Content-Length, clip readintos to the end
882 body = "HTTP/1.1 200 Ok\r\nContent-Length: 4\r\n\r\nText"
883 sock = FakeSocket(body)
884 resp = client.HTTPResponse(sock)
885 resp.begin()
886 b = bytearray(10)
887 n = resp.readinto(b)
888 self.assertEqual(n, 4)
889 self.assertEqual(bytes(b)[:4], b'Text')
890 self.assertTrue(resp.isclosed())
891 self.assertFalse(resp.closed)
892 resp.close()
893 self.assertTrue(resp.closed)
894
895 def test_partial_reads_no_content_length(self):
896 # when no length is present, the socket should be gracefully closed when
897 # all data was read
898 body = "HTTP/1.1 200 Ok\r\n\r\nText"
899 sock = FakeSocket(body)
900 resp = client.HTTPResponse(sock)
901 resp.begin()
902 self.assertEqual(resp.read(2), b'Te')
903 self.assertFalse(resp.isclosed())
904 self.assertEqual(resp.read(2), b'xt')
905 self.assertEqual(resp.read(1), b'')
906 self.assertTrue(resp.isclosed())
907 self.assertFalse(resp.closed)
908 resp.close()
909 self.assertTrue(resp.closed)
910
911 def test_partial_readintos_no_content_length(self):
912 # when no length is present, the socket should be gracefully closed when
913 # all data was read
914 body = "HTTP/1.1 200 Ok\r\n\r\nText"
915 sock = FakeSocket(body)
916 resp = client.HTTPResponse(sock)
917 resp.begin()
918 b = bytearray(2)
919 n = resp.readinto(b)
920 self.assertEqual(n, 2)
921 self.assertEqual(bytes(b), b'Te')
922 self.assertFalse(resp.isclosed())
923 n = resp.readinto(b)
924 self.assertEqual(n, 2)
925 self.assertEqual(bytes(b), b'xt')
926 n = resp.readinto(b)
927 self.assertEqual(n, 0)
928 self.assertTrue(resp.isclosed())
929
930 def test_partial_reads_incomplete_body(self):
931 # if the server shuts down the connection before the whole
932 # content-length is delivered, the socket is gracefully closed
933 body = "HTTP/1.1 200 Ok\r\nContent-Length: 10\r\n\r\nText"
934 sock = FakeSocket(body)
935 resp = client.HTTPResponse(sock)
936 resp.begin()
937 self.assertEqual(resp.read(2), b'Te')
938 self.assertFalse(resp.isclosed())
939 self.assertEqual(resp.read(2), b'xt')
940 self.assertEqual(resp.read(1), b'')
941 self.assertTrue(resp.isclosed())
942
943 def test_partial_readintos_incomplete_body(self):
944 # if the server shuts down the connection before the whole
945 # content-length is delivered, the socket is gracefully closed
946 body = "HTTP/1.1 200 Ok\r\nContent-Length: 10\r\n\r\nText"
947 sock = FakeSocket(body)
948 resp = client.HTTPResponse(sock)
949 resp.begin()
950 b = bytearray(2)
951 n = resp.readinto(b)
952 self.assertEqual(n, 2)
953 self.assertEqual(bytes(b), b'Te')
954 self.assertFalse(resp.isclosed())
955 n = resp.readinto(b)
956 self.assertEqual(n, 2)
957 self.assertEqual(bytes(b), b'xt')
958 n = resp.readinto(b)
959 self.assertEqual(n, 0)
960 self.assertTrue(resp.isclosed())
961 self.assertFalse(resp.closed)
962 resp.close()
963 self.assertTrue(resp.closed)
964
965 def test_host_port(self):
966 # Check invalid host_port
967
968 for hp in ("www.python.org:abc", "user:[email protected]"):
969 self.assertRaises(client.InvalidURL, client.HTTPConnection, hp)
970
971 for hp, h, p in (("[fe80::207:e9ff:fe9b]:8000",
972 "fe80::207:e9ff:fe9b", 8000),
973 ("www.python.org:80", "www.python.org", 80),
974 ("www.python.org:", "www.python.org", 80),
975 ("www.python.org", "www.python.org", 80),
976 ("[fe80::207:e9ff:fe9b]", "fe80::207:e9ff:fe9b", 80),
977 ("[fe80::207:e9ff:fe9b]:", "fe80::207:e9ff:fe9b", 80)):
978 c = client.HTTPConnection(hp)
979 self.assertEqual(h, c.host)
980 self.assertEqual(p, c.port)
981
982 def test_response_headers(self):
983 # test response with multiple message headers with the same field name.
984 text = ('HTTP/1.1 200 OK\r\n'
985 'Set-Cookie: Customer="WILE_E_COYOTE"; '
986 'Version="1"; Path="/acme"\r\n'
987 'Set-Cookie: Part_Number="Rocket_Launcher_0001"; Version="1";'
988 ' Path="/acme"\r\n'
989 '\r\n'
990 'No body\r\n')
991 hdr = ('Customer="WILE_E_COYOTE"; Version="1"; Path="/acme"'
992 ', '
993 'Part_Number="Rocket_Launcher_0001"; Version="1"; Path="/acme"')
994 s = FakeSocket(text)
995 r = client.HTTPResponse(s)
996 r.begin()
997 cookies = r.getheader("Set-Cookie")
998 self.assertEqual(cookies, hdr)
999
1000 def test_read_head(self):
1001 # Test that the library doesn't attempt to read any data
1002 # from a HEAD request. (Tickles SF bug #622042.)
1003 sock = FakeSocket(
1004 'HTTP/1.1 200 OK\r\n'
1005 'Content-Length: 14432\r\n'
1006 '\r\n',
1007 NoEOFBytesIO)
1008 resp = client.HTTPResponse(sock, method="HEAD")
1009 resp.begin()
1010 if resp.read():
1011 self.fail("Did not expect response from HEAD request")
1012
1013 def test_readinto_head(self):
1014 # Test that the library doesn't attempt to read any data
1015 # from a HEAD request. (Tickles SF bug #622042.)
1016 sock = FakeSocket(
1017 'HTTP/1.1 200 OK\r\n'
1018 'Content-Length: 14432\r\n'
1019 '\r\n',
1020 NoEOFBytesIO)
1021 resp = client.HTTPResponse(sock, method="HEAD")
1022 resp.begin()
1023 b = bytearray(5)
1024 if resp.readinto(b) != 0:
1025 self.fail("Did not expect response from HEAD request")
1026 self.assertEqual(bytes(b), b'\x00'*5)
1027
1028 def test_too_many_headers(self):
1029 headers = '\r\n'.join('Header%d: foo' % i
1030 for i in range(client._MAXHEADERS + 1)) + '\r\n'
1031 text = ('HTTP/1.1 200 OK\r\n' + headers)
1032 s = FakeSocket(text)
1033 r = client.HTTPResponse(s)
1034 self.assertRaisesRegex(client.HTTPException,
1035 r"got more than \d+ headers", r.begin)
1036
1037 def test_send_file(self):
1038 expected = (b'GET /foo HTTP/1.1\r\nHost: example.com\r\n'
1039 b'Accept-Encoding: identity\r\n'
1040 b'Transfer-Encoding: chunked\r\n'
1041 b'\r\n')
1042
1043 with open(__file__, 'rb') as body:
1044 conn = client.HTTPConnection('example.com')
1045 sock = FakeSocket(body)
1046 conn.sock = sock
1047 conn.request('GET', '/foo', body)
1048 self.assertStartsWith(sock.data, expected)
1049
1050 def test_send(self):
1051 expected = b'this is a test this is only a test'
1052 conn = client.HTTPConnection('example.com')
1053 sock = FakeSocket(None)
1054 conn.sock = sock
1055 conn.send(expected)
1056 self.assertEqual(expected, sock.data)
1057 sock.data = b''
1058 conn.send(array.array('b', expected))
1059 self.assertEqual(expected, sock.data)
1060 sock.data = b''
1061 conn.send(io.BytesIO(expected))
1062 self.assertEqual(expected, sock.data)
1063
1064 def test_send_updating_file(self):
1065 def data():
1066 yield 'data'
1067 yield None
1068 yield 'data_two'
1069
1070 class UpdatingFile(io.TextIOBase):
1071 mode = 'r'
1072 d = data()
1073 def read(self, blocksize=-1):
1074 return next(self.d)
1075
1076 expected = b'data'
1077
1078 conn = client.HTTPConnection('example.com')
1079 sock = FakeSocket("")
1080 conn.sock = sock
1081 conn.send(UpdatingFile())
1082 self.assertEqual(sock.data, expected)
1083
1084
1085 def test_send_iter(self):
1086 expected = b'GET /foo HTTP/1.1\r\nHost: example.com\r\n' \
1087 b'Accept-Encoding: identity\r\nContent-Length: 11\r\n' \
1088 b'\r\nonetwothree'
1089
1090 def body():
1091 yield b"one"
1092 yield b"two"
1093 yield b"three"
1094
1095 conn = client.HTTPConnection('example.com')
1096 sock = FakeSocket("")
1097 conn.sock = sock
1098 conn.request('GET', '/foo', body(), {'Content-Length': '11'})
1099 self.assertEqual(sock.data, expected)
1100
1101 def test_blocksize_request(self):
1102 """Check that request() respects the configured block size."""
1103 blocksize = 8 # For easy debugging.
1104 conn = client.HTTPConnection('example.com', blocksize=blocksize)
1105 sock = FakeSocket(None)
1106 conn.sock = sock
1107 expected = b"a" * blocksize + b"b"
1108 conn.request("PUT", "/", io.BytesIO(expected), {"Content-Length": "9"})
1109 self.assertEqual(sock.sendall_calls, 3)
1110 body = sock.data.split(b"\r\n\r\n", 1)[1]
1111 self.assertEqual(body, expected)
1112
1113 def test_blocksize_send(self):
1114 """Check that send() respects the configured block size."""
1115 blocksize = 8 # For easy debugging.
1116 conn = client.HTTPConnection('example.com', blocksize=blocksize)
1117 sock = FakeSocket(None)
1118 conn.sock = sock
1119 expected = b"a" * blocksize + b"b"
1120 conn.send(io.BytesIO(expected))
1121 self.assertEqual(sock.sendall_calls, 2)
1122 self.assertEqual(sock.data, expected)
1123
1124 def test_send_type_error(self):
1125 # See: Issue #12676
1126 conn = client.HTTPConnection('example.com')
1127 conn.sock = FakeSocket('')
1128 with self.assertRaises(TypeError):
1129 conn.request('POST', 'test', conn)
1130
1131 def test_chunked(self):
1132 expected = chunked_expected
1133 sock = FakeSocket(chunked_start + last_chunk + chunked_end)
1134 resp = client.HTTPResponse(sock, method="GET")
1135 resp.begin()
1136 self.assertEqual(resp.read(), expected)
1137 resp.close()
1138
1139 # Explicit full read
1140 for n in (-123, -1, None):
1141 with self.subTest('full read', n=n):
1142 sock = FakeSocket(chunked_start + last_chunk + chunked_end)
1143 resp = client.HTTPResponse(sock, method="GET")
1144 resp.begin()
1145 self.assertTrue(resp.chunked)
1146 self.assertEqual(resp.read(n), expected)
1147 resp.close()
1148
1149 # Read first chunk
1150 with self.subTest('read1(-1)'):
1151 sock = FakeSocket(chunked_start + last_chunk + chunked_end)
1152 resp = client.HTTPResponse(sock, method="GET")
1153 resp.begin()
1154 self.assertTrue(resp.chunked)
1155 self.assertEqual(resp.read1(-1), b"hello worl")
1156 resp.close()
1157
1158 # Various read sizes
1159 for n in range(1, 12):
1160 sock = FakeSocket(chunked_start + last_chunk + chunked_end)
1161 resp = client.HTTPResponse(sock, method="GET")
1162 resp.begin()
1163 self.assertEqual(resp.read(n) + resp.read(n) + resp.read(), expected)
1164 resp.close()
1165
1166 for x in ('', 'foo\r\n'):
1167 sock = FakeSocket(chunked_start + x)
1168 resp = client.HTTPResponse(sock, method="GET")
1169 resp.begin()
1170 try:
1171 resp.read()
1172 except client.IncompleteRead as i:
1173 self.assertEqual(i.partial, expected)
1174 expected_message = 'IncompleteRead(%d bytes read)' % len(expected)
1175 self.assertEqual(repr(i), expected_message)
1176 self.assertEqual(str(i), expected_message)
1177 else:
1178 self.fail('IncompleteRead expected')
1179 finally:
1180 resp.close()
1181
1182 def test_readinto_chunked(self):
1183
1184 expected = chunked_expected
1185 nexpected = len(expected)
1186 b = bytearray(128)
1187
1188 sock = FakeSocket(chunked_start + last_chunk + chunked_end)
1189 resp = client.HTTPResponse(sock, method="GET")
1190 resp.begin()
1191 n = resp.readinto(b)
1192 self.assertEqual(b[:nexpected], expected)
1193 self.assertEqual(n, nexpected)
1194 resp.close()
1195
1196 # Various read sizes
1197 for n in range(1, 12):
1198 sock = FakeSocket(chunked_start + last_chunk + chunked_end)
1199 resp = client.HTTPResponse(sock, method="GET")
1200 resp.begin()
1201 m = memoryview(b)
1202 i = resp.readinto(m[0:n])
1203 i += resp.readinto(m[i:n + i])
1204 i += resp.readinto(m[i:])
1205 self.assertEqual(b[:nexpected], expected)
1206 self.assertEqual(i, nexpected)
1207 resp.close()
1208
1209 for x in ('', 'foo\r\n'):
1210 sock = FakeSocket(chunked_start + x)
1211 resp = client.HTTPResponse(sock, method="GET")
1212 resp.begin()
1213 try:
1214 n = resp.readinto(b)
1215 except client.IncompleteRead as i:
1216 self.assertEqual(i.partial, expected)
1217 expected_message = 'IncompleteRead(%d bytes read)' % len(expected)
1218 self.assertEqual(repr(i), expected_message)
1219 self.assertEqual(str(i), expected_message)
1220 else:
1221 self.fail('IncompleteRead expected')
1222 finally:
1223 resp.close()
1224
1225 def test_chunked_head(self):
1226 chunked_start = (
1227 'HTTP/1.1 200 OK\r\n'
1228 'Transfer-Encoding: chunked\r\n\r\n'
1229 'a\r\n'
1230 'hello world\r\n'
1231 '1\r\n'
1232 'd\r\n'
1233 )
1234 sock = FakeSocket(chunked_start + last_chunk + chunked_end)
1235 resp = client.HTTPResponse(sock, method="HEAD")
1236 resp.begin()
1237 self.assertEqual(resp.read(), b'')
1238 self.assertEqual(resp.status, 200)
1239 self.assertEqual(resp.reason, 'OK')
1240 self.assertTrue(resp.isclosed())
1241 self.assertFalse(resp.closed)
1242 resp.close()
1243 self.assertTrue(resp.closed)
1244
1245 def test_readinto_chunked_head(self):
1246 chunked_start = (
1247 'HTTP/1.1 200 OK\r\n'
1248 'Transfer-Encoding: chunked\r\n\r\n'
1249 'a\r\n'
1250 'hello world\r\n'
1251 '1\r\n'
1252 'd\r\n'
1253 )
1254 sock = FakeSocket(chunked_start + last_chunk + chunked_end)
1255 resp = client.HTTPResponse(sock, method="HEAD")
1256 resp.begin()
1257 b = bytearray(5)
1258 n = resp.readinto(b)
1259 self.assertEqual(n, 0)
1260 self.assertEqual(bytes(b), b'\x00'*5)
1261 self.assertEqual(resp.status, 200)
1262 self.assertEqual(resp.reason, 'OK')
1263 self.assertTrue(resp.isclosed())
1264 self.assertFalse(resp.closed)
1265 resp.close()
1266 self.assertTrue(resp.closed)
1267
1268 def test_negative_content_length(self):
1269 sock = FakeSocket(
1270 'HTTP/1.1 200 OK\r\nContent-Length: -1\r\n\r\nHello\r\n')
1271 resp = client.HTTPResponse(sock, method="GET")
1272 resp.begin()
1273 self.assertEqual(resp.read(), b'Hello\r\n')
1274 self.assertTrue(resp.isclosed())
1275
1276 def test_incomplete_read(self):
1277 sock = FakeSocket('HTTP/1.1 200 OK\r\nContent-Length: 10\r\n\r\nHello\r\n')
1278 resp = client.HTTPResponse(sock, method="GET")
1279 resp.begin()
1280 try:
1281 resp.read()
1282 except client.IncompleteRead as i:
1283 self.assertEqual(i.partial, b'Hello\r\n')
1284 self.assertEqual(repr(i),
1285 "IncompleteRead(7 bytes read, 3 more expected)")
1286 self.assertEqual(str(i),
1287 "IncompleteRead(7 bytes read, 3 more expected)")
1288 self.assertTrue(resp.isclosed())
1289 else:
1290 self.fail('IncompleteRead expected')
1291
1292 def test_epipe(self):
1293 sock = EPipeSocket(
1294 "HTTP/1.0 401 Authorization Required\r\n"
1295 "Content-type: text/html\r\n"
1296 "WWW-Authenticate: Basic realm=\"example\"\r\n",
1297 b"Content-Length")
1298 conn = client.HTTPConnection("example.com")
1299 conn.sock = sock
1300 self.assertRaises(OSError,
1301 lambda: conn.request("PUT", "/url", "body"))
1302 resp = conn.getresponse()
1303 self.assertEqual(401, resp.status)
1304 self.assertEqual("Basic realm=\"example\"",
1305 resp.getheader("www-authenticate"))
1306
1307 # Test lines overflowing the max line size (_MAXLINE in http.client)
1308
1309 def test_overflowing_status_line(self):
1310 body = "HTTP/1.1 200 Ok" + "k" * 65536 + "\r\n"
1311 resp = client.HTTPResponse(FakeSocket(body))
1312 self.assertRaises((client.LineTooLong, client.BadStatusLine), resp.begin)
1313
1314 def test_overflowing_header_line(self):
1315 body = (
1316 'HTTP/1.1 200 OK\r\n'
1317 'X-Foo: bar' + 'r' * 65536 + '\r\n\r\n'
1318 )
1319 resp = client.HTTPResponse(FakeSocket(body))
1320 self.assertRaises(client.LineTooLong, resp.begin)
1321
1322 def test_overflowing_header_limit_after_100(self):
1323 body = (
1324 'HTTP/1.1 100 OK\r\n'
1325 'r\n' * 32768
1326 )
1327 resp = client.HTTPResponse(FakeSocket(body))
1328 with self.assertRaises(client.HTTPException) as cm:
1329 resp.begin()
1330 # We must assert more because other reasonable errors that we
1331 # do not want can also be HTTPException derived.
1332 self.assertIn('got more than ', str(cm.exception))
1333 self.assertIn('headers', str(cm.exception))
1334
1335 def test_overflowing_chunked_line(self):
1336 body = (
1337 'HTTP/1.1 200 OK\r\n'
1338 'Transfer-Encoding: chunked\r\n\r\n'
1339 + '0' * 65536 + 'a\r\n'
1340 'hello world\r\n'
1341 '0\r\n'
1342 '\r\n'
1343 )
1344 resp = client.HTTPResponse(FakeSocket(body))
1345 resp.begin()
1346 self.assertRaises(client.LineTooLong, resp.read)
1347
1348 def test_early_eof(self):
1349 # Test httpresponse with no \r\n termination,
1350 body = "HTTP/1.1 200 Ok"
1351 sock = FakeSocket(body)
1352 resp = client.HTTPResponse(sock)
1353 resp.begin()
1354 self.assertEqual(resp.read(), b'')
1355 self.assertTrue(resp.isclosed())
1356 self.assertFalse(resp.closed)
1357 resp.close()
1358 self.assertTrue(resp.closed)
1359
1360 def test_error_leak(self):
1361 # Test that the socket is not leaked if getresponse() fails
1362 conn = client.HTTPConnection('example.com')
1363 response = None
1364 class Response(client.HTTPResponse):
1365 def __init__(self, *pos, **kw):
1366 nonlocal response
1367 response = self # Avoid garbage collector closing the socket
1368 client.HTTPResponse.__init__(self, *pos, **kw)
1369 conn.response_class = Response
1370 conn.sock = FakeSocket('Invalid status line')
1371 conn.request('GET', '/')
1372 self.assertRaises(client.BadStatusLine, conn.getresponse)
1373 self.assertTrue(response.closed)
1374 self.assertTrue(conn.sock.file_closed)
1375
1376 def test_chunked_extension(self):
1377 extra = '3;foo=bar\r\n' + 'abc\r\n'
1378 expected = chunked_expected + b'abc'
1379
1380 sock = FakeSocket(chunked_start + extra + last_chunk_extended + chunked_end)
1381 resp = client.HTTPResponse(sock, method="GET")
1382 resp.begin()
1383 self.assertEqual(resp.read(), expected)
1384 resp.close()
1385
1386 def test_chunked_missing_end(self):
1387 """some servers may serve up a short chunked encoding stream"""
1388 expected = chunked_expected
1389 sock = FakeSocket(chunked_start + last_chunk) #no terminating crlf
1390 resp = client.HTTPResponse(sock, method="GET")
1391 resp.begin()
1392 self.assertEqual(resp.read(), expected)
1393 resp.close()
1394
1395 def test_chunked_trailers(self):
1396 """See that trailers are read and ignored"""
1397 expected = chunked_expected
1398 sock = FakeSocket(chunked_start + last_chunk + trailers + chunked_end)
1399 resp = client.HTTPResponse(sock, method="GET")
1400 resp.begin()
1401 self.assertEqual(resp.read(), expected)
1402 # we should have reached the end of the file
1403 self.assertEqual(sock.file.read(), b"") #we read to the end
1404 resp.close()
1405
1406 def test_chunked_sync(self):
1407 """Check that we don't read past the end of the chunked-encoding stream"""
1408 expected = chunked_expected
1409 extradata = "extradata"
1410 sock = FakeSocket(chunked_start + last_chunk + trailers + chunked_end + extradata)
1411 resp = client.HTTPResponse(sock, method="GET")
1412 resp.begin()
1413 self.assertEqual(resp.read(), expected)
1414 # the file should now have our extradata ready to be read
1415 self.assertEqual(sock.file.read(), extradata.encode("ascii")) #we read to the end
1416 resp.close()
1417
1418 def test_content_length_sync(self):
1419 """Check that we don't read past the end of the Content-Length stream"""
1420 extradata = b"extradata"
1421 expected = b"Hello123\r\n"
1422 sock = FakeSocket(b'HTTP/1.1 200 OK\r\nContent-Length: 10\r\n\r\n' + expected + extradata)
1423 resp = client.HTTPResponse(sock, method="GET")
1424 resp.begin()
1425 self.assertEqual(resp.read(), expected)
1426 # the file should now have our extradata ready to be read
1427 self.assertEqual(sock.file.read(), extradata) #we read to the end
1428 resp.close()
1429
1430 def test_readlines_content_length(self):
1431 extradata = b"extradata"
1432 expected = b"Hello123\r\n"
1433 sock = FakeSocket(b'HTTP/1.1 200 OK\r\nContent-Length: 10\r\n\r\n' + expected + extradata)
1434 resp = client.HTTPResponse(sock, method="GET")
1435 resp.begin()
1436 self.assertEqual(resp.readlines(2000), [expected])
1437 # the file should now have our extradata ready to be read
1438 self.assertEqual(sock.file.read(), extradata) #we read to the end
1439 resp.close()
1440
1441 def test_read1_content_length(self):
1442 extradata = b"extradata"
1443 expected = b"Hello123\r\n"
1444 sock = FakeSocket(b'HTTP/1.1 200 OK\r\nContent-Length: 10\r\n\r\n' + expected + extradata)
1445 resp = client.HTTPResponse(sock, method="GET")
1446 resp.begin()
1447 self.assertEqual(resp.read1(2000), expected)
1448 # the file should now have our extradata ready to be read
1449 self.assertEqual(sock.file.read(), extradata) #we read to the end
1450 resp.close()
1451
1452 def test_readline_bound_content_length(self):
1453 extradata = b"extradata"
1454 expected = b"Hello123\r\n"
1455 sock = FakeSocket(b'HTTP/1.1 200 OK\r\nContent-Length: 10\r\n\r\n' + expected + extradata)
1456 resp = client.HTTPResponse(sock, method="GET")
1457 resp.begin()
1458 self.assertEqual(resp.readline(10), expected)
1459 self.assertEqual(resp.readline(10), b"")
1460 # the file should now have our extradata ready to be read
1461 self.assertEqual(sock.file.read(), extradata) #we read to the end
1462 resp.close()
1463
1464 def test_read1_bound_content_length(self):
1465 extradata = b"extradata"
1466 expected = b"Hello123\r\n"
1467 sock = FakeSocket(b'HTTP/1.1 200 OK\r\nContent-Length: 30\r\n\r\n' + expected*3 + extradata)
1468 resp = client.HTTPResponse(sock, method="GET")
1469 resp.begin()
1470 self.assertEqual(resp.read1(20), expected*2)
1471 self.assertEqual(resp.read(), expected)
1472 # the file should now have our extradata ready to be read
1473 self.assertEqual(sock.file.read(), extradata) #we read to the end
1474 resp.close()
1475
1476 def test_response_fileno(self):
1477 # Make sure fd returned by fileno is valid.
1478 serv = socket.create_server((HOST, 0))
1479 self.addCleanup(serv.close)
1480
1481 result = None
1482 def run_server():
1483 [conn, address] = serv.accept()
1484 with conn, conn.makefile("rb") as reader:
1485 # Read the request header until a blank line
1486 while True:
1487 line = reader.readline()
1488 if not line.rstrip(b"\r\n"):
1489 break
1490 conn.sendall(b"HTTP/1.1 200 Connection established\r\n\r\n")
1491 nonlocal result
1492 result = reader.read()
1493
1494 thread = threading.Thread(target=run_server)
1495 thread.start()
1496 self.addCleanup(thread.join, float(1))
1497 conn = client.HTTPConnection(*serv.getsockname())
1498 conn.request("CONNECT", "dummy:1234")
1499 response = conn.getresponse()
1500 try:
1501 self.assertEqual(response.status, client.OK)
1502 s = socket.socket(fileno=response.fileno())
1503 try:
1504 s.sendall(b"proxied data\n")
1505 finally:
1506 s.detach()
1507 finally:
1508 response.close()
1509 conn.close()
1510 thread.join()
1511 self.assertEqual(result, b"proxied data\n")
1512
1513 def test_large_content_length(self):
1514 serv = socket.create_server((HOST, 0))
1515 self.addCleanup(serv.close)
1516
1517 def run_server():
1518 [conn, address] = serv.accept()
1519 with conn:
1520 while conn.recv(1024):
1521 conn.sendall(
1522 b"HTTP/1.1 200 Ok\r\n"
1523 b"Content-Length: %d\r\n"
1524 b"\r\n" % size)
1525 conn.sendall(b'A' * (size//3))
1526 conn.sendall(b'B' * (size - size//3))
1527
1528 thread = threading.Thread(target=run_server)
1529 thread.start()
1530 self.addCleanup(thread.join, 1.0)
1531
1532 conn = client.HTTPConnection(*serv.getsockname())
1533 try:
1534 for w in range(15, 27):
1535 size = 1 << w
1536 conn.request("GET", "/")
1537 with conn.getresponse() as response:
1538 self.assertEqual(len(response.read()), size)
1539 finally:
1540 conn.close()
1541 thread.join(1.0)
1542
1543 def test_large_content_length_truncated(self):
1544 serv = socket.create_server((HOST, 0))
1545 self.addCleanup(serv.close)
1546
1547 def run_server():
1548 while True:
1549 [conn, address] = serv.accept()
1550 with conn:
1551 conn.recv(1024)
1552 if not size:
1553 break
1554 conn.sendall(
1555 b"HTTP/1.1 200 Ok\r\n"
1556 b"Content-Length: %d\r\n"
1557 b"\r\n"
1558 b"Text" % size)
1559
1560 thread = threading.Thread(target=run_server)
1561 thread.start()
1562 self.addCleanup(thread.join, 1.0)
1563
1564 conn = client.HTTPConnection(*serv.getsockname())
1565 try:
1566 for w in range(18, 65):
1567 size = 1 << w
1568 conn.request("GET", "/")
1569 with conn.getresponse() as response:
1570 self.assertRaises(client.IncompleteRead, response.read)
1571 conn.close()
1572 finally:
1573 conn.close()
1574 size = 0
1575 conn.request("GET", "/")
1576 conn.close()
1577 thread.join(1.0)
1578
1579 def test_putrequest_override_domain_validation(self):
1580 """
1581 It should be possible to override the default validation
1582 behavior in putrequest (bpo-38216).
1583 """
1584 class UnsafeHTTPConnection(client.HTTPConnection):
1585 def _validate_path(self, url):
1586 pass
1587
1588 conn = UnsafeHTTPConnection('example.com')
1589 conn.sock = FakeSocket('')
1590 conn.putrequest('GET', '/\x00')
1591
1592 def test_putrequest_override_host_validation(self):
1593 class UnsafeHTTPConnection(client.HTTPConnection):
1594 def _validate_host(self, url):
1595 pass
1596
1597 conn = UnsafeHTTPConnection('example.com\r\n')
1598 conn.sock = FakeSocket('')
1599 # set skip_host so a ValueError is not raised upon adding the
1600 # invalid URL as the value of the "Host:" header
1601 conn.putrequest('GET', '/', skip_host=1)
1602
1603 def test_putrequest_override_encoding(self):
1604 """
1605 It should be possible to override the default encoding
1606 to transmit bytes in another encoding even if invalid
1607 (bpo-36274).
1608 """
1609 class UnsafeHTTPConnection(client.HTTPConnection):
1610 def _encode_request(self, str_url):
1611 return str_url.encode('utf-8')
1612
1613 conn = UnsafeHTTPConnection('example.com')
1614 conn.sock = FakeSocket('')
1615 conn.putrequest('GET', '/☃')
1616
1617
1618 class ExtendedReadTest(TestCase):
1619 """
1620 Test peek(), read1(), readline()
1621 """
1622 lines = (
1623 'HTTP/1.1 200 OK\r\n'
1624 '\r\n'
1625 'hello world!\n'
1626 'and now \n'
1627 'for something completely different\n'
1628 'foo'
1629 )
1630 lines_expected = lines[lines.find('hello'):].encode("ascii")
1631 lines_chunked = (
1632 'HTTP/1.1 200 OK\r\n'
1633 'Transfer-Encoding: chunked\r\n\r\n'
1634 'a\r\n'
1635 'hello worl\r\n'
1636 '3\r\n'
1637 'd!\n\r\n'
1638 '9\r\n'
1639 'and now \n\r\n'
1640 '23\r\n'
1641 'for something completely different\n\r\n'
1642 '3\r\n'
1643 'foo\r\n'
1644 '0\r\n' # terminating chunk
1645 '\r\n' # end of trailers
1646 )
1647
1648 def setUp(self):
1649 sock = FakeSocket(self.lines)
1650 resp = client.HTTPResponse(sock, method="GET")
1651 resp.begin()
1652 resp.fp = io.BufferedReader(resp.fp)
1653 self.resp = resp
1654
1655
1656
1657 def test_peek(self):
1658 resp = self.resp
1659 # patch up the buffered peek so that it returns not too much stuff
1660 oldpeek = resp.fp.peek
1661 def mypeek(n=-1):
1662 p = oldpeek(n)
1663 if n >= 0:
1664 return p[:n]
1665 return p[:10]
1666 resp.fp.peek = mypeek
1667
1668 all = []
1669 while True:
1670 # try a short peek
1671 p = resp.peek(3)
1672 if p:
1673 self.assertGreater(len(p), 0)
1674 # then unbounded peek
1675 p2 = resp.peek()
1676 self.assertGreaterEqual(len(p2), len(p))
1677 self.assertStartsWith(p2, p)
1678 next = resp.read(len(p2))
1679 self.assertEqual(next, p2)
1680 else:
1681 next = resp.read()
1682 self.assertFalse(next)
1683 all.append(next)
1684 if not next:
1685 break
1686 self.assertEqual(b"".join(all), self.lines_expected)
1687
1688 def test_readline(self):
1689 resp = self.resp
1690 self._verify_readline(self.resp.readline, self.lines_expected)
1691
1692 def test_readline_without_limit(self):
1693 self._verify_readline(self.resp.readline, self.lines_expected, limit=-1)
1694
1695 def _verify_readline(self, readline, expected, limit=5):
1696 all = []
1697 while True:
1698 # short readlines
1699 line = readline(limit)
1700 if line and line != b"foo":
1701 if len(line) < 5:
1702 self.assertEndsWith(line, b"\n")
1703 all.append(line)
1704 if not line:
1705 break
1706 self.assertEqual(b"".join(all), expected)
1707 self.assertTrue(self.resp.isclosed())
1708
1709 def test_read1(self):
1710 resp = self.resp
1711 def r():
1712 res = resp.read1(4)
1713 self.assertLessEqual(len(res), 4)
1714 return res
1715 readliner = Readliner(r)
1716 self._verify_readline(readliner.readline, self.lines_expected)
1717
1718 def test_read1_unbounded(self):
1719 resp = self.resp
1720 all = []
1721 while True:
1722 data = resp.read1()
1723 if not data:
1724 break
1725 all.append(data)
1726 self.assertEqual(b"".join(all), self.lines_expected)
1727 self.assertTrue(resp.isclosed())
1728
1729 def test_read1_bounded(self):
1730 resp = self.resp
1731 all = []
1732 while True:
1733 data = resp.read1(10)
1734 if not data:
1735 break
1736 self.assertLessEqual(len(data), 10)
1737 all.append(data)
1738 self.assertEqual(b"".join(all), self.lines_expected)
1739 self.assertTrue(resp.isclosed())
1740
1741 def test_read1_0(self):
1742 self.assertEqual(self.resp.read1(0), b"")
1743 self.assertFalse(self.resp.isclosed())
1744
1745 def test_peek_0(self):
1746 p = self.resp.peek(0)
1747 self.assertLessEqual(0, len(p))
1748
1749
1750 class ExtendedReadTestContentLengthKnown(ExtendedReadTest):
1751 _header, _body = ExtendedReadTest.lines.split('\r\n\r\n', 1)
1752 lines = _header + f'\r\nContent-Length: {len(_body)}\r\n\r\n' + _body
1753
1754
1755 class ExtendedReadTestChunked(ExtendedReadTest):
1756 """
1757 Test peek(), read1(), readline() in chunked mode
1758 """
1759 lines = (
1760 'HTTP/1.1 200 OK\r\n'
1761 'Transfer-Encoding: chunked\r\n\r\n'
1762 'a\r\n'
1763 'hello worl\r\n'
1764 '3\r\n'
1765 'd!\n\r\n'
1766 '9\r\n'
1767 'and now \n\r\n'
1768 '23\r\n'
1769 'for something completely different\n\r\n'
1770 '3\r\n'
1771 'foo\r\n'
1772 '0\r\n' # terminating chunk
1773 '\r\n' # end of trailers
1774 )
1775
1776
1777 class Readliner:
1778 """
1779 a simple readline class that uses an arbitrary read function and buffering
1780 """
1781 def __init__(self, readfunc):
1782 self.readfunc = readfunc
1783 self.remainder = b""
1784
1785 def readline(self, limit):
1786 data = []
1787 datalen = 0
1788 read = self.remainder
1789 try:
1790 while True:
1791 idx = read.find(b'\n')
1792 if idx != -1:
1793 break
1794 if datalen + len(read) >= limit:
1795 idx = limit - datalen - 1
1796 # read more data
1797 data.append(read)
1798 read = self.readfunc()
1799 if not read:
1800 idx = 0 #eof condition
1801 break
1802 idx += 1
1803 data.append(read[:idx])
1804 self.remainder = read[idx:]
1805 return b"".join(data)
1806 except:
1807 self.remainder = b"".join(data)
1808 raise
1809
1810
1811 class OfflineTest(TestCase):
1812 def test_all(self):
1813 # Documented objects defined in the module should be in __all__
1814 expected = {"responses"} # Allowlist documented dict() object
1815 # HTTPMessage, parse_headers(), and the HTTP status code constants are
1816 # intentionally omitted for simplicity
1817 denylist = {"HTTPMessage", "parse_headers"}
1818 for name in dir(client):
1819 if name.startswith("_") or name in denylist:
1820 continue
1821 module_object = getattr(client, name)
1822 if getattr(module_object, "__module__", None) == "http.client":
1823 expected.add(name)
1824 self.assertCountEqual(client.__all__, expected)
1825
1826 def test_responses(self):
1827 self.assertEqual(client.responses[client.NOT_FOUND], "Not Found")
1828
1829 def test_client_constants(self):
1830 # Make sure we don't break backward compatibility with 3.4
1831 expected = [
1832 'CONTINUE',
1833 'SWITCHING_PROTOCOLS',
1834 'PROCESSING',
1835 'OK',
1836 'CREATED',
1837 'ACCEPTED',
1838 'NON_AUTHORITATIVE_INFORMATION',
1839 'NO_CONTENT',
1840 'RESET_CONTENT',
1841 'PARTIAL_CONTENT',
1842 'MULTI_STATUS',
1843 'IM_USED',
1844 'MULTIPLE_CHOICES',
1845 'MOVED_PERMANENTLY',
1846 'FOUND',
1847 'SEE_OTHER',
1848 'NOT_MODIFIED',
1849 'USE_PROXY',
1850 'TEMPORARY_REDIRECT',
1851 'BAD_REQUEST',
1852 'UNAUTHORIZED',
1853 'PAYMENT_REQUIRED',
1854 'FORBIDDEN',
1855 'NOT_FOUND',
1856 'METHOD_NOT_ALLOWED',
1857 'NOT_ACCEPTABLE',
1858 'PROXY_AUTHENTICATION_REQUIRED',
1859 'REQUEST_TIMEOUT',
1860 'CONFLICT',
1861 'GONE',
1862 'LENGTH_REQUIRED',
1863 'PRECONDITION_FAILED',
1864 'CONTENT_TOO_LARGE',
1865 'REQUEST_ENTITY_TOO_LARGE',
1866 'URI_TOO_LONG',
1867 'REQUEST_URI_TOO_LONG',
1868 'UNSUPPORTED_MEDIA_TYPE',
1869 'RANGE_NOT_SATISFIABLE',
1870 'REQUESTED_RANGE_NOT_SATISFIABLE',
1871 'EXPECTATION_FAILED',
1872 'IM_A_TEAPOT',
1873 'MISDIRECTED_REQUEST',
1874 'UNPROCESSABLE_CONTENT',
1875 'UNPROCESSABLE_ENTITY',
1876 'LOCKED',
1877 'FAILED_DEPENDENCY',
1878 'UPGRADE_REQUIRED',
1879 'PRECONDITION_REQUIRED',
1880 'TOO_MANY_REQUESTS',
1881 'REQUEST_HEADER_FIELDS_TOO_LARGE',
1882 'UNAVAILABLE_FOR_LEGAL_REASONS',
1883 'INTERNAL_SERVER_ERROR',
1884 'NOT_IMPLEMENTED',
1885 'BAD_GATEWAY',
1886 'SERVICE_UNAVAILABLE',
1887 'GATEWAY_TIMEOUT',
1888 'HTTP_VERSION_NOT_SUPPORTED',
1889 'INSUFFICIENT_STORAGE',
1890 'NOT_EXTENDED',
1891 'NETWORK_AUTHENTICATION_REQUIRED',
1892 'EARLY_HINTS',
1893 'TOO_EARLY'
1894 ]
1895 for const in expected:
1896 with self.subTest(constant=const):
1897 self.assertHasAttr(client, const)
1898
1899
1900 class SourceAddressTest(TestCase):
1901 def setUp(self):
1902 self.serv = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
1903 self.port = socket_helper.bind_port(self.serv)
1904 self.source_port = socket_helper.find_unused_port()
1905 self.serv.listen()
1906 self.conn = None
1907
1908 def tearDown(self):
1909 if self.conn:
1910 self.conn.close()
1911 self.conn = None
1912 self.serv.close()
1913 self.serv = None
1914
1915 def testHTTPConnectionSourceAddress(self):
1916 self.conn = client.HTTPConnection(HOST, self.port,
1917 source_address=('', self.source_port))
1918 self.conn.connect()
1919 self.assertEqual(self.conn.sock.getsockname()[1], self.source_port)
1920
1921 @unittest.skipIf(not hasattr(client, 'HTTPSConnection'),
1922 'http.client.HTTPSConnection not defined')
1923 def testHTTPSConnectionSourceAddress(self):
1924 self.conn = client.HTTPSConnection(HOST, self.port,
1925 source_address=('', self.source_port))
1926 # We don't test anything here other than the constructor not barfing as
1927 # this code doesn't deal with setting up an active running SSL server
1928 # for an ssl_wrapped connect() to actually return from.
1929
1930
1931 class TimeoutTest(TestCase):
1932 PORT = None
1933
1934 def setUp(self):
1935 self.serv = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
1936 TimeoutTest.PORT = socket_helper.bind_port(self.serv)
1937 self.serv.listen()
1938
1939 def tearDown(self):
1940 self.serv.close()
1941 self.serv = None
1942
1943 def testTimeoutAttribute(self):
1944 # This will prove that the timeout gets through HTTPConnection
1945 # and into the socket.
1946
1947 # default -- use global socket timeout
1948 self.assertIsNone(socket.getdefaulttimeout())
1949 socket.setdefaulttimeout(30)
1950 try:
1951 httpConn = client.HTTPConnection(HOST, TimeoutTest.PORT)
1952 httpConn.connect()
1953 finally:
1954 socket.setdefaulttimeout(None)
1955 self.assertEqual(httpConn.sock.gettimeout(), 30)
1956 httpConn.close()
1957
1958 # no timeout -- do not use global socket default
1959 self.assertIsNone(socket.getdefaulttimeout())
1960 socket.setdefaulttimeout(30)
1961 try:
1962 httpConn = client.HTTPConnection(HOST, TimeoutTest.PORT,
1963 timeout=None)
1964 httpConn.connect()
1965 finally:
1966 socket.setdefaulttimeout(None)
1967 self.assertEqual(httpConn.sock.gettimeout(), None)
1968 httpConn.close()
1969
1970 # a value
1971 httpConn = client.HTTPConnection(HOST, TimeoutTest.PORT, timeout=30)
1972 httpConn.connect()
1973 self.assertEqual(httpConn.sock.gettimeout(), 30)
1974 httpConn.close()
1975
1976
1977 class PersistenceTest(TestCase):
1978
1979 def test_reuse_reconnect(self):
1980 # Should reuse or reconnect depending on header from server
1981 tests = (
1982 ('1.0', '', False),
1983 ('1.0', 'Connection: keep-alive\r\n', True),
1984 ('1.1', '', True),
1985 ('1.1', 'Connection: close\r\n', False),
1986 ('1.0', 'Connection: keep-ALIVE\r\n', True),
1987 ('1.1', 'Connection: cloSE\r\n', False),
1988 )
1989 for version, header, reuse in tests:
1990 with self.subTest(version=version, header=header):
1991 msg = (
1992 'HTTP/{} 200 OK\r\n'
1993 '{}'
1994 'Content-Length: 12\r\n'
1995 '\r\n'
1996 'Dummy body\r\n'
1997 ).format(version, header)
1998 conn = FakeSocketHTTPConnection(msg)
1999 self.assertIsNone(conn.sock)
2000 conn.request('GET', '/open-connection')
2001 with conn.getresponse() as response:
2002 self.assertEqual(conn.sock is None, not reuse)
2003 response.read()
2004 self.assertEqual(conn.sock is None, not reuse)
2005 self.assertEqual(conn.connections, 1)
2006 conn.request('GET', '/subsequent-request')
2007 self.assertEqual(conn.connections, 1 if reuse else 2)
2008
2009 def test_disconnected(self):
2010
2011 def make_reset_reader(text):
2012 """Return BufferedReader that raises ECONNRESET at EOF"""
2013 stream = io.BytesIO(text)
2014 def readinto(buffer):
2015 size = io.BytesIO.readinto(stream, buffer)
2016 if size == 0:
2017 raise ConnectionResetError()
2018 return size
2019 stream.readinto = readinto
2020 return io.BufferedReader(stream)
2021
2022 tests = (
2023 (io.BytesIO, client.RemoteDisconnected),
2024 (make_reset_reader, ConnectionResetError),
2025 )
2026 for stream_factory, exception in tests:
2027 with self.subTest(exception=exception):
2028 conn = FakeSocketHTTPConnection(b'', stream_factory)
2029 conn.request('GET', '/eof-response')
2030 self.assertRaises(exception, conn.getresponse)
2031 self.assertIsNone(conn.sock)
2032 # HTTPConnection.connect() should be automatically invoked
2033 conn.request('GET', '/reconnect')
2034 self.assertEqual(conn.connections, 2)
2035
2036 def test_100_close(self):
2037 conn = FakeSocketHTTPConnection(
2038 b'HTTP/1.1 100 Continue\r\n'
2039 b'\r\n'
2040 # Missing final response
2041 )
2042 conn.request('GET', '/', headers={'Expect': '100-continue'})
2043 self.assertRaises(client.RemoteDisconnected, conn.getresponse)
2044 self.assertIsNone(conn.sock)
2045 conn.request('GET', '/reconnect')
2046 self.assertEqual(conn.connections, 2)
2047
2048
2049 class HTTPSTest(TestCase):
2050
2051 def setUp(self):
2052 if not hasattr(client, 'HTTPSConnection'):
2053 self.skipTest('ssl support required')
2054
2055 def make_server(self, certfile):
2056 from test.ssl_servers import make_https_server
2057 return make_https_server(self, certfile=certfile)
2058
2059 def test_attributes(self):
2060 # simple test to check it's storing the timeout
2061 h = client.HTTPSConnection(HOST, TimeoutTest.PORT, timeout=30)
2062 self.assertEqual(h.timeout, 30)
2063
2064 def test_networked(self):
2065 # Default settings: requires a valid cert from a trusted CA
2066 import ssl
2067 support.requires('network')
2068 with socket_helper.transient_internet('self-signed.pythontest.net'):
2069 h = client.HTTPSConnection('self-signed.pythontest.net', 443)
2070 with self.assertRaises(ssl.SSLError) as exc_info:
2071 h.request('GET', '/')
2072 self.assertEqual(exc_info.exception.reason, 'CERTIFICATE_VERIFY_FAILED')
2073
2074 def test_networked_noverification(self):
2075 # Switch off cert verification
2076 import ssl
2077 support.requires('network')
2078 with socket_helper.transient_internet('self-signed.pythontest.net'):
2079 context = ssl._create_unverified_context()
2080 h = client.HTTPSConnection('self-signed.pythontest.net', 443,
2081 context=context)
2082 h.request('GET', '/')
2083 resp = h.getresponse()
2084 h.close()
2085 self.assertIn('nginx', resp.getheader('server'))
2086 resp.close()
2087
2088 @support.system_must_validate_cert
2089 def test_networked_trusted_by_default_cert(self):
2090 # Default settings: requires a valid cert from a trusted CA
2091 support.requires('network')
2092 with socket_helper.transient_internet('www.python.org'):
2093 h = client.HTTPSConnection('www.python.org', 443)
2094 h.request('GET', '/')
2095 resp = h.getresponse()
2096 content_type = resp.getheader('content-type')
2097 resp.close()
2098 h.close()
2099 self.assertIn('text/html', content_type)
2100
2101 def test_networked_good_cert(self):
2102 # We feed the server's cert as a validating cert
2103 import ssl
2104 support.requires('network')
2105 selfsigned_pythontestdotnet = 'self-signed.pythontest.net'
2106 with socket_helper.transient_internet(selfsigned_pythontestdotnet):
2107 context = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT)
2108 self.assertEqual(context.verify_mode, ssl.CERT_REQUIRED)
2109 self.assertEqual(context.check_hostname, True)
2110 context.load_verify_locations(CERT_selfsigned_pythontestdotnet)
2111 try:
2112 h = client.HTTPSConnection(selfsigned_pythontestdotnet, 443,
2113 context=context)
2114 h.request('GET', '/')
2115 resp = h.getresponse()
2116 except ssl.SSLError as ssl_err:
2117 ssl_err_str = str(ssl_err)
2118 # In the error message of [SSL: CERTIFICATE_VERIFY_FAILED] on
2119 # modern Linux distros (Debian Buster, etc) default OpenSSL
2120 # configurations it'll fail saying "key too weak" until we
2121 # address https://bugs.python.org/issue36816 to use a proper
2122 # key size on self-signed.pythontest.net.
2123 if re.search(r'(?i)key.too.weak', ssl_err_str):
2124 raise unittest.SkipTest(
2125 f'Got {ssl_err_str} trying to connect '
2126 f'to {selfsigned_pythontestdotnet}. '
2127 'See https://bugs.python.org/issue36816.')
2128 raise
2129 server_string = resp.getheader('server')
2130 resp.close()
2131 h.close()
2132 self.assertIn('nginx', server_string)
2133
2134 @support.requires_resource('walltime')
2135 def test_networked_bad_cert(self):
2136 # We feed a "CA" cert that is unrelated to the server's cert
2137 import ssl
2138 support.requires('network')
2139 with socket_helper.transient_internet('self-signed.pythontest.net'):
2140 context = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT)
2141 context.load_verify_locations(CERT_localhost)
2142 h = client.HTTPSConnection('self-signed.pythontest.net', 443, context=context)
2143 with self.assertRaises(ssl.SSLError) as exc_info:
2144 h.request('GET', '/')
2145 self.assertEqual(exc_info.exception.reason, 'CERTIFICATE_VERIFY_FAILED')
2146
2147 def test_local_unknown_cert(self):
2148 # The custom cert isn't known to the default trust bundle
2149 import ssl
2150 server = self.make_server(CERT_localhost)
2151 h = client.HTTPSConnection('localhost', server.port)
2152 with self.assertRaises(ssl.SSLError) as exc_info:
2153 h.request('GET', '/')
2154 self.assertEqual(exc_info.exception.reason, 'CERTIFICATE_VERIFY_FAILED')
2155
2156 def test_local_good_hostname(self):
2157 # The (valid) cert validates the HTTPS hostname
2158 import ssl
2159 server = self.make_server(CERT_localhost)
2160 context = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT)
2161 context.load_verify_locations(CERT_localhost)
2162 h = client.HTTPSConnection('localhost', server.port, context=context)
2163 self.addCleanup(h.close)
2164 h.request('GET', '/nonexistent')
2165 resp = h.getresponse()
2166 self.addCleanup(resp.close)
2167 self.assertEqual(resp.status, 404)
2168
2169 def test_local_bad_hostname(self):
2170 # The (valid) cert doesn't validate the HTTPS hostname
2171 import ssl
2172 server = self.make_server(CERT_fakehostname)
2173 context = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT)
2174 context.load_verify_locations(CERT_fakehostname)
2175 h = client.HTTPSConnection('localhost', server.port, context=context)
2176 with self.assertRaises(ssl.CertificateError):
2177 h.request('GET', '/')
2178
2179 # Same with explicit context.check_hostname=True
2180 context.check_hostname = True
2181 h = client.HTTPSConnection('localhost', server.port, context=context)
2182 with self.assertRaises(ssl.CertificateError):
2183 h.request('GET', '/')
2184
2185 # With context.check_hostname=False, the mismatching is ignored
2186 context.check_hostname = False
2187 h = client.HTTPSConnection('localhost', server.port, context=context)
2188 h.request('GET', '/nonexistent')
2189 resp = h.getresponse()
2190 resp.close()
2191 h.close()
2192 self.assertEqual(resp.status, 404)
2193
2194 @unittest.skipIf(not hasattr(client, 'HTTPSConnection'),
2195 'http.client.HTTPSConnection not available')
2196 def test_host_port(self):
2197 # Check invalid host_port
2198
2199 for hp in ("www.python.org:abc", "user:[email protected]"):
2200 self.assertRaises(client.InvalidURL, client.HTTPSConnection, hp)
2201
2202 for hp, h, p in (("[fe80::207:e9ff:fe9b]:8000",
2203 "fe80::207:e9ff:fe9b", 8000),
2204 ("www.python.org:443", "www.python.org", 443),
2205 ("www.python.org:", "www.python.org", 443),
2206 ("www.python.org", "www.python.org", 443),
2207 ("[fe80::207:e9ff:fe9b]", "fe80::207:e9ff:fe9b", 443),
2208 ("[fe80::207:e9ff:fe9b]:", "fe80::207:e9ff:fe9b",
2209 443)):
2210 c = client.HTTPSConnection(hp)
2211 self.assertEqual(h, c.host)
2212 self.assertEqual(p, c.port)
2213
2214 def test_tls13_pha(self):
2215 import ssl
2216 if not ssl.HAS_TLSv1_3 or not ssl.HAS_PHA:
2217 self.skipTest('TLS 1.3 PHA support required')
2218 # just check status of PHA flag
2219 h = client.HTTPSConnection('localhost', 443)
2220 self.assertTrue(h._context.post_handshake_auth)
2221
2222 context = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT)
2223 self.assertFalse(context.post_handshake_auth)
2224 h = client.HTTPSConnection('localhost', 443, context=context)
2225 self.assertIs(h._context, context)
2226 self.assertFalse(h._context.post_handshake_auth)
2227
2228 context = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT, cert_file=CERT_localhost)
2229 context.post_handshake_auth = True
2230 h = client.HTTPSConnection('localhost', 443, context=context)
2231 self.assertTrue(h._context.post_handshake_auth)
2232
2233
2234 class RequestBodyTest(TestCase):
2235 """Test cases where a request includes a message body."""
2236
2237 def setUp(self):
2238 self.conn = client.HTTPConnection('example.com')
2239 self.conn.sock = self.sock = FakeSocket("")
2240 self.conn.sock = self.sock
2241
2242 def get_headers_and_fp(self):
2243 f = io.BytesIO(self.sock.data)
2244 f.readline() # read the request line
2245 message = client.parse_headers(f)
2246 return message, f
2247
2248 def test_list_body(self):
2249 # Note that no content-length is automatically calculated for
2250 # an iterable. The request will fall back to send chunked
2251 # transfer encoding.
2252 cases = (
2253 ([b'foo', b'bar'], b'3\r\nfoo\r\n3\r\nbar\r\n0\r\n\r\n'),
2254 ((b'foo', b'bar'), b'3\r\nfoo\r\n3\r\nbar\r\n0\r\n\r\n'),
2255 )
2256 for body, expected in cases:
2257 with self.subTest(body):
2258 self.conn = client.HTTPConnection('example.com')
2259 self.conn.sock = self.sock = FakeSocket('')
2260
2261 self.conn.request('PUT', '/url', body)
2262 msg, f = self.get_headers_and_fp()
2263 self.assertNotIn('Content-Type', msg)
2264 self.assertNotIn('Content-Length', msg)
2265 self.assertEqual(msg.get('Transfer-Encoding'), 'chunked')
2266 self.assertEqual(expected, f.read())
2267
2268 def test_manual_content_length(self):
2269 # Set an incorrect content-length so that we can verify that
2270 # it will not be over-ridden by the library.
2271 self.conn.request("PUT", "/url", "body",
2272 {"Content-Length": "42"})
2273 message, f = self.get_headers_and_fp()
2274 self.assertEqual("42", message.get("content-length"))
2275 self.assertEqual(4, len(f.read()))
2276
2277 def test_ascii_body(self):
2278 self.conn.request("PUT", "/url", "body")
2279 message, f = self.get_headers_and_fp()
2280 self.assertEqual("text/plain", message.get_content_type())
2281 self.assertIsNone(message.get_charset())
2282 self.assertEqual("4", message.get("content-length"))
2283 self.assertEqual(b'body', f.read())
2284
2285 def test_latin1_body(self):
2286 self.conn.request("PUT", "/url", "body\xc1")
2287 message, f = self.get_headers_and_fp()
2288 self.assertEqual("text/plain", message.get_content_type())
2289 self.assertIsNone(message.get_charset())
2290 self.assertEqual("5", message.get("content-length"))
2291 self.assertEqual(b'body\xc1', f.read())
2292
2293 def test_bytes_body(self):
2294 self.conn.request("PUT", "/url", b"body\xc1")
2295 message, f = self.get_headers_and_fp()
2296 self.assertEqual("text/plain", message.get_content_type())
2297 self.assertIsNone(message.get_charset())
2298 self.assertEqual("5", message.get("content-length"))
2299 self.assertEqual(b'body\xc1', f.read())
2300
2301 def test_text_file_body(self):
2302 self.addCleanup(os_helper.unlink, os_helper.TESTFN)
2303 with open(os_helper.TESTFN, "w", encoding="utf-8") as f:
2304 f.write("body")
2305 with open(os_helper.TESTFN, encoding="utf-8") as f:
2306 self.conn.request("PUT", "/url", f)
2307 message, f = self.get_headers_and_fp()
2308 self.assertEqual("text/plain", message.get_content_type())
2309 self.assertIsNone(message.get_charset())
2310 # No content-length will be determined for files; the body
2311 # will be sent using chunked transfer encoding instead.
2312 self.assertIsNone(message.get("content-length"))
2313 self.assertEqual("chunked", message.get("transfer-encoding"))
2314 self.assertEqual(b'4\r\nbody\r\n0\r\n\r\n', f.read())
2315
2316 def test_binary_file_body(self):
2317 self.addCleanup(os_helper.unlink, os_helper.TESTFN)
2318 with open(os_helper.TESTFN, "wb") as f:
2319 f.write(b"body\xc1")
2320 with open(os_helper.TESTFN, "rb") as f:
2321 self.conn.request("PUT", "/url", f)
2322 message, f = self.get_headers_and_fp()
2323 self.assertEqual("text/plain", message.get_content_type())
2324 self.assertIsNone(message.get_charset())
2325 self.assertEqual("chunked", message.get("Transfer-Encoding"))
2326 self.assertNotIn("Content-Length", message)
2327 self.assertEqual(b'5\r\nbody\xc1\r\n0\r\n\r\n', f.read())
2328
2329
2330 class HTTPResponseTest(TestCase):
2331
2332 def setUp(self):
2333 body = "HTTP/1.1 200 Ok\r\nMy-Header: first-value\r\nMy-Header: \
2334 second-value\r\n\r\nText"
2335 sock = FakeSocket(body)
2336 self.resp = client.HTTPResponse(sock)
2337 self.resp.begin()
2338
2339 def test_getting_header(self):
2340 header = self.resp.getheader('My-Header')
2341 self.assertEqual(header, 'first-value, second-value')
2342
2343 header = self.resp.getheader('My-Header', 'some default')
2344 self.assertEqual(header, 'first-value, second-value')
2345
2346 def test_getting_nonexistent_header_with_string_default(self):
2347 header = self.resp.getheader('No-Such-Header', 'default-value')
2348 self.assertEqual(header, 'default-value')
2349
2350 def test_getting_nonexistent_header_with_iterable_default(self):
2351 header = self.resp.getheader('No-Such-Header', ['default', 'values'])
2352 self.assertEqual(header, 'default, values')
2353
2354 header = self.resp.getheader('No-Such-Header', ('default', 'values'))
2355 self.assertEqual(header, 'default, values')
2356
2357 def test_getting_nonexistent_header_without_default(self):
2358 header = self.resp.getheader('No-Such-Header')
2359 self.assertEqual(header, None)
2360
2361 def test_getting_header_defaultint(self):
2362 header = self.resp.getheader('No-Such-Header',default=42)
2363 self.assertEqual(header, 42)
2364
2365 class TunnelTests(TestCase):
2366 def setUp(self):
2367 response_text = (
2368 'HTTP/1.1 200 OK\r\n\r\n' # Reply to CONNECT
2369 'HTTP/1.1 200 OK\r\n' # Reply to HEAD
2370 'Content-Length: 42\r\n\r\n'
2371 )
2372 self.host = 'proxy.com'
2373 self.port = client.HTTP_PORT
2374 self.conn = client.HTTPConnection(self.host)
2375 self.conn._create_connection = self._create_connection(response_text)
2376
2377 def tearDown(self):
2378 self.conn.close()
2379
2380 def _create_connection(self, response_text):
2381 def create_connection(address, timeout=None, source_address=None):
2382 return FakeSocket(response_text, host=address[0], port=address[1])
2383 return create_connection
2384
2385 def test_set_tunnel_host_port_headers_add_host_missing(self):
2386 tunnel_host = 'destination.com'
2387 tunnel_port = 8888
2388 tunnel_headers = {'User-Agent': 'Mozilla/5.0 (compatible, MSIE 11)'}
2389 tunnel_headers_after = tunnel_headers.copy()
2390 tunnel_headers_after['Host'] = '%s:%d' % (tunnel_host, tunnel_port)
2391 self.conn.set_tunnel(tunnel_host, port=tunnel_port,
2392 headers=tunnel_headers)
2393 self.conn.request('HEAD', '/', '')
2394 self.assertEqual(self.conn.sock.host, self.host)
2395 self.assertEqual(self.conn.sock.port, self.port)
2396 self.assertEqual(self.conn._tunnel_host, tunnel_host)
2397 self.assertEqual(self.conn._tunnel_port, tunnel_port)
2398 self.assertEqual(self.conn._tunnel_headers, tunnel_headers_after)
2399
2400 def test_set_tunnel_host_port_headers_set_host_identical(self):
2401 tunnel_host = 'destination.com'
2402 tunnel_port = 8888
2403 tunnel_headers = {'User-Agent': 'Mozilla/5.0 (compatible, MSIE 11)',
2404 'Host': '%s:%d' % (tunnel_host, tunnel_port)}
2405 self.conn.set_tunnel(tunnel_host, port=tunnel_port,
2406 headers=tunnel_headers)
2407 self.conn.request('HEAD', '/', '')
2408 self.assertEqual(self.conn.sock.host, self.host)
2409 self.assertEqual(self.conn.sock.port, self.port)
2410 self.assertEqual(self.conn._tunnel_host, tunnel_host)
2411 self.assertEqual(self.conn._tunnel_port, tunnel_port)
2412 self.assertEqual(self.conn._tunnel_headers, tunnel_headers)
2413
2414 def test_set_tunnel_host_port_headers_set_host_different(self):
2415 tunnel_host = 'destination.com'
2416 tunnel_port = 8888
2417 tunnel_headers = {'User-Agent': 'Mozilla/5.0 (compatible, MSIE 11)',
2418 'Host': '%s:%d' % ('example.com', 4200)}
2419 self.conn.set_tunnel(tunnel_host, port=tunnel_port,
2420 headers=tunnel_headers)
2421 self.conn.request('HEAD', '/', '')
2422 self.assertEqual(self.conn.sock.host, self.host)
2423 self.assertEqual(self.conn.sock.port, self.port)
2424 self.assertEqual(self.conn._tunnel_host, tunnel_host)
2425 self.assertEqual(self.conn._tunnel_port, tunnel_port)
2426 self.assertEqual(self.conn._tunnel_headers, tunnel_headers)
2427
2428 def test_disallow_set_tunnel_after_connect(self):
2429 # Once connected, we shouldn't be able to tunnel anymore
2430 self.conn.connect()
2431 self.assertRaises(RuntimeError, self.conn.set_tunnel,
2432 'destination.com')
2433
2434 def test_connect_with_tunnel(self):
2435 d = {
2436 b'host': b'destination.com',
2437 b'port': client.HTTP_PORT,
2438 }
2439 self.conn.set_tunnel(d[b'host'].decode('ascii'))
2440 self.conn.request('HEAD', '/', '')
2441 self.assertEqual(self.conn.sock.host, self.host)
2442 self.assertEqual(self.conn.sock.port, self.port)
2443 self.assertIn(b'CONNECT %(host)s:%(port)d HTTP/1.1\r\n'
2444 b'Host: %(host)s:%(port)d\r\n\r\n' % d,
2445 self.conn.sock.data)
2446 self.assertIn(b'HEAD / HTTP/1.1\r\nHost: %(host)s\r\n' % d,
2447 self.conn.sock.data)
2448
2449 def test_connect_with_tunnel_with_default_port(self):
2450 d = {
2451 b'host': b'destination.com',
2452 b'port': client.HTTP_PORT,
2453 }
2454 self.conn.set_tunnel(d[b'host'].decode('ascii'), port=d[b'port'])
2455 self.conn.request('HEAD', '/', '')
2456 self.assertEqual(self.conn.sock.host, self.host)
2457 self.assertEqual(self.conn.sock.port, self.port)
2458 self.assertIn(b'CONNECT %(host)s:%(port)d HTTP/1.1\r\n'
2459 b'Host: %(host)s:%(port)d\r\n\r\n' % d,
2460 self.conn.sock.data)
2461 self.assertIn(b'HEAD / HTTP/1.1\r\nHost: %(host)s\r\n' % d,
2462 self.conn.sock.data)
2463
2464 def test_connect_with_tunnel_with_nonstandard_port(self):
2465 d = {
2466 b'host': b'destination.com',
2467 b'port': 8888,
2468 }
2469 self.conn.set_tunnel(d[b'host'].decode('ascii'), port=d[b'port'])
2470 self.conn.request('HEAD', '/', '')
2471 self.assertEqual(self.conn.sock.host, self.host)
2472 self.assertEqual(self.conn.sock.port, self.port)
2473 self.assertIn(b'CONNECT %(host)s:%(port)d HTTP/1.1\r\n'
2474 b'Host: %(host)s:%(port)d\r\n\r\n' % d,
2475 self.conn.sock.data)
2476 self.assertIn(b'HEAD / HTTP/1.1\r\nHost: %(host)s:%(port)d\r\n' % d,
2477 self.conn.sock.data)
2478
2479 # This request is not RFC-valid, but it's been possible with the library
2480 # for years, so don't break it unexpectedly... This also tests
2481 # case-insensitivity when injecting Host: headers if they're missing.
2482 def test_connect_with_tunnel_with_different_host_header(self):
2483 d = {
2484 b'host': b'destination.com',
2485 b'tunnel_host_header': b'example.com:9876',
2486 b'port': client.HTTP_PORT,
2487 }
2488 self.conn.set_tunnel(
2489 d[b'host'].decode('ascii'),
2490 headers={'HOST': d[b'tunnel_host_header'].decode('ascii')})
2491 self.conn.request('HEAD', '/', '')
2492 self.assertEqual(self.conn.sock.host, self.host)
2493 self.assertEqual(self.conn.sock.port, self.port)
2494 self.assertIn(b'CONNECT %(host)s:%(port)d HTTP/1.1\r\n'
2495 b'HOST: %(tunnel_host_header)s\r\n\r\n' % d,
2496 self.conn.sock.data)
2497 self.assertIn(b'HEAD / HTTP/1.1\r\nHost: %(host)s\r\n' % d,
2498 self.conn.sock.data)
2499
2500 def test_connect_with_tunnel_different_host(self):
2501 d = {
2502 b'host': b'destination.com',
2503 b'port': client.HTTP_PORT,
2504 }
2505 self.conn.set_tunnel(d[b'host'].decode('ascii'))
2506 self.conn.request('HEAD', '/', '')
2507 self.assertEqual(self.conn.sock.host, self.host)
2508 self.assertEqual(self.conn.sock.port, self.port)
2509 self.assertIn(b'CONNECT %(host)s:%(port)d HTTP/1.1\r\n'
2510 b'Host: %(host)s:%(port)d\r\n\r\n' % d,
2511 self.conn.sock.data)
2512 self.assertIn(b'HEAD / HTTP/1.1\r\nHost: %(host)s\r\n' % d,
2513 self.conn.sock.data)
2514
2515 def test_connect_with_tunnel_idna(self):
2516 dest = '\u03b4\u03c0\u03b8.gr'
2517 dest_port = b'%s:%d' % (dest.encode('idna'), client.HTTP_PORT)
2518 expected = b'CONNECT %s HTTP/1.1\r\nHost: %s\r\n\r\n' % (
2519 dest_port, dest_port)
2520 self.conn.set_tunnel(dest)
2521 self.conn.request('HEAD', '/', '')
2522 self.assertEqual(self.conn.sock.host, self.host)
2523 self.assertEqual(self.conn.sock.port, client.HTTP_PORT)
2524 self.assertIn(expected, self.conn.sock.data)
2525
2526 def test_tunnel_connect_single_send_connection_setup(self):
2527 """Regresstion test for https://bugs.python.org/issue43332."""
2528 with mock.patch.object(self.conn, 'send') as mock_send:
2529 self.conn.set_tunnel('destination.com')
2530 self.conn.connect()
2531 self.conn.request('GET', '/')
2532 mock_send.assert_called()
2533 # Likely 2, but this test only cares about the first.
2534 self.assertGreater(
2535 len(mock_send.mock_calls), 1,
2536 msg=f'unexpected number of send calls: {mock_send.mock_calls}')
2537 proxy_setup_data_sent = mock_send.mock_calls[0][1][0]
2538 self.assertIn(b'CONNECT destination.com', proxy_setup_data_sent)
2539 self.assertEndsWith(proxy_setup_data_sent, b'\r\n\r\n',
2540 msg=f'unexpected proxy data sent {proxy_setup_data_sent!r}')
2541
2542 def test_connect_put_request(self):
2543 d = {
2544 b'host': b'destination.com',
2545 b'port': client.HTTP_PORT,
2546 }
2547 self.conn.set_tunnel(d[b'host'].decode('ascii'))
2548 self.conn.request('PUT', '/', '')
2549 self.assertEqual(self.conn.sock.host, self.host)
2550 self.assertEqual(self.conn.sock.port, self.port)
2551 self.assertIn(b'CONNECT %(host)s:%(port)d HTTP/1.1\r\n'
2552 b'Host: %(host)s:%(port)d\r\n\r\n' % d,
2553 self.conn.sock.data)
2554 self.assertIn(b'PUT / HTTP/1.1\r\nHost: %(host)s\r\n' % d,
2555 self.conn.sock.data)
2556
2557 def test_connect_put_request_ipv6(self):
2558 self.conn.set_tunnel('[1:2:3::4]', 1234)
2559 self.conn.request('PUT', '/', '')
2560 self.assertEqual(self.conn.sock.host, self.host)
2561 self.assertEqual(self.conn.sock.port, client.HTTP_PORT)
2562 self.assertIn(b'CONNECT [1:2:3::4]:1234', self.conn.sock.data)
2563 self.assertIn(b'Host: [1:2:3::4]:1234', self.conn.sock.data)
2564
2565 def test_connect_put_request_ipv6_port(self):
2566 self.conn.set_tunnel('[1:2:3::4]:1234')
2567 self.conn.request('PUT', '/', '')
2568 self.assertEqual(self.conn.sock.host, self.host)
2569 self.assertEqual(self.conn.sock.port, client.HTTP_PORT)
2570 self.assertIn(b'CONNECT [1:2:3::4]:1234', self.conn.sock.data)
2571 self.assertIn(b'Host: [1:2:3::4]:1234', self.conn.sock.data)
2572
2573 def test_tunnel_debuglog(self):
2574 expected_header = 'X-Dummy: 1'
2575 response_text = 'HTTP/1.0 200 OK\r\n{}\r\n\r\n'.format(expected_header)
2576
2577 self.conn.set_debuglevel(1)
2578 self.conn._create_connection = self._create_connection(response_text)
2579 self.conn.set_tunnel('destination.com')
2580
2581 with support.captured_stdout() as output:
2582 self.conn.request('PUT', '/', '')
2583 lines = output.getvalue().splitlines()
2584 self.assertIn('header: {}'.format(expected_header), lines)
2585
2586 def test_proxy_response_headers(self):
2587 expected_header = ('X-Dummy', '1')
2588 response_text = (
2589 'HTTP/1.0 200 OK\r\n'
2590 '{0}\r\n\r\n'.format(':'.join(expected_header))
2591 )
2592
2593 self.conn._create_connection = self._create_connection(response_text)
2594 self.conn.set_tunnel('destination.com')
2595
2596 self.conn.request('PUT', '/', '')
2597 headers = self.conn.get_proxy_response_headers()
2598 self.assertIn(expected_header, headers.items())
2599
2600 def test_no_proxy_response_headers(self):
2601 expected_header = ('X-Dummy', '1')
2602 response_text = (
2603 'HTTP/1.0 200 OK\r\n'
2604 '{0}\r\n\r\n'.format(':'.join(expected_header))
2605 )
2606
2607 self.conn._create_connection = self._create_connection(response_text)
2608
2609 self.conn.request('PUT', '/', '')
2610 headers = self.conn.get_proxy_response_headers()
2611 self.assertIsNone(headers)
2612
2613 def test_tunnel_leak(self):
2614 sock = None
2615
2616 def _create_connection(address, timeout=None, source_address=None):
2617 nonlocal sock
2618 sock = FakeSocket(
2619 'HTTP/1.1 404 NOT FOUND\r\n\r\n',
2620 host=address[0],
2621 port=address[1],
2622 )
2623 return sock
2624
2625 self.conn._create_connection = _create_connection
2626 self.conn.set_tunnel('destination.com')
2627 exc = None
2628 try:
2629 self.conn.request('HEAD', '/', '')
2630 except OSError as e:
2631 # keeping a reference to exc keeps response alive in the traceback
2632 exc = e
2633 self.assertIsNotNone(exc)
2634 self.assertTrue(sock.file_closed)
2635
2636
2637 if __name__ == '__main__':
2638 unittest.main(verbosity=2)