Debian Codegraph @

Lib/test/test_asyncio/test_sendfile.py

raw
1 """Tests for sendfile functionality."""
2
3 import asyncio
4 import errno
5 import os
6 import socket
7 import sys
8 import tempfile
9 import unittest
10 from asyncio import base_events
11 from asyncio import constants
12 from unittest import mock
13 from test import support
14 from test.support import os_helper
15 from test.support import socket_helper
16 from test.test_asyncio import utils as test_utils
17
18 try:
19 import ssl
20 except ImportError:
21 ssl = None
22
23
24 def tearDownModule():
25 asyncio.events._set_event_loop_policy(None)
26
27
28 class MySendfileProto(asyncio.Protocol):
29
30 def __init__(self, loop=None, close_after=0):
31 self.transport = None
32 self.state = 'INITIAL'
33 self.nbytes = 0
34 if loop is not None:
35 self.connected = loop.create_future()
36 self.done = loop.create_future()
37 self.data = bytearray()
38 self.close_after = close_after
39
40 def _assert_state(self, *expected):
41 if self.state not in expected:
42 raise AssertionError(f'state: {self.state!r}, expected: {expected!r}')
43
44 def connection_made(self, transport):
45 self.transport = transport
46 self._assert_state('INITIAL')
47 self.state = 'CONNECTED'
48 if self.connected:
49 self.connected.set_result(None)
50
51 def eof_received(self):
52 self._assert_state('CONNECTED')
53 self.state = 'EOF'
54
55 def connection_lost(self, exc):
56 self._assert_state('CONNECTED', 'EOF')
57 self.state = 'CLOSED'
58 if self.done:
59 self.done.set_result(None)
60
61 def data_received(self, data):
62 self._assert_state('CONNECTED')
63 self.nbytes += len(data)
64 self.data.extend(data)
65 super().data_received(data)
66 if self.close_after and self.nbytes >= self.close_after:
67 self.transport.close()
68
69
70 class MyProto(asyncio.Protocol):
71
72 def __init__(self, loop):
73 self.started = False
74 self.closed = False
75 self.data = bytearray()
76 self.fut = loop.create_future()
77 self.transport = None
78
79 def connection_made(self, transport):
80 self.started = True
81 self.transport = transport
82
83 def data_received(self, data):
84 self.data.extend(data)
85
86 def connection_lost(self, exc):
87 self.closed = True
88 self.fut.set_result(None)
89
90 async def wait_closed(self):
91 await self.fut
92
93
94 class SendfileBase:
95
96 # Linux >= 6.10 seems buffering up to 17 pages of data.
97 # So DATA should be large enough to make this test reliable even with a
98 # 64 KiB page configuration.
99 DATA = b"x" * (1024 * 17 * 64 + 1)
100 # Reduce socket buffer size to test on relative small data sets.
101 BUF_SIZE = 4 * 1024 # 4 KiB
102
103 def create_event_loop(self):
104 raise NotImplementedError
105
106 @classmethod
107 def setUpClass(cls):
108 with open(os_helper.TESTFN, 'wb') as fp:
109 fp.write(cls.DATA)
110 super().setUpClass()
111
112 @classmethod
113 def tearDownClass(cls):
114 os_helper.unlink(os_helper.TESTFN)
115 super().tearDownClass()
116
117 def setUp(self):
118 self.file = open(os_helper.TESTFN, 'rb')
119 self.addCleanup(self.file.close)
120 self.loop = self.create_event_loop()
121 self.set_event_loop(self.loop)
122 super().setUp()
123
124 def tearDown(self):
125 # just in case if we have transport close callbacks
126 if not self.loop.is_closed():
127 test_utils.run_briefly(self.loop)
128
129 self.doCleanups()
130 support.gc_collect()
131 super().tearDown()
132
133 def run_loop(self, coro):
134 return self.loop.run_until_complete(coro)
135
136
137 class SockSendfileMixin(SendfileBase):
138
139 @classmethod
140 def setUpClass(cls):
141 cls.__old_bufsize = constants.SENDFILE_FALLBACK_READBUFFER_SIZE
142 constants.SENDFILE_FALLBACK_READBUFFER_SIZE = 1024 * 16
143 super().setUpClass()
144
145 @classmethod
146 def tearDownClass(cls):
147 constants.SENDFILE_FALLBACK_READBUFFER_SIZE = cls.__old_bufsize
148 super().tearDownClass()
149
150 def make_socket(self, cleanup=True):
151 sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
152 sock.setblocking(False)
153 if cleanup:
154 self.addCleanup(sock.close)
155 return sock
156
157 def reduce_receive_buffer_size(self, sock):
158 # Reduce receive socket buffer size to test on relative
159 # small data sets.
160 sock.setsockopt(socket.SOL_SOCKET, socket.SO_RCVBUF, self.BUF_SIZE)
161
162 def reduce_send_buffer_size(self, sock, transport=None):
163 # Reduce send socket buffer size to test on relative small data sets.
164
165 # On macOS, SO_SNDBUF is reset by connect(). So this method
166 # should be called after the socket is connected.
167 sock.setsockopt(socket.SOL_SOCKET, socket.SO_SNDBUF, self.BUF_SIZE)
168
169 if transport is not None:
170 transport.set_write_buffer_limits(high=self.BUF_SIZE)
171
172 def prepare_socksendfile(self):
173 proto = MyProto(self.loop)
174 port = socket_helper.find_unused_port()
175 srv_sock = self.make_socket(cleanup=False)
176 srv_sock.bind((socket_helper.HOST, port))
177 server = self.run_loop(self.loop.create_server(
178 lambda: proto, sock=srv_sock))
179 self.reduce_receive_buffer_size(srv_sock)
180
181 sock = self.make_socket()
182 self.run_loop(self.loop.sock_connect(sock, ('127.0.0.1', port)))
183 self.reduce_send_buffer_size(sock)
184
185 def cleanup():
186 if proto.transport is not None:
187 # can be None if the task was cancelled before
188 # connection_made callback
189 proto.transport.close()
190 self.run_loop(proto.wait_closed())
191
192 server.close()
193 self.run_loop(server.wait_closed())
194
195 self.addCleanup(cleanup)
196
197 return sock, proto
198
199 def test_sock_sendfile_success(self):
200 sock, proto = self.prepare_socksendfile()
201 ret = self.run_loop(self.loop.sock_sendfile(sock, self.file))
202 sock.close()
203 self.run_loop(proto.wait_closed())
204
205 self.assertEqual(ret, len(self.DATA))
206 self.assertEqual(proto.data, self.DATA)
207 self.assertEqual(self.file.tell(), len(self.DATA))
208
209 def test_sock_sendfile_with_offset_and_count(self):
210 sock, proto = self.prepare_socksendfile()
211 ret = self.run_loop(self.loop.sock_sendfile(sock, self.file,
212 1000, 2000))
213 sock.close()
214 self.run_loop(proto.wait_closed())
215
216 self.assertEqual(proto.data, self.DATA[1000:3000])
217 self.assertEqual(self.file.tell(), 3000)
218 self.assertEqual(ret, 2000)
219
220 def test_sock_sendfile_zero_size(self):
221 sock, proto = self.prepare_socksendfile()
222 with tempfile.TemporaryFile() as f:
223 ret = self.run_loop(self.loop.sock_sendfile(sock, f,
224 0, None))
225 sock.close()
226 self.run_loop(proto.wait_closed())
227
228 self.assertEqual(ret, 0)
229 self.assertEqual(self.file.tell(), 0)
230
231 def test_sock_sendfile_mix_with_regular_send(self):
232 buf = b"mix_regular_send" * (4 * 1024) # 64 KiB
233 sock, proto = self.prepare_socksendfile()
234 self.run_loop(self.loop.sock_sendall(sock, buf))
235 ret = self.run_loop(self.loop.sock_sendfile(sock, self.file))
236 self.run_loop(self.loop.sock_sendall(sock, buf))
237 sock.close()
238 self.run_loop(proto.wait_closed())
239
240 self.assertEqual(ret, len(self.DATA))
241 expected = buf + self.DATA + buf
242 self.assertEqual(proto.data, expected)
243 self.assertEqual(self.file.tell(), len(self.DATA))
244
245
246 class SendfileMixin(SendfileBase):
247
248 # Note: sendfile via SSL transport is equal to sendfile fallback
249
250 def prepare_sendfile(self, *, is_ssl=False, close_after=0):
251 port = socket_helper.find_unused_port()
252 srv_proto = MySendfileProto(loop=self.loop,
253 close_after=close_after)
254 if is_ssl:
255 if not ssl:
256 self.skipTest("No ssl module")
257 srv_ctx = test_utils.simple_server_sslcontext()
258 cli_ctx = test_utils.simple_client_sslcontext()
259 else:
260 srv_ctx = None
261 cli_ctx = None
262 srv_sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
263 srv_sock.bind((socket_helper.HOST, port))
264 server = self.run_loop(self.loop.create_server(
265 lambda: srv_proto, sock=srv_sock, ssl=srv_ctx))
266 self.reduce_receive_buffer_size(srv_sock)
267
268 if is_ssl:
269 server_hostname = socket_helper.HOST
270 else:
271 server_hostname = None
272 cli_sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
273 cli_sock.connect((socket_helper.HOST, port))
274
275 cli_proto = MySendfileProto(loop=self.loop)
276 tr, pr = self.run_loop(self.loop.create_connection(
277 lambda: cli_proto, sock=cli_sock,
278 ssl=cli_ctx, server_hostname=server_hostname))
279 self.reduce_send_buffer_size(cli_sock, transport=tr)
280
281 def cleanup():
282 srv_proto.transport.close()
283 cli_proto.transport.close()
284 self.run_loop(srv_proto.done)
285 self.run_loop(cli_proto.done)
286
287 server.close()
288 self.run_loop(server.wait_closed())
289
290 self.addCleanup(cleanup)
291 return srv_proto, cli_proto
292
293 @unittest.skipIf(sys.platform == 'win32', "UDP sockets are not supported")
294 def test_sendfile_not_supported(self):
295 tr, pr = self.run_loop(
296 self.loop.create_datagram_endpoint(
297 asyncio.DatagramProtocol,
298 family=socket.AF_INET))
299 try:
300 with self.assertRaisesRegex(RuntimeError, "not supported"):
301 self.run_loop(
302 self.loop.sendfile(tr, self.file))
303 self.assertEqual(0, self.file.tell())
304 finally:
305 # don't use self.addCleanup because it produces resource warning
306 tr.close()
307
308 def test_sendfile(self):
309 srv_proto, cli_proto = self.prepare_sendfile()
310 ret = self.run_loop(
311 self.loop.sendfile(cli_proto.transport, self.file))
312 cli_proto.transport.close()
313 self.run_loop(srv_proto.done)
314 self.assertEqual(ret, len(self.DATA))
315 self.assertEqual(srv_proto.nbytes, len(self.DATA))
316 self.assertEqual(srv_proto.data, self.DATA)
317 self.assertEqual(self.file.tell(), len(self.DATA))
318
319 def test_sendfile_force_fallback(self):
320 srv_proto, cli_proto = self.prepare_sendfile()
321
322 def sendfile_native(transp, file, offset, count):
323 # to raise SendfileNotAvailableError
324 return base_events.BaseEventLoop._sendfile_native(
325 self.loop, transp, file, offset, count)
326
327 self.loop._sendfile_native = sendfile_native
328
329 ret = self.run_loop(
330 self.loop.sendfile(cli_proto.transport, self.file))
331 cli_proto.transport.close()
332 self.run_loop(srv_proto.done)
333 self.assertEqual(ret, len(self.DATA))
334 self.assertEqual(srv_proto.nbytes, len(self.DATA))
335 self.assertEqual(srv_proto.data, self.DATA)
336 self.assertEqual(self.file.tell(), len(self.DATA))
337
338 def test_sendfile_force_unsupported_native(self):
339 if sys.platform == 'win32':
340 if isinstance(self.loop, asyncio.ProactorEventLoop):
341 self.skipTest("Fails on proactor event loop")
342 srv_proto, cli_proto = self.prepare_sendfile()
343
344 def sendfile_native(transp, file, offset, count):
345 # to raise SendfileNotAvailableError
346 return base_events.BaseEventLoop._sendfile_native(
347 self.loop, transp, file, offset, count)
348
349 self.loop._sendfile_native = sendfile_native
350
351 with self.assertRaisesRegex(asyncio.SendfileNotAvailableError,
352 "not supported"):
353 self.run_loop(
354 self.loop.sendfile(cli_proto.transport, self.file,
355 fallback=False))
356
357 cli_proto.transport.close()
358 self.run_loop(srv_proto.done)
359 self.assertEqual(srv_proto.nbytes, 0)
360 self.assertEqual(self.file.tell(), 0)
361
362 def test_sendfile_ssl(self):
363 srv_proto, cli_proto = self.prepare_sendfile(is_ssl=True)
364 ret = self.run_loop(
365 self.loop.sendfile(cli_proto.transport, self.file))
366 cli_proto.transport.close()
367 self.run_loop(srv_proto.done)
368 self.assertEqual(ret, len(self.DATA))
369 self.assertEqual(srv_proto.nbytes, len(self.DATA))
370 self.assertEqual(srv_proto.data, self.DATA)
371 self.assertEqual(self.file.tell(), len(self.DATA))
372
373 def test_sendfile_for_closing_transp(self):
374 srv_proto, cli_proto = self.prepare_sendfile()
375 cli_proto.transport.close()
376 with self.assertRaisesRegex(RuntimeError, "is closing"):
377 self.run_loop(self.loop.sendfile(cli_proto.transport, self.file))
378 self.run_loop(srv_proto.done)
379 self.assertEqual(srv_proto.nbytes, 0)
380 self.assertEqual(self.file.tell(), 0)
381
382 def test_sendfile_pre_and_post_data(self):
383 srv_proto, cli_proto = self.prepare_sendfile()
384 PREFIX = b'PREFIX__' * 1024 # 8 KiB
385 SUFFIX = b'--SUFFIX' * 1024 # 8 KiB
386 cli_proto.transport.write(PREFIX)
387 ret = self.run_loop(
388 self.loop.sendfile(cli_proto.transport, self.file))
389 cli_proto.transport.write(SUFFIX)
390 cli_proto.transport.close()
391 self.run_loop(srv_proto.done)
392 self.assertEqual(ret, len(self.DATA))
393 self.assertEqual(srv_proto.data, PREFIX + self.DATA + SUFFIX)
394 self.assertEqual(self.file.tell(), len(self.DATA))
395
396 def test_sendfile_ssl_pre_and_post_data(self):
397 srv_proto, cli_proto = self.prepare_sendfile(is_ssl=True)
398 PREFIX = b'zxcvbnm' * 1024
399 SUFFIX = b'0987654321' * 1024
400 cli_proto.transport.write(PREFIX)
401 ret = self.run_loop(
402 self.loop.sendfile(cli_proto.transport, self.file))
403 cli_proto.transport.write(SUFFIX)
404 cli_proto.transport.close()
405 self.run_loop(srv_proto.done)
406 self.assertEqual(ret, len(self.DATA))
407 self.assertEqual(srv_proto.data, PREFIX + self.DATA + SUFFIX)
408 self.assertEqual(self.file.tell(), len(self.DATA))
409
410 def test_sendfile_partial(self):
411 srv_proto, cli_proto = self.prepare_sendfile()
412 ret = self.run_loop(
413 self.loop.sendfile(cli_proto.transport, self.file, 1000, 100))
414 cli_proto.transport.close()
415 self.run_loop(srv_proto.done)
416 self.assertEqual(ret, 100)
417 self.assertEqual(srv_proto.nbytes, 100)
418 self.assertEqual(srv_proto.data, self.DATA[1000:1100])
419 self.assertEqual(self.file.tell(), 1100)
420
421 def test_sendfile_ssl_partial(self):
422 srv_proto, cli_proto = self.prepare_sendfile(is_ssl=True)
423 ret = self.run_loop(
424 self.loop.sendfile(cli_proto.transport, self.file, 1000, 100))
425 cli_proto.transport.close()
426 self.run_loop(srv_proto.done)
427 self.assertEqual(ret, 100)
428 self.assertEqual(srv_proto.nbytes, 100)
429 self.assertEqual(srv_proto.data, self.DATA[1000:1100])
430 self.assertEqual(self.file.tell(), 1100)
431
432 def test_sendfile_close_peer_after_receiving(self):
433 srv_proto, cli_proto = self.prepare_sendfile(
434 close_after=len(self.DATA))
435 ret = self.run_loop(
436 self.loop.sendfile(cli_proto.transport, self.file))
437 cli_proto.transport.close()
438 self.run_loop(srv_proto.done)
439 self.assertEqual(ret, len(self.DATA))
440 self.assertEqual(srv_proto.nbytes, len(self.DATA))
441 self.assertEqual(srv_proto.data, self.DATA)
442 self.assertEqual(self.file.tell(), len(self.DATA))
443
444 def test_sendfile_ssl_close_peer_after_receiving(self):
445 srv_proto, cli_proto = self.prepare_sendfile(
446 is_ssl=True, close_after=len(self.DATA))
447 ret = self.run_loop(
448 self.loop.sendfile(cli_proto.transport, self.file))
449 self.run_loop(srv_proto.done)
450 self.assertEqual(ret, len(self.DATA))
451 self.assertEqual(srv_proto.nbytes, len(self.DATA))
452 self.assertEqual(srv_proto.data, self.DATA)
453 self.assertEqual(self.file.tell(), len(self.DATA))
454
455 # On Solaris, lowering SO_RCVBUF on a TCP connection after it has been
456 # established has no effect. Due to its age, this bug affects both Oracle
457 # Solaris as well as all other OpenSolaris forks (unless they fixed it
458 # themselves).
459 @unittest.skipIf(sys.platform.startswith('sunos'),
460 "Doesn't work on Solaris")
461 def test_sendfile_close_peer_in_the_middle_of_receiving(self):
462 srv_proto, cli_proto = self.prepare_sendfile(close_after=1024)
463 with self.assertRaises(ConnectionError):
464 self.run_loop(
465 self.loop.sendfile(cli_proto.transport, self.file))
466 self.run_loop(srv_proto.done)
467
468 self.assertTrue(1024 <= srv_proto.nbytes < len(self.DATA),
469 srv_proto.nbytes)
470 if not (sys.platform == 'win32'
471 and isinstance(self.loop, asyncio.ProactorEventLoop)):
472 # On Windows, Proactor uses transmitFile, which does not update tell()
473 self.assertTrue(1024 <= self.file.tell() < len(self.DATA),
474 self.file.tell())
475 self.assertTrue(cli_proto.transport.is_closing())
476
477 def test_sendfile_fallback_close_peer_in_the_middle_of_receiving(self):
478
479 def sendfile_native(transp, file, offset, count):
480 # to raise SendfileNotAvailableError
481 return base_events.BaseEventLoop._sendfile_native(
482 self.loop, transp, file, offset, count)
483
484 self.loop._sendfile_native = sendfile_native
485
486 srv_proto, cli_proto = self.prepare_sendfile(close_after=1024)
487 with self.assertRaises(ConnectionError):
488 try:
489 self.run_loop(
490 self.loop.sendfile(cli_proto.transport, self.file))
491 except OSError as e:
492 # macOS may raise OSError of EPROTOTYPE when writing to a
493 # socket that is in the process of closing down.
494 if e.errno == errno.EPROTOTYPE and sys.platform == "darwin":
495 raise ConnectionError
496 else:
497 raise
498
499 self.run_loop(srv_proto.done)
500
501 self.assertTrue(1024 <= srv_proto.nbytes < len(self.DATA),
502 srv_proto.nbytes)
503 self.assertTrue(1024 <= self.file.tell() < len(self.DATA),
504 self.file.tell())
505
506 @unittest.skipIf(not hasattr(os, 'sendfile'),
507 "Don't have native sendfile support")
508 def test_sendfile_prevents_bare_write(self):
509 srv_proto, cli_proto = self.prepare_sendfile()
510 fut = self.loop.create_future()
511
512 async def coro():
513 fut.set_result(None)
514 return await self.loop.sendfile(cli_proto.transport, self.file)
515
516 t = self.loop.create_task(coro())
517 self.run_loop(fut)
518 with self.assertRaisesRegex(RuntimeError,
519 "sendfile is in progress"):
520 cli_proto.transport.write(b'data')
521 ret = self.run_loop(t)
522 self.assertEqual(ret, len(self.DATA))
523
524 def test_sendfile_no_fallback_for_fallback_transport(self):
525 transport = mock.Mock()
526 transport.is_closing.side_effect = lambda: False
527 transport._sendfile_compatible = constants._SendfileMode.FALLBACK
528 with self.assertRaisesRegex(RuntimeError, 'fallback is disabled'):
529 self.loop.run_until_complete(
530 self.loop.sendfile(transport, None, fallback=False))
531
532
533 class SendfileTestsBase(SendfileMixin, SockSendfileMixin):
534 pass
535
536
537 if sys.platform == 'win32':
538
539 class SelectEventLoopTests(SendfileTestsBase,
540 test_utils.TestCase):
541
542 def create_event_loop(self):
543 return asyncio.SelectorEventLoop()
544
545 class ProactorEventLoopTests(SendfileTestsBase,
546 test_utils.TestCase):
547
548 def create_event_loop(self):
549 return asyncio.ProactorEventLoop()
550
551 else:
552 import selectors
553
554 if hasattr(selectors, 'KqueueSelector'):
555 class KqueueEventLoopTests(SendfileTestsBase,
556 test_utils.TestCase):
557
558 def create_event_loop(self):
559 return asyncio.SelectorEventLoop(
560 selectors.KqueueSelector())
561
562 if hasattr(selectors, 'EpollSelector'):
563 class EPollEventLoopTests(SendfileTestsBase,
564 test_utils.TestCase):
565
566 def create_event_loop(self):
567 return asyncio.SelectorEventLoop(selectors.EpollSelector())
568
569 if hasattr(selectors, 'PollSelector'):
570 class PollEventLoopTests(SendfileTestsBase,
571 test_utils.TestCase):
572
573 def create_event_loop(self):
574 return asyncio.SelectorEventLoop(selectors.PollSelector())
575
576 # Should always exist.
577 class SelectEventLoopTests(SendfileTestsBase,
578 test_utils.TestCase):
579
580 def create_event_loop(self):
581 return asyncio.SelectorEventLoop(selectors.SelectSelector())
582
583
584 if __name__ == '__main__':
585 unittest.main()