Debian Codegraph @

Lib/asyncio/selector_events.py

raw
1 """Event loop using a selector and related classes.
2
3 A selector is a "notify-when-ready" multiplexer. For a subclass which
4 also includes support for signal handling, see the unix_events sub-module.
5 """
6
7 __all__ = 'BaseSelectorEventLoop',
8
9 import collections
10 import errno
11 import functools
12 import itertools
13 import os
14 import selectors
15 import socket
16 import warnings
17 import weakref
18 try:
19 import ssl
20 except ImportError: # pragma: no cover
21 ssl = None
22
23 from . import base_events
24 from . import constants
25 from . import events
26 from . import futures
27 from . import protocols
28 from . import sslproto
29 from . import transports
30 from . import trsock
31 from .log import logger
32
33 _HAS_SENDMSG = hasattr(socket.socket, 'sendmsg')
34
35 if _HAS_SENDMSG:
36 try:
37 SC_IOV_MAX = os.sysconf('SC_IOV_MAX')
38 except OSError:
39 # Fallback to send
40 _HAS_SENDMSG = False
41
42 def _test_selector_event(selector, fd, event):
43 # Test if the selector is monitoring 'event' events
44 # for the file descriptor 'fd'.
45 try:
46 key = selector.get_key(fd)
47 except KeyError:
48 return False
49 else:
50 return bool(key.events & event)
51
52
53 class BaseSelectorEventLoop(base_events.BaseEventLoop):
54 """Selector event loop.
55
56 See events.EventLoop for API specification.
57 """
58
59 def __init__(self, selector=None):
60 super().__init__()
61
62 if selector is None:
63 selector = selectors.DefaultSelector()
64 logger.debug('Using selector: %s', selector.__class__.__name__)
65 self._selector = selector
66 self._make_self_pipe()
67 self._transports = weakref.WeakValueDictionary()
68
69 def _make_socket_transport(self, sock, protocol, waiter=None, *,
70 extra=None, server=None):
71 self._ensure_fd_no_transport(sock)
72 return _SelectorSocketTransport(self, sock, protocol, waiter,
73 extra, server)
74
75 def _make_ssl_transport(
76 self, rawsock, protocol, sslcontext, waiter=None,
77 *, server_side=False, server_hostname=None,
78 extra=None, server=None,
79 ssl_handshake_timeout=constants.SSL_HANDSHAKE_TIMEOUT,
80 ssl_shutdown_timeout=constants.SSL_SHUTDOWN_TIMEOUT,
81 ):
82 self._ensure_fd_no_transport(rawsock)
83 ssl_protocol = sslproto.SSLProtocol(
84 self, protocol, sslcontext, waiter,
85 server_side, server_hostname,
86 ssl_handshake_timeout=ssl_handshake_timeout,
87 ssl_shutdown_timeout=ssl_shutdown_timeout
88 )
89 _SelectorSocketTransport(self, rawsock, ssl_protocol,
90 extra=extra, server=server)
91 return ssl_protocol._app_transport
92
93 def _make_datagram_transport(self, sock, protocol,
94 address=None, waiter=None, extra=None):
95 self._ensure_fd_no_transport(sock)
96 return _SelectorDatagramTransport(self, sock, protocol,
97 address, waiter, extra)
98
99 def close(self):
100 if self.is_running():
101 raise RuntimeError("Cannot close a running event loop")
102 if self.is_closed():
103 return
104 self._close_self_pipe()
105 super().close()
106 if self._selector is not None:
107 self._selector.close()
108 self._selector = None
109
110 def _close_self_pipe(self):
111 self._remove_reader(self._ssock.fileno())
112 self._ssock.close()
113 self._ssock = None
114 self._csock.close()
115 self._csock = None
116 self._internal_fds -= 1
117
118 def _make_self_pipe(self):
119 # A self-socket, really. :-)
120 self._ssock, self._csock = socket.socketpair()
121 self._ssock.setblocking(False)
122 self._csock.setblocking(False)
123 self._internal_fds += 1
124 self._add_reader(self._ssock.fileno(), self._read_from_self)
125
126 def _process_self_data(self, data):
127 pass
128
129 def _read_from_self(self):
130 while True:
131 try:
132 data = self._ssock.recv(4096)
133 if not data:
134 break
135 self._process_self_data(data)
136 except InterruptedError:
137 continue
138 except BlockingIOError:
139 break
140
141 def _write_to_self(self):
142 # This may be called from a different thread, possibly after
143 # _close_self_pipe() has been called or even while it is
144 # running. Guard for self._csock being None or closed. When
145 # a socket is closed, send() raises OSError (with errno set to
146 # EBADF, but let's not rely on the exact error code).
147 csock = self._csock
148 if csock is None:
149 return
150
151 try:
152 csock.send(b'\0')
153 except OSError:
154 if self._debug:
155 logger.debug("Fail to write a null byte into the "
156 "self-pipe socket",
157 exc_info=True)
158
159 def _start_serving(self, protocol_factory, sock,
160 sslcontext=None, server=None, backlog=100,
161 ssl_handshake_timeout=constants.SSL_HANDSHAKE_TIMEOUT,
162 ssl_shutdown_timeout=constants.SSL_SHUTDOWN_TIMEOUT):
163 self._add_reader(sock.fileno(), self._accept_connection,
164 protocol_factory, sock, sslcontext, server, backlog,
165 ssl_handshake_timeout, ssl_shutdown_timeout)
166
167 def _accept_connection(
168 self, protocol_factory, sock,
169 sslcontext=None, server=None, backlog=100,
170 ssl_handshake_timeout=constants.SSL_HANDSHAKE_TIMEOUT,
171 ssl_shutdown_timeout=constants.SSL_SHUTDOWN_TIMEOUT):
172 # This method is only called once for each event loop tick where the
173 # listening socket has triggered an EVENT_READ. There may be multiple
174 # connections waiting for an .accept() so it is called in a loop.
175 # See https://bugs.python.org/issue27906 for more details.
176 for _ in range(backlog + 1):
177 try:
178 conn, addr = sock.accept()
179 if self._debug:
180 logger.debug("%r got a new connection from %r: %r",
181 server, addr, conn)
182 conn.setblocking(False)
183 except ConnectionAbortedError:
184 # Discard connections that were aborted before accept().
185 continue
186 except (BlockingIOError, InterruptedError):
187 # Early exit because of a signal or
188 # the socket accept buffer is empty.
189 return
190 except OSError as exc:
191 # There's nowhere to send the error, so just log it.
192 if exc.errno in (errno.EMFILE, errno.ENFILE,
193 errno.ENOBUFS, errno.ENOMEM):
194 # Some platforms (e.g. Linux keep reporting the FD as
195 # ready, so we remove the read handler temporarily.
196 # We'll try again in a while.
197 self.call_exception_handler({
198 'message': 'socket.accept() out of system resource',
199 'exception': exc,
200 'socket': trsock.TransportSocket(sock),
201 })
202 self._remove_reader(sock.fileno())
203 self.call_later(constants.ACCEPT_RETRY_DELAY,
204 self._start_serving,
205 protocol_factory, sock, sslcontext, server,
206 backlog, ssl_handshake_timeout,
207 ssl_shutdown_timeout)
208 else:
209 raise # The event loop will catch, log and ignore it.
210 else:
211 extra = {'peername': addr}
212 accept = self._accept_connection2(
213 protocol_factory, conn, extra, sslcontext, server,
214 ssl_handshake_timeout, ssl_shutdown_timeout)
215 self.create_task(accept)
216
217 async def _accept_connection2(
218 self, protocol_factory, conn, extra,
219 sslcontext=None, server=None,
220 ssl_handshake_timeout=constants.SSL_HANDSHAKE_TIMEOUT,
221 ssl_shutdown_timeout=constants.SSL_SHUTDOWN_TIMEOUT):
222 protocol = None
223 transport = None
224 try:
225 protocol = protocol_factory()
226 waiter = self.create_future()
227 if sslcontext:
228 transport = self._make_ssl_transport(
229 conn, protocol, sslcontext, waiter=waiter,
230 server_side=True, extra=extra, server=server,
231 ssl_handshake_timeout=ssl_handshake_timeout,
232 ssl_shutdown_timeout=ssl_shutdown_timeout)
233 else:
234 transport = self._make_socket_transport(
235 conn, protocol, waiter=waiter, extra=extra,
236 server=server)
237
238 try:
239 await waiter
240 except BaseException:
241 transport.close()
242 # gh-109534: When an exception is raised by the SSLProtocol object the
243 # exception set in this future can keep the protocol object alive and
244 # cause a reference cycle.
245 waiter = None
246 raise
247 # It's now up to the protocol to handle the connection.
248
249 except (SystemExit, KeyboardInterrupt):
250 raise
251 except BaseException as exc:
252 if self._debug:
253 context = {
254 'message':
255 'Error on transport creation for incoming connection',
256 'exception': exc,
257 }
258 if protocol is not None:
259 context['protocol'] = protocol
260 if transport is not None:
261 context['transport'] = transport
262 self.call_exception_handler(context)
263
264 def _ensure_fd_no_transport(self, fd):
265 fileno = fd
266 if not isinstance(fileno, int):
267 try:
268 fileno = int(fileno.fileno())
269 except (AttributeError, TypeError, ValueError):
270 # This code matches selectors._fileobj_to_fd function.
271 raise ValueError(f"Invalid file object: {fd!r}") from None
272 transport = self._transports.get(fileno)
273 if transport and not transport.is_closing():
274 raise RuntimeError(
275 f'File descriptor {fd!r} is used by transport '
276 f'{transport!r}')
277
278 def _add_reader(self, fd, callback, *args):
279 self._check_closed()
280 handle = events.Handle(callback, args, self, None)
281 key = self._selector.get_map().get(fd)
282 if key is None:
283 self._selector.register(fd, selectors.EVENT_READ,
284 (handle, None))
285 else:
286 mask, (reader, writer) = key.events, key.data
287 self._selector.modify(fd, mask | selectors.EVENT_READ,
288 (handle, writer))
289 if reader is not None:
290 reader.cancel()
291 return handle
292
293 def _remove_reader(self, fd):
294 if self.is_closed():
295 return False
296 key = self._selector.get_map().get(fd)
297 if key is None:
298 return False
299 mask, (reader, writer) = key.events, key.data
300 mask &= ~selectors.EVENT_READ
301 if not mask:
302 self._selector.unregister(fd)
303 else:
304 self._selector.modify(fd, mask, (None, writer))
305
306 if reader is not None:
307 reader.cancel()
308 return True
309 else:
310 return False
311
312 def _add_writer(self, fd, callback, *args):
313 self._check_closed()
314 handle = events.Handle(callback, args, self, None)
315 key = self._selector.get_map().get(fd)
316 if key is None:
317 self._selector.register(fd, selectors.EVENT_WRITE,
318 (None, handle))
319 else:
320 mask, (reader, writer) = key.events, key.data
321 self._selector.modify(fd, mask | selectors.EVENT_WRITE,
322 (reader, handle))
323 if writer is not None:
324 writer.cancel()
325 return handle
326
327 def _remove_writer(self, fd):
328 """Remove a writer callback."""
329 if self.is_closed():
330 return False
331 key = self._selector.get_map().get(fd)
332 if key is None:
333 return False
334 mask, (reader, writer) = key.events, key.data
335 # Remove both writer and connector.
336 mask &= ~selectors.EVENT_WRITE
337 if not mask:
338 self._selector.unregister(fd)
339 else:
340 self._selector.modify(fd, mask, (reader, None))
341
342 if writer is not None:
343 writer.cancel()
344 return True
345 else:
346 return False
347
348 def add_reader(self, fd, callback, *args):
349 """Add a reader callback."""
350 self._ensure_fd_no_transport(fd)
351 self._add_reader(fd, callback, *args)
352
353 def remove_reader(self, fd):
354 """Remove a reader callback."""
355 self._ensure_fd_no_transport(fd)
356 return self._remove_reader(fd)
357
358 def add_writer(self, fd, callback, *args):
359 """Add a writer callback.."""
360 self._ensure_fd_no_transport(fd)
361 self._add_writer(fd, callback, *args)
362
363 def remove_writer(self, fd):
364 """Remove a writer callback."""
365 self._ensure_fd_no_transport(fd)
366 return self._remove_writer(fd)
367
368 async def sock_recv(self, sock, n):
369 """Receive data from the socket.
370
371 The return value is a bytes object representing the data received.
372 The maximum amount of data to be received at once is specified by
373 nbytes.
374 """
375 base_events._check_ssl_socket(sock)
376 if self._debug and sock.gettimeout() != 0:
377 raise ValueError("the socket must be non-blocking")
378 try:
379 return sock.recv(n)
380 except (BlockingIOError, InterruptedError):
381 pass
382 fut = self.create_future()
383 fd = sock.fileno()
384 self._ensure_fd_no_transport(fd)
385 handle = self._add_reader(fd, self._sock_recv, fut, sock, n)
386 fut.add_done_callback(
387 functools.partial(self._sock_read_done, fd, handle=handle))
388 return await fut
389
390 def _sock_read_done(self, fd, fut, handle=None):
391 if handle is None or not handle.cancelled():
392 self.remove_reader(fd)
393
394 def _sock_recv(self, fut, sock, n):
395 # _sock_recv() can add itself as an I/O callback if the operation can't
396 # be done immediately. Don't use it directly, call sock_recv().
397 if fut.done():
398 return
399 try:
400 data = sock.recv(n)
401 except (BlockingIOError, InterruptedError):
402 return # try again next time
403 except (SystemExit, KeyboardInterrupt):
404 raise
405 except BaseException as exc:
406 fut.set_exception(exc)
407 else:
408 fut.set_result(data)
409
410 async def sock_recv_into(self, sock, buf):
411 """Receive data from the socket.
412
413 The received data is written into *buf* (a writable buffer).
414 The return value is the number of bytes written.
415 """
416 base_events._check_ssl_socket(sock)
417 if self._debug and sock.gettimeout() != 0:
418 raise ValueError("the socket must be non-blocking")
419 try:
420 return sock.recv_into(buf)
421 except (BlockingIOError, InterruptedError):
422 pass
423 fut = self.create_future()
424 fd = sock.fileno()
425 self._ensure_fd_no_transport(fd)
426 handle = self._add_reader(fd, self._sock_recv_into, fut, sock, buf)
427 fut.add_done_callback(
428 functools.partial(self._sock_read_done, fd, handle=handle))
429 return await fut
430
431 def _sock_recv_into(self, fut, sock, buf):
432 # _sock_recv_into() can add itself as an I/O callback if the operation
433 # can't be done immediately. Don't use it directly, call
434 # sock_recv_into().
435 if fut.done():
436 return
437 try:
438 nbytes = sock.recv_into(buf)
439 except (BlockingIOError, InterruptedError):
440 return # try again next time
441 except (SystemExit, KeyboardInterrupt):
442 raise
443 except BaseException as exc:
444 fut.set_exception(exc)
445 else:
446 fut.set_result(nbytes)
447
448 async def sock_recvfrom(self, sock, bufsize):
449 """Receive a datagram from a datagram socket.
450
451 The return value is a tuple of (bytes, address) representing the
452 datagram received and the address it came from.
453 The maximum amount of data to be received at once is specified by
454 nbytes.
455 """
456 base_events._check_ssl_socket(sock)
457 if self._debug and sock.gettimeout() != 0:
458 raise ValueError("the socket must be non-blocking")
459 try:
460 return sock.recvfrom(bufsize)
461 except (BlockingIOError, InterruptedError):
462 pass
463 fut = self.create_future()
464 fd = sock.fileno()
465 self._ensure_fd_no_transport(fd)
466 handle = self._add_reader(fd, self._sock_recvfrom, fut, sock, bufsize)
467 fut.add_done_callback(
468 functools.partial(self._sock_read_done, fd, handle=handle))
469 return await fut
470
471 def _sock_recvfrom(self, fut, sock, bufsize):
472 # _sock_recvfrom() can add itself as an I/O callback if the operation
473 # can't be done immediately. Don't use it directly, call
474 # sock_recvfrom().
475 if fut.done():
476 return
477 try:
478 result = sock.recvfrom(bufsize)
479 except (BlockingIOError, InterruptedError):
480 return # try again next time
481 except (SystemExit, KeyboardInterrupt):
482 raise
483 except BaseException as exc:
484 fut.set_exception(exc)
485 else:
486 fut.set_result(result)
487
488 async def sock_recvfrom_into(self, sock, buf, nbytes=0):
489 """Receive data from the socket.
490
491 The received data is written into *buf* (a writable buffer).
492 The return value is a tuple of (number of bytes written, address).
493 """
494 base_events._check_ssl_socket(sock)
495 if self._debug and sock.gettimeout() != 0:
496 raise ValueError("the socket must be non-blocking")
497 if not nbytes:
498 nbytes = len(buf)
499
500 try:
501 return sock.recvfrom_into(buf, nbytes)
502 except (BlockingIOError, InterruptedError):
503 pass
504 fut = self.create_future()
505 fd = sock.fileno()
506 self._ensure_fd_no_transport(fd)
507 handle = self._add_reader(fd, self._sock_recvfrom_into, fut, sock, buf,
508 nbytes)
509 fut.add_done_callback(
510 functools.partial(self._sock_read_done, fd, handle=handle))
511 return await fut
512
513 def _sock_recvfrom_into(self, fut, sock, buf, bufsize):
514 # _sock_recv_into() can add itself as an I/O callback if the operation
515 # can't be done immediately. Don't use it directly, call
516 # sock_recv_into().
517 if fut.done():
518 return
519 try:
520 result = sock.recvfrom_into(buf, bufsize)
521 except (BlockingIOError, InterruptedError):
522 return # try again next time
523 except (SystemExit, KeyboardInterrupt):
524 raise
525 except BaseException as exc:
526 fut.set_exception(exc)
527 else:
528 fut.set_result(result)
529
530 async def sock_sendall(self, sock, data):
531 """Send data to the socket.
532
533 The socket must be connected to a remote socket. This method
534 continues to send data from data until either all data has been
535 sent or an error occurs. None is returned on success. On error,
536 an exception is raised, and there is no way to determine how much
537 data, if any, was successfully processed by the receiving end of
538 the connection.
539 """
540 base_events._check_ssl_socket(sock)
541 if self._debug and sock.gettimeout() != 0:
542 raise ValueError("the socket must be non-blocking")
543 try:
544 n = sock.send(data)
545 except (BlockingIOError, InterruptedError):
546 n = 0
547
548 if n == len(data):
549 # all data sent
550 return
551
552 fut = self.create_future()
553 fd = sock.fileno()
554 self._ensure_fd_no_transport(fd)
555 # use a trick with a list in closure to store a mutable state
556 handle = self._add_writer(fd, self._sock_sendall, fut, sock,
557 memoryview(data), [n])
558 fut.add_done_callback(
559 functools.partial(self._sock_write_done, fd, handle=handle))
560 return await fut
561
562 def _sock_sendall(self, fut, sock, view, pos):
563 if fut.done():
564 # Future cancellation can be scheduled on previous loop iteration
565 return
566 start = pos[0]
567 try:
568 n = sock.send(view[start:])
569 except (BlockingIOError, InterruptedError):
570 return
571 except (SystemExit, KeyboardInterrupt):
572 raise
573 except BaseException as exc:
574 fut.set_exception(exc)
575 return
576
577 start += n
578
579 if start == len(view):
580 fut.set_result(None)
581 else:
582 pos[0] = start
583
584 async def sock_sendto(self, sock, data, address):
585 """Send a datagram from sock to address.
586
587 The socket does not have to be connected. This method sends the
588 whole datagram in a single call. Return the number of bytes sent.
589 """
590 base_events._check_ssl_socket(sock)
591 if self._debug and sock.gettimeout() != 0:
592 raise ValueError("the socket must be non-blocking")
593 try:
594 return sock.sendto(data, address)
595 except (BlockingIOError, InterruptedError):
596 pass
597
598 fut = self.create_future()
599 fd = sock.fileno()
600 self._ensure_fd_no_transport(fd)
601 # use a trick with a list in closure to store a mutable state
602 handle = self._add_writer(fd, self._sock_sendto, fut, sock, data,
603 address)
604 fut.add_done_callback(
605 functools.partial(self._sock_write_done, fd, handle=handle))
606 return await fut
607
608 def _sock_sendto(self, fut, sock, data, address):
609 if fut.done():
610 # Future cancellation can be scheduled on previous loop iteration
611 return
612 try:
613 n = sock.sendto(data, 0, address)
614 except (BlockingIOError, InterruptedError):
615 return
616 except (SystemExit, KeyboardInterrupt):
617 raise
618 except BaseException as exc:
619 fut.set_exception(exc)
620 else:
621 fut.set_result(n)
622
623 async def sock_connect(self, sock, address):
624 """Connect to a remote socket at address.
625
626 This method is a coroutine.
627 """
628 base_events._check_ssl_socket(sock)
629 if self._debug and sock.gettimeout() != 0:
630 raise ValueError("the socket must be non-blocking")
631
632 if sock.family == socket.AF_INET or (
633 base_events._HAS_IPv6 and sock.family == socket.AF_INET6):
634 resolved = await self._ensure_resolved(
635 address, family=sock.family, type=sock.type, proto=sock.proto,
636 loop=self,
637 )
638 _, _, _, _, address = resolved[0]
639
640 fut = self.create_future()
641 self._sock_connect(fut, sock, address)
642 try:
643 return await fut
644 finally:
645 # Needed to break cycles when an exception occurs.
646 fut = None
647
648 def _sock_connect(self, fut, sock, address):
649 fd = sock.fileno()
650 try:
651 sock.connect(address)
652 except (BlockingIOError, InterruptedError):
653 # Issue #23618: When the C function connect() fails with EINTR, the
654 # connection runs in background. We have to wait until the socket
655 # becomes writable to be notified when the connection succeed or
656 # fails.
657 self._ensure_fd_no_transport(fd)
658 handle = self._add_writer(
659 fd, self._sock_connect_cb, fut, sock, address)
660 fut.add_done_callback(
661 functools.partial(self._sock_write_done, fd, handle=handle))
662 except (SystemExit, KeyboardInterrupt):
663 raise
664 except BaseException as exc:
665 fut.set_exception(exc)
666 else:
667 fut.set_result(None)
668 finally:
669 fut = None
670
671 def _sock_write_done(self, fd, fut, handle=None):
672 if handle is None or not handle.cancelled():
673 self.remove_writer(fd)
674
675 def _sock_connect_cb(self, fut, sock, address):
676 if fut.done():
677 return
678
679 try:
680 err = sock.getsockopt(socket.SOL_SOCKET, socket.SO_ERROR)
681 if err != 0:
682 # Jump to any except clause below.
683 raise OSError(err, f'Connect call failed {address}')
684 except (BlockingIOError, InterruptedError):
685 # socket is still registered, the callback will be retried later
686 pass
687 except (SystemExit, KeyboardInterrupt):
688 raise
689 except BaseException as exc:
690 fut.set_exception(exc)
691 else:
692 fut.set_result(None)
693 finally:
694 fut = None
695
696 async def sock_accept(self, sock):
697 """Accept a connection.
698
699 The socket must be bound to an address and listening for
700 connections. The return value is a pair (conn, address) where
701 conn is a new socket object usable to send and receive data on the
702 connection, and address is the address bound to the socket on the
703 other end of the connection.
704 """
705 base_events._check_ssl_socket(sock)
706 if self._debug and sock.gettimeout() != 0:
707 raise ValueError("the socket must be non-blocking")
708 fut = self.create_future()
709 self._sock_accept(fut, sock)
710 return await fut
711
712 def _sock_accept(self, fut, sock):
713 # gh-153761: _sock_accept must not scheduled with already cancelled future
714 if fut.done():
715 return
716 fd = sock.fileno()
717 try:
718 conn, address = sock.accept()
719 conn.setblocking(False)
720 except (BlockingIOError, InterruptedError):
721 self._ensure_fd_no_transport(fd)
722 handle = self._add_reader(fd, self._sock_accept, fut, sock)
723 fut.add_done_callback(
724 functools.partial(self._sock_read_done, fd, handle=handle))
725 except (SystemExit, KeyboardInterrupt):
726 raise
727 except BaseException as exc:
728 fut.set_exception(exc)
729 else:
730 fut.set_result((conn, address))
731
732 async def _sendfile_native(self, transp, file, offset, count):
733 del self._transports[transp._sock_fd]
734 resume_reading = transp.is_reading()
735 transp.pause_reading()
736 await transp._make_empty_waiter()
737 try:
738 return await self.sock_sendfile(transp._sock, file, offset, count,
739 fallback=False)
740 finally:
741 transp._reset_empty_waiter()
742 if resume_reading:
743 transp.resume_reading()
744 self._transports[transp._sock_fd] = transp
745
746 def _process_events(self, event_list):
747 for key, mask in event_list:
748 fileobj, (reader, writer) = key.fileobj, key.data
749 if mask & selectors.EVENT_READ and reader is not None:
750 if reader._cancelled:
751 self._remove_reader(fileobj)
752 else:
753 self._add_callback(reader)
754 if mask & selectors.EVENT_WRITE and writer is not None:
755 if writer._cancelled:
756 self._remove_writer(fileobj)
757 else:
758 self._add_callback(writer)
759
760 def _stop_serving(self, sock):
761 self._remove_reader(sock.fileno())
762 sock.close()
763
764
765 class _SelectorTransport(transports._FlowControlMixin,
766 transports.Transport):
767
768 max_size = 256 * 1024 # Buffer size passed to recv().
769
770 # Attribute used in the destructor: it must be set even if the constructor
771 # is not called (see _SelectorSslTransport which may start by raising an
772 # exception)
773 _sock = None
774
775 def __init__(self, loop, sock, protocol, extra=None, server=None):
776 super().__init__(extra, loop)
777 self._extra['socket'] = trsock.TransportSocket(sock)
778 try:
779 self._extra['sockname'] = sock.getsockname()
780 except OSError:
781 self._extra['sockname'] = None
782 if 'peername' not in self._extra:
783 try:
784 self._extra['peername'] = sock.getpeername()
785 except socket.error:
786 self._extra['peername'] = None
787 self._sock = sock
788 self._sock_fd = sock.fileno()
789
790 self._protocol_connected = False
791 self.set_protocol(protocol)
792
793 self._server = server
794 self._buffer = collections.deque()
795 self._conn_lost = 0 # Set when call to connection_lost scheduled.
796 self._closing = False # Set when close() called.
797 self._paused = False # Set when pause_reading() called
798
799 if self._server is not None:
800 self._server._attach(self)
801 loop._transports[self._sock_fd] = self
802
803 def __repr__(self):
804 info = [self.__class__.__name__]
805 if self._sock is None:
806 info.append('closed')
807 elif self._closing:
808 info.append('closing')
809 info.append(f'fd={self._sock_fd}')
810 # test if the transport was closed
811 if self._loop is not None and not self._loop.is_closed():
812 polling = _test_selector_event(self._loop._selector,
813 self._sock_fd, selectors.EVENT_READ)
814 if polling:
815 info.append('read=polling')
816 else:
817 info.append('read=idle')
818
819 polling = _test_selector_event(self._loop._selector,
820 self._sock_fd,
821 selectors.EVENT_WRITE)
822 if polling:
823 state = 'polling'
824 else:
825 state = 'idle'
826
827 bufsize = self.get_write_buffer_size()
828 info.append(f'write=<{state}, bufsize={bufsize}>')
829 return '<{}>'.format(' '.join(info))
830
831 def abort(self):
832 self._force_close(None)
833
834 def set_protocol(self, protocol):
835 self._protocol = protocol
836 self._protocol_connected = True
837
838 def get_protocol(self):
839 return self._protocol
840
841 def is_closing(self):
842 return self._closing
843
844 def is_reading(self):
845 return not self.is_closing() and not self._paused
846
847 def pause_reading(self):
848 if not self.is_reading():
849 return
850 self._paused = True
851 self._loop._remove_reader(self._sock_fd)
852 if self._loop.get_debug():
853 logger.debug("%r pauses reading", self)
854
855 def resume_reading(self):
856 if self._closing or not self._paused:
857 return
858 self._paused = False
859 self._add_reader(self._sock_fd, self._read_ready)
860 if self._loop.get_debug():
861 logger.debug("%r resumes reading", self)
862
863 def close(self):
864 if self._closing:
865 return
866 self._closing = True
867 self._loop._remove_reader(self._sock_fd)
868 if not self._buffer:
869 self._conn_lost += 1
870 self._loop._remove_writer(self._sock_fd)
871 self._loop.call_soon(self._call_connection_lost, None)
872
873 def __del__(self, _warn=warnings.warn):
874 if self._sock is not None:
875 _warn(f"unclosed transport {self!r}", ResourceWarning, source=self)
876 self._sock.close()
877 if self._server is not None:
878 self._server._detach(self)
879
880 def _fatal_error(self, exc, message='Fatal error on transport'):
881 # Should be called from exception handler only.
882 if isinstance(exc, OSError):
883 if self._loop.get_debug():
884 logger.debug("%r: %s", self, message, exc_info=True)
885 else:
886 self._loop.call_exception_handler({
887 'message': message,
888 'exception': exc,
889 'transport': self,
890 'protocol': self._protocol,
891 })
892 self._force_close(exc)
893
894 def _force_close(self, exc):
895 if self._conn_lost:
896 return
897 if self._buffer:
898 self._buffer.clear()
899 self._loop._remove_writer(self._sock_fd)
900 if not self._closing:
901 self._closing = True
902 self._loop._remove_reader(self._sock_fd)
903 self._conn_lost += 1
904 self._loop.call_soon(self._call_connection_lost, exc)
905
906 def _call_connection_lost(self, exc):
907 try:
908 if self._protocol_connected:
909 self._protocol.connection_lost(exc)
910 finally:
911 self._sock.close()
912 self._sock = None
913 self._protocol = None
914 self._loop = None
915 server = self._server
916 if server is not None:
917 server._detach(self)
918 self._server = None
919
920 def get_write_buffer_size(self):
921 return sum(map(len, self._buffer))
922
923 def _add_reader(self, fd, callback, *args):
924 if not self.is_reading():
925 return
926 self._loop._add_reader(fd, callback, *args)
927
928
929 class _SelectorSocketTransport(_SelectorTransport):
930
931 _start_tls_compatible = True
932 _sendfile_compatible = constants._SendfileMode.TRY_NATIVE
933
934 def __init__(self, loop, sock, protocol, waiter=None,
935 extra=None, server=None):
936
937 self._read_ready_cb = None
938 super().__init__(loop, sock, protocol, extra, server)
939 self._eof = False
940 self._empty_waiter = None
941 if _HAS_SENDMSG:
942 self._write_ready = self._write_sendmsg
943 else:
944 self._write_ready = self._write_send
945 # Disable the Nagle algorithm -- small writes will be
946 # sent without waiting for the TCP ACK. This generally
947 # decreases the latency (in some cases significantly.)
948 base_events._set_nodelay(self._sock)
949
950 self._loop.call_soon(self._protocol.connection_made, self)
951 # only start reading when connection_made() has been called
952 self._loop.call_soon(self._add_reader,
953 self._sock_fd, self._read_ready)
954 if waiter is not None:
955 # only wake up the waiter when connection_made() has been called
956 self._loop.call_soon(futures._set_result_unless_cancelled,
957 waiter, None)
958
959 def set_protocol(self, protocol):
960 if isinstance(protocol, protocols.BufferedProtocol):
961 self._read_ready_cb = self._read_ready__get_buffer
962 else:
963 self._read_ready_cb = self._read_ready__data_received
964
965 super().set_protocol(protocol)
966
967 def _read_ready(self):
968 self._read_ready_cb()
969
970 def _read_ready__get_buffer(self):
971 if self._conn_lost:
972 return
973
974 try:
975 buf = self._protocol.get_buffer(-1)
976 if not len(buf):
977 raise RuntimeError('get_buffer() returned an empty buffer')
978 except (SystemExit, KeyboardInterrupt):
979 raise
980 except BaseException as exc:
981 self._fatal_error(
982 exc, 'Fatal error: protocol.get_buffer() call failed.')
983 return
984
985 try:
986 nbytes = self._sock.recv_into(buf)
987 except (BlockingIOError, InterruptedError):
988 return
989 except (SystemExit, KeyboardInterrupt):
990 raise
991 except BaseException as exc:
992 self._fatal_error(exc, 'Fatal read error on socket transport')
993 return
994
995 if not nbytes:
996 self._read_ready__on_eof()
997 return
998
999 try:
1000 self._protocol.buffer_updated(nbytes)
1001 except (SystemExit, KeyboardInterrupt):
1002 raise
1003 except BaseException as exc:
1004 self._fatal_error(
1005 exc, 'Fatal error: protocol.buffer_updated() call failed.')
1006
1007 def _read_ready__data_received(self):
1008 if self._conn_lost:
1009 return
1010 try:
1011 data = self._sock.recv(self.max_size)
1012 except (BlockingIOError, InterruptedError):
1013 return
1014 except (SystemExit, KeyboardInterrupt):
1015 raise
1016 except BaseException as exc:
1017 self._fatal_error(exc, 'Fatal read error on socket transport')
1018 return
1019
1020 if not data:
1021 self._read_ready__on_eof()
1022 return
1023
1024 try:
1025 self._protocol.data_received(data)
1026 except (SystemExit, KeyboardInterrupt):
1027 raise
1028 except BaseException as exc:
1029 self._fatal_error(
1030 exc, 'Fatal error: protocol.data_received() call failed.')
1031
1032 def _read_ready__on_eof(self):
1033 if self._loop.get_debug():
1034 logger.debug("%r received EOF", self)
1035
1036 try:
1037 keep_open = self._protocol.eof_received()
1038 except (SystemExit, KeyboardInterrupt):
1039 raise
1040 except BaseException as exc:
1041 self._fatal_error(
1042 exc, 'Fatal error: protocol.eof_received() call failed.')
1043 return
1044
1045 if keep_open:
1046 # We're keeping the connection open so the
1047 # protocol can write more, but we still can't
1048 # receive more, so remove the reader callback.
1049 self._loop._remove_reader(self._sock_fd)
1050 else:
1051 self.close()
1052
1053 def write(self, data):
1054 if not isinstance(data, (bytes, bytearray, memoryview)):
1055 raise TypeError(f'data argument must be a bytes, bytearray, or memoryview '
1056 f'object, not {type(data).__name__!r}')
1057 if self._eof:
1058 raise RuntimeError('Cannot call write() after write_eof()')
1059 if self._empty_waiter is not None:
1060 raise RuntimeError('unable to write; sendfile is in progress')
1061 if not data:
1062 return
1063
1064 if self._conn_lost:
1065 if self._conn_lost >= constants.LOG_THRESHOLD_FOR_CONNLOST_WRITES:
1066 logger.warning('socket.send() raised exception.')
1067 self._conn_lost += 1
1068 return
1069
1070 if not self._buffer:
1071 # Optimization: try to send now.
1072 try:
1073 n = self._sock.send(data)
1074 except (BlockingIOError, InterruptedError):
1075 pass
1076 except (SystemExit, KeyboardInterrupt):
1077 raise
1078 except BaseException as exc:
1079 self._fatal_error(exc, 'Fatal write error on socket transport')
1080 return
1081 else:
1082 data = memoryview(data)[n:]
1083 if not data:
1084 return
1085 # Not all was written; register write handler.
1086 self._loop._add_writer(self._sock_fd, self._write_ready)
1087
1088 # Add it to the buffer.
1089 self._buffer.append(data)
1090 self._maybe_pause_protocol()
1091
1092 def _get_sendmsg_buffer(self):
1093 return itertools.islice(self._buffer, SC_IOV_MAX)
1094
1095 def _write_sendmsg(self):
1096 assert self._buffer, 'Data should not be empty'
1097 if self._conn_lost:
1098 return
1099 try:
1100 nbytes = self._sock.sendmsg(self._get_sendmsg_buffer())
1101 self._adjust_leftover_buffer(nbytes)
1102 except (BlockingIOError, InterruptedError):
1103 pass
1104 except (SystemExit, KeyboardInterrupt):
1105 raise
1106 except BaseException as exc:
1107 self._loop._remove_writer(self._sock_fd)
1108 self._buffer.clear()
1109 self._fatal_error(exc, 'Fatal write error on socket transport')
1110 if self._empty_waiter is not None:
1111 self._empty_waiter.set_exception(exc)
1112 else:
1113 self._maybe_resume_protocol() # May append to buffer.
1114 if not self._buffer:
1115 self._loop._remove_writer(self._sock_fd)
1116 if self._empty_waiter is not None:
1117 self._empty_waiter.set_result(None)
1118 if self._closing:
1119 self._call_connection_lost(None)
1120 elif self._eof:
1121 self._sock.shutdown(socket.SHUT_WR)
1122
1123 def _adjust_leftover_buffer(self, nbytes: int) -> None:
1124 buffer = self._buffer
1125 while nbytes:
1126 b = buffer.popleft()
1127 b_len = len(b)
1128 if b_len <= nbytes:
1129 nbytes -= b_len
1130 else:
1131 buffer.appendleft(b[nbytes:])
1132 break
1133
1134 def _write_send(self):
1135 assert self._buffer, 'Data should not be empty'
1136 if self._conn_lost:
1137 return
1138 try:
1139 buffer = self._buffer.popleft()
1140 n = self._sock.send(buffer)
1141 if n != len(buffer):
1142 # Not all data was written
1143 self._buffer.appendleft(buffer[n:])
1144 except (BlockingIOError, InterruptedError):
1145 pass
1146 except (SystemExit, KeyboardInterrupt):
1147 raise
1148 except BaseException as exc:
1149 self._loop._remove_writer(self._sock_fd)
1150 self._buffer.clear()
1151 self._fatal_error(exc, 'Fatal write error on socket transport')
1152 if self._empty_waiter is not None:
1153 self._empty_waiter.set_exception(exc)
1154 else:
1155 self._maybe_resume_protocol() # May append to buffer.
1156 if not self._buffer:
1157 self._loop._remove_writer(self._sock_fd)
1158 if self._empty_waiter is not None:
1159 self._empty_waiter.set_result(None)
1160 if self._closing:
1161 self._call_connection_lost(None)
1162 elif self._eof:
1163 self._sock.shutdown(socket.SHUT_WR)
1164
1165 def write_eof(self):
1166 if self._closing or self._eof:
1167 return
1168 self._eof = True
1169 if not self._buffer:
1170 self._sock.shutdown(socket.SHUT_WR)
1171
1172 def writelines(self, list_of_data):
1173 if self._eof:
1174 raise RuntimeError('Cannot call writelines() after write_eof()')
1175 if self._empty_waiter is not None:
1176 raise RuntimeError('unable to writelines; sendfile is in progress')
1177 if not list_of_data:
1178 return
1179
1180 if self._conn_lost:
1181 if self._conn_lost >= constants.LOG_THRESHOLD_FOR_CONNLOST_WRITES:
1182 logger.warning('socket.send() raised exception.')
1183 self._conn_lost += 1
1184 return
1185
1186 self._buffer.extend([memoryview(data) for data in list_of_data])
1187 self._write_ready()
1188 # If the entire buffer couldn't be written, register a write handler
1189 if self._buffer:
1190 self._loop._add_writer(self._sock_fd, self._write_ready)
1191 self._maybe_pause_protocol()
1192
1193 def can_write_eof(self):
1194 return True
1195
1196 def _call_connection_lost(self, exc):
1197 try:
1198 super()._call_connection_lost(exc)
1199 finally:
1200 self._write_ready = None
1201 if self._empty_waiter is not None:
1202 self._empty_waiter.set_exception(
1203 ConnectionError("Connection is closed by peer"))
1204
1205 def _make_empty_waiter(self):
1206 if self._empty_waiter is not None:
1207 raise RuntimeError("Empty waiter is already set")
1208 self._empty_waiter = self._loop.create_future()
1209 if not self._buffer:
1210 self._empty_waiter.set_result(None)
1211 return self._empty_waiter
1212
1213 def _reset_empty_waiter(self):
1214 self._empty_waiter = None
1215
1216 def close(self):
1217 self._read_ready_cb = None
1218 super().close()
1219
1220
1221 class _SelectorDatagramTransport(_SelectorTransport, transports.DatagramTransport):
1222
1223 _buffer_factory = collections.deque
1224 _header_size = 8
1225
1226 def __init__(self, loop, sock, protocol, address=None,
1227 waiter=None, extra=None):
1228 super().__init__(loop, sock, protocol, extra)
1229 self._address = address
1230 self._buffer_size = 0
1231 self._loop.call_soon(self._protocol.connection_made, self)
1232 # only start reading when connection_made() has been called
1233 self._loop.call_soon(self._add_reader,
1234 self._sock_fd, self._read_ready)
1235 if waiter is not None:
1236 # only wake up the waiter when connection_made() has been called
1237 self._loop.call_soon(futures._set_result_unless_cancelled,
1238 waiter, None)
1239
1240 def get_write_buffer_size(self):
1241 return self._buffer_size
1242
1243 def _read_ready(self):
1244 if self._conn_lost:
1245 return
1246 try:
1247 data, addr = self._sock.recvfrom(self.max_size)
1248 except (BlockingIOError, InterruptedError):
1249 pass
1250 except OSError as exc:
1251 self._protocol.error_received(exc)
1252 except (SystemExit, KeyboardInterrupt):
1253 raise
1254 except BaseException as exc:
1255 self._fatal_error(exc, 'Fatal read error on datagram transport')
1256 else:
1257 self._protocol.datagram_received(data, addr)
1258
1259 def sendto(self, data, addr=None):
1260 if not isinstance(data, (bytes, bytearray, memoryview)):
1261 raise TypeError(f'data argument must be a bytes-like object, '
1262 f'not {type(data).__name__!r}')
1263
1264 if self._address:
1265 if addr not in (None, self._address):
1266 raise ValueError(
1267 f'Invalid address: must be None or {self._address}')
1268 addr = self._address
1269
1270 if self._conn_lost and self._address:
1271 if self._conn_lost >= constants.LOG_THRESHOLD_FOR_CONNLOST_WRITES:
1272 logger.warning('socket.send() raised exception.')
1273 self._conn_lost += 1
1274 return
1275
1276 if not self._buffer:
1277 # Attempt to send it right away first.
1278 try:
1279 if self._extra['peername']:
1280 self._sock.send(data)
1281 else:
1282 self._sock.sendto(data, addr)
1283 return
1284 except (BlockingIOError, InterruptedError):
1285 self._loop._add_writer(self._sock_fd, self._sendto_ready)
1286 except OSError as exc:
1287 self._protocol.error_received(exc)
1288 return
1289 except (SystemExit, KeyboardInterrupt):
1290 raise
1291 except BaseException as exc:
1292 self._fatal_error(
1293 exc, 'Fatal write error on datagram transport')
1294 return
1295
1296 # Ensure that what we buffer is immutable.
1297 self._buffer.append((bytes(data), addr))
1298 self._buffer_size += len(data) + self._header_size
1299 self._maybe_pause_protocol()
1300
1301 def _sendto_ready(self):
1302 while self._buffer:
1303 data, addr = self._buffer.popleft()
1304 self._buffer_size -= len(data) + self._header_size
1305 try:
1306 if self._extra['peername']:
1307 self._sock.send(data)
1308 else:
1309 self._sock.sendto(data, addr)
1310 except (BlockingIOError, InterruptedError):
1311 self._buffer.appendleft((data, addr)) # Try again later.
1312 self._buffer_size += len(data) + self._header_size
1313 break
1314 except OSError as exc:
1315 self._protocol.error_received(exc)
1316 return
1317 except (SystemExit, KeyboardInterrupt):
1318 raise
1319 except BaseException as exc:
1320 self._fatal_error(
1321 exc, 'Fatal write error on datagram transport')
1322 return
1323
1324 self._maybe_resume_protocol() # May append to buffer.
1325 if not self._buffer:
1326 self._loop._remove_writer(self._sock_fd)
1327 if self._closing:
1328 self._call_connection_lost(None)