Debian Codegraph @

Lib/asyncio/selector_events.py

raw
1 """Event loop using a selector and related classes.
2
3 A selector is a "notify-when-ready" multiplexer. For a subclass which
4 also includes support for signal handling, see the unix_events sub-module.
5 """
6
7 __all__ = 'BaseSelectorEventLoop',
8
9 import collections
10 import errno
11 import functools
12 import itertools
13 import os
14 import selectors
15 import socket
16 import warnings
17 import weakref
18 try:
19 import ssl
20 except ImportError: # pragma: no cover
21 ssl = None
22
23 from . import base_events
24 from . import constants
25 from . import events
26 from . import futures
27 from . import protocols
28 from . import sslproto
29 from . import transports
30 from . import trsock
31 from .log import logger
32
33 _HAS_SENDMSG = hasattr(socket.socket, 'sendmsg')
34
35 if _HAS_SENDMSG:
36 try:
37 SC_IOV_MAX = os.sysconf('SC_IOV_MAX')
38 except OSError:
39 # Fallback to send
40 _HAS_SENDMSG = False
41
42 def _test_selector_event(selector, fd, event):
43 # Test if the selector is monitoring 'event' events
44 # for the file descriptor 'fd'.
45 try:
46 key = selector.get_key(fd)
47 except KeyError:
48 return False
49 else:
50 return bool(key.events & event)
51
52
53 class BaseSelectorEventLoop(base_events.BaseEventLoop):
54 """Selector event loop.
55
56 See events.EventLoop for API specification.
57 """
58
59 def __init__(self, selector=None):
60 super().__init__()
61
62 if selector is None:
63 selector = selectors.DefaultSelector()
64 logger.debug('Using selector: %s', selector.__class__.__name__)
65 self._selector = selector
66 self._make_self_pipe()
67 self._transports = weakref.WeakValueDictionary()
68
69 def _make_socket_transport(self, sock, protocol, waiter=None, *,
70 extra=None, server=None):
71 self._ensure_fd_no_transport(sock)
72 return _SelectorSocketTransport(self, sock, protocol, waiter,
73 extra, server)
74
75 def _make_ssl_transport(
76 self, rawsock, protocol, sslcontext, waiter=None,
77 *, server_side=False, server_hostname=None,
78 extra=None, server=None,
79 ssl_handshake_timeout=constants.SSL_HANDSHAKE_TIMEOUT,
80 ssl_shutdown_timeout=constants.SSL_SHUTDOWN_TIMEOUT,
81 ):
82 self._ensure_fd_no_transport(rawsock)
83 ssl_protocol = sslproto.SSLProtocol(
84 self, protocol, sslcontext, waiter,
85 server_side, server_hostname,
86 ssl_handshake_timeout=ssl_handshake_timeout,
87 ssl_shutdown_timeout=ssl_shutdown_timeout
88 )
89 _SelectorSocketTransport(self, rawsock, ssl_protocol,
90 extra=extra, server=server)
91 return ssl_protocol._app_transport
92
93 def _make_datagram_transport(self, sock, protocol,
94 address=None, waiter=None, extra=None):
95 self._ensure_fd_no_transport(sock)
96 return _SelectorDatagramTransport(self, sock, protocol,
97 address, waiter, extra)
98
99 def close(self):
100 if self.is_running():
101 raise RuntimeError("Cannot close a running event loop")
102 if self.is_closed():
103 return
104 self._close_self_pipe()
105 super().close()
106 if self._selector is not None:
107 self._selector.close()
108 self._selector = None
109
110 def _close_self_pipe(self):
111 self._remove_reader(self._ssock.fileno())
112 self._ssock.close()
113 self._ssock = None
114 self._csock.close()
115 self._csock = None
116 self._internal_fds -= 1
117
118 def _make_self_pipe(self):
119 # A self-socket, really. :-)
120 self._ssock, self._csock = socket.socketpair()
121 self._ssock.setblocking(False)
122 self._csock.setblocking(False)
123 self._internal_fds += 1
124 self._add_reader(self._ssock.fileno(), self._read_from_self)
125
126 def _process_self_data(self, data):
127 pass
128
129 def _read_from_self(self):
130 while True:
131 try:
132 data = self._ssock.recv(4096)
133 if not data:
134 break
135 self._process_self_data(data)
136 except InterruptedError:
137 continue
138 except BlockingIOError:
139 break
140
141 def _write_to_self(self):
142 # This may be called from a different thread, possibly after
143 # _close_self_pipe() has been called or even while it is
144 # running. Guard for self._csock being None or closed. When
145 # a socket is closed, send() raises OSError (with errno set to
146 # EBADF, but let's not rely on the exact error code).
147 csock = self._csock
148 if csock is None:
149 return
150
151 try:
152 csock.send(b'\0')
153 except OSError:
154 if self._debug:
155 logger.debug("Fail to write a null byte into the "
156 "self-pipe socket",
157 exc_info=True)
158
159 def _start_serving(self, protocol_factory, sock,
160 sslcontext=None, server=None, backlog=100,
161 ssl_handshake_timeout=constants.SSL_HANDSHAKE_TIMEOUT,
162 ssl_shutdown_timeout=constants.SSL_SHUTDOWN_TIMEOUT):
163 self._add_reader(sock.fileno(), self._accept_connection,
164 protocol_factory, sock, sslcontext, server, backlog,
165 ssl_handshake_timeout, ssl_shutdown_timeout)
166
167 def _accept_connection(
168 self, protocol_factory, sock,
169 sslcontext=None, server=None, backlog=100,
170 ssl_handshake_timeout=constants.SSL_HANDSHAKE_TIMEOUT,
171 ssl_shutdown_timeout=constants.SSL_SHUTDOWN_TIMEOUT):
172 # This method is only called once for each event loop tick where the
173 # listening socket has triggered an EVENT_READ. There may be multiple
174 # connections waiting for an .accept() so it is called in a loop.
175 # See https://bugs.python.org/issue27906 for more details.
176 for _ in range(backlog + 1):
177 try:
178 conn, addr = sock.accept()
179 if self._debug:
180 logger.debug("%r got a new connection from %r: %r",
181 server, addr, conn)
182 conn.setblocking(False)
183 except ConnectionAbortedError:
184 # Discard connections that were aborted before accept().
185 continue
186 except (BlockingIOError, InterruptedError):
187 # Early exit because of a signal or
188 # the socket accept buffer is empty.
189 return
190 except OSError as exc:
191 # There's nowhere to send the error, so just log it.
192 if exc.errno in (errno.EMFILE, errno.ENFILE,
193 errno.ENOBUFS, errno.ENOMEM):
194 # Some platforms (e.g. Linux keep reporting the FD as
195 # ready, so we remove the read handler temporarily.
196 # We'll try again in a while.
197 self.call_exception_handler({
198 'message': 'socket.accept() out of system resource',
199 'exception': exc,
200 'socket': trsock.TransportSocket(sock),
201 })
202 self._remove_reader(sock.fileno())
203 self.call_later(constants.ACCEPT_RETRY_DELAY,
204 self._start_serving,
205 protocol_factory, sock, sslcontext, server,
206 backlog, ssl_handshake_timeout,
207 ssl_shutdown_timeout)
208 else:
209 raise # The event loop will catch, log and ignore it.
210 else:
211 extra = {'peername': addr}
212 accept = self._accept_connection2(
213 protocol_factory, conn, extra, sslcontext, server,
214 ssl_handshake_timeout, ssl_shutdown_timeout)
215 self.create_task(accept)
216
217 async def _accept_connection2(
218 self, protocol_factory, conn, extra,
219 sslcontext=None, server=None,
220 ssl_handshake_timeout=constants.SSL_HANDSHAKE_TIMEOUT,
221 ssl_shutdown_timeout=constants.SSL_SHUTDOWN_TIMEOUT):
222 protocol = None
223 transport = None
224 try:
225 protocol = protocol_factory()
226 waiter = self.create_future()
227 if sslcontext:
228 transport = self._make_ssl_transport(
229 conn, protocol, sslcontext, waiter=waiter,
230 server_side=True, extra=extra, server=server,
231 ssl_handshake_timeout=ssl_handshake_timeout,
232 ssl_shutdown_timeout=ssl_shutdown_timeout)
233 else:
234 transport = self._make_socket_transport(
235 conn, protocol, waiter=waiter, extra=extra,
236 server=server)
237
238 try:
239 await waiter
240 except BaseException:
241 transport.close()
242 # gh-109534: When an exception is raised by the SSLProtocol object the
243 # exception set in this future can keep the protocol object alive and
244 # cause a reference cycle.
245 waiter = None
246 raise
247 # It's now up to the protocol to handle the connection.
248
249 except (SystemExit, KeyboardInterrupt):
250 raise
251 except BaseException as exc:
252 if self._debug:
253 context = {
254 'message':
255 'Error on transport creation for incoming connection',
256 'exception': exc,
257 }
258 if protocol is not None:
259 context['protocol'] = protocol
260 if transport is not None:
261 context['transport'] = transport
262 self.call_exception_handler(context)
263
264 def _ensure_fd_no_transport(self, fd):
265 fileno = fd
266 if not isinstance(fileno, int):
267 try:
268 fileno = int(fileno.fileno())
269 except (AttributeError, TypeError, ValueError):
270 # This code matches selectors._fileobj_to_fd function.
271 raise ValueError(f"Invalid file object: {fd!r}") from None
272 transport = self._transports.get(fileno)
273 if transport and not transport.is_closing():
274 raise RuntimeError(
275 f'File descriptor {fd!r} is used by transport '
276 f'{transport!r}')
277
278 def _add_reader(self, fd, callback, *args):
279 self._check_closed()
280 handle = events.Handle(callback, args, self, None)
281 key = self._selector.get_map().get(fd)
282 if key is None:
283 self._selector.register(fd, selectors.EVENT_READ,
284 (handle, None))
285 else:
286 mask, (reader, writer) = key.events, key.data
287 self._selector.modify(fd, mask | selectors.EVENT_READ,
288 (handle, writer))
289 if reader is not None:
290 reader.cancel()
291 return handle
292
293 def _remove_reader(self, fd):
294 if self.is_closed():
295 return False
296 key = self._selector.get_map().get(fd)
297 if key is None:
298 return False
299 mask, (reader, writer) = key.events, key.data
300 mask &= ~selectors.EVENT_READ
301 if not mask:
302 self._selector.unregister(fd)
303 else:
304 self._selector.modify(fd, mask, (None, writer))
305
306 if reader is not None:
307 reader.cancel()
308 return True
309 else:
310 return False
311
312 def _add_writer(self, fd, callback, *args):
313 self._check_closed()
314 handle = events.Handle(callback, args, self, None)
315 key = self._selector.get_map().get(fd)
316 if key is None:
317 self._selector.register(fd, selectors.EVENT_WRITE,
318 (None, handle))
319 else:
320 mask, (reader, writer) = key.events, key.data
321 self._selector.modify(fd, mask | selectors.EVENT_WRITE,
322 (reader, handle))
323 if writer is not None:
324 writer.cancel()
325 return handle
326
327 def _remove_writer(self, fd):
328 """Remove a writer callback."""
329 if self.is_closed():
330 return False
331 key = self._selector.get_map().get(fd)
332 if key is None:
333 return False
334 mask, (reader, writer) = key.events, key.data
335 # Remove both writer and connector.
336 mask &= ~selectors.EVENT_WRITE
337 if not mask:
338 self._selector.unregister(fd)
339 else:
340 self._selector.modify(fd, mask, (reader, None))
341
342 if writer is not None:
343 writer.cancel()
344 return True
345 else:
346 return False
347
348 def add_reader(self, fd, callback, *args):
349 """Add a reader callback."""
350 self._ensure_fd_no_transport(fd)
351 self._add_reader(fd, callback, *args)
352
353 def remove_reader(self, fd):
354 """Remove a reader callback."""
355 self._ensure_fd_no_transport(fd)
356 return self._remove_reader(fd)
357
358 def add_writer(self, fd, callback, *args):
359 """Add a writer callback.."""
360 self._ensure_fd_no_transport(fd)
361 self._add_writer(fd, callback, *args)
362
363 def remove_writer(self, fd):
364 """Remove a writer callback."""
365 self._ensure_fd_no_transport(fd)
366 return self._remove_writer(fd)
367
368 async def sock_recv(self, sock, n):
369 """Receive data from the socket.
370
371 The return value is a bytes object representing the data received.
372 The maximum amount of data to be received at once is specified by
373 nbytes.
374 """
375 base_events._check_ssl_socket(sock)
376 if self._debug and sock.gettimeout() != 0:
377 raise ValueError("the socket must be non-blocking")
378 try:
379 return sock.recv(n)
380 except (BlockingIOError, InterruptedError):
381 pass
382 fut = self.create_future()
383 fd = sock.fileno()
384 self._ensure_fd_no_transport(fd)
385 handle = self._add_reader(fd, self._sock_recv, fut, sock, n)
386 fut.add_done_callback(
387 functools.partial(self._sock_read_done, fd, handle=handle))
388 return await fut
389
390 def _sock_read_done(self, fd, fut, handle=None):
391 if handle is None or not handle.cancelled():
392 self.remove_reader(fd)
393
394 def _sock_recv(self, fut, sock, n):
395 # _sock_recv() can add itself as an I/O callback if the operation can't
396 # be done immediately. Don't use it directly, call sock_recv().
397 if fut.done():
398 return
399 try:
400 data = sock.recv(n)
401 except (BlockingIOError, InterruptedError):
402 return # try again next time
403 except (SystemExit, KeyboardInterrupt):
404 raise
405 except BaseException as exc:
406 fut.set_exception(exc)
407 else:
408 fut.set_result(data)
409
410 async def sock_recv_into(self, sock, buf):
411 """Receive data from the socket.
412
413 The received data is written into *buf* (a writable buffer).
414 The return value is the number of bytes written.
415 """
416 base_events._check_ssl_socket(sock)
417 if self._debug and sock.gettimeout() != 0:
418 raise ValueError("the socket must be non-blocking")
419 try:
420 return sock.recv_into(buf)
421 except (BlockingIOError, InterruptedError):
422 pass
423 fut = self.create_future()
424 fd = sock.fileno()
425 self._ensure_fd_no_transport(fd)
426 handle = self._add_reader(fd, self._sock_recv_into, fut, sock, buf)
427 fut.add_done_callback(
428 functools.partial(self._sock_read_done, fd, handle=handle))
429 return await fut
430
431 def _sock_recv_into(self, fut, sock, buf):
432 # _sock_recv_into() can add itself as an I/O callback if the operation
433 # can't be done immediately. Don't use it directly, call
434 # sock_recv_into().
435 if fut.done():
436 return
437 try:
438 nbytes = sock.recv_into(buf)
439 except (BlockingIOError, InterruptedError):
440 return # try again next time
441 except (SystemExit, KeyboardInterrupt):
442 raise
443 except BaseException as exc:
444 fut.set_exception(exc)
445 else:
446 fut.set_result(nbytes)
447
448 async def sock_recvfrom(self, sock, bufsize):
449 """Receive a datagram from a datagram socket.
450
451 The return value is a tuple of (bytes, address) representing the
452 datagram received and the address it came from.
453 The maximum amount of data to be received at once is specified by
454 nbytes.
455 """
456 base_events._check_ssl_socket(sock)
457 if self._debug and sock.gettimeout() != 0:
458 raise ValueError("the socket must be non-blocking")
459 try:
460 return sock.recvfrom(bufsize)
461 except (BlockingIOError, InterruptedError):
462 pass
463 fut = self.create_future()
464 fd = sock.fileno()
465 self._ensure_fd_no_transport(fd)
466 handle = self._add_reader(fd, self._sock_recvfrom, fut, sock, bufsize)
467 fut.add_done_callback(
468 functools.partial(self._sock_read_done, fd, handle=handle))
469 return await fut
470
471 def _sock_recvfrom(self, fut, sock, bufsize):
472 # _sock_recvfrom() can add itself as an I/O callback if the operation
473 # can't be done immediately. Don't use it directly, call
474 # sock_recvfrom().
475 if fut.done():
476 return
477 try:
478 result = sock.recvfrom(bufsize)
479 except (BlockingIOError, InterruptedError):
480 return # try again next time
481 except (SystemExit, KeyboardInterrupt):
482 raise
483 except BaseException as exc:
484 fut.set_exception(exc)
485 else:
486 fut.set_result(result)
487
488 async def sock_recvfrom_into(self, sock, buf, nbytes=0):
489 """Receive data from the socket.
490
491 The received data is written into *buf* (a writable buffer).
492 The return value is a tuple of (number of bytes written, address).
493 """
494 base_events._check_ssl_socket(sock)
495 if self._debug and sock.gettimeout() != 0:
496 raise ValueError("the socket must be non-blocking")
497 if not nbytes:
498 nbytes = len(buf)
499
500 try:
501 return sock.recvfrom_into(buf, nbytes)
502 except (BlockingIOError, InterruptedError):
503 pass
504 fut = self.create_future()
505 fd = sock.fileno()
506 self._ensure_fd_no_transport(fd)
507 handle = self._add_reader(fd, self._sock_recvfrom_into, fut, sock, buf,
508 nbytes)
509 fut.add_done_callback(
510 functools.partial(self._sock_read_done, fd, handle=handle))
511 return await fut
512
513 def _sock_recvfrom_into(self, fut, sock, buf, bufsize):
514 # _sock_recv_into() can add itself as an I/O callback if the operation
515 # can't be done immediately. Don't use it directly, call
516 # sock_recv_into().
517 if fut.done():
518 return
519 try:
520 result = sock.recvfrom_into(buf, bufsize)
521 except (BlockingIOError, InterruptedError):
522 return # try again next time
523 except (SystemExit, KeyboardInterrupt):
524 raise
525 except BaseException as exc:
526 fut.set_exception(exc)
527 else:
528 fut.set_result(result)
529
530 async def sock_sendall(self, sock, data):
531 """Send data to the socket.
532
533 The socket must be connected to a remote socket. This method
534 continues to send data from data until either all data has been
535 sent or an error occurs. None is returned on success. On error,
536 an exception is raised, and there is no way to determine how much
537 data, if any, was successfully processed by the receiving end of
538 the connection.
539 """
540 base_events._check_ssl_socket(sock)
541 if self._debug and sock.gettimeout() != 0:
542 raise ValueError("the socket must be non-blocking")
543 try:
544 n = sock.send(data)
545 except (BlockingIOError, InterruptedError):
546 n = 0
547
548 if n == len(data):
549 # all data sent
550 return
551
552 fut = self.create_future()
553 fd = sock.fileno()
554 self._ensure_fd_no_transport(fd)
555 # use a trick with a list in closure to store a mutable state
556 handle = self._add_writer(fd, self._sock_sendall, fut, sock,
557 memoryview(data), [n])
558 fut.add_done_callback(
559 functools.partial(self._sock_write_done, fd, handle=handle))
560 return await fut
561
562 def _sock_sendall(self, fut, sock, view, pos):
563 if fut.done():
564 # Future cancellation can be scheduled on previous loop iteration
565 return
566 start = pos[0]
567 try:
568 n = sock.send(view[start:])
569 except (BlockingIOError, InterruptedError):
570 return
571 except (SystemExit, KeyboardInterrupt):
572 raise
573 except BaseException as exc:
574 fut.set_exception(exc)
575 return
576
577 start += n
578
579 if start == len(view):
580 fut.set_result(None)
581 else:
582 pos[0] = start
583
584 async def sock_sendto(self, sock, data, address):
585 """Send data to the socket.
586
587 The socket must be connected to a remote socket. This method
588 continues to send data from data until either all data has been
589 sent or an error occurs. None is returned on success. On error,
590 an exception is raised, and there is no way to determine how much
591 data, if any, was successfully processed by the receiving end of
592 the connection.
593 """
594 base_events._check_ssl_socket(sock)
595 if self._debug and sock.gettimeout() != 0:
596 raise ValueError("the socket must be non-blocking")
597 try:
598 return sock.sendto(data, address)
599 except (BlockingIOError, InterruptedError):
600 pass
601
602 fut = self.create_future()
603 fd = sock.fileno()
604 self._ensure_fd_no_transport(fd)
605 # use a trick with a list in closure to store a mutable state
606 handle = self._add_writer(fd, self._sock_sendto, fut, sock, data,
607 address)
608 fut.add_done_callback(
609 functools.partial(self._sock_write_done, fd, handle=handle))
610 return await fut
611
612 def _sock_sendto(self, fut, sock, data, address):
613 if fut.done():
614 # Future cancellation can be scheduled on previous loop iteration
615 return
616 try:
617 n = sock.sendto(data, 0, address)
618 except (BlockingIOError, InterruptedError):
619 return
620 except (SystemExit, KeyboardInterrupt):
621 raise
622 except BaseException as exc:
623 fut.set_exception(exc)
624 else:
625 fut.set_result(n)
626
627 async def sock_connect(self, sock, address):
628 """Connect to a remote socket at address.
629
630 This method is a coroutine.
631 """
632 base_events._check_ssl_socket(sock)
633 if self._debug and sock.gettimeout() != 0:
634 raise ValueError("the socket must be non-blocking")
635
636 if sock.family == socket.AF_INET or (
637 base_events._HAS_IPv6 and sock.family == socket.AF_INET6):
638 resolved = await self._ensure_resolved(
639 address, family=sock.family, type=sock.type, proto=sock.proto,
640 loop=self,
641 )
642 _, _, _, _, address = resolved[0]
643
644 fut = self.create_future()
645 self._sock_connect(fut, sock, address)
646 try:
647 return await fut
648 finally:
649 # Needed to break cycles when an exception occurs.
650 fut = None
651
652 def _sock_connect(self, fut, sock, address):
653 fd = sock.fileno()
654 try:
655 sock.connect(address)
656 except (BlockingIOError, InterruptedError):
657 # Issue #23618: When the C function connect() fails with EINTR, the
658 # connection runs in background. We have to wait until the socket
659 # becomes writable to be notified when the connection succeed or
660 # fails.
661 self._ensure_fd_no_transport(fd)
662 handle = self._add_writer(
663 fd, self._sock_connect_cb, fut, sock, address)
664 fut.add_done_callback(
665 functools.partial(self._sock_write_done, fd, handle=handle))
666 except (SystemExit, KeyboardInterrupt):
667 raise
668 except BaseException as exc:
669 fut.set_exception(exc)
670 else:
671 fut.set_result(None)
672 finally:
673 fut = None
674
675 def _sock_write_done(self, fd, fut, handle=None):
676 if handle is None or not handle.cancelled():
677 self.remove_writer(fd)
678
679 def _sock_connect_cb(self, fut, sock, address):
680 if fut.done():
681 return
682
683 try:
684 err = sock.getsockopt(socket.SOL_SOCKET, socket.SO_ERROR)
685 if err != 0:
686 # Jump to any except clause below.
687 raise OSError(err, f'Connect call failed {address}')
688 except (BlockingIOError, InterruptedError):
689 # socket is still registered, the callback will be retried later
690 pass
691 except (SystemExit, KeyboardInterrupt):
692 raise
693 except BaseException as exc:
694 fut.set_exception(exc)
695 else:
696 fut.set_result(None)
697 finally:
698 fut = None
699
700 async def sock_accept(self, sock):
701 """Accept a connection.
702
703 The socket must be bound to an address and listening for
704 connections. The return value is a pair (conn, address) where
705 conn is a new socket object usable to send and receive data on the
706 connection, and address is the address bound to the socket on the
707 other end of the connection.
708 """
709 base_events._check_ssl_socket(sock)
710 if self._debug and sock.gettimeout() != 0:
711 raise ValueError("the socket must be non-blocking")
712 fut = self.create_future()
713 self._sock_accept(fut, sock)
714 return await fut
715
716 def _sock_accept(self, fut, sock):
717 fd = sock.fileno()
718 try:
719 conn, address = sock.accept()
720 conn.setblocking(False)
721 except (BlockingIOError, InterruptedError):
722 self._ensure_fd_no_transport(fd)
723 handle = self._add_reader(fd, self._sock_accept, fut, sock)
724 fut.add_done_callback(
725 functools.partial(self._sock_read_done, fd, handle=handle))
726 except (SystemExit, KeyboardInterrupt):
727 raise
728 except BaseException as exc:
729 fut.set_exception(exc)
730 else:
731 fut.set_result((conn, address))
732
733 async def _sendfile_native(self, transp, file, offset, count):
734 del self._transports[transp._sock_fd]
735 resume_reading = transp.is_reading()
736 transp.pause_reading()
737 await transp._make_empty_waiter()
738 try:
739 return await self.sock_sendfile(transp._sock, file, offset, count,
740 fallback=False)
741 finally:
742 transp._reset_empty_waiter()
743 if resume_reading:
744 transp.resume_reading()
745 self._transports[transp._sock_fd] = transp
746
747 def _process_events(self, event_list):
748 for key, mask in event_list:
749 fileobj, (reader, writer) = key.fileobj, key.data
750 if mask & selectors.EVENT_READ and reader is not None:
751 if reader._cancelled:
752 self._remove_reader(fileobj)
753 else:
754 self._add_callback(reader)
755 if mask & selectors.EVENT_WRITE and writer is not None:
756 if writer._cancelled:
757 self._remove_writer(fileobj)
758 else:
759 self._add_callback(writer)
760
761 def _stop_serving(self, sock):
762 self._remove_reader(sock.fileno())
763 sock.close()
764
765
766 class _SelectorTransport(transports._FlowControlMixin,
767 transports.Transport):
768
769 max_size = 256 * 1024 # Buffer size passed to recv().
770
771 # Attribute used in the destructor: it must be set even if the constructor
772 # is not called (see _SelectorSslTransport which may start by raising an
773 # exception)
774 _sock = None
775
776 def __init__(self, loop, sock, protocol, extra=None, server=None):
777 super().__init__(extra, loop)
778 self._extra['socket'] = trsock.TransportSocket(sock)
779 try:
780 self._extra['sockname'] = sock.getsockname()
781 except OSError:
782 self._extra['sockname'] = None
783 if 'peername' not in self._extra:
784 try:
785 self._extra['peername'] = sock.getpeername()
786 except socket.error:
787 self._extra['peername'] = None
788 self._sock = sock
789 self._sock_fd = sock.fileno()
790
791 self._protocol_connected = False
792 self.set_protocol(protocol)
793
794 self._server = server
795 self._buffer = collections.deque()
796 self._conn_lost = 0 # Set when call to connection_lost scheduled.
797 self._closing = False # Set when close() called.
798 self._paused = False # Set when pause_reading() called
799
800 if self._server is not None:
801 self._server._attach(self)
802 loop._transports[self._sock_fd] = self
803
804 def __repr__(self):
805 info = [self.__class__.__name__]
806 if self._sock is None:
807 info.append('closed')
808 elif self._closing:
809 info.append('closing')
810 info.append(f'fd={self._sock_fd}')
811 # test if the transport was closed
812 if self._loop is not None and not self._loop.is_closed():
813 polling = _test_selector_event(self._loop._selector,
814 self._sock_fd, selectors.EVENT_READ)
815 if polling:
816 info.append('read=polling')
817 else:
818 info.append('read=idle')
819
820 polling = _test_selector_event(self._loop._selector,
821 self._sock_fd,
822 selectors.EVENT_WRITE)
823 if polling:
824 state = 'polling'
825 else:
826 state = 'idle'
827
828 bufsize = self.get_write_buffer_size()
829 info.append(f'write=<{state}, bufsize={bufsize}>')
830 return '<{}>'.format(' '.join(info))
831
832 def abort(self):
833 self._force_close(None)
834
835 def set_protocol(self, protocol):
836 self._protocol = protocol
837 self._protocol_connected = True
838
839 def get_protocol(self):
840 return self._protocol
841
842 def is_closing(self):
843 return self._closing
844
845 def is_reading(self):
846 return not self.is_closing() and not self._paused
847
848 def pause_reading(self):
849 if not self.is_reading():
850 return
851 self._paused = True
852 self._loop._remove_reader(self._sock_fd)
853 if self._loop.get_debug():
854 logger.debug("%r pauses reading", self)
855
856 def resume_reading(self):
857 if self._closing or not self._paused:
858 return
859 self._paused = False
860 self._add_reader(self._sock_fd, self._read_ready)
861 if self._loop.get_debug():
862 logger.debug("%r resumes reading", self)
863
864 def close(self):
865 if self._closing:
866 return
867 self._closing = True
868 self._loop._remove_reader(self._sock_fd)
869 if not self._buffer:
870 self._conn_lost += 1
871 self._loop._remove_writer(self._sock_fd)
872 self._loop.call_soon(self._call_connection_lost, None)
873
874 def __del__(self, _warn=warnings.warn):
875 if self._sock is not None:
876 _warn(f"unclosed transport {self!r}", ResourceWarning, source=self)
877 self._sock.close()
878 if self._server is not None:
879 self._server._detach(self)
880
881 def _fatal_error(self, exc, message='Fatal error on transport'):
882 # Should be called from exception handler only.
883 if isinstance(exc, OSError):
884 if self._loop.get_debug():
885 logger.debug("%r: %s", self, message, exc_info=True)
886 else:
887 self._loop.call_exception_handler({
888 'message': message,
889 'exception': exc,
890 'transport': self,
891 'protocol': self._protocol,
892 })
893 self._force_close(exc)
894
895 def _force_close(self, exc):
896 if self._conn_lost:
897 return
898 if self._buffer:
899 self._buffer.clear()
900 self._loop._remove_writer(self._sock_fd)
901 if not self._closing:
902 self._closing = True
903 self._loop._remove_reader(self._sock_fd)
904 self._conn_lost += 1
905 self._loop.call_soon(self._call_connection_lost, exc)
906
907 def _call_connection_lost(self, exc):
908 try:
909 if self._protocol_connected:
910 self._protocol.connection_lost(exc)
911 finally:
912 self._sock.close()
913 self._sock = None
914 self._protocol = None
915 self._loop = None
916 server = self._server
917 if server is not None:
918 server._detach(self)
919 self._server = None
920
921 def get_write_buffer_size(self):
922 return sum(map(len, self._buffer))
923
924 def _add_reader(self, fd, callback, *args):
925 if not self.is_reading():
926 return
927 self._loop._add_reader(fd, callback, *args)
928
929
930 class _SelectorSocketTransport(_SelectorTransport):
931
932 _start_tls_compatible = True
933 _sendfile_compatible = constants._SendfileMode.TRY_NATIVE
934
935 def __init__(self, loop, sock, protocol, waiter=None,
936 extra=None, server=None):
937
938 self._read_ready_cb = None
939 super().__init__(loop, sock, protocol, extra, server)
940 self._eof = False
941 self._empty_waiter = None
942 if _HAS_SENDMSG:
943 self._write_ready = self._write_sendmsg
944 else:
945 self._write_ready = self._write_send
946 # Disable the Nagle algorithm -- small writes will be
947 # sent without waiting for the TCP ACK. This generally
948 # decreases the latency (in some cases significantly.)
949 base_events._set_nodelay(self._sock)
950
951 self._loop.call_soon(self._protocol.connection_made, self)
952 # only start reading when connection_made() has been called
953 self._loop.call_soon(self._add_reader,
954 self._sock_fd, self._read_ready)
955 if waiter is not None:
956 # only wake up the waiter when connection_made() has been called
957 self._loop.call_soon(futures._set_result_unless_cancelled,
958 waiter, None)
959
960 def set_protocol(self, protocol):
961 if isinstance(protocol, protocols.BufferedProtocol):
962 self._read_ready_cb = self._read_ready__get_buffer
963 else:
964 self._read_ready_cb = self._read_ready__data_received
965
966 super().set_protocol(protocol)
967
968 def _read_ready(self):
969 self._read_ready_cb()
970
971 def _read_ready__get_buffer(self):
972 if self._conn_lost:
973 return
974
975 try:
976 buf = self._protocol.get_buffer(-1)
977 if not len(buf):
978 raise RuntimeError('get_buffer() returned an empty buffer')
979 except (SystemExit, KeyboardInterrupt):
980 raise
981 except BaseException as exc:
982 self._fatal_error(
983 exc, 'Fatal error: protocol.get_buffer() call failed.')
984 return
985
986 try:
987 nbytes = self._sock.recv_into(buf)
988 except (BlockingIOError, InterruptedError):
989 return
990 except (SystemExit, KeyboardInterrupt):
991 raise
992 except BaseException as exc:
993 self._fatal_error(exc, 'Fatal read error on socket transport')
994 return
995
996 if not nbytes:
997 self._read_ready__on_eof()
998 return
999
1000 try:
1001 self._protocol.buffer_updated(nbytes)
1002 except (SystemExit, KeyboardInterrupt):
1003 raise
1004 except BaseException as exc:
1005 self._fatal_error(
1006 exc, 'Fatal error: protocol.buffer_updated() call failed.')
1007
1008 def _read_ready__data_received(self):
1009 if self._conn_lost:
1010 return
1011 try:
1012 data = self._sock.recv(self.max_size)
1013 except (BlockingIOError, InterruptedError):
1014 return
1015 except (SystemExit, KeyboardInterrupt):
1016 raise
1017 except BaseException as exc:
1018 self._fatal_error(exc, 'Fatal read error on socket transport')
1019 return
1020
1021 if not data:
1022 self._read_ready__on_eof()
1023 return
1024
1025 try:
1026 self._protocol.data_received(data)
1027 except (SystemExit, KeyboardInterrupt):
1028 raise
1029 except BaseException as exc:
1030 self._fatal_error(
1031 exc, 'Fatal error: protocol.data_received() call failed.')
1032
1033 def _read_ready__on_eof(self):
1034 if self._loop.get_debug():
1035 logger.debug("%r received EOF", self)
1036
1037 try:
1038 keep_open = self._protocol.eof_received()
1039 except (SystemExit, KeyboardInterrupt):
1040 raise
1041 except BaseException as exc:
1042 self._fatal_error(
1043 exc, 'Fatal error: protocol.eof_received() call failed.')
1044 return
1045
1046 if keep_open:
1047 # We're keeping the connection open so the
1048 # protocol can write more, but we still can't
1049 # receive more, so remove the reader callback.
1050 self._loop._remove_reader(self._sock_fd)
1051 else:
1052 self.close()
1053
1054 def write(self, data):
1055 if not isinstance(data, (bytes, bytearray, memoryview)):
1056 raise TypeError(f'data argument must be a bytes, bytearray, or memoryview '
1057 f'object, not {type(data).__name__!r}')
1058 if self._eof:
1059 raise RuntimeError('Cannot call write() after write_eof()')
1060 if self._empty_waiter is not None:
1061 raise RuntimeError('unable to write; sendfile is in progress')
1062 if not data:
1063 return
1064
1065 if self._conn_lost:
1066 if self._conn_lost >= constants.LOG_THRESHOLD_FOR_CONNLOST_WRITES:
1067 logger.warning('socket.send() raised exception.')
1068 self._conn_lost += 1
1069 return
1070
1071 if not self._buffer:
1072 # Optimization: try to send now.
1073 try:
1074 n = self._sock.send(data)
1075 except (BlockingIOError, InterruptedError):
1076 pass
1077 except (SystemExit, KeyboardInterrupt):
1078 raise
1079 except BaseException as exc:
1080 self._fatal_error(exc, 'Fatal write error on socket transport')
1081 return
1082 else:
1083 data = memoryview(data)[n:]
1084 if not data:
1085 return
1086 # Not all was written; register write handler.
1087 self._loop._add_writer(self._sock_fd, self._write_ready)
1088
1089 # Add it to the buffer.
1090 self._buffer.append(data)
1091 self._maybe_pause_protocol()
1092
1093 def _get_sendmsg_buffer(self):
1094 return itertools.islice(self._buffer, SC_IOV_MAX)
1095
1096 def _write_sendmsg(self):
1097 assert self._buffer, 'Data should not be empty'
1098 if self._conn_lost:
1099 return
1100 try:
1101 nbytes = self._sock.sendmsg(self._get_sendmsg_buffer())
1102 self._adjust_leftover_buffer(nbytes)
1103 except (BlockingIOError, InterruptedError):
1104 pass
1105 except (SystemExit, KeyboardInterrupt):
1106 raise
1107 except BaseException as exc:
1108 self._loop._remove_writer(self._sock_fd)
1109 self._buffer.clear()
1110 self._fatal_error(exc, 'Fatal write error on socket transport')
1111 if self._empty_waiter is not None:
1112 self._empty_waiter.set_exception(exc)
1113 else:
1114 self._maybe_resume_protocol() # May append to buffer.
1115 if not self._buffer:
1116 self._loop._remove_writer(self._sock_fd)
1117 if self._empty_waiter is not None:
1118 self._empty_waiter.set_result(None)
1119 if self._closing:
1120 self._call_connection_lost(None)
1121 elif self._eof:
1122 self._sock.shutdown(socket.SHUT_WR)
1123
1124 def _adjust_leftover_buffer(self, nbytes: int) -> None:
1125 buffer = self._buffer
1126 while nbytes:
1127 b = buffer.popleft()
1128 b_len = len(b)
1129 if b_len <= nbytes:
1130 nbytes -= b_len
1131 else:
1132 buffer.appendleft(b[nbytes:])
1133 break
1134
1135 def _write_send(self):
1136 assert self._buffer, 'Data should not be empty'
1137 if self._conn_lost:
1138 return
1139 try:
1140 buffer = self._buffer.popleft()
1141 n = self._sock.send(buffer)
1142 if n != len(buffer):
1143 # Not all data was written
1144 self._buffer.appendleft(buffer[n:])
1145 except (BlockingIOError, InterruptedError):
1146 pass
1147 except (SystemExit, KeyboardInterrupt):
1148 raise
1149 except BaseException as exc:
1150 self._loop._remove_writer(self._sock_fd)
1151 self._buffer.clear()
1152 self._fatal_error(exc, 'Fatal write error on socket transport')
1153 if self._empty_waiter is not None:
1154 self._empty_waiter.set_exception(exc)
1155 else:
1156 self._maybe_resume_protocol() # May append to buffer.
1157 if not self._buffer:
1158 self._loop._remove_writer(self._sock_fd)
1159 if self._empty_waiter is not None:
1160 self._empty_waiter.set_result(None)
1161 if self._closing:
1162 self._call_connection_lost(None)
1163 elif self._eof:
1164 self._sock.shutdown(socket.SHUT_WR)
1165
1166 def write_eof(self):
1167 if self._closing or self._eof:
1168 return
1169 self._eof = True
1170 if not self._buffer:
1171 self._sock.shutdown(socket.SHUT_WR)
1172
1173 def writelines(self, list_of_data):
1174 if self._eof:
1175 raise RuntimeError('Cannot call writelines() after write_eof()')
1176 if self._empty_waiter is not None:
1177 raise RuntimeError('unable to writelines; sendfile is in progress')
1178 if not list_of_data:
1179 return
1180
1181 if self._conn_lost:
1182 if self._conn_lost >= constants.LOG_THRESHOLD_FOR_CONNLOST_WRITES:
1183 logger.warning('socket.send() raised exception.')
1184 self._conn_lost += 1
1185 return
1186
1187 self._buffer.extend([memoryview(data) for data in list_of_data])
1188 self._write_ready()
1189 # If the entire buffer couldn't be written, register a write handler
1190 if self._buffer:
1191 self._loop._add_writer(self._sock_fd, self._write_ready)
1192 self._maybe_pause_protocol()
1193
1194 def can_write_eof(self):
1195 return True
1196
1197 def _call_connection_lost(self, exc):
1198 try:
1199 super()._call_connection_lost(exc)
1200 finally:
1201 self._write_ready = None
1202 if self._empty_waiter is not None:
1203 self._empty_waiter.set_exception(
1204 ConnectionError("Connection is closed by peer"))
1205
1206 def _make_empty_waiter(self):
1207 if self._empty_waiter is not None:
1208 raise RuntimeError("Empty waiter is already set")
1209 self._empty_waiter = self._loop.create_future()
1210 if not self._buffer:
1211 self._empty_waiter.set_result(None)
1212 return self._empty_waiter
1213
1214 def _reset_empty_waiter(self):
1215 self._empty_waiter = None
1216
1217 def close(self):
1218 self._read_ready_cb = None
1219 super().close()
1220
1221
1222 class _SelectorDatagramTransport(_SelectorTransport, transports.DatagramTransport):
1223
1224 _buffer_factory = collections.deque
1225 _header_size = 8
1226
1227 def __init__(self, loop, sock, protocol, address=None,
1228 waiter=None, extra=None):
1229 super().__init__(loop, sock, protocol, extra)
1230 self._address = address
1231 self._buffer_size = 0
1232 self._loop.call_soon(self._protocol.connection_made, self)
1233 # only start reading when connection_made() has been called
1234 self._loop.call_soon(self._add_reader,
1235 self._sock_fd, self._read_ready)
1236 if waiter is not None:
1237 # only wake up the waiter when connection_made() has been called
1238 self._loop.call_soon(futures._set_result_unless_cancelled,
1239 waiter, None)
1240
1241 def get_write_buffer_size(self):
1242 return self._buffer_size
1243
1244 def _read_ready(self):
1245 if self._conn_lost:
1246 return
1247 try:
1248 data, addr = self._sock.recvfrom(self.max_size)
1249 except (BlockingIOError, InterruptedError):
1250 pass
1251 except OSError as exc:
1252 self._protocol.error_received(exc)
1253 except (SystemExit, KeyboardInterrupt):
1254 raise
1255 except BaseException as exc:
1256 self._fatal_error(exc, 'Fatal read error on datagram transport')
1257 else:
1258 self._protocol.datagram_received(data, addr)
1259
1260 def sendto(self, data, addr=None):
1261 if not isinstance(data, (bytes, bytearray, memoryview)):
1262 raise TypeError(f'data argument must be a bytes-like object, '
1263 f'not {type(data).__name__!r}')
1264
1265 if self._address:
1266 if addr not in (None, self._address):
1267 raise ValueError(
1268 f'Invalid address: must be None or {self._address}')
1269 addr = self._address
1270
1271 if self._conn_lost and self._address:
1272 if self._conn_lost >= constants.LOG_THRESHOLD_FOR_CONNLOST_WRITES:
1273 logger.warning('socket.send() raised exception.')
1274 self._conn_lost += 1
1275 return
1276
1277 if not self._buffer:
1278 # Attempt to send it right away first.
1279 try:
1280 if self._extra['peername']:
1281 self._sock.send(data)
1282 else:
1283 self._sock.sendto(data, addr)
1284 return
1285 except (BlockingIOError, InterruptedError):
1286 self._loop._add_writer(self._sock_fd, self._sendto_ready)
1287 except OSError as exc:
1288 self._protocol.error_received(exc)
1289 return
1290 except (SystemExit, KeyboardInterrupt):
1291 raise
1292 except BaseException as exc:
1293 self._fatal_error(
1294 exc, 'Fatal write error on datagram transport')
1295 return
1296
1297 # Ensure that what we buffer is immutable.
1298 self._buffer.append((bytes(data), addr))
1299 self._buffer_size += len(data) + self._header_size
1300 self._maybe_pause_protocol()
1301
1302 def _sendto_ready(self):
1303 while self._buffer:
1304 data, addr = self._buffer.popleft()
1305 self._buffer_size -= len(data) + self._header_size
1306 try:
1307 if self._extra['peername']:
1308 self._sock.send(data)
1309 else:
1310 self._sock.sendto(data, addr)
1311 except (BlockingIOError, InterruptedError):
1312 self._buffer.appendleft((data, addr)) # Try again later.
1313 self._buffer_size += len(data) + self._header_size
1314 break
1315 except OSError as exc:
1316 self._protocol.error_received(exc)
1317 return
1318 except (SystemExit, KeyboardInterrupt):
1319 raise
1320 except BaseException as exc:
1321 self._fatal_error(
1322 exc, 'Fatal write error on datagram transport')
1323 return
1324
1325 self._maybe_resume_protocol() # May append to buffer.
1326 if not self._buffer:
1327 self._loop._remove_writer(self._sock_fd)
1328 if self._closing:
1329 self._call_connection_lost(None)