Debian Codegraph @

Lib/asyncio/base_events.py

raw
1 """Base implementation of event loop.
2
3 The event loop can be broken up into a multiplexer (the part
4 responsible for notifying us of I/O events) and the event loop proper,
5 which wraps a multiplexer with functionality for scheduling callbacks,
6 immediately or at a given time in the future.
7
8 Whenever a public API takes a callback, subsequent positional
9 arguments will be passed to the callback if/when it is called. This
10 avoids the proliferation of trivial lambdas implementing closures.
11 Keyword arguments for the callback are not supported; this is a
12 conscious design decision, leaving the door open for keyword arguments
13 to modify the meaning of the API call itself.
14 """
15
16 import collections
17 import collections.abc
18 import concurrent.futures
19 import errno
20 import heapq
21 import itertools
22 import os
23 import socket
24 import stat
25 import subprocess
26 import threading
27 import time
28 import traceback
29 import sys
30 import warnings
31 import weakref
32
33 try:
34 import ssl
35 except ImportError: # pragma: no cover
36 ssl = None
37
38 from . import constants
39 from . import coroutines
40 from . import events
41 from . import exceptions
42 from . import futures
43 from . import protocols
44 from . import sslproto
45 from . import staggered
46 from . import tasks
47 from . import timeouts
48 from . import transports
49 from . import trsock
50 from .log import logger
51
52
53 __all__ = 'BaseEventLoop','Server',
54
55
56 # Minimum number of _scheduled timer handles before cleanup of
57 # cancelled handles is performed.
58 _MIN_SCHEDULED_TIMER_HANDLES = 100
59
60 # Minimum fraction of _scheduled timer handles that are cancelled
61 # before cleanup of cancelled handles is performed.
62 _MIN_CANCELLED_TIMER_HANDLES_FRACTION = 0.5
63
64
65 _HAS_IPv6 = hasattr(socket, 'AF_INET6')
66
67 # Maximum timeout passed to select to avoid OS limitations
68 MAXIMUM_SELECT_TIMEOUT = 24 * 3600
69
70
71 def _format_handle(handle):
72 cb = handle._callback
73 if isinstance(getattr(cb, '__self__', None), tasks.Task):
74 # format the task
75 return repr(cb.__self__)
76 else:
77 return str(handle)
78
79
80 def _format_pipe(fd):
81 if fd == subprocess.PIPE:
82 return '<pipe>'
83 elif fd == subprocess.STDOUT:
84 return '<stdout>'
85 else:
86 return repr(fd)
87
88
89 def _set_reuseport(sock):
90 if not hasattr(socket, 'SO_REUSEPORT'):
91 raise ValueError('reuse_port not supported by socket module')
92 else:
93 try:
94 sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEPORT, 1)
95 except OSError:
96 raise ValueError('reuse_port not supported by socket module, '
97 'SO_REUSEPORT defined but not implemented.')
98
99
100 def _ipaddr_info(host, port, family, type, proto, flowinfo=0, scopeid=0):
101 # Try to skip getaddrinfo if "host" is already an IP. Users might have
102 # handled name resolution in their own code and pass in resolved IPs.
103 if not hasattr(socket, 'inet_pton'):
104 return
105
106 if proto not in {0, socket.IPPROTO_TCP, socket.IPPROTO_UDP} or \
107 host is None:
108 return None
109
110 if type == socket.SOCK_STREAM:
111 proto = socket.IPPROTO_TCP
112 elif type == socket.SOCK_DGRAM:
113 proto = socket.IPPROTO_UDP
114 else:
115 return None
116
117 if port is None:
118 port = 0
119 elif isinstance(port, bytes) and port == b'':
120 port = 0
121 elif isinstance(port, str) and port == '':
122 port = 0
123 else:
124 # If port's a service name like "http", don't skip getaddrinfo.
125 try:
126 port = int(port)
127 except (TypeError, ValueError):
128 return None
129
130 if family == socket.AF_UNSPEC:
131 afs = [socket.AF_INET]
132 if _HAS_IPv6:
133 afs.append(socket.AF_INET6)
134 else:
135 afs = [family]
136
137 if isinstance(host, bytes):
138 host = host.decode('idna')
139 if '%' in host:
140 # Linux's inet_pton doesn't accept an IPv6 zone index after host,
141 # like '::1%lo0'.
142 return None
143
144 for af in afs:
145 try:
146 socket.inet_pton(af, host)
147 # The host has already been resolved.
148 if _HAS_IPv6 and af == socket.AF_INET6:
149 return af, type, proto, '', (host, port, flowinfo, scopeid)
150 else:
151 return af, type, proto, '', (host, port)
152 except OSError:
153 pass
154
155 # "host" is not an IP address.
156 return None
157
158
159 def _interleave_addrinfos(addrinfos, first_address_family_count=1):
160 """Interleave list of addrinfo tuples by family."""
161 # Group addresses by family
162 addrinfos_by_family = collections.OrderedDict()
163 for addr in addrinfos:
164 family = addr[0]
165 if family not in addrinfos_by_family:
166 addrinfos_by_family[family] = []
167 addrinfos_by_family[family].append(addr)
168 addrinfos_lists = list(addrinfos_by_family.values())
169
170 reordered = []
171 if first_address_family_count > 1:
172 reordered.extend(addrinfos_lists[0][:first_address_family_count - 1])
173 del addrinfos_lists[0][:first_address_family_count - 1]
174 reordered.extend(
175 a for a in itertools.chain.from_iterable(
176 itertools.zip_longest(*addrinfos_lists)
177 ) if a is not None)
178 return reordered
179
180
181 def _run_until_complete_cb(fut):
182 if not fut.cancelled():
183 exc = fut.exception()
184 if isinstance(exc, (SystemExit, KeyboardInterrupt)):
185 # Issue #22429: run_forever() already finished, no need to
186 # stop it.
187 return
188 futures._get_loop(fut).stop()
189
190
191 if hasattr(socket, 'TCP_NODELAY'):
192 def _set_nodelay(sock):
193 if (sock.family in {socket.AF_INET, socket.AF_INET6} and
194 sock.type == socket.SOCK_STREAM and
195 sock.proto == socket.IPPROTO_TCP):
196 sock.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1)
197 else:
198 def _set_nodelay(sock):
199 pass
200
201
202 def _check_ssl_socket(sock):
203 if ssl is not None and isinstance(sock, ssl.SSLSocket):
204 raise TypeError("Socket cannot be of type SSLSocket")
205
206
207 class _SendfileFallbackProtocol(protocols.Protocol):
208 def __init__(self, transp):
209 if not isinstance(transp, transports._FlowControlMixin):
210 raise TypeError("transport should be _FlowControlMixin instance")
211 self._transport = transp
212 self._proto = transp.get_protocol()
213 self._should_resume_reading = transp.is_reading()
214 self._should_resume_writing = transp._protocol_paused
215 transp.pause_reading()
216 transp.set_protocol(self)
217 if self._should_resume_writing:
218 self._write_ready_fut = self._transport._loop.create_future()
219 else:
220 self._write_ready_fut = None
221
222 async def drain(self):
223 if self._transport.is_closing():
224 raise ConnectionError("Connection closed by peer")
225 fut = self._write_ready_fut
226 if fut is None:
227 return
228 await fut
229
230 def connection_made(self, transport):
231 raise RuntimeError("Invalid state: "
232 "connection should have been established already.")
233
234 def connection_lost(self, exc):
235 if self._write_ready_fut is not None:
236 # Never happens if peer disconnects after sending the whole content
237 # Thus disconnection is always an exception from user perspective
238 if exc is None:
239 self._write_ready_fut.set_exception(
240 ConnectionError("Connection is closed by peer"))
241 else:
242 self._write_ready_fut.set_exception(exc)
243 self._proto.connection_lost(exc)
244
245 def pause_writing(self):
246 if self._write_ready_fut is not None:
247 return
248 self._write_ready_fut = self._transport._loop.create_future()
249
250 def resume_writing(self):
251 if self._write_ready_fut is None:
252 return
253 self._write_ready_fut.set_result(False)
254 self._write_ready_fut = None
255
256 def data_received(self, data):
257 raise RuntimeError("Invalid state: reading should be paused")
258
259 def eof_received(self):
260 raise RuntimeError("Invalid state: reading should be paused")
261
262 async def restore(self):
263 self._transport.set_protocol(self._proto)
264 if self._should_resume_reading:
265 self._transport.resume_reading()
266 if self._write_ready_fut is not None:
267 # Cancel the future.
268 # Basically it has no effect because protocol is switched back,
269 # no code should wait for it anymore.
270 self._write_ready_fut.cancel()
271 if self._should_resume_writing:
272 self._proto.resume_writing()
273
274
275 class Server(events.AbstractServer):
276
277 def __init__(self, loop, sockets, protocol_factory, ssl_context, backlog,
278 ssl_handshake_timeout, ssl_shutdown_timeout=None):
279 self._loop = loop
280 self._sockets = sockets
281 # Weak references so we don't break Transport's ability to
282 # detect abandoned transports
283 self._clients = weakref.WeakSet()
284 self._waiters = []
285 self._protocol_factory = protocol_factory
286 self._backlog = backlog
287 self._ssl_context = ssl_context
288 self._ssl_handshake_timeout = ssl_handshake_timeout
289 self._ssl_shutdown_timeout = ssl_shutdown_timeout
290 self._serving = False
291 self._serving_forever_fut = None
292
293 def __repr__(self):
294 return f'<{self.__class__.__name__} sockets={self.sockets!r}>'
295
296 def _attach(self, transport):
297 assert self._sockets is not None
298 self._clients.add(transport)
299
300 def _detach(self, transport):
301 self._clients.discard(transport)
302 if len(self._clients) == 0 and self._sockets is None:
303 self._wakeup()
304
305 def _wakeup(self):
306 waiters = self._waiters
307 self._waiters = None
308 for waiter in waiters:
309 if not waiter.done():
310 waiter.set_result(None)
311
312 def _start_serving(self):
313 if self._serving:
314 return
315 self._serving = True
316 for sock in self._sockets:
317 sock.listen(self._backlog)
318 self._loop._start_serving(
319 self._protocol_factory, sock, self._ssl_context,
320 self, self._backlog, self._ssl_handshake_timeout,
321 self._ssl_shutdown_timeout)
322
323 def get_loop(self):
324 return self._loop
325
326 def is_serving(self):
327 return self._serving
328
329 @property
330 def sockets(self):
331 if self._sockets is None:
332 return ()
333 return tuple(trsock.TransportSocket(s) for s in self._sockets)
334
335 def close(self):
336 sockets = self._sockets
337 if sockets is None:
338 return
339 self._sockets = None
340
341 for sock in sockets:
342 self._loop._stop_serving(sock)
343
344 self._serving = False
345
346 if (self._serving_forever_fut is not None and
347 not self._serving_forever_fut.done()):
348 self._serving_forever_fut.cancel()
349 self._serving_forever_fut = None
350
351 if len(self._clients) == 0:
352 self._wakeup()
353
354 def close_clients(self):
355 for transport in self._clients.copy():
356 transport.close()
357
358 def abort_clients(self):
359 for transport in self._clients.copy():
360 transport.abort()
361
362 async def start_serving(self):
363 self._start_serving()
364 # Skip one loop iteration so that all 'loop.add_reader'
365 # go through.
366 await tasks.sleep(0)
367
368 async def serve_forever(self):
369 if self._serving_forever_fut is not None:
370 raise RuntimeError(
371 f'server {self!r} is already being awaited on serve_forever()')
372 if self._sockets is None:
373 raise RuntimeError(f'server {self!r} is closed')
374
375 self._start_serving()
376 self._serving_forever_fut = self._loop.create_future()
377
378 try:
379 await self._serving_forever_fut
380 except exceptions.CancelledError:
381 try:
382 self.close()
383 self.close_clients()
384 await self.wait_closed()
385 finally:
386 raise
387 finally:
388 self._serving_forever_fut = None
389
390 async def wait_closed(self):
391 """Wait until server is closed and all connections are dropped.
392
393 - If the server is not closed, wait.
394 - If it is closed, but there are still active connections, wait.
395
396 Anyone waiting here will be unblocked once both conditions
397 (server is closed and all connections have been dropped)
398 have become true, in either order.
399
400 Historical note: In 3.11 and before, this was broken, returning
401 immediately if the server was already closed, even if there
402 were still active connections. An attempted fix in 3.12.0 was
403 still broken, returning immediately if the server was still
404 open and there were no active connections. Hopefully in 3.12.1
405 we have it right.
406 """
407 # Waiters are unblocked by self._wakeup(), which is called
408 # from two places: self.close() and self._detach(), but only
409 # when both conditions have become true. To signal that this
410 # has happened, self._wakeup() sets self._waiters to None.
411 if self._waiters is None:
412 return
413 waiter = self._loop.create_future()
414 self._waiters.append(waiter)
415 await waiter
416
417
418 class BaseEventLoop(events.AbstractEventLoop):
419
420 def __init__(self):
421 self._timer_cancelled_count = 0
422 self._closed = False
423 self._stopping = False
424 self._ready = collections.deque()
425 self._scheduled = []
426 self._default_executor = None
427 self._internal_fds = 0
428 # Identifier of the thread running the event loop, or None if the
429 # event loop is not running
430 self._thread_id = None
431 self._clock_resolution = time.get_clock_info('monotonic').resolution
432 self._exception_handler = None
433 self.set_debug(coroutines._is_debug_mode())
434 # The preserved state of async generator hooks.
435 self._old_agen_hooks = None
436 # In debug mode, if the execution of a callback or a step of a task
437 # exceed this duration in seconds, the slow callback/task is logged.
438 self.slow_callback_duration = 0.1
439 self._current_handle = None
440 self._task_factory = None
441 self._coroutine_origin_tracking_enabled = False
442 self._coroutine_origin_tracking_saved_depth = None
443
444 # A weak set of all asynchronous generators that are
445 # being iterated by the loop.
446 self._asyncgens = weakref.WeakSet()
447 # Set to True when `loop.shutdown_asyncgens` is called.
448 self._asyncgens_shutdown_called = False
449 # Set to True when `loop.shutdown_default_executor` is called.
450 self._executor_shutdown_called = False
451
452 def __repr__(self):
453 return (
454 f'<{self.__class__.__name__} running={self.is_running()} '
455 f'closed={self.is_closed()} debug={self.get_debug()}>'
456 )
457
458 def create_future(self):
459 """Create a Future object attached to the loop."""
460 return futures.Future(loop=self)
461
462 def create_task(self, coro, *, name=None, context=None, **kwargs):
463 """Schedule a coroutine object.
464
465 Return a task object.
466 """
467 self._check_closed()
468 if self._task_factory is not None:
469 if context is not None:
470 kwargs["context"] = context
471
472 task = self._task_factory(self, coro, **kwargs)
473 task.set_name(name)
474
475 else:
476 task = tasks.Task(coro, loop=self, name=name, context=context, **kwargs)
477 if task._source_traceback:
478 del task._source_traceback[-1]
479
480 try:
481 return task
482 finally:
483 # gh-128552: prevent a refcycle of
484 # task.exception().__traceback__->BaseEventLoop.create_task->task
485 del task
486
487 def set_task_factory(self, factory):
488 """Set a task factory that will be used by loop.create_task().
489
490 If factory is None the default task factory will be set.
491
492 If factory is a callable, it should have a signature matching
493 '(loop, coro, **kwargs)', where 'loop' will be a reference to the
494 active event loop, 'coro' will be a coroutine object, and **kwargs
495 will be arbitrary keyword arguments that should be passed on to
496 Task. The callable must return a Task.
497 """
498 if factory is not None and not callable(factory):
499 raise TypeError('task factory must be a callable or None')
500 self._task_factory = factory
501
502 def get_task_factory(self):
503 """Return a task factory, or None if the default one is in use."""
504 return self._task_factory
505
506 def _make_socket_transport(self, sock, protocol, waiter=None, *,
507 extra=None, server=None):
508 """Create socket transport."""
509 raise NotImplementedError
510
511 def _make_ssl_transport(
512 self, rawsock, protocol, sslcontext, waiter=None,
513 *, server_side=False, server_hostname=None,
514 extra=None, server=None,
515 ssl_handshake_timeout=None,
516 ssl_shutdown_timeout=None,
517 call_connection_made=True):
518 """Create SSL transport."""
519 raise NotImplementedError
520
521 def _make_datagram_transport(self, sock, protocol,
522 address=None, waiter=None, extra=None):
523 """Create datagram transport."""
524 raise NotImplementedError
525
526 def _make_read_pipe_transport(self, pipe, protocol, waiter=None,
527 extra=None):
528 """Create read pipe transport."""
529 raise NotImplementedError
530
531 def _make_write_pipe_transport(self, pipe, protocol, waiter=None,
532 extra=None):
533 """Create write pipe transport."""
534 raise NotImplementedError
535
536 async def _make_subprocess_transport(self, protocol, args, shell,
537 stdin, stdout, stderr, bufsize,
538 extra=None, **kwargs):
539 """Create subprocess transport."""
540 raise NotImplementedError
541
542 def _write_to_self(self):
543 """Write a byte to self-pipe, to wake up the event loop.
544
545 This may be called from a different thread.
546
547 The subclass is responsible for implementing the self-pipe.
548 """
549 raise NotImplementedError
550
551 def _process_events(self, event_list):
552 """Process selector events."""
553 raise NotImplementedError
554
555 def _check_closed(self):
556 if self._closed:
557 raise RuntimeError('Event loop is closed')
558
559 def _check_default_executor(self):
560 if self._executor_shutdown_called:
561 raise RuntimeError('Executor shutdown has been called')
562
563 def _asyncgen_finalizer_hook(self, agen):
564 self._asyncgens.discard(agen)
565 if not self.is_closed():
566 self.call_soon_threadsafe(self.create_task, agen.aclose())
567
568 def _asyncgen_firstiter_hook(self, agen):
569 if self._asyncgens_shutdown_called:
570 warnings.warn(
571 f"asynchronous generator {agen!r} was scheduled after "
572 f"loop.shutdown_asyncgens() call",
573 ResourceWarning, source=self)
574
575 self._asyncgens.add(agen)
576
577 async def shutdown_asyncgens(self):
578 """Shutdown all active asynchronous generators."""
579 self._asyncgens_shutdown_called = True
580
581 if not len(self._asyncgens):
582 # If Python version is <3.6 or we don't have any asynchronous
583 # generators alive.
584 return
585
586 closing_agens = list(self._asyncgens)
587 self._asyncgens.clear()
588
589 results = await tasks.gather(
590 *[ag.aclose() for ag in closing_agens],
591 return_exceptions=True)
592
593 for result, agen in zip(results, closing_agens):
594 if isinstance(result, Exception):
595 self.call_exception_handler({
596 'message': f'an error occurred during closing of '
597 f'asynchronous generator {agen!r}',
598 'exception': result,
599 'asyncgen': agen
600 })
601
602 async def shutdown_default_executor(self, timeout=None):
603 """Schedule the shutdown of the default executor.
604
605 The timeout parameter specifies the amount of time the executor will
606 be given to finish joining. The default value is None, which means
607 that the executor will be given an unlimited amount of time.
608 """
609 self._executor_shutdown_called = True
610 if self._default_executor is None:
611 return
612 future = self.create_future()
613 thread = threading.Thread(target=self._do_shutdown, args=(future,))
614 thread.start()
615 try:
616 async with timeouts.timeout(timeout):
617 await future
618 except TimeoutError:
619 warnings.warn("The executor did not finishing joining "
620 f"its threads within {timeout} seconds.",
621 RuntimeWarning, stacklevel=2)
622 self._default_executor.shutdown(wait=False)
623 else:
624 thread.join()
625
626 def _do_shutdown(self, future):
627 try:
628 self._default_executor.shutdown(wait=True)
629 if not self.is_closed():
630 self.call_soon_threadsafe(futures._set_result_unless_cancelled,
631 future, None)
632 except Exception as ex:
633 if not self.is_closed() and not future.cancelled():
634 self.call_soon_threadsafe(future.set_exception, ex)
635
636 def _check_running(self):
637 if self.is_running():
638 raise RuntimeError('This event loop is already running')
639 if events._get_running_loop() is not None:
640 raise RuntimeError(
641 'Cannot run the event loop while another loop is running')
642
643 def _run_forever_setup(self):
644 """Prepare the run loop to process events.
645
646 This method exists so that custom event loop subclasses (e.g., event loops
647 that integrate a GUI event loop with Python's event loop) have access to all the
648 loop setup logic.
649 """
650 self._check_closed()
651 self._check_running()
652 self._set_coroutine_origin_tracking(self._debug)
653
654 self._old_agen_hooks = sys.get_asyncgen_hooks()
655 self._thread_id = threading.get_ident()
656 sys.set_asyncgen_hooks(
657 firstiter=self._asyncgen_firstiter_hook,
658 finalizer=self._asyncgen_finalizer_hook
659 )
660
661 events._set_running_loop(self)
662
663 def _run_forever_cleanup(self):
664 """Clean up after an event loop finishes the looping over events.
665
666 This method exists so that custom event loop subclasses (e.g., event loops
667 that integrate a GUI event loop with Python's event loop) have access to all the
668 loop cleanup logic.
669 """
670 self._stopping = False
671 self._thread_id = None
672 events._set_running_loop(None)
673 self._set_coroutine_origin_tracking(False)
674 # Restore any pre-existing async generator hooks.
675 if self._old_agen_hooks is not None:
676 sys.set_asyncgen_hooks(*self._old_agen_hooks)
677 self._old_agen_hooks = None
678
679 def run_forever(self):
680 """Run until stop() is called."""
681 self._run_forever_setup()
682 try:
683 while True:
684 self._run_once()
685 if self._stopping:
686 break
687 finally:
688 self._run_forever_cleanup()
689
690 def run_until_complete(self, future):
691 """Run until the Future is done.
692
693 If the argument is a coroutine, it is wrapped in a Task.
694
695 WARNING: It would be disastrous to call run_until_complete()
696 with the same coroutine twice -- it would wrap it in two
697 different Tasks and that can't be good.
698
699 Return the Future's result, or raise its exception.
700 """
701 self._check_closed()
702 self._check_running()
703
704 new_task = not futures.isfuture(future)
705 future = tasks.ensure_future(future, loop=self)
706 if new_task:
707 # An exception is raised if the future didn't complete, so there
708 # is no need to log the "destroy pending task" message
709 future._log_destroy_pending = False
710
711 future.add_done_callback(_run_until_complete_cb)
712 try:
713 self.run_forever()
714 except:
715 if new_task and future.done() and not future.cancelled():
716 # The coroutine raised a BaseException. Consume the exception
717 # to not log a warning, the caller doesn't have access to the
718 # local task.
719 future.exception()
720 raise
721 finally:
722 future.remove_done_callback(_run_until_complete_cb)
723 if not future.done():
724 raise RuntimeError('Event loop stopped before Future completed.')
725
726 return future.result()
727
728 def stop(self):
729 """Stop running the event loop.
730
731 Every callback already scheduled will still run. This simply
732 informs run_forever to stop looping after a complete iteration.
733 """
734 self._stopping = True
735
736 def close(self):
737 """Close the event loop.
738
739 This clears the queues and shuts down the executor,
740 but does not wait for the executor to finish.
741
742 The event loop must not be running.
743 """
744 if self.is_running():
745 raise RuntimeError("Cannot close a running event loop")
746 if self._closed:
747 return
748 if self._debug:
749 logger.debug("Close %r", self)
750 self._closed = True
751 self._ready.clear()
752 self._scheduled.clear()
753 self._executor_shutdown_called = True
754 executor = self._default_executor
755 if executor is not None:
756 self._default_executor = None
757 executor.shutdown(wait=False)
758
759 def is_closed(self):
760 """Returns True if the event loop was closed."""
761 return self._closed
762
763 def __del__(self, _warn=warnings.warn):
764 if not self.is_closed():
765 _warn(f"unclosed event loop {self!r}", ResourceWarning, source=self)
766 if not self.is_running():
767 self.close()
768
769 def is_running(self):
770 """Returns True if the event loop is running."""
771 return (self._thread_id is not None)
772
773 def time(self):
774 """Return the time according to the event loop's clock.
775
776 This is a float expressed in seconds since an epoch, but the
777 epoch, precision, accuracy and drift are unspecified and may
778 differ per event loop.
779 """
780 return time.monotonic()
781
782 def call_later(self, delay, callback, *args, context=None):
783 """Arrange for a callback to be called at a given time.
784
785 Return a Handle: an opaque object with a cancel() method that
786 can be used to cancel the call.
787
788 The delay can be an int or float, expressed in seconds. It is
789 always relative to the current time.
790
791 Each callback will be called exactly once. If two callbacks
792 are scheduled for exactly the same time, it is undefined which
793 will be called first.
794
795 Any positional arguments after the callback will be passed to
796 the callback when it is called.
797 """
798 if delay is None:
799 raise TypeError('delay must not be None')
800 timer = self.call_at(self.time() + delay, callback, *args,
801 context=context)
802 if timer._source_traceback:
803 del timer._source_traceback[-1]
804 return timer
805
806 def call_at(self, when, callback, *args, context=None):
807 """Like call_later(), but uses an absolute time.
808
809 Absolute time corresponds to the event loop's time() method.
810 """
811 if when is None:
812 raise TypeError("when cannot be None")
813 self._check_closed()
814 if self._debug:
815 self._check_thread()
816 self._check_callback(callback, 'call_at')
817 timer = events.TimerHandle(when, callback, args, self, context)
818 if timer._source_traceback:
819 del timer._source_traceback[-1]
820 heapq.heappush(self._scheduled, timer)
821 timer._scheduled = True
822 return timer
823
824 def call_soon(self, callback, *args, context=None):
825 """Arrange for a callback to be called as soon as possible.
826
827 This operates as a FIFO queue: callbacks are called in the
828 order in which they are registered. Each callback will be
829 called exactly once.
830
831 Any positional arguments after the callback will be passed to
832 the callback when it is called.
833 """
834 self._check_closed()
835 if self._debug:
836 self._check_thread()
837 self._check_callback(callback, 'call_soon')
838 handle = self._call_soon(callback, args, context)
839 if handle._source_traceback:
840 del handle._source_traceback[-1]
841 return handle
842
843 def _check_callback(self, callback, method):
844 if (coroutines.iscoroutine(callback) or
845 coroutines.iscoroutinefunction(callback)):
846 raise TypeError(
847 f"coroutines cannot be used with {method}()")
848 if not callable(callback):
849 raise TypeError(
850 f'a callable object was expected by {method}(), '
851 f'got {callback!r}')
852
853 def _call_soon(self, callback, args, context):
854 handle = events.Handle(callback, args, self, context)
855 if handle._source_traceback:
856 del handle._source_traceback[-1]
857 self._ready.append(handle)
858 return handle
859
860 def _check_thread(self):
861 """Check that the current thread is the thread running the event loop.
862
863 Non-thread-safe methods of this class make this assumption and will
864 likely behave incorrectly when the assumption is violated.
865
866 Should only be called when (self._debug == True). The caller is
867 responsible for checking this condition for performance reasons.
868 """
869 if self._thread_id is None:
870 return
871 thread_id = threading.get_ident()
872 if thread_id != self._thread_id:
873 raise RuntimeError(
874 "Non-thread-safe operation invoked on an event loop other "
875 "than the current one")
876
877 def call_soon_threadsafe(self, callback, *args, context=None):
878 """Like call_soon(), but thread-safe."""
879 self._check_closed()
880 if self._debug:
881 self._check_callback(callback, 'call_soon_threadsafe')
882 handle = self._call_soon(callback, args, context)
883 if handle._source_traceback:
884 del handle._source_traceback[-1]
885 self._write_to_self()
886 return handle
887
888 def run_in_executor(self, executor, func, *args):
889 self._check_closed()
890 if self._debug:
891 self._check_callback(func, 'run_in_executor')
892 if executor is None:
893 executor = self._default_executor
894 # Only check when the default executor is being used
895 self._check_default_executor()
896 if executor is None:
897 executor = concurrent.futures.ThreadPoolExecutor(
898 thread_name_prefix='asyncio'
899 )
900 self._default_executor = executor
901 return futures.wrap_future(
902 executor.submit(func, *args), loop=self)
903
904 def set_default_executor(self, executor):
905 if not isinstance(executor, concurrent.futures.ThreadPoolExecutor):
906 raise TypeError('executor must be ThreadPoolExecutor instance')
907 self._default_executor = executor
908
909 def _getaddrinfo_debug(self, host, port, family, type, proto, flags):
910 msg = [f"{host}:{port!r}"]
911 if family:
912 msg.append(f'family={family!r}')
913 if type:
914 msg.append(f'type={type!r}')
915 if proto:
916 msg.append(f'proto={proto!r}')
917 if flags:
918 msg.append(f'flags={flags!r}')
919 msg = ', '.join(msg)
920 logger.debug('Get address info %s', msg)
921
922 t0 = self.time()
923 addrinfo = socket.getaddrinfo(host, port, family, type, proto, flags)
924 dt = self.time() - t0
925
926 msg = f'Getting address info {msg} took {dt * 1e3:.3f}ms: {addrinfo!r}'
927 if dt >= self.slow_callback_duration:
928 logger.info(msg)
929 else:
930 logger.debug(msg)
931 return addrinfo
932
933 async def getaddrinfo(self, host, port, *,
934 family=0, type=0, proto=0, flags=0):
935 if self._debug:
936 getaddr_func = self._getaddrinfo_debug
937 else:
938 getaddr_func = socket.getaddrinfo
939
940 return await self.run_in_executor(
941 None, getaddr_func, host, port, family, type, proto, flags)
942
943 async def getnameinfo(self, sockaddr, flags=0):
944 return await self.run_in_executor(
945 None, socket.getnameinfo, sockaddr, flags)
946
947 async def sock_sendfile(self, sock, file, offset=0, count=None,
948 *, fallback=True):
949 if self._debug and sock.gettimeout() != 0:
950 raise ValueError("the socket must be non-blocking")
951 _check_ssl_socket(sock)
952 self._check_sendfile_params(sock, file, offset, count)
953 try:
954 return await self._sock_sendfile_native(sock, file,
955 offset, count)
956 except exceptions.SendfileNotAvailableError as exc:
957 if not fallback:
958 raise
959 return await self._sock_sendfile_fallback(sock, file,
960 offset, count)
961
962 async def _sock_sendfile_native(self, sock, file, offset, count):
963 # NB: sendfile syscall is not supported for SSL sockets and
964 # non-mmap files even if sendfile is supported by OS
965 raise exceptions.SendfileNotAvailableError(
966 f"syscall sendfile is not available for socket {sock!r} "
967 f"and file {file!r} combination")
968
969 async def _sock_sendfile_fallback(self, sock, file, offset, count):
970 if hasattr(file, 'seek'):
971 file.seek(offset)
972 blocksize = (
973 min(count, constants.SENDFILE_FALLBACK_READBUFFER_SIZE)
974 if count else constants.SENDFILE_FALLBACK_READBUFFER_SIZE
975 )
976 buf = bytearray(blocksize)
977 total_sent = 0
978 try:
979 while True:
980 if count:
981 blocksize = min(count - total_sent, blocksize)
982 if blocksize <= 0:
983 break
984 view = memoryview(buf)[:blocksize]
985 read = await self.run_in_executor(None, file.readinto, view)
986 if not read:
987 break # EOF
988 await self.sock_sendall(sock, view[:read])
989 total_sent += read
990 return total_sent
991 finally:
992 if total_sent > 0 and hasattr(file, 'seek'):
993 file.seek(offset + total_sent)
994
995 def _check_sendfile_params(self, sock, file, offset, count):
996 if 'b' not in getattr(file, 'mode', 'b'):
997 raise ValueError("file should be opened in binary mode")
998 if not sock.type == socket.SOCK_STREAM:
999 raise ValueError("only SOCK_STREAM type sockets are supported")
1000 if count is not None:
1001 if not isinstance(count, int):
1002 raise TypeError(
1003 "count must be a positive integer (got {!r})".format(count))
1004 if count <= 0:
1005 raise ValueError(
1006 "count must be a positive integer (got {!r})".format(count))
1007 if not isinstance(offset, int):
1008 raise TypeError(
1009 "offset must be a non-negative integer (got {!r})".format(
1010 offset))
1011 if offset < 0:
1012 raise ValueError(
1013 "offset must be a non-negative integer (got {!r})".format(
1014 offset))
1015
1016 async def _connect_sock(self, exceptions, addr_info, local_addr_infos=None):
1017 """Create, bind and connect one socket."""
1018 my_exceptions = []
1019 exceptions.append(my_exceptions)
1020 family, type_, proto, _, address = addr_info
1021 sock = None
1022 try:
1023 try:
1024 sock = socket.socket(family=family, type=type_, proto=proto)
1025 sock.setblocking(False)
1026 if local_addr_infos is not None:
1027 for lfamily, _, _, _, laddr in local_addr_infos:
1028 # skip local addresses of different family
1029 if lfamily != family:
1030 continue
1031 try:
1032 sock.bind(laddr)
1033 break
1034 except OSError as exc:
1035 msg = (
1036 f'error while attempting to bind on '
1037 f'address {laddr!r}: {str(exc).lower()}'
1038 )
1039 exc = OSError(exc.errno, msg)
1040 my_exceptions.append(exc)
1041 else: # all bind attempts failed
1042 if my_exceptions:
1043 raise my_exceptions.pop()
1044 else:
1045 raise OSError(f"no matching local address with {family=} found")
1046 await self.sock_connect(sock, address)
1047 return sock
1048 except OSError as exc:
1049 my_exceptions.append(exc)
1050 raise
1051 except:
1052 if sock is not None:
1053 try:
1054 sock.close()
1055 except OSError:
1056 # An error when closing a newly created socket is
1057 # not important, but it can overwrite more important
1058 # non-OSError error. So ignore it.
1059 pass
1060 raise
1061 finally:
1062 exceptions = my_exceptions = None
1063
1064 async def create_connection(
1065 self, protocol_factory, host=None, port=None,
1066 *, ssl=None, family=0,
1067 proto=0, flags=0, sock=None,
1068 local_addr=None, server_hostname=None,
1069 ssl_handshake_timeout=None,
1070 ssl_shutdown_timeout=None,
1071 happy_eyeballs_delay=None, interleave=None,
1072 all_errors=False):
1073 """Connect to a TCP server.
1074
1075 Create a streaming transport connection to a given internet host and
1076 port: socket family AF_INET or socket.AF_INET6 depending on host (or
1077 family if specified), socket type SOCK_STREAM. protocol_factory must
1078 be a callable returning a protocol instance.
1079
1080 This method is a coroutine which will try to establish the
1081 connection in the background. When successful, the coroutine
1082 returns a (transport, protocol) pair.
1083 """
1084 if server_hostname is not None and not ssl:
1085 raise ValueError('server_hostname is only meaningful with ssl')
1086
1087 if server_hostname is None and ssl:
1088 # Use host as default for server_hostname. It is an error
1089 # if host is empty or not set, e.g. when an
1090 # already-connected socket was passed or when only a port
1091 # is given. To avoid this error, you can pass
1092 # server_hostname='' -- this will bypass the hostname
1093 # check. (This also means that if host is a numeric
1094 # IP/IPv6 address, we will attempt to verify that exact
1095 # address; this will probably fail, but it is possible to
1096 # create a certificate for a specific IP address, so we
1097 # don't judge it here.)
1098 if not host:
1099 raise ValueError('You must set server_hostname '
1100 'when using ssl without a host')
1101 server_hostname = host
1102
1103 if ssl_handshake_timeout is not None and not ssl:
1104 raise ValueError(
1105 'ssl_handshake_timeout is only meaningful with ssl')
1106
1107 if ssl_shutdown_timeout is not None and not ssl:
1108 raise ValueError(
1109 'ssl_shutdown_timeout is only meaningful with ssl')
1110
1111 if sock is not None:
1112 _check_ssl_socket(sock)
1113
1114 if happy_eyeballs_delay is not None and interleave is None:
1115 # If using happy eyeballs, default to interleave addresses by family
1116 interleave = 1
1117
1118 if host is not None or port is not None:
1119 if sock is not None:
1120 raise ValueError(
1121 'host/port and sock can not be specified at the same time')
1122
1123 infos = await self._ensure_resolved(
1124 (host, port), family=family,
1125 type=socket.SOCK_STREAM, proto=proto, flags=flags, loop=self)
1126 if not infos:
1127 raise OSError('getaddrinfo() returned empty list')
1128
1129 if local_addr is not None:
1130 laddr_infos = await self._ensure_resolved(
1131 local_addr, family=family,
1132 type=socket.SOCK_STREAM, proto=proto,
1133 flags=flags, loop=self)
1134 if not laddr_infos:
1135 raise OSError('getaddrinfo() returned empty list')
1136 else:
1137 laddr_infos = None
1138
1139 if interleave:
1140 infos = _interleave_addrinfos(infos, interleave)
1141
1142 exceptions = []
1143 if happy_eyeballs_delay is None:
1144 # not using happy eyeballs
1145 for addrinfo in infos:
1146 try:
1147 sock = await self._connect_sock(
1148 exceptions, addrinfo, laddr_infos)
1149 break
1150 except OSError:
1151 continue
1152 else: # using happy eyeballs
1153 sock = (await staggered.staggered_race(
1154 (
1155 # can't use functools.partial as it keeps a reference
1156 # to exceptions
1157 lambda addrinfo=addrinfo: self._connect_sock(
1158 exceptions, addrinfo, laddr_infos
1159 )
1160 for addrinfo in infos
1161 ),
1162 happy_eyeballs_delay,
1163 loop=self,
1164 ))[0] # can't use sock, _, _ as it keeks a reference to exceptions
1165
1166 if sock is None:
1167 exceptions = [exc for sub in exceptions for exc in sub]
1168 try:
1169 if all_errors:
1170 raise ExceptionGroup("create_connection failed", exceptions)
1171 if len(exceptions) == 1:
1172 raise exceptions[0]
1173 elif exceptions:
1174 # If they all have the same str(), raise one.
1175 model = str(exceptions[0])
1176 if all(str(exc) == model for exc in exceptions):
1177 raise exceptions[0]
1178 # Raise a combined exception so the user can see all
1179 # the various error messages.
1180 raise OSError('Multiple exceptions: {}'.format(
1181 ', '.join(str(exc) for exc in exceptions)))
1182 else:
1183 # No exceptions were collected, raise a timeout error
1184 raise TimeoutError('create_connection failed')
1185 finally:
1186 exceptions = None
1187
1188 else:
1189 if sock is None:
1190 raise ValueError(
1191 'host and port was not specified and no sock specified')
1192 if sock.type != socket.SOCK_STREAM:
1193 # We allow AF_INET, AF_INET6, AF_UNIX as long as they
1194 # are SOCK_STREAM.
1195 # We support passing AF_UNIX sockets even though we have
1196 # a dedicated API for that: create_unix_connection.
1197 # Disallowing AF_UNIX in this method, breaks backwards
1198 # compatibility.
1199 raise ValueError(
1200 f'A Stream Socket was expected, got {sock!r}')
1201
1202 transport, protocol = await self._create_connection_transport(
1203 sock, protocol_factory, ssl, server_hostname,
1204 ssl_handshake_timeout=ssl_handshake_timeout,
1205 ssl_shutdown_timeout=ssl_shutdown_timeout)
1206 if self._debug:
1207 # Get the socket from the transport because SSL transport closes
1208 # the old socket and creates a new SSL socket
1209 sock = transport.get_extra_info('socket')
1210 logger.debug("%r connected to %s:%r: (%r, %r)",
1211 sock, host, port, transport, protocol)
1212 return transport, protocol
1213
1214 async def _create_connection_transport(
1215 self, sock, protocol_factory, ssl,
1216 server_hostname, server_side=False,
1217 ssl_handshake_timeout=None,
1218 ssl_shutdown_timeout=None):
1219
1220 sock.setblocking(False)
1221
1222 protocol = protocol_factory()
1223 waiter = self.create_future()
1224 if ssl:
1225 sslcontext = None if isinstance(ssl, bool) else ssl
1226 transport = self._make_ssl_transport(
1227 sock, protocol, sslcontext, waiter,
1228 server_side=server_side, server_hostname=server_hostname,
1229 ssl_handshake_timeout=ssl_handshake_timeout,
1230 ssl_shutdown_timeout=ssl_shutdown_timeout)
1231 else:
1232 transport = self._make_socket_transport(sock, protocol, waiter)
1233
1234 try:
1235 await waiter
1236 except:
1237 transport.close()
1238 raise
1239
1240 return transport, protocol
1241
1242 async def sendfile(self, transport, file, offset=0, count=None,
1243 *, fallback=True):
1244 """Send a file to transport.
1245
1246 Return the total number of bytes which were sent.
1247
1248 The method uses high-performance os.sendfile if available.
1249
1250 file must be a regular file object opened in binary mode.
1251
1252 offset tells from where to start reading the file. If specified,
1253 count is the total number of bytes to transmit as opposed to
1254 sending the file until EOF is reached. File position is updated on
1255 return or also in case of error in which case file.tell()
1256 can be used to figure out the number of bytes
1257 which were sent.
1258
1259 fallback set to True makes asyncio to manually read and send
1260 the file when the platform does not support the sendfile syscall
1261 (e.g. Windows or SSL socket on Unix).
1262
1263 Raise SendfileNotAvailableError if the system does not support
1264 sendfile syscall and fallback is False.
1265 """
1266 if transport.is_closing():
1267 raise RuntimeError("Transport is closing")
1268 mode = getattr(transport, '_sendfile_compatible',
1269 constants._SendfileMode.UNSUPPORTED)
1270 if mode is constants._SendfileMode.UNSUPPORTED:
1271 raise RuntimeError(
1272 f"sendfile is not supported for transport {transport!r}")
1273 if mode is constants._SendfileMode.TRY_NATIVE:
1274 try:
1275 return await self._sendfile_native(transport, file,
1276 offset, count)
1277 except exceptions.SendfileNotAvailableError as exc:
1278 if not fallback:
1279 raise
1280
1281 if not fallback:
1282 raise RuntimeError(
1283 f"fallback is disabled and native sendfile is not "
1284 f"supported for transport {transport!r}")
1285 return await self._sendfile_fallback(transport, file,
1286 offset, count)
1287
1288 async def _sendfile_native(self, transp, file, offset, count):
1289 raise exceptions.SendfileNotAvailableError(
1290 "sendfile syscall is not supported")
1291
1292 async def _sendfile_fallback(self, transp, file, offset, count):
1293 if hasattr(file, 'seek'):
1294 file.seek(offset)
1295 blocksize = min(count, 16384) if count else 16384
1296 buf = bytearray(blocksize)
1297 total_sent = 0
1298 proto = _SendfileFallbackProtocol(transp)
1299 try:
1300 while True:
1301 if count:
1302 blocksize = min(count - total_sent, blocksize)
1303 if blocksize <= 0:
1304 return total_sent
1305 view = memoryview(buf)[:blocksize]
1306 read = await self.run_in_executor(None, file.readinto, view)
1307 if not read:
1308 return total_sent # EOF
1309 transp.write(view[:read])
1310 await proto.drain()
1311 total_sent += read
1312 finally:
1313 if total_sent > 0 and hasattr(file, 'seek'):
1314 file.seek(offset + total_sent)
1315 await proto.restore()
1316
1317 async def start_tls(self, transport, protocol, sslcontext, *,
1318 server_side=False,
1319 server_hostname=None,
1320 ssl_handshake_timeout=None,
1321 ssl_shutdown_timeout=None):
1322 """Upgrade transport to TLS.
1323
1324 Return a new transport that *protocol* should start using
1325 immediately.
1326 """
1327 if ssl is None:
1328 raise RuntimeError('Python ssl module is not available')
1329
1330 if not isinstance(sslcontext, ssl.SSLContext):
1331 raise TypeError(
1332 f'sslcontext is expected to be an instance of ssl.SSLContext, '
1333 f'got {sslcontext!r}')
1334
1335 if not getattr(transport, '_start_tls_compatible', False):
1336 raise TypeError(
1337 f'transport {transport!r} is not supported by start_tls()')
1338
1339 waiter = self.create_future()
1340 ssl_protocol = sslproto.SSLProtocol(
1341 self, protocol, sslcontext, waiter,
1342 server_side, server_hostname,
1343 ssl_handshake_timeout=ssl_handshake_timeout,
1344 ssl_shutdown_timeout=ssl_shutdown_timeout,
1345 call_connection_made=False)
1346
1347 # Pause early so that "ssl_protocol.data_received()" doesn't
1348 # have a chance to get called before "ssl_protocol.connection_made()".
1349 transport.pause_reading()
1350
1351 # gh-142352: move buffered StreamReader data to SSLProtocol
1352 if server_side:
1353 from .streams import StreamReaderProtocol
1354 if isinstance(protocol, StreamReaderProtocol):
1355 stream_reader = getattr(protocol, '_stream_reader', None)
1356 if stream_reader is not None:
1357 buffer = stream_reader._buffer
1358 if buffer:
1359 ssl_protocol._incoming.write(buffer)
1360 buffer.clear()
1361
1362 transport.set_protocol(ssl_protocol)
1363 conmade_cb = self.call_soon(ssl_protocol.connection_made, transport)
1364 resume_cb = self.call_soon(transport.resume_reading)
1365
1366 try:
1367 await waiter
1368 except BaseException:
1369 transport.close()
1370 conmade_cb.cancel()
1371 resume_cb.cancel()
1372 raise
1373
1374 return ssl_protocol._app_transport
1375
1376 async def create_datagram_endpoint(self, protocol_factory,
1377 local_addr=None, remote_addr=None, *,
1378 family=0, proto=0, flags=0,
1379 reuse_port=None,
1380 allow_broadcast=None, sock=None):
1381 """Create datagram connection."""
1382 if sock is not None:
1383 if sock.type == socket.SOCK_STREAM:
1384 raise ValueError(
1385 f'A datagram socket was expected, got {sock!r}')
1386 if (local_addr or remote_addr or
1387 family or proto or flags or
1388 reuse_port or allow_broadcast):
1389 # show the problematic kwargs in exception msg
1390 opts = dict(local_addr=local_addr, remote_addr=remote_addr,
1391 family=family, proto=proto, flags=flags,
1392 reuse_port=reuse_port,
1393 allow_broadcast=allow_broadcast)
1394 problems = ', '.join(f'{k}={v}' for k, v in opts.items() if v)
1395 raise ValueError(
1396 f'socket modifier keyword arguments can not be used '
1397 f'when sock is specified. ({problems})')
1398 sock.setblocking(False)
1399 r_addr = None
1400 else:
1401 if not (local_addr or remote_addr):
1402 if family == 0:
1403 raise ValueError('unexpected address family')
1404 addr_pairs_info = (((family, proto), (None, None)),)
1405 elif hasattr(socket, 'AF_UNIX') and family == socket.AF_UNIX:
1406 for addr in (local_addr, remote_addr):
1407 if addr is not None and not isinstance(addr, str):
1408 raise TypeError('string is expected')
1409
1410 if local_addr and local_addr[0] not in (0, '\x00'):
1411 try:
1412 if stat.S_ISSOCK(os.stat(local_addr).st_mode):
1413 os.remove(local_addr)
1414 except FileNotFoundError:
1415 pass
1416 except OSError as err:
1417 # Directory may have permissions only to create socket.
1418 logger.error('Unable to check or remove stale UNIX '
1419 'socket %r: %r',
1420 local_addr, err)
1421
1422 addr_pairs_info = (((family, proto),
1423 (local_addr, remote_addr)), )
1424 else:
1425 # join address by (family, protocol)
1426 addr_infos = {} # Using order preserving dict
1427 for idx, addr in ((0, local_addr), (1, remote_addr)):
1428 if addr is not None:
1429 if not (isinstance(addr, tuple) and len(addr) == 2):
1430 raise TypeError('2-tuple is expected')
1431
1432 infos = await self._ensure_resolved(
1433 addr, family=family, type=socket.SOCK_DGRAM,
1434 proto=proto, flags=flags, loop=self)
1435 if not infos:
1436 raise OSError('getaddrinfo() returned empty list')
1437
1438 for fam, _, pro, _, address in infos:
1439 key = (fam, pro)
1440 if key not in addr_infos:
1441 addr_infos[key] = [None, None]
1442 addr_infos[key][idx] = address
1443
1444 # each addr has to have info for each (family, proto) pair
1445 addr_pairs_info = [
1446 (key, addr_pair) for key, addr_pair in addr_infos.items()
1447 if not ((local_addr and addr_pair[0] is None) or
1448 (remote_addr and addr_pair[1] is None))]
1449
1450 if not addr_pairs_info:
1451 raise ValueError('can not get address information')
1452
1453 exceptions = []
1454
1455 for ((family, proto),
1456 (local_address, remote_address)) in addr_pairs_info:
1457 sock = None
1458 r_addr = None
1459 try:
1460 sock = socket.socket(
1461 family=family, type=socket.SOCK_DGRAM, proto=proto)
1462 if reuse_port:
1463 _set_reuseport(sock)
1464 if allow_broadcast:
1465 sock.setsockopt(
1466 socket.SOL_SOCKET, socket.SO_BROADCAST, 1)
1467 sock.setblocking(False)
1468
1469 if local_addr:
1470 sock.bind(local_address)
1471 if remote_addr:
1472 if not allow_broadcast:
1473 await self.sock_connect(sock, remote_address)
1474 r_addr = remote_address
1475 except OSError as exc:
1476 if sock is not None:
1477 sock.close()
1478 exceptions.append(exc)
1479 except:
1480 if sock is not None:
1481 sock.close()
1482 raise
1483 else:
1484 break
1485 else:
1486 raise exceptions[0]
1487
1488 protocol = protocol_factory()
1489 waiter = self.create_future()
1490 transport = self._make_datagram_transport(
1491 sock, protocol, r_addr, waiter)
1492 if self._debug:
1493 if local_addr:
1494 logger.info("Datagram endpoint local_addr=%r remote_addr=%r "
1495 "created: (%r, %r)",
1496 local_addr, remote_addr, transport, protocol)
1497 else:
1498 logger.debug("Datagram endpoint remote_addr=%r created: "
1499 "(%r, %r)",
1500 remote_addr, transport, protocol)
1501
1502 try:
1503 await waiter
1504 except:
1505 transport.close()
1506 raise
1507
1508 return transport, protocol
1509
1510 async def _ensure_resolved(self, address, *,
1511 family=0, type=socket.SOCK_STREAM,
1512 proto=0, flags=0, loop):
1513 host, port = address[:2]
1514 info = _ipaddr_info(host, port, family, type, proto, *address[2:])
1515 if info is not None:
1516 # "host" is already a resolved IP.
1517 return [info]
1518 else:
1519 return await loop.getaddrinfo(host, port, family=family, type=type,
1520 proto=proto, flags=flags)
1521
1522 async def _create_server_getaddrinfo(self, host, port, family, flags):
1523 infos = await self._ensure_resolved((host, port), family=family,
1524 type=socket.SOCK_STREAM,
1525 flags=flags, loop=self)
1526 if not infos:
1527 raise OSError(f'getaddrinfo({host!r}) returned empty list')
1528 return infos
1529
1530 async def create_server(
1531 self, protocol_factory, host=None, port=None,
1532 *,
1533 family=socket.AF_UNSPEC,
1534 flags=socket.AI_PASSIVE,
1535 sock=None,
1536 backlog=100,
1537 ssl=None,
1538 reuse_address=None,
1539 reuse_port=None,
1540 keep_alive=None,
1541 ssl_handshake_timeout=None,
1542 ssl_shutdown_timeout=None,
1543 start_serving=True):
1544 """Create a TCP server.
1545
1546 The host parameter can be a string, in that case the TCP server is
1547 bound to host and port.
1548
1549 The host parameter can also be a sequence of strings and in that
1550 case the TCP server is bound to all hosts of the sequence. If
1551 a host appears multiple times (possibly indirectly e.g. when
1552 hostnames resolve to the same IP address), the server is only bound
1553 once to that host.
1554
1555 Return a Server object which can be used to stop the service.
1556
1557 This method is a coroutine.
1558 """
1559 if isinstance(ssl, bool):
1560 raise TypeError('ssl argument must be an SSLContext or None')
1561
1562 if ssl_handshake_timeout is not None and ssl is None:
1563 raise ValueError(
1564 'ssl_handshake_timeout is only meaningful with ssl')
1565
1566 if ssl_shutdown_timeout is not None and ssl is None:
1567 raise ValueError(
1568 'ssl_shutdown_timeout is only meaningful with ssl')
1569
1570 if sock is not None:
1571 _check_ssl_socket(sock)
1572
1573 if host is not None or port is not None:
1574 if sock is not None:
1575 raise ValueError(
1576 'host/port and sock can not be specified at the same time')
1577
1578 if reuse_address is None:
1579 reuse_address = os.name == "posix" and sys.platform != "cygwin"
1580 sockets = []
1581 if host == '':
1582 hosts = [None]
1583 elif (isinstance(host, str) or
1584 not isinstance(host, collections.abc.Iterable)):
1585 hosts = [host]
1586 else:
1587 hosts = host
1588
1589 fs = [self._create_server_getaddrinfo(host, port, family=family,
1590 flags=flags)
1591 for host in hosts]
1592 infos = await tasks.gather(*fs)
1593 infos = set(itertools.chain.from_iterable(infos))
1594
1595 completed = False
1596 try:
1597 for res in infos:
1598 af, socktype, proto, canonname, sa = res
1599 try:
1600 sock = socket.socket(af, socktype, proto)
1601 except socket.error:
1602 # Assume it's a bad family/type/protocol combination.
1603 if self._debug:
1604 logger.warning('create_server() failed to create '
1605 'socket.socket(%r, %r, %r)',
1606 af, socktype, proto, exc_info=True)
1607 continue
1608 sockets.append(sock)
1609 if reuse_address:
1610 sock.setsockopt(
1611 socket.SOL_SOCKET, socket.SO_REUSEADDR, True)
1612 # Since Linux 6.12.9, SO_REUSEPORT is not allowed
1613 # on other address families than AF_INET/AF_INET6.
1614 if reuse_port and af in (socket.AF_INET, socket.AF_INET6):
1615 _set_reuseport(sock)
1616 if keep_alive:
1617 sock.setsockopt(
1618 socket.SOL_SOCKET, socket.SO_KEEPALIVE, True)
1619 # Disable IPv4/IPv6 dual stack support (enabled by
1620 # default on Linux) which makes a single socket
1621 # listen on both address families.
1622 if (_HAS_IPv6 and
1623 af == socket.AF_INET6 and
1624 hasattr(socket, 'IPPROTO_IPV6')):
1625 sock.setsockopt(socket.IPPROTO_IPV6,
1626 socket.IPV6_V6ONLY,
1627 True)
1628 try:
1629 sock.bind(sa)
1630 except OSError as err:
1631 msg = ('error while attempting '
1632 'to bind on address %r: %s'
1633 % (sa, str(err).lower()))
1634 if err.errno == errno.EADDRNOTAVAIL:
1635 # Assume the family is not enabled (bpo-30945)
1636 sockets.pop()
1637 sock.close()
1638 if self._debug:
1639 logger.warning(msg)
1640 continue
1641 raise OSError(err.errno, msg) from None
1642
1643 if not sockets:
1644 raise OSError('could not bind on any address out of %r'
1645 % ([info[4] for info in infos],))
1646
1647 completed = True
1648 finally:
1649 if not completed:
1650 for sock in sockets:
1651 sock.close()
1652 else:
1653 if sock is None:
1654 raise ValueError('Neither host/port nor sock were specified')
1655 if sock.type != socket.SOCK_STREAM:
1656 raise ValueError(f'A Stream Socket was expected, got {sock!r}')
1657 sockets = [sock]
1658
1659 for sock in sockets:
1660 sock.setblocking(False)
1661
1662 server = Server(self, sockets, protocol_factory,
1663 ssl, backlog, ssl_handshake_timeout,
1664 ssl_shutdown_timeout)
1665 if start_serving:
1666 server._start_serving()
1667 # Skip one loop iteration so that all 'loop.add_reader'
1668 # go through.
1669 await tasks.sleep(0)
1670
1671 if self._debug:
1672 logger.info("%r is serving", server)
1673 return server
1674
1675 async def connect_accepted_socket(
1676 self, protocol_factory, sock,
1677 *, ssl=None,
1678 ssl_handshake_timeout=None,
1679 ssl_shutdown_timeout=None):
1680 if sock.type != socket.SOCK_STREAM:
1681 raise ValueError(f'A Stream Socket was expected, got {sock!r}')
1682
1683 if ssl_handshake_timeout is not None and not ssl:
1684 raise ValueError(
1685 'ssl_handshake_timeout is only meaningful with ssl')
1686
1687 if ssl_shutdown_timeout is not None and not ssl:
1688 raise ValueError(
1689 'ssl_shutdown_timeout is only meaningful with ssl')
1690
1691 if sock is not None:
1692 _check_ssl_socket(sock)
1693
1694 transport, protocol = await self._create_connection_transport(
1695 sock, protocol_factory, ssl, '', server_side=True,
1696 ssl_handshake_timeout=ssl_handshake_timeout,
1697 ssl_shutdown_timeout=ssl_shutdown_timeout)
1698 if self._debug:
1699 # Get the socket from the transport because SSL transport closes
1700 # the old socket and creates a new SSL socket
1701 sock = transport.get_extra_info('socket')
1702 logger.debug("%r handled: (%r, %r)", sock, transport, protocol)
1703 return transport, protocol
1704
1705 async def connect_read_pipe(self, protocol_factory, pipe):
1706 protocol = protocol_factory()
1707 waiter = self.create_future()
1708 transport = self._make_read_pipe_transport(pipe, protocol, waiter)
1709
1710 try:
1711 await waiter
1712 except:
1713 transport.close()
1714 raise
1715
1716 if self._debug:
1717 logger.debug('Read pipe %r connected: (%r, %r)',
1718 pipe.fileno(), transport, protocol)
1719 return transport, protocol
1720
1721 async def connect_write_pipe(self, protocol_factory, pipe):
1722 protocol = protocol_factory()
1723 waiter = self.create_future()
1724 transport = self._make_write_pipe_transport(pipe, protocol, waiter)
1725
1726 try:
1727 await waiter
1728 except:
1729 transport.close()
1730 raise
1731
1732 if self._debug:
1733 logger.debug('Write pipe %r connected: (%r, %r)',
1734 pipe.fileno(), transport, protocol)
1735 return transport, protocol
1736
1737 def _log_subprocess(self, msg, stdin, stdout, stderr):
1738 info = [msg]
1739 if stdin is not None:
1740 info.append(f'stdin={_format_pipe(stdin)}')
1741 if stdout is not None and stderr == subprocess.STDOUT:
1742 info.append(f'stdout=stderr={_format_pipe(stdout)}')
1743 else:
1744 if stdout is not None:
1745 info.append(f'stdout={_format_pipe(stdout)}')
1746 if stderr is not None:
1747 info.append(f'stderr={_format_pipe(stderr)}')
1748 logger.debug(' '.join(info))
1749
1750 async def subprocess_shell(self, protocol_factory, cmd, *,
1751 stdin=subprocess.PIPE,
1752 stdout=subprocess.PIPE,
1753 stderr=subprocess.PIPE,
1754 universal_newlines=False,
1755 shell=True, bufsize=0,
1756 encoding=None, errors=None, text=None,
1757 **kwargs):
1758 if not isinstance(cmd, (bytes, str)):
1759 raise ValueError("cmd must be a string")
1760 if universal_newlines:
1761 raise ValueError("universal_newlines must be False")
1762 if not shell:
1763 raise ValueError("shell must be True")
1764 if bufsize != 0:
1765 raise ValueError("bufsize must be 0")
1766 if text:
1767 raise ValueError("text must be False")
1768 if encoding is not None:
1769 raise ValueError("encoding must be None")
1770 if errors is not None:
1771 raise ValueError("errors must be None")
1772
1773 protocol = protocol_factory()
1774 debug_log = None
1775 if self._debug:
1776 # don't log parameters: they may contain sensitive information
1777 # (password) and may be too long
1778 debug_log = 'run shell command %r' % cmd
1779 self._log_subprocess(debug_log, stdin, stdout, stderr)
1780 transport = await self._make_subprocess_transport(
1781 protocol, cmd, True, stdin, stdout, stderr, bufsize, **kwargs)
1782 if self._debug and debug_log is not None:
1783 logger.info('%s: %r', debug_log, transport)
1784 return transport, protocol
1785
1786 async def subprocess_exec(self, protocol_factory, program, *args,
1787 stdin=subprocess.PIPE, stdout=subprocess.PIPE,
1788 stderr=subprocess.PIPE, universal_newlines=False,
1789 shell=False, bufsize=0,
1790 encoding=None, errors=None, text=None,
1791 **kwargs):
1792 if universal_newlines:
1793 raise ValueError("universal_newlines must be False")
1794 if shell:
1795 raise ValueError("shell must be False")
1796 if bufsize != 0:
1797 raise ValueError("bufsize must be 0")
1798 if text:
1799 raise ValueError("text must be False")
1800 if encoding is not None:
1801 raise ValueError("encoding must be None")
1802 if errors is not None:
1803 raise ValueError("errors must be None")
1804
1805 popen_args = (program,) + args
1806 protocol = protocol_factory()
1807 debug_log = None
1808 if self._debug:
1809 # don't log parameters: they may contain sensitive information
1810 # (password) and may be too long
1811 debug_log = f'execute program {program!r}'
1812 self._log_subprocess(debug_log, stdin, stdout, stderr)
1813 transport = await self._make_subprocess_transport(
1814 protocol, popen_args, False, stdin, stdout, stderr,
1815 bufsize, **kwargs)
1816 if self._debug and debug_log is not None:
1817 logger.info('%s: %r', debug_log, transport)
1818 return transport, protocol
1819
1820 def get_exception_handler(self):
1821 """Return an exception handler, or None if the default one is in use.
1822 """
1823 return self._exception_handler
1824
1825 def set_exception_handler(self, handler):
1826 """Set handler as the new event loop exception handler.
1827
1828 If handler is None, the default exception handler will
1829 be set.
1830
1831 If handler is a callable object, it should have a
1832 signature matching '(loop, context)', where 'loop'
1833 will be a reference to the active event loop, 'context'
1834 will be a dict object (see `call_exception_handler()`
1835 documentation for details about context).
1836 """
1837 if handler is not None and not callable(handler):
1838 raise TypeError(f'A callable object or None is expected, '
1839 f'got {handler!r}')
1840 self._exception_handler = handler
1841
1842 def default_exception_handler(self, context):
1843 """Default exception handler.
1844
1845 This is called when an exception occurs and no exception
1846 handler is set, and can be called by a custom exception
1847 handler that wants to defer to the default behavior.
1848
1849 This default handler logs the error message and other
1850 context-dependent information. In debug mode, a truncated
1851 stack trace is also appended showing where the given object
1852 (e.g. a handle or future or task) was created, if any.
1853
1854 The context parameter has the same meaning as in
1855 `call_exception_handler()`.
1856 """
1857 message = context.get('message')
1858 if not message:
1859 message = 'Unhandled exception in event loop'
1860
1861 exception = context.get('exception')
1862 if exception is not None:
1863 exc_info = (type(exception), exception, exception.__traceback__)
1864 else:
1865 exc_info = False
1866
1867 if ('source_traceback' not in context and
1868 self._current_handle is not None and
1869 self._current_handle._source_traceback):
1870 context['handle_traceback'] = \
1871 self._current_handle._source_traceback
1872
1873 log_lines = [message]
1874 for key in sorted(context):
1875 if key in {'message', 'exception'}:
1876 continue
1877 value = context[key]
1878 if key == 'source_traceback':
1879 tb = ''.join(traceback.format_list(value))
1880 value = 'Object created at (most recent call last):\n'
1881 value += tb.rstrip()
1882 elif key == 'handle_traceback':
1883 tb = ''.join(traceback.format_list(value))
1884 value = 'Handle created at (most recent call last):\n'
1885 value += tb.rstrip()
1886 else:
1887 value = repr(value)
1888 log_lines.append(f'{key}: {value}')
1889
1890 logger.error('\n'.join(log_lines), exc_info=exc_info)
1891
1892 def call_exception_handler(self, context):
1893 """Call the current event loop's exception handler.
1894
1895 The context argument is a dict containing the following keys:
1896
1897 - 'message': Error message;
1898 - 'exception' (optional): Exception object;
1899 - 'future' (optional): Future instance;
1900 - 'task' (optional): Task instance;
1901 - 'handle' (optional): Handle instance;
1902 - 'protocol' (optional): Protocol instance;
1903 - 'transport' (optional): Transport instance;
1904 - 'socket' (optional): Socket instance;
1905 - 'source_traceback' (optional): Traceback of the source;
1906 - 'handle_traceback' (optional): Traceback of the handle;
1907 - 'asyncgen' (optional): Asynchronous generator that caused
1908 the exception.
1909
1910 New keys maybe introduced in the future.
1911
1912 Note: do not overload this method in an event loop subclass.
1913 For custom exception handling, use the
1914 `set_exception_handler()` method.
1915 """
1916 if self._exception_handler is None:
1917 try:
1918 self.default_exception_handler(context)
1919 except (SystemExit, KeyboardInterrupt):
1920 raise
1921 except BaseException:
1922 # Second protection layer for unexpected errors
1923 # in the default implementation, as well as for subclassed
1924 # event loops with overloaded "default_exception_handler".
1925 logger.error('Exception in default exception handler',
1926 exc_info=True)
1927 else:
1928 try:
1929 ctx = None
1930 thing = context.get("task")
1931 if thing is None:
1932 # Even though Futures don't have a context,
1933 # Task is a subclass of Future,
1934 # and sometimes the 'future' key holds a Task.
1935 thing = context.get("future")
1936 if thing is None:
1937 # Handles also have a context.
1938 thing = context.get("handle")
1939 if thing is not None and hasattr(thing, "get_context"):
1940 ctx = thing.get_context()
1941 if ctx is not None and hasattr(ctx, "run"):
1942 ctx.run(self._exception_handler, self, context)
1943 else:
1944 self._exception_handler(self, context)
1945 except (SystemExit, KeyboardInterrupt):
1946 raise
1947 except BaseException as exc:
1948 # Exception in the user set custom exception handler.
1949 try:
1950 # Let's try default handler.
1951 self.default_exception_handler({
1952 'message': 'Unhandled error in exception handler',
1953 'exception': exc,
1954 'context': context,
1955 })
1956 except (SystemExit, KeyboardInterrupt):
1957 raise
1958 except BaseException:
1959 # Guard 'default_exception_handler' in case it is
1960 # overloaded.
1961 logger.error('Exception in default exception handler '
1962 'while handling an unexpected error '
1963 'in custom exception handler',
1964 exc_info=True)
1965
1966 def _add_callback(self, handle):
1967 """Add a Handle to _ready."""
1968 if not handle._cancelled:
1969 self._ready.append(handle)
1970
1971 def _add_callback_signalsafe(self, handle):
1972 """Like _add_callback() but called from a signal handler."""
1973 self._add_callback(handle)
1974 self._write_to_self()
1975
1976 def _timer_handle_cancelled(self, handle):
1977 """Notification that a TimerHandle has been cancelled."""
1978 if handle._scheduled:
1979 self._timer_cancelled_count += 1
1980
1981 def _run_once(self):
1982 """Run one full iteration of the event loop.
1983
1984 This calls all currently ready callbacks, polls for I/O,
1985 schedules the resulting callbacks, and finally schedules
1986 'call_later' callbacks.
1987 """
1988
1989 sched_count = len(self._scheduled)
1990 if (sched_count > _MIN_SCHEDULED_TIMER_HANDLES and
1991 self._timer_cancelled_count / sched_count >
1992 _MIN_CANCELLED_TIMER_HANDLES_FRACTION):
1993 # Remove delayed calls that were cancelled if their number
1994 # is too high
1995 new_scheduled = []
1996 for handle in self._scheduled:
1997 if handle._cancelled:
1998 handle._scheduled = False
1999 else:
2000 new_scheduled.append(handle)
2001
2002 heapq.heapify(new_scheduled)
2003 self._scheduled = new_scheduled
2004 self._timer_cancelled_count = 0
2005 else:
2006 # Remove delayed calls that were cancelled from head of queue.
2007 while self._scheduled and self._scheduled[0]._cancelled:
2008 self._timer_cancelled_count -= 1
2009 handle = heapq.heappop(self._scheduled)
2010 handle._scheduled = False
2011
2012 timeout = None
2013 if self._ready or self._stopping:
2014 timeout = 0
2015 elif self._scheduled:
2016 # Compute the desired timeout.
2017 timeout = self._scheduled[0]._when - self.time()
2018 if timeout > MAXIMUM_SELECT_TIMEOUT:
2019 timeout = MAXIMUM_SELECT_TIMEOUT
2020 elif timeout < 0:
2021 timeout = 0
2022
2023 event_list = self._selector.select(timeout)
2024 self._process_events(event_list)
2025 # Needed to break cycles when an exception occurs.
2026 event_list = None
2027
2028 # Handle 'later' callbacks that are ready.
2029 end_time = self.time() + self._clock_resolution
2030 while self._scheduled:
2031 handle = self._scheduled[0]
2032 if handle._when >= end_time:
2033 break
2034 handle = heapq.heappop(self._scheduled)
2035 handle._scheduled = False
2036 self._ready.append(handle)
2037
2038 # This is the only place where callbacks are actually *called*.
2039 # All other places just add them to ready.
2040 # Note: We run all currently scheduled callbacks, but not any
2041 # callbacks scheduled by callbacks run this time around --
2042 # they will be run the next time (after another I/O poll).
2043 # Use an idiom that is thread-safe without using locks.
2044 ntodo = len(self._ready)
2045 for i in range(ntodo):
2046 handle = self._ready.popleft()
2047 if handle._cancelled:
2048 continue
2049 if self._debug:
2050 try:
2051 self._current_handle = handle
2052 t0 = self.time()
2053 handle._run()
2054 dt = self.time() - t0
2055 if dt >= self.slow_callback_duration:
2056 logger.warning('Executing %s took %.3f seconds',
2057 _format_handle(handle), dt)
2058 finally:
2059 self._current_handle = None
2060 else:
2061 handle._run()
2062 handle = None # Needed to break cycles when an exception occurs.
2063
2064 def _set_coroutine_origin_tracking(self, enabled):
2065 if bool(enabled) == bool(self._coroutine_origin_tracking_enabled):
2066 return
2067
2068 if enabled:
2069 self._coroutine_origin_tracking_saved_depth = (
2070 sys.get_coroutine_origin_tracking_depth())
2071 sys.set_coroutine_origin_tracking_depth(
2072 constants.DEBUG_STACK_DEPTH)
2073 else:
2074 sys.set_coroutine_origin_tracking_depth(
2075 self._coroutine_origin_tracking_saved_depth)
2076
2077 self._coroutine_origin_tracking_enabled = enabled
2078
2079 def get_debug(self):
2080 return self._debug
2081
2082 def set_debug(self, enabled):
2083 self._debug = enabled
2084
2085 if self.is_running():
2086 self.call_soon_threadsafe(self._set_coroutine_origin_tracking, enabled)