Debian Codegraph @

Lib/asyncio/base_events.py

raw
1 """Base implementation of event loop.
2
3 The event loop can be broken up into a multiplexer (the part
4 responsible for notifying us of I/O events) and the event loop proper,
5 which wraps a multiplexer with functionality for scheduling callbacks,
6 immediately or at a given time in the future.
7
8 Whenever a public API takes a callback, subsequent positional
9 arguments will be passed to the callback if/when it is called. This
10 avoids the proliferation of trivial lambdas implementing closures.
11 Keyword arguments for the callback are not supported; this is a
12 conscious design decision, leaving the door open for keyword arguments
13 to modify the meaning of the API call itself.
14 """
15
16 import collections
17 import collections.abc
18 import concurrent.futures
19 import errno
20 import heapq
21 import itertools
22 import os
23 import socket
24 import stat
25 import subprocess
26 import threading
27 import time
28 import traceback
29 import sys
30 import warnings
31 import weakref
32
33 try:
34 import ssl
35 except ImportError: # pragma: no cover
36 ssl = None
37
38 from . import constants
39 from . import coroutines
40 from . import events
41 from . import exceptions
42 from . import futures
43 from . import protocols
44 from . import sslproto
45 from . import staggered
46 from . import tasks
47 from . import timeouts
48 from . import transports
49 from . import trsock
50 from .log import logger
51
52
53 __all__ = 'BaseEventLoop','Server',
54
55
56 # Minimum number of _scheduled timer handles before cleanup of
57 # cancelled handles is performed.
58 _MIN_SCHEDULED_TIMER_HANDLES = 100
59
60 # Minimum fraction of _scheduled timer handles that are cancelled
61 # before cleanup of cancelled handles is performed.
62 _MIN_CANCELLED_TIMER_HANDLES_FRACTION = 0.5
63
64
65 _HAS_IPv6 = hasattr(socket, 'AF_INET6')
66
67 # Maximum timeout passed to select to avoid OS limitations
68 MAXIMUM_SELECT_TIMEOUT = 24 * 3600
69
70
71 def _format_handle(handle):
72 cb = handle._callback
73 if isinstance(getattr(cb, '__self__', None), tasks.Task):
74 # format the task
75 return repr(cb.__self__)
76 else:
77 return str(handle)
78
79
80 def _format_pipe(fd):
81 if fd == subprocess.PIPE:
82 return '<pipe>'
83 elif fd == subprocess.STDOUT:
84 return '<stdout>'
85 else:
86 return repr(fd)
87
88
89 def _set_reuseport(sock):
90 if not hasattr(socket, 'SO_REUSEPORT'):
91 raise ValueError('reuse_port not supported by socket module')
92 else:
93 try:
94 sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEPORT, 1)
95 except OSError:
96 raise ValueError('reuse_port not supported by socket module, '
97 'SO_REUSEPORT defined but not implemented.')
98
99
100 def _ipaddr_info(host, port, family, type, proto, flowinfo=0, scopeid=0):
101 # Try to skip getaddrinfo if "host" is already an IP. Users might have
102 # handled name resolution in their own code and pass in resolved IPs.
103 if not hasattr(socket, 'inet_pton'):
104 return
105
106 if proto not in {0, socket.IPPROTO_TCP, socket.IPPROTO_UDP} or \
107 host is None:
108 return None
109
110 if type == socket.SOCK_STREAM:
111 proto = socket.IPPROTO_TCP
112 elif type == socket.SOCK_DGRAM:
113 proto = socket.IPPROTO_UDP
114 else:
115 return None
116
117 if port is None:
118 port = 0
119 elif isinstance(port, bytes) and port == b'':
120 port = 0
121 elif isinstance(port, str) and port == '':
122 port = 0
123 else:
124 # If port's a service name like "http", don't skip getaddrinfo.
125 try:
126 port = int(port)
127 except (TypeError, ValueError):
128 return None
129
130 if family == socket.AF_UNSPEC:
131 afs = [socket.AF_INET]
132 if _HAS_IPv6:
133 afs.append(socket.AF_INET6)
134 else:
135 afs = [family]
136
137 if isinstance(host, bytes):
138 host = host.decode('idna')
139 if '%' in host:
140 # Linux's inet_pton doesn't accept an IPv6 zone index after host,
141 # like '::1%lo0'.
142 return None
143
144 for af in afs:
145 try:
146 socket.inet_pton(af, host)
147 # The host has already been resolved.
148 if _HAS_IPv6 and af == socket.AF_INET6:
149 return af, type, proto, '', (host, port, flowinfo, scopeid)
150 else:
151 return af, type, proto, '', (host, port)
152 except OSError:
153 pass
154
155 # "host" is not an IP address.
156 return None
157
158
159 def _interleave_addrinfos(addrinfos, first_address_family_count=1):
160 """Interleave list of addrinfo tuples by family."""
161 # Group addresses by family
162 addrinfos_by_family = collections.OrderedDict()
163 for addr in addrinfos:
164 family = addr[0]
165 if family not in addrinfos_by_family:
166 addrinfos_by_family[family] = []
167 addrinfos_by_family[family].append(addr)
168 addrinfos_lists = list(addrinfos_by_family.values())
169
170 reordered = []
171 if first_address_family_count > 1:
172 reordered.extend(addrinfos_lists[0][:first_address_family_count - 1])
173 del addrinfos_lists[0][:first_address_family_count - 1]
174 reordered.extend(
175 a for a in itertools.chain.from_iterable(
176 itertools.zip_longest(*addrinfos_lists)
177 ) if a is not None)
178 return reordered
179
180
181 def _run_until_complete_cb(fut):
182 if not fut.cancelled():
183 exc = fut.exception()
184 if isinstance(exc, (SystemExit, KeyboardInterrupt)):
185 # Issue #22429: run_forever() already finished, no need to
186 # stop it.
187 return
188 futures._get_loop(fut).stop()
189
190
191 if hasattr(socket, 'TCP_NODELAY'):
192 def _set_nodelay(sock):
193 if (sock.family in {socket.AF_INET, socket.AF_INET6} and
194 sock.type == socket.SOCK_STREAM and
195 sock.proto == socket.IPPROTO_TCP):
196 sock.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1)
197 else:
198 def _set_nodelay(sock):
199 pass
200
201
202 def _check_ssl_socket(sock):
203 if ssl is not None and isinstance(sock, ssl.SSLSocket):
204 raise TypeError("Socket cannot be of type SSLSocket")
205
206
207 class _SendfileFallbackProtocol(protocols.Protocol):
208 def __init__(self, transp):
209 if not isinstance(transp, transports._FlowControlMixin):
210 raise TypeError("transport should be _FlowControlMixin instance")
211 self._transport = transp
212 self._proto = transp.get_protocol()
213 self._should_resume_reading = transp.is_reading()
214 self._should_resume_writing = transp._protocol_paused
215 transp.pause_reading()
216 transp.set_protocol(self)
217 if self._should_resume_writing:
218 self._write_ready_fut = self._transport._loop.create_future()
219 else:
220 self._write_ready_fut = None
221
222 async def drain(self):
223 if self._transport.is_closing():
224 raise ConnectionError("Connection closed by peer")
225 fut = self._write_ready_fut
226 if fut is None:
227 return
228 await fut
229
230 def connection_made(self, transport):
231 raise RuntimeError("Invalid state: "
232 "connection should have been established already.")
233
234 def connection_lost(self, exc):
235 if self._write_ready_fut is not None:
236 # Never happens if peer disconnects after sending the whole content
237 # Thus disconnection is always an exception from user perspective
238 if exc is None:
239 self._write_ready_fut.set_exception(
240 ConnectionError("Connection is closed by peer"))
241 else:
242 self._write_ready_fut.set_exception(exc)
243 self._proto.connection_lost(exc)
244
245 def pause_writing(self):
246 if self._write_ready_fut is not None:
247 return
248 self._write_ready_fut = self._transport._loop.create_future()
249
250 def resume_writing(self):
251 if self._write_ready_fut is None:
252 return
253 self._write_ready_fut.set_result(False)
254 self._write_ready_fut = None
255
256 def data_received(self, data):
257 raise RuntimeError("Invalid state: reading should be paused")
258
259 def eof_received(self):
260 raise RuntimeError("Invalid state: reading should be paused")
261
262 async def restore(self):
263 self._transport.set_protocol(self._proto)
264 if self._should_resume_reading:
265 self._transport.resume_reading()
266 if self._write_ready_fut is not None:
267 # Cancel the future.
268 # Basically it has no effect because protocol is switched back,
269 # no code should wait for it anymore.
270 self._write_ready_fut.cancel()
271 if self._should_resume_writing:
272 self._proto.resume_writing()
273
274
275 class Server(events.AbstractServer):
276
277 def __init__(self, loop, sockets, protocol_factory, ssl_context, backlog,
278 ssl_handshake_timeout, ssl_shutdown_timeout=None):
279 self._loop = loop
280 self._sockets = sockets
281 # Weak references so we don't break Transport's ability to
282 # detect abandoned transports
283 self._clients = weakref.WeakSet()
284 self._waiters = []
285 self._protocol_factory = protocol_factory
286 self._backlog = backlog
287 self._ssl_context = ssl_context
288 self._ssl_handshake_timeout = ssl_handshake_timeout
289 self._ssl_shutdown_timeout = ssl_shutdown_timeout
290 self._serving = False
291 self._serving_forever_fut = None
292
293 def __repr__(self):
294 return f'<{self.__class__.__name__} sockets={self.sockets!r}>'
295
296 def _attach(self, transport):
297 assert self._sockets is not None
298 self._clients.add(transport)
299
300 def _detach(self, transport):
301 self._clients.discard(transport)
302 if len(self._clients) == 0 and self._sockets is None:
303 self._wakeup()
304
305 def _wakeup(self):
306 waiters = self._waiters
307 self._waiters = None
308 for waiter in waiters:
309 if not waiter.done():
310 waiter.set_result(None)
311
312 def _start_serving(self):
313 if self._serving:
314 return
315 self._serving = True
316 for sock in self._sockets:
317 sock.listen(self._backlog)
318 self._loop._start_serving(
319 self._protocol_factory, sock, self._ssl_context,
320 self, self._backlog, self._ssl_handshake_timeout,
321 self._ssl_shutdown_timeout)
322
323 def get_loop(self):
324 return self._loop
325
326 def is_serving(self):
327 return self._serving
328
329 @property
330 def sockets(self):
331 if self._sockets is None:
332 return ()
333 return tuple(trsock.TransportSocket(s) for s in self._sockets)
334
335 def close(self):
336 sockets = self._sockets
337 if sockets is None:
338 return
339 self._sockets = None
340
341 for sock in sockets:
342 self._loop._stop_serving(sock)
343
344 self._serving = False
345
346 if (self._serving_forever_fut is not None and
347 not self._serving_forever_fut.done()):
348 self._serving_forever_fut.cancel()
349 self._serving_forever_fut = None
350
351 if len(self._clients) == 0:
352 self._wakeup()
353
354 def close_clients(self):
355 for transport in self._clients.copy():
356 transport.close()
357
358 def abort_clients(self):
359 for transport in self._clients.copy():
360 transport.abort()
361
362 async def start_serving(self):
363 self._start_serving()
364 # Skip one loop iteration so that all 'loop.add_reader'
365 # go through.
366 await tasks.sleep(0)
367
368 async def serve_forever(self):
369 if self._serving_forever_fut is not None:
370 raise RuntimeError(
371 f'server {self!r} is already being awaited on serve_forever()')
372 if self._sockets is None:
373 raise RuntimeError(f'server {self!r} is closed')
374
375 self._start_serving()
376 self._serving_forever_fut = self._loop.create_future()
377
378 try:
379 await self._serving_forever_fut
380 except exceptions.CancelledError:
381 try:
382 self.close()
383 self.close_clients()
384 await self.wait_closed()
385 finally:
386 raise
387 finally:
388 self._serving_forever_fut = None
389
390 async def wait_closed(self):
391 """Wait until server is closed and all connections are dropped.
392
393 - If the server is not closed, wait.
394 - If it is closed, but there are still active connections, wait.
395
396 Anyone waiting here will be unblocked once both conditions
397 (server is closed and all connections have been dropped)
398 have become true, in either order.
399
400 Historical note: In 3.11 and before, this was broken, returning
401 immediately if the server was already closed, even if there
402 were still active connections. An attempted fix in 3.12.0 was
403 still broken, returning immediately if the server was still
404 open and there were no active connections. Hopefully in 3.12.1
405 we have it right.
406 """
407 # Waiters are unblocked by self._wakeup(), which is called
408 # from two places: self.close() and self._detach(), but only
409 # when both conditions have become true. To signal that this
410 # has happened, self._wakeup() sets self._waiters to None.
411 if self._waiters is None:
412 return
413 waiter = self._loop.create_future()
414 self._waiters.append(waiter)
415 await waiter
416
417
418 class BaseEventLoop(events.AbstractEventLoop):
419
420 def __init__(self):
421 self._timer_cancelled_count = 0
422 self._closed = False
423 self._stopping = False
424 self._ready = collections.deque()
425 self._scheduled = []
426 self._default_executor = None
427 self._internal_fds = 0
428 # Identifier of the thread running the event loop, or None if the
429 # event loop is not running
430 self._thread_id = None
431 self._clock_resolution = time.get_clock_info('monotonic').resolution
432 self._exception_handler = None
433 self.set_debug(coroutines._is_debug_mode())
434 # The preserved state of async generator hooks.
435 self._old_agen_hooks = None
436 # In debug mode, if the execution of a callback or a step of a task
437 # exceed this duration in seconds, the slow callback/task is logged.
438 self.slow_callback_duration = 0.1
439 self._current_handle = None
440 self._task_factory = None
441 self._coroutine_origin_tracking_enabled = False
442 self._coroutine_origin_tracking_saved_depth = None
443
444 # A weak set of all asynchronous generators that are
445 # being iterated by the loop.
446 self._asyncgens = weakref.WeakSet()
447 # Set to True when `loop.shutdown_asyncgens` is called.
448 self._asyncgens_shutdown_called = False
449 # Set to True when `loop.shutdown_default_executor` is called.
450 self._executor_shutdown_called = False
451
452 def __repr__(self):
453 return (
454 f'<{self.__class__.__name__} running={self.is_running()} '
455 f'closed={self.is_closed()} debug={self.get_debug()}>'
456 )
457
458 def create_future(self):
459 """Create a Future object attached to the loop."""
460 return futures.Future(loop=self)
461
462 def create_task(self, coro, **kwargs):
463 """Schedule or begin executing a coroutine object.
464
465 Return a task object.
466 """
467 self._check_closed()
468 if self._task_factory is not None:
469 return self._task_factory(self, coro, **kwargs)
470
471 task = tasks.Task(coro, loop=self, **kwargs)
472 if task._source_traceback:
473 del task._source_traceback[-1]
474 try:
475 return task
476 finally:
477 # gh-128552: prevent a refcycle of
478 # task.exception().__traceback__->BaseEventLoop.create_task->task
479 del task
480
481 def set_task_factory(self, factory):
482 """Set a task factory that will be used by loop.create_task().
483
484 If factory is None the default task factory will be set.
485
486 If factory is a callable, it should have a signature matching
487 '(loop, coro, **kwargs)', where 'loop' will be a reference to the
488 active event loop, 'coro' will be a coroutine object, and **kwargs
489 will be arbitrary keyword arguments that should be passed on to
490 Task. The callable must return a Task.
491 """
492 if factory is not None and not callable(factory):
493 raise TypeError('task factory must be a callable or None')
494 self._task_factory = factory
495
496 def get_task_factory(self):
497 """Return a task factory, or None if the default one is in use."""
498 return self._task_factory
499
500 def _make_socket_transport(self, sock, protocol, waiter=None, *,
501 extra=None, server=None):
502 """Create socket transport."""
503 raise NotImplementedError
504
505 def _make_ssl_transport(
506 self, rawsock, protocol, sslcontext, waiter=None,
507 *, server_side=False, server_hostname=None,
508 extra=None, server=None,
509 ssl_handshake_timeout=None,
510 ssl_shutdown_timeout=None,
511 call_connection_made=True):
512 """Create SSL transport."""
513 raise NotImplementedError
514
515 def _make_datagram_transport(self, sock, protocol,
516 address=None, waiter=None, extra=None):
517 """Create datagram transport."""
518 raise NotImplementedError
519
520 def _make_read_pipe_transport(self, pipe, protocol, waiter=None,
521 extra=None):
522 """Create read pipe transport."""
523 raise NotImplementedError
524
525 def _make_write_pipe_transport(self, pipe, protocol, waiter=None,
526 extra=None):
527 """Create write pipe transport."""
528 raise NotImplementedError
529
530 async def _make_subprocess_transport(self, protocol, args, shell,
531 stdin, stdout, stderr, bufsize,
532 extra=None, **kwargs):
533 """Create subprocess transport."""
534 raise NotImplementedError
535
536 def _write_to_self(self):
537 """Write a byte to self-pipe, to wake up the event loop.
538
539 This may be called from a different thread.
540
541 The subclass is responsible for implementing the self-pipe.
542 """
543 raise NotImplementedError
544
545 def _process_events(self, event_list):
546 """Process selector events."""
547 raise NotImplementedError
548
549 def _check_closed(self):
550 if self._closed:
551 raise RuntimeError('Event loop is closed')
552
553 def _check_default_executor(self):
554 if self._executor_shutdown_called:
555 raise RuntimeError('Executor shutdown has been called')
556
557 def _asyncgen_finalizer_hook(self, agen):
558 self._asyncgens.discard(agen)
559 if not self.is_closed():
560 self.call_soon_threadsafe(self.create_task, agen.aclose())
561
562 def _asyncgen_firstiter_hook(self, agen):
563 if self._asyncgens_shutdown_called:
564 warnings.warn(
565 f"asynchronous generator {agen!r} was scheduled after "
566 f"loop.shutdown_asyncgens() call",
567 ResourceWarning, source=self)
568
569 self._asyncgens.add(agen)
570
571 async def shutdown_asyncgens(self):
572 """Shutdown all active asynchronous generators."""
573 self._asyncgens_shutdown_called = True
574
575 if not len(self._asyncgens):
576 # If Python version is <3.6 or we don't have any asynchronous
577 # generators alive.
578 return
579
580 closing_agens = list(self._asyncgens)
581 self._asyncgens.clear()
582
583 results = await tasks.gather(
584 *[ag.aclose() for ag in closing_agens],
585 return_exceptions=True)
586
587 for result, agen in zip(results, closing_agens):
588 if isinstance(result, Exception):
589 self.call_exception_handler({
590 'message': f'an error occurred during closing of '
591 f'asynchronous generator {agen!r}',
592 'exception': result,
593 'asyncgen': agen
594 })
595
596 async def shutdown_default_executor(self, timeout=None):
597 """Schedule the shutdown of the default executor.
598
599 The timeout parameter specifies the amount of time the executor will
600 be given to finish joining. The default value is None, which means
601 that the executor will be given an unlimited amount of time.
602 """
603 self._executor_shutdown_called = True
604 if self._default_executor is None:
605 return
606 future = self.create_future()
607 thread = threading.Thread(target=self._do_shutdown, args=(future,))
608 thread.start()
609 try:
610 async with timeouts.timeout(timeout):
611 await future
612 except TimeoutError:
613 warnings.warn("The executor did not finishing joining "
614 f"its threads within {timeout} seconds.",
615 RuntimeWarning, stacklevel=2)
616 self._default_executor.shutdown(wait=False)
617 else:
618 thread.join()
619
620 def _do_shutdown(self, future):
621 try:
622 self._default_executor.shutdown(wait=True)
623 if not self.is_closed():
624 self.call_soon_threadsafe(futures._set_result_unless_cancelled,
625 future, None)
626 except Exception as ex:
627 if not self.is_closed() and not future.cancelled():
628 self.call_soon_threadsafe(future.set_exception, ex)
629
630 def _check_running(self):
631 if self.is_running():
632 raise RuntimeError('This event loop is already running')
633 if events._get_running_loop() is not None:
634 raise RuntimeError(
635 'Cannot run the event loop while another loop is running')
636
637 def _run_forever_setup(self):
638 """Prepare the run loop to process events.
639
640 This method exists so that custom event loop subclasses (e.g., event loops
641 that integrate a GUI event loop with Python's event loop) have access to all the
642 loop setup logic.
643 """
644 self._check_closed()
645 self._check_running()
646 self._set_coroutine_origin_tracking(self._debug)
647
648 self._old_agen_hooks = sys.get_asyncgen_hooks()
649 self._thread_id = threading.get_ident()
650 sys.set_asyncgen_hooks(
651 firstiter=self._asyncgen_firstiter_hook,
652 finalizer=self._asyncgen_finalizer_hook
653 )
654
655 events._set_running_loop(self)
656
657 def _run_forever_cleanup(self):
658 """Clean up after an event loop finishes the looping over events.
659
660 This method exists so that custom event loop subclasses (e.g., event loops
661 that integrate a GUI event loop with Python's event loop) have access to all the
662 loop cleanup logic.
663 """
664 self._stopping = False
665 self._thread_id = None
666 events._set_running_loop(None)
667 self._set_coroutine_origin_tracking(False)
668 # Restore any pre-existing async generator hooks.
669 if self._old_agen_hooks is not None:
670 sys.set_asyncgen_hooks(*self._old_agen_hooks)
671 self._old_agen_hooks = None
672
673 def run_forever(self):
674 """Run until stop() is called."""
675 self._run_forever_setup()
676 try:
677 while True:
678 self._run_once()
679 if self._stopping:
680 break
681 finally:
682 self._run_forever_cleanup()
683
684 def run_until_complete(self, future):
685 """Run until the Future is done.
686
687 If the argument is a coroutine, it is wrapped in a Task.
688
689 WARNING: It would be disastrous to call run_until_complete()
690 with the same coroutine twice -- it would wrap it in two
691 different Tasks and that can't be good.
692
693 Return the Future's result, or raise its exception.
694 """
695 self._check_closed()
696 self._check_running()
697
698 new_task = not futures.isfuture(future)
699 future = tasks.ensure_future(future, loop=self)
700 if new_task:
701 # An exception is raised if the future didn't complete, so there
702 # is no need to log the "destroy pending task" message
703 future._log_destroy_pending = False
704
705 future.add_done_callback(_run_until_complete_cb)
706 try:
707 self.run_forever()
708 except:
709 if new_task and future.done() and not future.cancelled():
710 # The coroutine raised a BaseException. Consume the exception
711 # to not log a warning, the caller doesn't have access to the
712 # local task.
713 future.exception()
714 raise
715 finally:
716 future.remove_done_callback(_run_until_complete_cb)
717 if not future.done():
718 raise RuntimeError('Event loop stopped before Future completed.')
719
720 return future.result()
721
722 def stop(self):
723 """Stop running the event loop.
724
725 Every callback already scheduled will still run. This simply
726 informs run_forever to stop looping after a complete iteration.
727 """
728 self._stopping = True
729
730 def close(self):
731 """Close the event loop.
732
733 This clears the queues and shuts down the executor,
734 but does not wait for the executor to finish.
735
736 The event loop must not be running.
737 """
738 if self.is_running():
739 raise RuntimeError("Cannot close a running event loop")
740 if self._closed:
741 return
742 if self._debug:
743 logger.debug("Close %r", self)
744 self._closed = True
745 self._ready.clear()
746 self._scheduled.clear()
747 self._executor_shutdown_called = True
748 executor = self._default_executor
749 if executor is not None:
750 self._default_executor = None
751 executor.shutdown(wait=False)
752
753 def is_closed(self):
754 """Returns True if the event loop was closed."""
755 return self._closed
756
757 def __del__(self, _warn=warnings.warn):
758 if not self.is_closed():
759 _warn(f"unclosed event loop {self!r}", ResourceWarning, source=self)
760 if not self.is_running():
761 self.close()
762
763 def is_running(self):
764 """Returns True if the event loop is running."""
765 return (self._thread_id is not None)
766
767 def time(self):
768 """Return the time according to the event loop's clock.
769
770 This is a float expressed in seconds since an epoch, but the
771 epoch, precision, accuracy and drift are unspecified and may
772 differ per event loop.
773 """
774 return time.monotonic()
775
776 def call_later(self, delay, callback, *args, context=None):
777 """Arrange for a callback to be called at a given time.
778
779 Return a Handle: an opaque object with a cancel() method that
780 can be used to cancel the call.
781
782 The delay can be an int or float, expressed in seconds. It is
783 always relative to the current time.
784
785 Each callback will be called exactly once. If two callbacks
786 are scheduled for exactly the same time, it is undefined which
787 will be called first.
788
789 Any positional arguments after the callback will be passed to
790 the callback when it is called.
791 """
792 if delay is None:
793 raise TypeError('delay must not be None')
794 timer = self.call_at(self.time() + delay, callback, *args,
795 context=context)
796 if timer._source_traceback:
797 del timer._source_traceback[-1]
798 return timer
799
800 def call_at(self, when, callback, *args, context=None):
801 """Like call_later(), but uses an absolute time.
802
803 Absolute time corresponds to the event loop's time() method.
804 """
805 if when is None:
806 raise TypeError("when cannot be None")
807 self._check_closed()
808 if self._debug:
809 self._check_thread()
810 self._check_callback(callback, 'call_at')
811 timer = events.TimerHandle(when, callback, args, self, context)
812 if timer._source_traceback:
813 del timer._source_traceback[-1]
814 heapq.heappush(self._scheduled, timer)
815 timer._scheduled = True
816 return timer
817
818 def call_soon(self, callback, *args, context=None):
819 """Arrange for a callback to be called as soon as possible.
820
821 This operates as a FIFO queue: callbacks are called in the
822 order in which they are registered. Each callback will be
823 called exactly once.
824
825 Any positional arguments after the callback will be passed to
826 the callback when it is called.
827 """
828 self._check_closed()
829 if self._debug:
830 self._check_thread()
831 self._check_callback(callback, 'call_soon')
832 handle = self._call_soon(callback, args, context)
833 if handle._source_traceback:
834 del handle._source_traceback[-1]
835 return handle
836
837 def _check_callback(self, callback, method):
838 if (coroutines.iscoroutine(callback) or
839 coroutines._iscoroutinefunction(callback)):
840 raise TypeError(
841 f"coroutines cannot be used with {method}()")
842 if not callable(callback):
843 raise TypeError(
844 f'a callable object was expected by {method}(), '
845 f'got {callback!r}')
846
847 def _call_soon(self, callback, args, context):
848 handle = events.Handle(callback, args, self, context)
849 if handle._source_traceback:
850 del handle._source_traceback[-1]
851 self._ready.append(handle)
852 return handle
853
854 def _check_thread(self):
855 """Check that the current thread is the thread running the event loop.
856
857 Non-thread-safe methods of this class make this assumption and will
858 likely behave incorrectly when the assumption is violated.
859
860 Should only be called when (self._debug == True). The caller is
861 responsible for checking this condition for performance reasons.
862 """
863 if self._thread_id is None:
864 return
865 thread_id = threading.get_ident()
866 if thread_id != self._thread_id:
867 raise RuntimeError(
868 "Non-thread-safe operation invoked on an event loop other "
869 "than the current one")
870
871 def call_soon_threadsafe(self, callback, *args, context=None):
872 """Like call_soon(), but thread-safe."""
873 self._check_closed()
874 if self._debug:
875 self._check_callback(callback, 'call_soon_threadsafe')
876 handle = events._ThreadSafeHandle(callback, args, self, context)
877 self._ready.append(handle)
878 if handle._source_traceback:
879 del handle._source_traceback[-1]
880 if handle._source_traceback:
881 del handle._source_traceback[-1]
882 self._write_to_self()
883 return handle
884
885 def run_in_executor(self, executor, func, *args):
886 self._check_closed()
887 if self._debug:
888 self._check_callback(func, 'run_in_executor')
889 if executor is None:
890 executor = self._default_executor
891 # Only check when the default executor is being used
892 self._check_default_executor()
893 if executor is None:
894 executor = concurrent.futures.ThreadPoolExecutor(
895 thread_name_prefix='asyncio'
896 )
897 self._default_executor = executor
898 return futures.wrap_future(
899 executor.submit(func, *args), loop=self)
900
901 def set_default_executor(self, executor):
902 if not isinstance(executor, concurrent.futures.ThreadPoolExecutor):
903 raise TypeError('executor must be ThreadPoolExecutor instance')
904 self._default_executor = executor
905
906 def _getaddrinfo_debug(self, host, port, family, type, proto, flags):
907 msg = [f"{host}:{port!r}"]
908 if family:
909 msg.append(f'family={family!r}')
910 if type:
911 msg.append(f'type={type!r}')
912 if proto:
913 msg.append(f'proto={proto!r}')
914 if flags:
915 msg.append(f'flags={flags!r}')
916 msg = ', '.join(msg)
917 logger.debug('Get address info %s', msg)
918
919 t0 = self.time()
920 addrinfo = socket.getaddrinfo(host, port, family, type, proto, flags)
921 dt = self.time() - t0
922
923 msg = f'Getting address info {msg} took {dt * 1e3:.3f}ms: {addrinfo!r}'
924 if dt >= self.slow_callback_duration:
925 logger.info(msg)
926 else:
927 logger.debug(msg)
928 return addrinfo
929
930 async def getaddrinfo(self, host, port, *,
931 family=0, type=0, proto=0, flags=0):
932 if self._debug:
933 getaddr_func = self._getaddrinfo_debug
934 else:
935 getaddr_func = socket.getaddrinfo
936
937 return await self.run_in_executor(
938 None, getaddr_func, host, port, family, type, proto, flags)
939
940 async def getnameinfo(self, sockaddr, flags=0):
941 return await self.run_in_executor(
942 None, socket.getnameinfo, sockaddr, flags)
943
944 async def sock_sendfile(self, sock, file, offset=0, count=None,
945 *, fallback=True):
946 if self._debug and sock.gettimeout() != 0:
947 raise ValueError("the socket must be non-blocking")
948 _check_ssl_socket(sock)
949 self._check_sendfile_params(sock, file, offset, count)
950 try:
951 return await self._sock_sendfile_native(sock, file,
952 offset, count)
953 except exceptions.SendfileNotAvailableError as exc:
954 if not fallback:
955 raise
956 return await self._sock_sendfile_fallback(sock, file,
957 offset, count)
958
959 async def _sock_sendfile_native(self, sock, file, offset, count):
960 # NB: sendfile syscall is not supported for SSL sockets and
961 # non-mmap files even if sendfile is supported by OS
962 raise exceptions.SendfileNotAvailableError(
963 f"syscall sendfile is not available for socket {sock!r} "
964 f"and file {file!r} combination")
965
966 async def _sock_sendfile_fallback(self, sock, file, offset, count):
967 if hasattr(file, 'seek'):
968 file.seek(offset)
969 blocksize = (
970 min(count, constants.SENDFILE_FALLBACK_READBUFFER_SIZE)
971 if count else constants.SENDFILE_FALLBACK_READBUFFER_SIZE
972 )
973 buf = bytearray(blocksize)
974 total_sent = 0
975 try:
976 while True:
977 if count:
978 blocksize = min(count - total_sent, blocksize)
979 if blocksize <= 0:
980 break
981 view = memoryview(buf)[:blocksize]
982 read = await self.run_in_executor(None, file.readinto, view)
983 if not read:
984 break # EOF
985 await self.sock_sendall(sock, view[:read])
986 total_sent += read
987 return total_sent
988 finally:
989 if total_sent > 0 and hasattr(file, 'seek'):
990 file.seek(offset + total_sent)
991
992 def _check_sendfile_params(self, sock, file, offset, count):
993 if 'b' not in getattr(file, 'mode', 'b'):
994 raise ValueError("file should be opened in binary mode")
995 if not sock.type == socket.SOCK_STREAM:
996 raise ValueError("only SOCK_STREAM type sockets are supported")
997 if count is not None:
998 if not isinstance(count, int):
999 raise TypeError(
1000 "count must be a positive integer (got {!r})".format(count))
1001 if count <= 0:
1002 raise ValueError(
1003 "count must be a positive integer (got {!r})".format(count))
1004 if not isinstance(offset, int):
1005 raise TypeError(
1006 "offset must be a non-negative integer (got {!r})".format(
1007 offset))
1008 if offset < 0:
1009 raise ValueError(
1010 "offset must be a non-negative integer (got {!r})".format(
1011 offset))
1012
1013 async def _connect_sock(self, exceptions, addr_info, local_addr_infos=None):
1014 """Create, bind and connect one socket."""
1015 my_exceptions = []
1016 exceptions.append(my_exceptions)
1017 family, type_, proto, _, address = addr_info
1018 sock = None
1019 try:
1020 try:
1021 sock = socket.socket(family=family, type=type_, proto=proto)
1022 sock.setblocking(False)
1023 if local_addr_infos is not None:
1024 for lfamily, _, _, _, laddr in local_addr_infos:
1025 # skip local addresses of different family
1026 if lfamily != family:
1027 continue
1028 try:
1029 sock.bind(laddr)
1030 break
1031 except OSError as exc:
1032 msg = (
1033 f'error while attempting to bind on '
1034 f'address {laddr!r}: {str(exc).lower()}'
1035 )
1036 exc = OSError(exc.errno, msg)
1037 my_exceptions.append(exc)
1038 else: # all bind attempts failed
1039 if my_exceptions:
1040 raise my_exceptions.pop()
1041 else:
1042 raise OSError(f"no matching local address with {family=} found")
1043 await self.sock_connect(sock, address)
1044 return sock
1045 except OSError as exc:
1046 my_exceptions.append(exc)
1047 raise
1048 except:
1049 if sock is not None:
1050 try:
1051 sock.close()
1052 except OSError:
1053 # An error when closing a newly created socket is
1054 # not important, but it can overwrite more important
1055 # non-OSError error. So ignore it.
1056 pass
1057 raise
1058 finally:
1059 exceptions = my_exceptions = None
1060
1061 async def create_connection(
1062 self, protocol_factory, host=None, port=None,
1063 *, ssl=None, family=0,
1064 proto=0, flags=0, sock=None,
1065 local_addr=None, server_hostname=None,
1066 ssl_handshake_timeout=None,
1067 ssl_shutdown_timeout=None,
1068 happy_eyeballs_delay=None, interleave=None,
1069 all_errors=False):
1070 """Connect to a TCP server.
1071
1072 Create a streaming transport connection to a given internet host and
1073 port: socket family AF_INET or socket.AF_INET6 depending on host (or
1074 family if specified), socket type SOCK_STREAM. protocol_factory must
1075 be a callable returning a protocol instance.
1076
1077 This method is a coroutine which will try to establish the
1078 connection in the background. When successful, the coroutine
1079 returns a (transport, protocol) pair.
1080 """
1081 if server_hostname is not None and not ssl:
1082 raise ValueError('server_hostname is only meaningful with ssl')
1083
1084 if server_hostname is None and ssl:
1085 # Use host as default for server_hostname. It is an error
1086 # if host is empty or not set, e.g. when an
1087 # already-connected socket was passed or when only a port
1088 # is given. To avoid this error, you can pass
1089 # server_hostname='' -- this will bypass the hostname
1090 # check. (This also means that if host is a numeric
1091 # IP/IPv6 address, we will attempt to verify that exact
1092 # address; this will probably fail, but it is possible to
1093 # create a certificate for a specific IP address, so we
1094 # don't judge it here.)
1095 if not host:
1096 raise ValueError('You must set server_hostname '
1097 'when using ssl without a host')
1098 server_hostname = host
1099
1100 if ssl_handshake_timeout is not None and not ssl:
1101 raise ValueError(
1102 'ssl_handshake_timeout is only meaningful with ssl')
1103
1104 if ssl_shutdown_timeout is not None and not ssl:
1105 raise ValueError(
1106 'ssl_shutdown_timeout is only meaningful with ssl')
1107
1108 if sock is not None:
1109 _check_ssl_socket(sock)
1110
1111 if happy_eyeballs_delay is not None and interleave is None:
1112 # If using happy eyeballs, default to interleave addresses by family
1113 interleave = 1
1114
1115 if host is not None or port is not None:
1116 if sock is not None:
1117 raise ValueError(
1118 'host/port and sock can not be specified at the same time')
1119
1120 infos = await self._ensure_resolved(
1121 (host, port), family=family,
1122 type=socket.SOCK_STREAM, proto=proto, flags=flags, loop=self)
1123 if not infos:
1124 raise OSError('getaddrinfo() returned empty list')
1125
1126 if local_addr is not None:
1127 laddr_infos = await self._ensure_resolved(
1128 local_addr, family=family,
1129 type=socket.SOCK_STREAM, proto=proto,
1130 flags=flags, loop=self)
1131 if not laddr_infos:
1132 raise OSError('getaddrinfo() returned empty list')
1133 else:
1134 laddr_infos = None
1135
1136 if interleave:
1137 infos = _interleave_addrinfos(infos, interleave)
1138
1139 exceptions = []
1140 if happy_eyeballs_delay is None:
1141 # not using happy eyeballs
1142 for addrinfo in infos:
1143 try:
1144 sock = await self._connect_sock(
1145 exceptions, addrinfo, laddr_infos)
1146 break
1147 except OSError:
1148 continue
1149 else: # using happy eyeballs
1150 sock = (await staggered.staggered_race(
1151 (
1152 # can't use functools.partial as it keeps a reference
1153 # to exceptions
1154 lambda addrinfo=addrinfo: self._connect_sock(
1155 exceptions, addrinfo, laddr_infos
1156 )
1157 for addrinfo in infos
1158 ),
1159 happy_eyeballs_delay,
1160 loop=self,
1161 ))[0] # can't use sock, _, _ as it keeks a reference to exceptions
1162
1163 if sock is None:
1164 exceptions = [exc for sub in exceptions for exc in sub]
1165 try:
1166 if all_errors:
1167 raise ExceptionGroup("create_connection failed", exceptions)
1168 if len(exceptions) == 1:
1169 raise exceptions[0]
1170 elif exceptions:
1171 # If they all have the same str(), raise one.
1172 model = str(exceptions[0])
1173 if all(str(exc) == model for exc in exceptions):
1174 raise exceptions[0]
1175 # Raise a combined exception so the user can see all
1176 # the various error messages.
1177 raise OSError('Multiple exceptions: {}'.format(
1178 ', '.join(str(exc) for exc in exceptions)))
1179 else:
1180 # No exceptions were collected, raise a timeout error
1181 raise TimeoutError('create_connection failed')
1182 finally:
1183 exceptions = None
1184
1185 else:
1186 if sock is None:
1187 raise ValueError(
1188 'host and port was not specified and no sock specified')
1189 if sock.type != socket.SOCK_STREAM:
1190 # We allow AF_INET, AF_INET6, AF_UNIX as long as they
1191 # are SOCK_STREAM.
1192 # We support passing AF_UNIX sockets even though we have
1193 # a dedicated API for that: create_unix_connection.
1194 # Disallowing AF_UNIX in this method, breaks backwards
1195 # compatibility.
1196 raise ValueError(
1197 f'A Stream Socket was expected, got {sock!r}')
1198
1199 transport, protocol = await self._create_connection_transport(
1200 sock, protocol_factory, ssl, server_hostname,
1201 ssl_handshake_timeout=ssl_handshake_timeout,
1202 ssl_shutdown_timeout=ssl_shutdown_timeout)
1203 if self._debug:
1204 # Get the socket from the transport because SSL transport closes
1205 # the old socket and creates a new SSL socket
1206 sock = transport.get_extra_info('socket')
1207 logger.debug("%r connected to %s:%r: (%r, %r)",
1208 sock, host, port, transport, protocol)
1209 return transport, protocol
1210
1211 async def _create_connection_transport(
1212 self, sock, protocol_factory, ssl,
1213 server_hostname, server_side=False,
1214 ssl_handshake_timeout=None,
1215 ssl_shutdown_timeout=None):
1216
1217 sock.setblocking(False)
1218
1219 protocol = protocol_factory()
1220 waiter = self.create_future()
1221 if ssl:
1222 sslcontext = None if isinstance(ssl, bool) else ssl
1223 transport = self._make_ssl_transport(
1224 sock, protocol, sslcontext, waiter,
1225 server_side=server_side, server_hostname=server_hostname,
1226 ssl_handshake_timeout=ssl_handshake_timeout,
1227 ssl_shutdown_timeout=ssl_shutdown_timeout)
1228 else:
1229 transport = self._make_socket_transport(sock, protocol, waiter)
1230
1231 try:
1232 await waiter
1233 except:
1234 transport.close()
1235 raise
1236
1237 return transport, protocol
1238
1239 async def sendfile(self, transport, file, offset=0, count=None,
1240 *, fallback=True):
1241 """Send a file to transport.
1242
1243 Return the total number of bytes which were sent.
1244
1245 The method uses high-performance os.sendfile if available.
1246
1247 file must be a regular file object opened in binary mode.
1248
1249 offset tells from where to start reading the file. If specified,
1250 count is the total number of bytes to transmit as opposed to
1251 sending the file until EOF is reached. File position is updated on
1252 return or also in case of error in which case file.tell()
1253 can be used to figure out the number of bytes
1254 which were sent.
1255
1256 fallback set to True makes asyncio to manually read and send
1257 the file when the platform does not support the sendfile syscall
1258 (e.g. Windows or SSL socket on Unix).
1259
1260 Raise SendfileNotAvailableError if the system does not support
1261 sendfile syscall and fallback is False.
1262 """
1263 if transport.is_closing():
1264 raise RuntimeError("Transport is closing")
1265 mode = getattr(transport, '_sendfile_compatible',
1266 constants._SendfileMode.UNSUPPORTED)
1267 if mode is constants._SendfileMode.UNSUPPORTED:
1268 raise RuntimeError(
1269 f"sendfile is not supported for transport {transport!r}")
1270 if mode is constants._SendfileMode.TRY_NATIVE:
1271 try:
1272 return await self._sendfile_native(transport, file,
1273 offset, count)
1274 except exceptions.SendfileNotAvailableError as exc:
1275 if not fallback:
1276 raise
1277
1278 if not fallback:
1279 raise RuntimeError(
1280 f"fallback is disabled and native sendfile is not "
1281 f"supported for transport {transport!r}")
1282 return await self._sendfile_fallback(transport, file,
1283 offset, count)
1284
1285 async def _sendfile_native(self, transp, file, offset, count):
1286 raise exceptions.SendfileNotAvailableError(
1287 "sendfile syscall is not supported")
1288
1289 async def _sendfile_fallback(self, transp, file, offset, count):
1290 if hasattr(file, 'seek'):
1291 file.seek(offset)
1292 blocksize = min(count, 16384) if count else 16384
1293 buf = bytearray(blocksize)
1294 total_sent = 0
1295 proto = _SendfileFallbackProtocol(transp)
1296 try:
1297 while True:
1298 if count:
1299 blocksize = min(count - total_sent, blocksize)
1300 if blocksize <= 0:
1301 return total_sent
1302 view = memoryview(buf)[:blocksize]
1303 read = await self.run_in_executor(None, file.readinto, view)
1304 if not read:
1305 return total_sent # EOF
1306 transp.write(view[:read])
1307 await proto.drain()
1308 total_sent += read
1309 finally:
1310 if total_sent > 0 and hasattr(file, 'seek'):
1311 file.seek(offset + total_sent)
1312 await proto.restore()
1313
1314 async def start_tls(self, transport, protocol, sslcontext, *,
1315 server_side=False,
1316 server_hostname=None,
1317 ssl_handshake_timeout=None,
1318 ssl_shutdown_timeout=None):
1319 """Upgrade transport to TLS.
1320
1321 Return a new transport that *protocol* should start using
1322 immediately.
1323 """
1324 if ssl is None:
1325 raise RuntimeError('Python ssl module is not available')
1326
1327 if not isinstance(sslcontext, ssl.SSLContext):
1328 raise TypeError(
1329 f'sslcontext is expected to be an instance of ssl.SSLContext, '
1330 f'got {sslcontext!r}')
1331
1332 if not getattr(transport, '_start_tls_compatible', False):
1333 raise TypeError(
1334 f'transport {transport!r} is not supported by start_tls()')
1335
1336 waiter = self.create_future()
1337 ssl_protocol = sslproto.SSLProtocol(
1338 self, protocol, sslcontext, waiter,
1339 server_side, server_hostname,
1340 ssl_handshake_timeout=ssl_handshake_timeout,
1341 ssl_shutdown_timeout=ssl_shutdown_timeout,
1342 call_connection_made=False)
1343
1344 # Pause early so that "ssl_protocol.data_received()" doesn't
1345 # have a chance to get called before "ssl_protocol.connection_made()".
1346 transport.pause_reading()
1347
1348 # gh-142352: move buffered StreamReader data to SSLProtocol
1349 if server_side:
1350 from .streams import StreamReaderProtocol
1351 if isinstance(protocol, StreamReaderProtocol):
1352 stream_reader = getattr(protocol, '_stream_reader', None)
1353 if stream_reader is not None:
1354 buffer = stream_reader._buffer
1355 if buffer:
1356 ssl_protocol._incoming.write(buffer)
1357 buffer.clear()
1358
1359 transport.set_protocol(ssl_protocol)
1360 conmade_cb = self.call_soon(ssl_protocol.connection_made, transport)
1361 resume_cb = self.call_soon(transport.resume_reading)
1362
1363 try:
1364 await waiter
1365 except BaseException:
1366 transport.close()
1367 conmade_cb.cancel()
1368 resume_cb.cancel()
1369 raise
1370
1371 return ssl_protocol._app_transport
1372
1373 async def create_datagram_endpoint(self, protocol_factory,
1374 local_addr=None, remote_addr=None, *,
1375 family=0, proto=0, flags=0,
1376 reuse_port=None,
1377 allow_broadcast=None, sock=None):
1378 """Create datagram connection."""
1379 if sock is not None:
1380 if sock.type == socket.SOCK_STREAM:
1381 raise ValueError(
1382 f'A datagram socket was expected, got {sock!r}')
1383 if (local_addr or remote_addr or
1384 family or proto or flags or
1385 reuse_port or allow_broadcast):
1386 # show the problematic kwargs in exception msg
1387 opts = dict(local_addr=local_addr, remote_addr=remote_addr,
1388 family=family, proto=proto, flags=flags,
1389 reuse_port=reuse_port,
1390 allow_broadcast=allow_broadcast)
1391 problems = ', '.join(f'{k}={v}' for k, v in opts.items() if v)
1392 raise ValueError(
1393 f'socket modifier keyword arguments can not be used '
1394 f'when sock is specified. ({problems})')
1395 sock.setblocking(False)
1396 r_addr = None
1397 else:
1398 if not (local_addr or remote_addr):
1399 if family == 0:
1400 raise ValueError('unexpected address family')
1401 addr_pairs_info = (((family, proto), (None, None)),)
1402 elif hasattr(socket, 'AF_UNIX') and family == socket.AF_UNIX:
1403 for addr in (local_addr, remote_addr):
1404 if addr is not None and not isinstance(addr, str):
1405 raise TypeError('string is expected')
1406
1407 if local_addr and local_addr[0] not in (0, '\x00'):
1408 try:
1409 if stat.S_ISSOCK(os.stat(local_addr).st_mode):
1410 os.remove(local_addr)
1411 except FileNotFoundError:
1412 pass
1413 except OSError as err:
1414 # Directory may have permissions only to create socket.
1415 logger.error('Unable to check or remove stale UNIX '
1416 'socket %r: %r',
1417 local_addr, err)
1418
1419 addr_pairs_info = (((family, proto),
1420 (local_addr, remote_addr)), )
1421 else:
1422 # join address by (family, protocol)
1423 addr_infos = {} # Using order preserving dict
1424 for idx, addr in ((0, local_addr), (1, remote_addr)):
1425 if addr is not None:
1426 if not (isinstance(addr, tuple) and len(addr) == 2):
1427 raise TypeError('2-tuple is expected')
1428
1429 infos = await self._ensure_resolved(
1430 addr, family=family, type=socket.SOCK_DGRAM,
1431 proto=proto, flags=flags, loop=self)
1432 if not infos:
1433 raise OSError('getaddrinfo() returned empty list')
1434
1435 for fam, _, pro, _, address in infos:
1436 key = (fam, pro)
1437 if key not in addr_infos:
1438 addr_infos[key] = [None, None]
1439 addr_infos[key][idx] = address
1440
1441 # each addr has to have info for each (family, proto) pair
1442 addr_pairs_info = [
1443 (key, addr_pair) for key, addr_pair in addr_infos.items()
1444 if not ((local_addr and addr_pair[0] is None) or
1445 (remote_addr and addr_pair[1] is None))]
1446
1447 if not addr_pairs_info:
1448 raise ValueError('can not get address information')
1449
1450 exceptions = []
1451
1452 for ((family, proto),
1453 (local_address, remote_address)) in addr_pairs_info:
1454 sock = None
1455 r_addr = None
1456 try:
1457 sock = socket.socket(
1458 family=family, type=socket.SOCK_DGRAM, proto=proto)
1459 if reuse_port:
1460 _set_reuseport(sock)
1461 if allow_broadcast:
1462 sock.setsockopt(
1463 socket.SOL_SOCKET, socket.SO_BROADCAST, 1)
1464 sock.setblocking(False)
1465
1466 if local_addr:
1467 sock.bind(local_address)
1468 if remote_addr:
1469 if not allow_broadcast:
1470 await self.sock_connect(sock, remote_address)
1471 r_addr = remote_address
1472 except OSError as exc:
1473 if sock is not None:
1474 sock.close()
1475 exceptions.append(exc)
1476 except:
1477 if sock is not None:
1478 sock.close()
1479 raise
1480 else:
1481 break
1482 else:
1483 raise exceptions[0]
1484
1485 protocol = protocol_factory()
1486 waiter = self.create_future()
1487 transport = self._make_datagram_transport(
1488 sock, protocol, r_addr, waiter)
1489 if self._debug:
1490 if local_addr:
1491 logger.info("Datagram endpoint local_addr=%r remote_addr=%r "
1492 "created: (%r, %r)",
1493 local_addr, remote_addr, transport, protocol)
1494 else:
1495 logger.debug("Datagram endpoint remote_addr=%r created: "
1496 "(%r, %r)",
1497 remote_addr, transport, protocol)
1498
1499 try:
1500 await waiter
1501 except:
1502 transport.close()
1503 raise
1504
1505 return transport, protocol
1506
1507 async def _ensure_resolved(self, address, *,
1508 family=0, type=socket.SOCK_STREAM,
1509 proto=0, flags=0, loop):
1510 host, port = address[:2]
1511 info = _ipaddr_info(host, port, family, type, proto, *address[2:])
1512 if info is not None:
1513 # "host" is already a resolved IP.
1514 return [info]
1515 else:
1516 return await loop.getaddrinfo(host, port, family=family, type=type,
1517 proto=proto, flags=flags)
1518
1519 async def _create_server_getaddrinfo(self, host, port, family, flags):
1520 infos = await self._ensure_resolved((host, port), family=family,
1521 type=socket.SOCK_STREAM,
1522 flags=flags, loop=self)
1523 if not infos:
1524 raise OSError(f'getaddrinfo({host!r}) returned empty list')
1525 return infos
1526
1527 async def create_server(
1528 self, protocol_factory, host=None, port=None,
1529 *,
1530 family=socket.AF_UNSPEC,
1531 flags=socket.AI_PASSIVE,
1532 sock=None,
1533 backlog=100,
1534 ssl=None,
1535 reuse_address=None,
1536 reuse_port=None,
1537 keep_alive=None,
1538 ssl_handshake_timeout=None,
1539 ssl_shutdown_timeout=None,
1540 start_serving=True):
1541 """Create a TCP server.
1542
1543 The host parameter can be a string, in that case the TCP server is
1544 bound to host and port.
1545
1546 The host parameter can also be a sequence of strings and in that
1547 case the TCP server is bound to all hosts of the sequence. If
1548 a host appears multiple times (possibly indirectly e.g. when
1549 hostnames resolve to the same IP address), the server is only bound
1550 once to that host.
1551
1552 Return a Server object which can be used to stop the service.
1553
1554 This method is a coroutine.
1555 """
1556 if isinstance(ssl, bool):
1557 raise TypeError('ssl argument must be an SSLContext or None')
1558
1559 if ssl_handshake_timeout is not None and ssl is None:
1560 raise ValueError(
1561 'ssl_handshake_timeout is only meaningful with ssl')
1562
1563 if ssl_shutdown_timeout is not None and ssl is None:
1564 raise ValueError(
1565 'ssl_shutdown_timeout is only meaningful with ssl')
1566
1567 if sock is not None:
1568 _check_ssl_socket(sock)
1569
1570 if host is not None or port is not None:
1571 if sock is not None:
1572 raise ValueError(
1573 'host/port and sock can not be specified at the same time')
1574
1575 if reuse_address is None:
1576 reuse_address = os.name == "posix" and sys.platform != "cygwin"
1577 sockets = []
1578 if host == '':
1579 hosts = [None]
1580 elif (isinstance(host, str) or
1581 not isinstance(host, collections.abc.Iterable)):
1582 hosts = [host]
1583 else:
1584 hosts = host
1585
1586 fs = [self._create_server_getaddrinfo(host, port, family=family,
1587 flags=flags)
1588 for host in hosts]
1589 infos = await tasks.gather(*fs)
1590 infos = set(itertools.chain.from_iterable(infos))
1591
1592 completed = False
1593 try:
1594 for res in infos:
1595 af, socktype, proto, canonname, sa = res
1596 try:
1597 sock = socket.socket(af, socktype, proto)
1598 except socket.error:
1599 # Assume it's a bad family/type/protocol combination.
1600 if self._debug:
1601 logger.warning('create_server() failed to create '
1602 'socket.socket(%r, %r, %r)',
1603 af, socktype, proto, exc_info=True)
1604 continue
1605 sockets.append(sock)
1606 if reuse_address:
1607 sock.setsockopt(
1608 socket.SOL_SOCKET, socket.SO_REUSEADDR, True)
1609 # Since Linux 6.12.9, SO_REUSEPORT is not allowed
1610 # on other address families than AF_INET/AF_INET6.
1611 if reuse_port and af in (socket.AF_INET, socket.AF_INET6):
1612 _set_reuseport(sock)
1613 if keep_alive:
1614 sock.setsockopt(
1615 socket.SOL_SOCKET, socket.SO_KEEPALIVE, True)
1616 # Disable IPv4/IPv6 dual stack support (enabled by
1617 # default on Linux) which makes a single socket
1618 # listen on both address families.
1619 if (_HAS_IPv6 and
1620 af == socket.AF_INET6 and
1621 hasattr(socket, 'IPPROTO_IPV6')):
1622 sock.setsockopt(socket.IPPROTO_IPV6,
1623 socket.IPV6_V6ONLY,
1624 True)
1625 try:
1626 sock.bind(sa)
1627 except OSError as err:
1628 msg = ('error while attempting '
1629 'to bind on address %r: %s'
1630 % (sa, str(err).lower()))
1631 if err.errno == errno.EADDRNOTAVAIL:
1632 # Assume the family is not enabled (bpo-30945)
1633 sockets.pop()
1634 sock.close()
1635 if self._debug:
1636 logger.warning(msg)
1637 continue
1638 raise OSError(err.errno, msg) from None
1639
1640 if not sockets:
1641 raise OSError('could not bind on any address out of %r'
1642 % ([info[4] for info in infos],))
1643
1644 completed = True
1645 finally:
1646 if not completed:
1647 for sock in sockets:
1648 sock.close()
1649 else:
1650 if sock is None:
1651 raise ValueError('Neither host/port nor sock were specified')
1652 if sock.type != socket.SOCK_STREAM:
1653 raise ValueError(f'A Stream Socket was expected, got {sock!r}')
1654 sockets = [sock]
1655
1656 for sock in sockets:
1657 sock.setblocking(False)
1658
1659 server = Server(self, sockets, protocol_factory,
1660 ssl, backlog, ssl_handshake_timeout,
1661 ssl_shutdown_timeout)
1662 if start_serving:
1663 server._start_serving()
1664 # Skip one loop iteration so that all 'loop.add_reader'
1665 # go through.
1666 await tasks.sleep(0)
1667
1668 if self._debug:
1669 logger.info("%r is serving", server)
1670 return server
1671
1672 async def connect_accepted_socket(
1673 self, protocol_factory, sock,
1674 *, ssl=None,
1675 ssl_handshake_timeout=None,
1676 ssl_shutdown_timeout=None):
1677 if sock.type != socket.SOCK_STREAM:
1678 raise ValueError(f'A Stream Socket was expected, got {sock!r}')
1679
1680 if ssl_handshake_timeout is not None and not ssl:
1681 raise ValueError(
1682 'ssl_handshake_timeout is only meaningful with ssl')
1683
1684 if ssl_shutdown_timeout is not None and not ssl:
1685 raise ValueError(
1686 'ssl_shutdown_timeout is only meaningful with ssl')
1687
1688 _check_ssl_socket(sock)
1689
1690 transport, protocol = await self._create_connection_transport(
1691 sock, protocol_factory, ssl, '', server_side=True,
1692 ssl_handshake_timeout=ssl_handshake_timeout,
1693 ssl_shutdown_timeout=ssl_shutdown_timeout)
1694 if self._debug:
1695 # Get the socket from the transport because SSL transport closes
1696 # the old socket and creates a new SSL socket
1697 sock = transport.get_extra_info('socket')
1698 logger.debug("%r handled: (%r, %r)", sock, transport, protocol)
1699 return transport, protocol
1700
1701 async def connect_read_pipe(self, protocol_factory, pipe):
1702 protocol = protocol_factory()
1703 waiter = self.create_future()
1704 transport = self._make_read_pipe_transport(pipe, protocol, waiter)
1705
1706 try:
1707 await waiter
1708 except:
1709 transport.close()
1710 raise
1711
1712 if self._debug:
1713 logger.debug('Read pipe %r connected: (%r, %r)',
1714 pipe.fileno(), transport, protocol)
1715 return transport, protocol
1716
1717 async def connect_write_pipe(self, protocol_factory, pipe):
1718 protocol = protocol_factory()
1719 waiter = self.create_future()
1720 transport = self._make_write_pipe_transport(pipe, protocol, waiter)
1721
1722 try:
1723 await waiter
1724 except:
1725 transport.close()
1726 raise
1727
1728 if self._debug:
1729 logger.debug('Write pipe %r connected: (%r, %r)',
1730 pipe.fileno(), transport, protocol)
1731 return transport, protocol
1732
1733 def _log_subprocess(self, msg, stdin, stdout, stderr):
1734 info = [msg]
1735 if stdin is not None:
1736 info.append(f'stdin={_format_pipe(stdin)}')
1737 if stdout is not None and stderr == subprocess.STDOUT:
1738 info.append(f'stdout=stderr={_format_pipe(stdout)}')
1739 else:
1740 if stdout is not None:
1741 info.append(f'stdout={_format_pipe(stdout)}')
1742 if stderr is not None:
1743 info.append(f'stderr={_format_pipe(stderr)}')
1744 logger.debug(' '.join(info))
1745
1746 async def subprocess_shell(self, protocol_factory, cmd, *,
1747 stdin=subprocess.PIPE,
1748 stdout=subprocess.PIPE,
1749 stderr=subprocess.PIPE,
1750 universal_newlines=False,
1751 shell=True, bufsize=0,
1752 encoding=None, errors=None, text=None,
1753 **kwargs):
1754 if not isinstance(cmd, (bytes, str)):
1755 raise ValueError("cmd must be a string")
1756 if universal_newlines:
1757 raise ValueError("universal_newlines must be False")
1758 if not shell:
1759 raise ValueError("shell must be True")
1760 if bufsize != 0:
1761 raise ValueError("bufsize must be 0")
1762 if text:
1763 raise ValueError("text must be False")
1764 if encoding is not None:
1765 raise ValueError("encoding must be None")
1766 if errors is not None:
1767 raise ValueError("errors must be None")
1768
1769 protocol = protocol_factory()
1770 debug_log = None
1771 if self._debug:
1772 # don't log parameters: they may contain sensitive information
1773 # (password) and may be too long
1774 debug_log = 'run shell command %r' % cmd
1775 self._log_subprocess(debug_log, stdin, stdout, stderr)
1776 transport = await self._make_subprocess_transport(
1777 protocol, cmd, True, stdin, stdout, stderr, bufsize, **kwargs)
1778 if self._debug and debug_log is not None:
1779 logger.info('%s: %r', debug_log, transport)
1780 return transport, protocol
1781
1782 async def subprocess_exec(self, protocol_factory, program, *args,
1783 stdin=subprocess.PIPE, stdout=subprocess.PIPE,
1784 stderr=subprocess.PIPE, universal_newlines=False,
1785 shell=False, bufsize=0,
1786 encoding=None, errors=None, text=None,
1787 **kwargs):
1788 if universal_newlines:
1789 raise ValueError("universal_newlines must be False")
1790 if shell:
1791 raise ValueError("shell must be False")
1792 if bufsize != 0:
1793 raise ValueError("bufsize must be 0")
1794 if text:
1795 raise ValueError("text must be False")
1796 if encoding is not None:
1797 raise ValueError("encoding must be None")
1798 if errors is not None:
1799 raise ValueError("errors must be None")
1800
1801 popen_args = (program,) + args
1802 protocol = protocol_factory()
1803 debug_log = None
1804 if self._debug:
1805 # don't log parameters: they may contain sensitive information
1806 # (password) and may be too long
1807 debug_log = f'execute program {program!r}'
1808 self._log_subprocess(debug_log, stdin, stdout, stderr)
1809 transport = await self._make_subprocess_transport(
1810 protocol, popen_args, False, stdin, stdout, stderr,
1811 bufsize, **kwargs)
1812 if self._debug and debug_log is not None:
1813 logger.info('%s: %r', debug_log, transport)
1814 return transport, protocol
1815
1816 def get_exception_handler(self):
1817 """Return an exception handler, or None if the default one is in use.
1818 """
1819 return self._exception_handler
1820
1821 def set_exception_handler(self, handler):
1822 """Set handler as the new event loop exception handler.
1823
1824 If handler is None, the default exception handler will
1825 be set.
1826
1827 If handler is a callable object, it should have a
1828 signature matching '(loop, context)', where 'loop'
1829 will be a reference to the active event loop, 'context'
1830 will be a dict object (see `call_exception_handler()`
1831 documentation for details about context).
1832 """
1833 if handler is not None and not callable(handler):
1834 raise TypeError(f'A callable object or None is expected, '
1835 f'got {handler!r}')
1836 self._exception_handler = handler
1837
1838 def default_exception_handler(self, context):
1839 """Default exception handler.
1840
1841 This is called when an exception occurs and no exception
1842 handler is set, and can be called by a custom exception
1843 handler that wants to defer to the default behavior.
1844
1845 This default handler logs the error message and other
1846 context-dependent information. In debug mode, a truncated
1847 stack trace is also appended showing where the given object
1848 (e.g. a handle or future or task) was created, if any.
1849
1850 The context parameter has the same meaning as in
1851 `call_exception_handler()`.
1852 """
1853 message = context.get('message')
1854 if not message:
1855 message = 'Unhandled exception in event loop'
1856
1857 exception = context.get('exception')
1858 if exception is not None:
1859 exc_info = (type(exception), exception, exception.__traceback__)
1860 else:
1861 exc_info = False
1862
1863 if ('source_traceback' not in context and
1864 self._current_handle is not None and
1865 self._current_handle._source_traceback):
1866 context['handle_traceback'] = \
1867 self._current_handle._source_traceback
1868
1869 log_lines = [message]
1870 for key in sorted(context):
1871 if key in {'message', 'exception'}:
1872 continue
1873 value = context[key]
1874 if key == 'source_traceback':
1875 tb = ''.join(traceback.format_list(value))
1876 value = 'Object created at (most recent call last):\n'
1877 value += tb.rstrip()
1878 elif key == 'handle_traceback':
1879 tb = ''.join(traceback.format_list(value))
1880 value = 'Handle created at (most recent call last):\n'
1881 value += tb.rstrip()
1882 else:
1883 value = repr(value)
1884 log_lines.append(f'{key}: {value}')
1885
1886 logger.error('\n'.join(log_lines), exc_info=exc_info)
1887
1888 def call_exception_handler(self, context):
1889 """Call the current event loop's exception handler.
1890
1891 The context argument is a dict containing the following keys:
1892
1893 - 'message': Error message;
1894 - 'exception' (optional): Exception object;
1895 - 'future' (optional): Future instance;
1896 - 'task' (optional): Task instance;
1897 - 'handle' (optional): Handle instance;
1898 - 'protocol' (optional): Protocol instance;
1899 - 'transport' (optional): Transport instance;
1900 - 'socket' (optional): Socket instance;
1901 - 'source_traceback' (optional): Traceback of the source;
1902 - 'handle_traceback' (optional): Traceback of the handle;
1903 - 'asyncgen' (optional): Asynchronous generator that caused
1904 the exception.
1905
1906 New keys maybe introduced in the future.
1907
1908 Note: do not overload this method in an event loop subclass.
1909 For custom exception handling, use the
1910 `set_exception_handler()` method.
1911 """
1912 if self._exception_handler is None:
1913 try:
1914 self.default_exception_handler(context)
1915 except (SystemExit, KeyboardInterrupt):
1916 raise
1917 except BaseException:
1918 # Second protection layer for unexpected errors
1919 # in the default implementation, as well as for subclassed
1920 # event loops with overloaded "default_exception_handler".
1921 logger.error('Exception in default exception handler',
1922 exc_info=True)
1923 else:
1924 try:
1925 ctx = None
1926 thing = context.get("task")
1927 if thing is None:
1928 # Even though Futures don't have a context,
1929 # Task is a subclass of Future,
1930 # and sometimes the 'future' key holds a Task.
1931 thing = context.get("future")
1932 if thing is None:
1933 # Handles also have a context.
1934 thing = context.get("handle")
1935 if thing is not None and hasattr(thing, "get_context"):
1936 ctx = thing.get_context()
1937 if ctx is not None and hasattr(ctx, "run"):
1938 ctx.run(self._exception_handler, self, context)
1939 else:
1940 self._exception_handler(self, context)
1941 except (SystemExit, KeyboardInterrupt):
1942 raise
1943 except BaseException as exc:
1944 # Exception in the user set custom exception handler.
1945 try:
1946 # Let's try default handler.
1947 self.default_exception_handler({
1948 'message': 'Unhandled error in exception handler',
1949 'exception': exc,
1950 'context': context,
1951 })
1952 except (SystemExit, KeyboardInterrupt):
1953 raise
1954 except BaseException:
1955 # Guard 'default_exception_handler' in case it is
1956 # overloaded.
1957 logger.error('Exception in default exception handler '
1958 'while handling an unexpected error '
1959 'in custom exception handler',
1960 exc_info=True)
1961
1962 def _add_callback(self, handle):
1963 """Add a Handle to _ready."""
1964 if not handle._cancelled:
1965 self._ready.append(handle)
1966
1967 def _add_callback_signalsafe(self, handle):
1968 """Like _add_callback() but called from a signal handler."""
1969 self._add_callback(handle)
1970 self._write_to_self()
1971
1972 def _timer_handle_cancelled(self, handle):
1973 """Notification that a TimerHandle has been cancelled."""
1974 if handle._scheduled:
1975 self._timer_cancelled_count += 1
1976
1977 def _run_once(self):
1978 """Run one full iteration of the event loop.
1979
1980 This calls all currently ready callbacks, polls for I/O,
1981 schedules the resulting callbacks, and finally schedules
1982 'call_later' callbacks.
1983 """
1984
1985 sched_count = len(self._scheduled)
1986 if (sched_count > _MIN_SCHEDULED_TIMER_HANDLES and
1987 self._timer_cancelled_count / sched_count >
1988 _MIN_CANCELLED_TIMER_HANDLES_FRACTION):
1989 # Remove delayed calls that were cancelled if their number
1990 # is too high
1991 new_scheduled = []
1992 for handle in self._scheduled:
1993 if handle._cancelled:
1994 handle._scheduled = False
1995 else:
1996 new_scheduled.append(handle)
1997
1998 heapq.heapify(new_scheduled)
1999 self._scheduled = new_scheduled
2000 self._timer_cancelled_count = 0
2001 else:
2002 # Remove delayed calls that were cancelled from head of queue.
2003 while self._scheduled and self._scheduled[0]._cancelled:
2004 self._timer_cancelled_count -= 1
2005 handle = heapq.heappop(self._scheduled)
2006 handle._scheduled = False
2007
2008 timeout = None
2009 if self._ready or self._stopping:
2010 timeout = 0
2011 elif self._scheduled:
2012 # Compute the desired timeout.
2013 timeout = self._scheduled[0]._when - self.time()
2014 if timeout > MAXIMUM_SELECT_TIMEOUT:
2015 timeout = MAXIMUM_SELECT_TIMEOUT
2016 elif timeout < 0:
2017 timeout = 0
2018
2019 event_list = self._selector.select(timeout)
2020 self._process_events(event_list)
2021 # Needed to break cycles when an exception occurs.
2022 event_list = None
2023
2024 # Handle 'later' callbacks that are ready.
2025 end_time = self.time() + self._clock_resolution
2026 while self._scheduled:
2027 handle = self._scheduled[0]
2028 if handle._when >= end_time:
2029 break
2030 handle = heapq.heappop(self._scheduled)
2031 handle._scheduled = False
2032 self._ready.append(handle)
2033
2034 # This is the only place where callbacks are actually *called*.
2035 # All other places just add them to ready.
2036 # Note: We run all currently scheduled callbacks, but not any
2037 # callbacks scheduled by callbacks run this time around --
2038 # they will be run the next time (after another I/O poll).
2039 # Use an idiom that is thread-safe without using locks.
2040 ntodo = len(self._ready)
2041 for i in range(ntodo):
2042 handle = self._ready.popleft()
2043 if handle._cancelled:
2044 continue
2045 if self._debug:
2046 try:
2047 self._current_handle = handle
2048 t0 = self.time()
2049 handle._run()
2050 dt = self.time() - t0
2051 if dt >= self.slow_callback_duration:
2052 logger.warning('Executing %s took %.3f seconds',
2053 _format_handle(handle), dt)
2054 finally:
2055 self._current_handle = None
2056 else:
2057 handle._run()
2058 handle = None # Needed to break cycles when an exception occurs.
2059
2060 def _set_coroutine_origin_tracking(self, enabled):
2061 if bool(enabled) == bool(self._coroutine_origin_tracking_enabled):
2062 return
2063
2064 if enabled:
2065 self._coroutine_origin_tracking_saved_depth = (
2066 sys.get_coroutine_origin_tracking_depth())
2067 sys.set_coroutine_origin_tracking_depth(
2068 constants.DEBUG_STACK_DEPTH)
2069 else:
2070 sys.set_coroutine_origin_tracking_depth(
2071 self._coroutine_origin_tracking_saved_depth)
2072
2073 self._coroutine_origin_tracking_enabled = enabled
2074
2075 def get_debug(self):
2076 return self._debug
2077
2078 def set_debug(self, enabled):
2079 self._debug = enabled
2080
2081 if self.is_running():
2082 self.call_soon_threadsafe(self._set_coroutine_origin_tracking, enabled)