Debian Codegraph @

Lib/asyncio/base_events.py

raw
1 """Base implementation of event loop.
2
3 The event loop can be broken up into a multiplexer (the part
4 responsible for notifying us of I/O events) and the event loop proper,
5 which wraps a multiplexer with functionality for scheduling callbacks,
6 immediately or at a given time in the future.
7
8 Whenever a public API takes a callback, subsequent positional
9 arguments will be passed to the callback if/when it is called. This
10 avoids the proliferation of trivial lambdas implementing closures.
11 Keyword arguments for the callback are not supported; this is a
12 conscious design decision, leaving the door open for keyword arguments
13 to modify the meaning of the API call itself.
14 """
15
16 import collections
17 import collections.abc
18 import concurrent.futures
19 import errno
20 import heapq
21 import itertools
22 import os
23 import socket
24 import stat
25 import subprocess
26 import threading
27 import time
28 import traceback
29 import sys
30 import warnings
31 import weakref
32
33 try:
34 import ssl
35 except ImportError: # pragma: no cover
36 ssl = None
37
38 from . import constants
39 from . import coroutines
40 from . import events
41 from . import exceptions
42 from . import futures
43 from . import protocols
44 from . import sslproto
45 from . import staggered
46 from . import tasks
47 from . import timeouts
48 from . import transports
49 from . import trsock
50 from .log import logger
51
52
53 __all__ = 'BaseEventLoop','Server',
54
55
56 # Minimum number of _scheduled timer handles before cleanup of
57 # cancelled handles is performed.
58 _MIN_SCHEDULED_TIMER_HANDLES = 100
59
60 # Minimum fraction of _scheduled timer handles that are cancelled
61 # before cleanup of cancelled handles is performed.
62 _MIN_CANCELLED_TIMER_HANDLES_FRACTION = 0.5
63
64
65 _HAS_IPv6 = hasattr(socket, 'AF_INET6')
66
67 # Maximum timeout passed to select to avoid OS limitations
68 MAXIMUM_SELECT_TIMEOUT = 24 * 3600
69
70
71 def _format_handle(handle):
72 cb = handle._callback
73 if isinstance(getattr(cb, '__self__', None), tasks.Task):
74 # format the task
75 return repr(cb.__self__)
76 else:
77 return str(handle)
78
79
80 def _format_pipe(fd):
81 if fd == subprocess.PIPE:
82 return '<pipe>'
83 elif fd == subprocess.STDOUT:
84 return '<stdout>'
85 else:
86 return repr(fd)
87
88
89 def _set_reuseport(sock):
90 if not hasattr(socket, 'SO_REUSEPORT'):
91 raise ValueError('reuse_port not supported by socket module')
92 else:
93 try:
94 sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEPORT, 1)
95 except OSError:
96 raise ValueError('reuse_port not supported by socket module, '
97 'SO_REUSEPORT defined but not implemented.')
98
99
100 def _ipaddr_info(host, port, family, type, proto, flowinfo=0, scopeid=0):
101 # Try to skip getaddrinfo if "host" is already an IP. Users might have
102 # handled name resolution in their own code and pass in resolved IPs.
103 if not hasattr(socket, 'inet_pton'):
104 return
105
106 if proto not in {0, socket.IPPROTO_TCP, socket.IPPROTO_UDP} or \
107 host is None:
108 return None
109
110 if type == socket.SOCK_STREAM:
111 proto = socket.IPPROTO_TCP
112 elif type == socket.SOCK_DGRAM:
113 proto = socket.IPPROTO_UDP
114 else:
115 return None
116
117 if port is None:
118 port = 0
119 elif isinstance(port, bytes) and port == b'':
120 port = 0
121 elif isinstance(port, str) and port == '':
122 port = 0
123 else:
124 # If port's a service name like "http", don't skip getaddrinfo.
125 try:
126 port = int(port)
127 except (TypeError, ValueError):
128 return None
129
130 if family == socket.AF_UNSPEC:
131 afs = [socket.AF_INET]
132 if _HAS_IPv6:
133 afs.append(socket.AF_INET6)
134 else:
135 afs = [family]
136
137 if isinstance(host, bytes):
138 host = host.decode('idna')
139 if '%' in host:
140 # Linux's inet_pton doesn't accept an IPv6 zone index after host,
141 # like '::1%lo0'.
142 return None
143
144 for af in afs:
145 try:
146 socket.inet_pton(af, host)
147 # The host has already been resolved.
148 if _HAS_IPv6 and af == socket.AF_INET6:
149 return af, type, proto, '', (host, port, flowinfo, scopeid)
150 else:
151 return af, type, proto, '', (host, port)
152 except OSError:
153 pass
154
155 # "host" is not an IP address.
156 return None
157
158
159 def _interleave_addrinfos(addrinfos, first_address_family_count=1):
160 """Interleave list of addrinfo tuples by family."""
161 # Group addresses by family
162 addrinfos_by_family = collections.OrderedDict()
163 for addr in addrinfos:
164 family = addr[0]
165 if family not in addrinfos_by_family:
166 addrinfos_by_family[family] = []
167 addrinfos_by_family[family].append(addr)
168 addrinfos_lists = list(addrinfos_by_family.values())
169
170 reordered = []
171 if first_address_family_count > 1:
172 reordered.extend(addrinfos_lists[0][:first_address_family_count - 1])
173 del addrinfos_lists[0][:first_address_family_count - 1]
174 reordered.extend(
175 a for a in itertools.chain.from_iterable(
176 itertools.zip_longest(*addrinfos_lists)
177 ) if a is not None)
178 return reordered
179
180
181 def _run_until_complete_cb(fut):
182 if not fut.cancelled():
183 exc = fut.exception()
184 if isinstance(exc, (SystemExit, KeyboardInterrupt)):
185 # Issue #22429: run_forever() already finished, no need to
186 # stop it.
187 return
188 futures._get_loop(fut).stop()
189
190
191 if hasattr(socket, 'TCP_NODELAY'):
192 def _set_nodelay(sock):
193 if (sock.family in {socket.AF_INET, socket.AF_INET6} and
194 sock.type == socket.SOCK_STREAM and
195 sock.proto == socket.IPPROTO_TCP):
196 sock.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1)
197 else:
198 def _set_nodelay(sock):
199 pass
200
201
202 def _check_ssl_socket(sock):
203 if ssl is not None and isinstance(sock, ssl.SSLSocket):
204 raise TypeError("Socket cannot be of type SSLSocket")
205
206
207 class _SendfileFallbackProtocol(protocols.Protocol):
208 def __init__(self, transp):
209 if not isinstance(transp, transports._FlowControlMixin):
210 raise TypeError("transport should be _FlowControlMixin instance")
211 self._transport = transp
212 self._proto = transp.get_protocol()
213 self._should_resume_reading = transp.is_reading()
214 self._should_resume_writing = transp._protocol_paused
215 transp.pause_reading()
216 transp.set_protocol(self)
217 if self._should_resume_writing:
218 self._write_ready_fut = self._transport._loop.create_future()
219 else:
220 self._write_ready_fut = None
221
222 async def drain(self):
223 if self._transport.is_closing():
224 raise ConnectionError("Connection closed by peer")
225 fut = self._write_ready_fut
226 if fut is None:
227 return
228 await fut
229
230 def connection_made(self, transport):
231 raise RuntimeError("Invalid state: "
232 "connection should have been established already.")
233
234 def connection_lost(self, exc):
235 if self._write_ready_fut is not None:
236 # Never happens if peer disconnects after sending the whole content
237 # Thus disconnection is always an exception from user perspective
238 if exc is None:
239 self._write_ready_fut.set_exception(
240 ConnectionError("Connection is closed by peer"))
241 else:
242 self._write_ready_fut.set_exception(exc)
243 self._proto.connection_lost(exc)
244
245 def pause_writing(self):
246 if self._write_ready_fut is not None:
247 return
248 self._write_ready_fut = self._transport._loop.create_future()
249
250 def resume_writing(self):
251 if self._write_ready_fut is None:
252 return
253 self._write_ready_fut.set_result(False)
254 self._write_ready_fut = None
255
256 def data_received(self, data):
257 raise RuntimeError("Invalid state: reading should be paused")
258
259 def eof_received(self):
260 raise RuntimeError("Invalid state: reading should be paused")
261
262 async def restore(self):
263 self._transport.set_protocol(self._proto)
264 if self._should_resume_reading:
265 self._transport.resume_reading()
266 if self._write_ready_fut is not None:
267 # Cancel the future.
268 # Basically it has no effect because protocol is switched back,
269 # no code should wait for it anymore.
270 self._write_ready_fut.cancel()
271 if self._should_resume_writing:
272 self._proto.resume_writing()
273
274
275 class Server(events.AbstractServer):
276
277 def __init__(self, loop, sockets, protocol_factory, ssl_context, backlog,
278 ssl_handshake_timeout, ssl_shutdown_timeout=None):
279 self._loop = loop
280 self._sockets = sockets
281 # Weak references so we don't break Transport's ability to
282 # detect abandoned transports
283 self._clients = weakref.WeakSet()
284 self._waiters = []
285 self._protocol_factory = protocol_factory
286 self._backlog = backlog
287 self._ssl_context = ssl_context
288 self._ssl_handshake_timeout = ssl_handshake_timeout
289 self._ssl_shutdown_timeout = ssl_shutdown_timeout
290 self._serving = False
291 self._serving_forever_fut = None
292
293 def __repr__(self):
294 return f'<{self.__class__.__name__} sockets={self.sockets!r}>'
295
296 def _attach(self, transport):
297 assert self._sockets is not None
298 self._clients.add(transport)
299
300 def _detach(self, transport):
301 self._clients.discard(transport)
302 if len(self._clients) == 0 and self._sockets is None:
303 self._wakeup()
304
305 def _wakeup(self):
306 waiters = self._waiters
307 self._waiters = None
308 for waiter in waiters:
309 if not waiter.done():
310 waiter.set_result(None)
311
312 def _start_serving(self):
313 if self._serving:
314 return
315 self._serving = True
316 for sock in self._sockets:
317 sock.listen(self._backlog)
318 self._loop._start_serving(
319 self._protocol_factory, sock, self._ssl_context,
320 self, self._backlog, self._ssl_handshake_timeout,
321 self._ssl_shutdown_timeout)
322
323 def get_loop(self):
324 return self._loop
325
326 def is_serving(self):
327 return self._serving
328
329 @property
330 def sockets(self):
331 if self._sockets is None:
332 return ()
333 return tuple(trsock.TransportSocket(s) for s in self._sockets)
334
335 def close(self):
336 sockets = self._sockets
337 if sockets is None:
338 return
339 self._sockets = None
340
341 for sock in sockets:
342 self._loop._stop_serving(sock)
343
344 self._serving = False
345
346 if (self._serving_forever_fut is not None and
347 not self._serving_forever_fut.done()):
348 self._serving_forever_fut.cancel()
349 self._serving_forever_fut = None
350
351 if len(self._clients) == 0:
352 self._wakeup()
353
354 def close_clients(self):
355 for transport in self._clients.copy():
356 transport.close()
357
358 def abort_clients(self):
359 for transport in self._clients.copy():
360 transport.abort()
361
362 async def start_serving(self):
363 self._start_serving()
364 # Skip one loop iteration so that all 'loop.add_reader'
365 # go through.
366 await tasks.sleep(0)
367
368 async def serve_forever(self):
369 if self._serving_forever_fut is not None:
370 raise RuntimeError(
371 f'server {self!r} is already being awaited on serve_forever()')
372 if self._sockets is None:
373 raise RuntimeError(f'server {self!r} is closed')
374
375 self._start_serving()
376 self._serving_forever_fut = self._loop.create_future()
377
378 try:
379 await self._serving_forever_fut
380 except exceptions.CancelledError:
381 try:
382 self.close()
383 await self.wait_closed()
384 finally:
385 raise
386 finally:
387 self._serving_forever_fut = None
388
389 async def wait_closed(self):
390 """Wait until server is closed and all connections are dropped.
391
392 - If the server is not closed, wait.
393 - If it is closed, but there are still active connections, wait.
394
395 Anyone waiting here will be unblocked once both conditions
396 (server is closed and all connections have been dropped)
397 have become true, in either order.
398
399 Historical note: In 3.11 and before, this was broken, returning
400 immediately if the server was already closed, even if there
401 were still active connections. An attempted fix in 3.12.0 was
402 still broken, returning immediately if the server was still
403 open and there were no active connections. Hopefully in 3.12.1
404 we have it right.
405 """
406 # Waiters are unblocked by self._wakeup(), which is called
407 # from two places: self.close() and self._detach(), but only
408 # when both conditions have become true. To signal that this
409 # has happened, self._wakeup() sets self._waiters to None.
410 if self._waiters is None:
411 return
412 waiter = self._loop.create_future()
413 self._waiters.append(waiter)
414 await waiter
415
416
417 class BaseEventLoop(events.AbstractEventLoop):
418
419 def __init__(self):
420 self._timer_cancelled_count = 0
421 self._closed = False
422 self._stopping = False
423 self._ready = collections.deque()
424 self._scheduled = []
425 self._default_executor = None
426 self._internal_fds = 0
427 # Identifier of the thread running the event loop, or None if the
428 # event loop is not running
429 self._thread_id = None
430 self._clock_resolution = time.get_clock_info('monotonic').resolution
431 self._exception_handler = None
432 self.set_debug(coroutines._is_debug_mode())
433 # The preserved state of async generator hooks.
434 self._old_agen_hooks = None
435 # In debug mode, if the execution of a callback or a step of a task
436 # exceed this duration in seconds, the slow callback/task is logged.
437 self.slow_callback_duration = 0.1
438 self._current_handle = None
439 self._task_factory = None
440 self._coroutine_origin_tracking_enabled = False
441 self._coroutine_origin_tracking_saved_depth = None
442
443 # A weak set of all asynchronous generators that are
444 # being iterated by the loop.
445 self._asyncgens = weakref.WeakSet()
446 # Set to True when `loop.shutdown_asyncgens` is called.
447 self._asyncgens_shutdown_called = False
448 # Set to True when `loop.shutdown_default_executor` is called.
449 self._executor_shutdown_called = False
450
451 def __repr__(self):
452 return (
453 f'<{self.__class__.__name__} running={self.is_running()} '
454 f'closed={self.is_closed()} debug={self.get_debug()}>'
455 )
456
457 def create_future(self):
458 """Create a Future object attached to the loop."""
459 return futures.Future(loop=self)
460
461 def create_task(self, coro, **kwargs):
462 """Schedule or begin executing a coroutine object.
463
464 Return a task object.
465 """
466 self._check_closed()
467 if self._task_factory is not None:
468 return self._task_factory(self, coro, **kwargs)
469
470 task = tasks.Task(coro, loop=self, **kwargs)
471 if task._source_traceback:
472 del task._source_traceback[-1]
473 try:
474 return task
475 finally:
476 # gh-128552: prevent a refcycle of
477 # task.exception().__traceback__->BaseEventLoop.create_task->task
478 del task
479
480 def set_task_factory(self, factory):
481 """Set a task factory that will be used by loop.create_task().
482
483 If factory is None the default task factory will be set.
484
485 If factory is a callable, it should have a signature matching
486 '(loop, coro, **kwargs)', where 'loop' will be a reference to the active
487 event loop, 'coro' will be a coroutine object, and **kwargs will be
488 arbitrary keyword arguments that should be passed on to Task.
489 The callable must return a Task.
490 """
491 if factory is not None and not callable(factory):
492 raise TypeError('task factory must be a callable or None')
493 self._task_factory = factory
494
495 def get_task_factory(self):
496 """Return a task factory, or None if the default one is in use."""
497 return self._task_factory
498
499 def _make_socket_transport(self, sock, protocol, waiter=None, *,
500 extra=None, server=None):
501 """Create socket transport."""
502 raise NotImplementedError
503
504 def _make_ssl_transport(
505 self, rawsock, protocol, sslcontext, waiter=None,
506 *, server_side=False, server_hostname=None,
507 extra=None, server=None,
508 ssl_handshake_timeout=None,
509 ssl_shutdown_timeout=None,
510 call_connection_made=True):
511 """Create SSL transport."""
512 raise NotImplementedError
513
514 def _make_datagram_transport(self, sock, protocol,
515 address=None, waiter=None, extra=None):
516 """Create datagram transport."""
517 raise NotImplementedError
518
519 def _make_read_pipe_transport(self, pipe, protocol, waiter=None,
520 extra=None):
521 """Create read pipe transport."""
522 raise NotImplementedError
523
524 def _make_write_pipe_transport(self, pipe, protocol, waiter=None,
525 extra=None):
526 """Create write pipe transport."""
527 raise NotImplementedError
528
529 async def _make_subprocess_transport(self, protocol, args, shell,
530 stdin, stdout, stderr, bufsize,
531 extra=None, **kwargs):
532 """Create subprocess transport."""
533 raise NotImplementedError
534
535 def _write_to_self(self):
536 """Write a byte to self-pipe, to wake up the event loop.
537
538 This may be called from a different thread.
539
540 The subclass is responsible for implementing the self-pipe.
541 """
542 raise NotImplementedError
543
544 def _process_events(self, event_list):
545 """Process selector events."""
546 raise NotImplementedError
547
548 def _check_closed(self):
549 if self._closed:
550 raise RuntimeError('Event loop is closed')
551
552 def _check_default_executor(self):
553 if self._executor_shutdown_called:
554 raise RuntimeError('Executor shutdown has been called')
555
556 def _asyncgen_finalizer_hook(self, agen):
557 self._asyncgens.discard(agen)
558 if not self.is_closed():
559 self.call_soon_threadsafe(self.create_task, agen.aclose())
560
561 def _asyncgen_firstiter_hook(self, agen):
562 if self._asyncgens_shutdown_called:
563 warnings.warn(
564 f"asynchronous generator {agen!r} was scheduled after "
565 f"loop.shutdown_asyncgens() call",
566 ResourceWarning, source=self)
567
568 self._asyncgens.add(agen)
569
570 async def shutdown_asyncgens(self):
571 """Shutdown all active asynchronous generators."""
572 self._asyncgens_shutdown_called = True
573
574 if not len(self._asyncgens):
575 # If Python version is <3.6 or we don't have any asynchronous
576 # generators alive.
577 return
578
579 closing_agens = list(self._asyncgens)
580 self._asyncgens.clear()
581
582 results = await tasks.gather(
583 *[ag.aclose() for ag in closing_agens],
584 return_exceptions=True)
585
586 for result, agen in zip(results, closing_agens):
587 if isinstance(result, Exception):
588 self.call_exception_handler({
589 'message': f'an error occurred during closing of '
590 f'asynchronous generator {agen!r}',
591 'exception': result,
592 'asyncgen': agen
593 })
594
595 async def shutdown_default_executor(self, timeout=None):
596 """Schedule the shutdown of the default executor.
597
598 The timeout parameter specifies the amount of time the executor will
599 be given to finish joining. The default value is None, which means
600 that the executor will be given an unlimited amount of time.
601 """
602 self._executor_shutdown_called = True
603 if self._default_executor is None:
604 return
605 future = self.create_future()
606 thread = threading.Thread(target=self._do_shutdown, args=(future,))
607 thread.start()
608 try:
609 async with timeouts.timeout(timeout):
610 await future
611 except TimeoutError:
612 warnings.warn("The executor did not finishing joining "
613 f"its threads within {timeout} seconds.",
614 RuntimeWarning, stacklevel=2)
615 self._default_executor.shutdown(wait=False)
616 else:
617 thread.join()
618
619 def _do_shutdown(self, future):
620 try:
621 self._default_executor.shutdown(wait=True)
622 if not self.is_closed():
623 self.call_soon_threadsafe(futures._set_result_unless_cancelled,
624 future, None)
625 except Exception as ex:
626 if not self.is_closed() and not future.cancelled():
627 self.call_soon_threadsafe(future.set_exception, ex)
628
629 def _check_running(self):
630 if self.is_running():
631 raise RuntimeError('This event loop is already running')
632 if events._get_running_loop() is not None:
633 raise RuntimeError(
634 'Cannot run the event loop while another loop is running')
635
636 def _run_forever_setup(self):
637 """Prepare the run loop to process events.
638
639 This method exists so that custom event loop subclasses (e.g., event loops
640 that integrate a GUI event loop with Python's event loop) have access to all the
641 loop setup logic.
642 """
643 self._check_closed()
644 self._check_running()
645 self._set_coroutine_origin_tracking(self._debug)
646
647 self._old_agen_hooks = sys.get_asyncgen_hooks()
648 self._thread_id = threading.get_ident()
649 sys.set_asyncgen_hooks(
650 firstiter=self._asyncgen_firstiter_hook,
651 finalizer=self._asyncgen_finalizer_hook
652 )
653
654 events._set_running_loop(self)
655
656 def _run_forever_cleanup(self):
657 """Clean up after an event loop finishes the looping over events.
658
659 This method exists so that custom event loop subclasses (e.g., event loops
660 that integrate a GUI event loop with Python's event loop) have access to all the
661 loop cleanup logic.
662 """
663 self._stopping = False
664 self._thread_id = None
665 events._set_running_loop(None)
666 self._set_coroutine_origin_tracking(False)
667 # Restore any pre-existing async generator hooks.
668 if self._old_agen_hooks is not None:
669 sys.set_asyncgen_hooks(*self._old_agen_hooks)
670 self._old_agen_hooks = None
671
672 def run_forever(self):
673 """Run until stop() is called."""
674 self._run_forever_setup()
675 try:
676 while True:
677 self._run_once()
678 if self._stopping:
679 break
680 finally:
681 self._run_forever_cleanup()
682
683 def run_until_complete(self, future):
684 """Run until the Future is done.
685
686 If the argument is a coroutine, it is wrapped in a Task.
687
688 WARNING: It would be disastrous to call run_until_complete()
689 with the same coroutine twice -- it would wrap it in two
690 different Tasks and that can't be good.
691
692 Return the Future's result, or raise its exception.
693 """
694 self._check_closed()
695 self._check_running()
696
697 new_task = not futures.isfuture(future)
698 future = tasks.ensure_future(future, loop=self)
699 if new_task:
700 # An exception is raised if the future didn't complete, so there
701 # is no need to log the "destroy pending task" message
702 future._log_destroy_pending = False
703
704 future.add_done_callback(_run_until_complete_cb)
705 try:
706 self.run_forever()
707 except:
708 if new_task and future.done() and not future.cancelled():
709 # The coroutine raised a BaseException. Consume the exception
710 # to not log a warning, the caller doesn't have access to the
711 # local task.
712 future.exception()
713 raise
714 finally:
715 future.remove_done_callback(_run_until_complete_cb)
716 if not future.done():
717 raise RuntimeError('Event loop stopped before Future completed.')
718
719 return future.result()
720
721 def stop(self):
722 """Stop running the event loop.
723
724 Every callback already scheduled will still run. This simply informs
725 run_forever to stop looping after a complete iteration.
726 """
727 self._stopping = True
728
729 def close(self):
730 """Close the event loop.
731
732 This clears the queues and shuts down the executor,
733 but does not wait for the executor to finish.
734
735 The event loop must not be running.
736 """
737 if self.is_running():
738 raise RuntimeError("Cannot close a running event loop")
739 if self._closed:
740 return
741 if self._debug:
742 logger.debug("Close %r", self)
743 self._closed = True
744 self._ready.clear()
745 self._scheduled.clear()
746 self._executor_shutdown_called = True
747 executor = self._default_executor
748 if executor is not None:
749 self._default_executor = None
750 executor.shutdown(wait=False)
751
752 def is_closed(self):
753 """Returns True if the event loop was closed."""
754 return self._closed
755
756 def __del__(self, _warn=warnings.warn):
757 if not self.is_closed():
758 _warn(f"unclosed event loop {self!r}", ResourceWarning, source=self)
759 if not self.is_running():
760 self.close()
761
762 def is_running(self):
763 """Returns True if the event loop is running."""
764 return (self._thread_id is not None)
765
766 def time(self):
767 """Return the time according to the event loop's clock.
768
769 This is a float expressed in seconds since an epoch, but the
770 epoch, precision, accuracy and drift are unspecified and may
771 differ per event loop.
772 """
773 return time.monotonic()
774
775 def call_later(self, delay, callback, *args, context=None):
776 """Arrange for a callback to be called at a given time.
777
778 Return a Handle: an opaque object with a cancel() method that
779 can be used to cancel the call.
780
781 The delay can be an int or float, expressed in seconds. It is
782 always relative to the current time.
783
784 Each callback will be called exactly once. If two callbacks
785 are scheduled for exactly the same time, it is undefined which
786 will be called first.
787
788 Any positional arguments after the callback will be passed to
789 the callback when it is called.
790 """
791 if delay is None:
792 raise TypeError('delay must not be None')
793 timer = self.call_at(self.time() + delay, callback, *args,
794 context=context)
795 if timer._source_traceback:
796 del timer._source_traceback[-1]
797 return timer
798
799 def call_at(self, when, callback, *args, context=None):
800 """Like call_later(), but uses an absolute time.
801
802 Absolute time corresponds to the event loop's time() method.
803 """
804 if when is None:
805 raise TypeError("when cannot be None")
806 self._check_closed()
807 if self._debug:
808 self._check_thread()
809 self._check_callback(callback, 'call_at')
810 timer = events.TimerHandle(when, callback, args, self, context)
811 if timer._source_traceback:
812 del timer._source_traceback[-1]
813 heapq.heappush(self._scheduled, timer)
814 timer._scheduled = True
815 return timer
816
817 def call_soon(self, callback, *args, context=None):
818 """Arrange for a callback to be called as soon as possible.
819
820 This operates as a FIFO queue: callbacks are called in the
821 order in which they are registered. Each callback will be
822 called exactly once.
823
824 Any positional arguments after the callback will be passed to
825 the callback when it is called.
826 """
827 self._check_closed()
828 if self._debug:
829 self._check_thread()
830 self._check_callback(callback, 'call_soon')
831 handle = self._call_soon(callback, args, context)
832 if handle._source_traceback:
833 del handle._source_traceback[-1]
834 return handle
835
836 def _check_callback(self, callback, method):
837 if (coroutines.iscoroutine(callback) or
838 coroutines._iscoroutinefunction(callback)):
839 raise TypeError(
840 f"coroutines cannot be used with {method}()")
841 if not callable(callback):
842 raise TypeError(
843 f'a callable object was expected by {method}(), '
844 f'got {callback!r}')
845
846 def _call_soon(self, callback, args, context):
847 handle = events.Handle(callback, args, self, context)
848 if handle._source_traceback:
849 del handle._source_traceback[-1]
850 self._ready.append(handle)
851 return handle
852
853 def _check_thread(self):
854 """Check that the current thread is the thread running the event loop.
855
856 Non-thread-safe methods of this class make this assumption and will
857 likely behave incorrectly when the assumption is violated.
858
859 Should only be called when (self._debug == True). The caller is
860 responsible for checking this condition for performance reasons.
861 """
862 if self._thread_id is None:
863 return
864 thread_id = threading.get_ident()
865 if thread_id != self._thread_id:
866 raise RuntimeError(
867 "Non-thread-safe operation invoked on an event loop other "
868 "than the current one")
869
870 def call_soon_threadsafe(self, callback, *args, context=None):
871 """Like call_soon(), but thread-safe."""
872 self._check_closed()
873 if self._debug:
874 self._check_callback(callback, 'call_soon_threadsafe')
875 handle = events._ThreadSafeHandle(callback, args, self, context)
876 self._ready.append(handle)
877 if handle._source_traceback:
878 del handle._source_traceback[-1]
879 if handle._source_traceback:
880 del handle._source_traceback[-1]
881 self._write_to_self()
882 return handle
883
884 def run_in_executor(self, executor, func, *args):
885 self._check_closed()
886 if self._debug:
887 self._check_callback(func, 'run_in_executor')
888 if executor is None:
889 executor = self._default_executor
890 # Only check when the default executor is being used
891 self._check_default_executor()
892 if executor is None:
893 executor = concurrent.futures.ThreadPoolExecutor(
894 thread_name_prefix='asyncio'
895 )
896 self._default_executor = executor
897 return futures.wrap_future(
898 executor.submit(func, *args), loop=self)
899
900 def set_default_executor(self, executor):
901 if not isinstance(executor, concurrent.futures.ThreadPoolExecutor):
902 raise TypeError('executor must be ThreadPoolExecutor instance')
903 self._default_executor = executor
904
905 def _getaddrinfo_debug(self, host, port, family, type, proto, flags):
906 msg = [f"{host}:{port!r}"]
907 if family:
908 msg.append(f'family={family!r}')
909 if type:
910 msg.append(f'type={type!r}')
911 if proto:
912 msg.append(f'proto={proto!r}')
913 if flags:
914 msg.append(f'flags={flags!r}')
915 msg = ', '.join(msg)
916 logger.debug('Get address info %s', msg)
917
918 t0 = self.time()
919 addrinfo = socket.getaddrinfo(host, port, family, type, proto, flags)
920 dt = self.time() - t0
921
922 msg = f'Getting address info {msg} took {dt * 1e3:.3f}ms: {addrinfo!r}'
923 if dt >= self.slow_callback_duration:
924 logger.info(msg)
925 else:
926 logger.debug(msg)
927 return addrinfo
928
929 async def getaddrinfo(self, host, port, *,
930 family=0, type=0, proto=0, flags=0):
931 if self._debug:
932 getaddr_func = self._getaddrinfo_debug
933 else:
934 getaddr_func = socket.getaddrinfo
935
936 return await self.run_in_executor(
937 None, getaddr_func, host, port, family, type, proto, flags)
938
939 async def getnameinfo(self, sockaddr, flags=0):
940 return await self.run_in_executor(
941 None, socket.getnameinfo, sockaddr, flags)
942
943 async def sock_sendfile(self, sock, file, offset=0, count=None,
944 *, fallback=True):
945 if self._debug and sock.gettimeout() != 0:
946 raise ValueError("the socket must be non-blocking")
947 _check_ssl_socket(sock)
948 self._check_sendfile_params(sock, file, offset, count)
949 try:
950 return await self._sock_sendfile_native(sock, file,
951 offset, count)
952 except exceptions.SendfileNotAvailableError as exc:
953 if not fallback:
954 raise
955 return await self._sock_sendfile_fallback(sock, file,
956 offset, count)
957
958 async def _sock_sendfile_native(self, sock, file, offset, count):
959 # NB: sendfile syscall is not supported for SSL sockets and
960 # non-mmap files even if sendfile is supported by OS
961 raise exceptions.SendfileNotAvailableError(
962 f"syscall sendfile is not available for socket {sock!r} "
963 f"and file {file!r} combination")
964
965 async def _sock_sendfile_fallback(self, sock, file, offset, count):
966 if offset:
967 file.seek(offset)
968 blocksize = (
969 min(count, constants.SENDFILE_FALLBACK_READBUFFER_SIZE)
970 if count else constants.SENDFILE_FALLBACK_READBUFFER_SIZE
971 )
972 buf = bytearray(blocksize)
973 total_sent = 0
974 try:
975 while True:
976 if count:
977 blocksize = min(count - total_sent, blocksize)
978 if blocksize <= 0:
979 break
980 view = memoryview(buf)[:blocksize]
981 read = await self.run_in_executor(None, file.readinto, view)
982 if not read:
983 break # EOF
984 await self.sock_sendall(sock, view[:read])
985 total_sent += read
986 return total_sent
987 finally:
988 if total_sent > 0 and hasattr(file, 'seek'):
989 file.seek(offset + total_sent)
990
991 def _check_sendfile_params(self, sock, file, offset, count):
992 if 'b' not in getattr(file, 'mode', 'b'):
993 raise ValueError("file should be opened in binary mode")
994 if not sock.type == socket.SOCK_STREAM:
995 raise ValueError("only SOCK_STREAM type sockets are supported")
996 if count is not None:
997 if not isinstance(count, int):
998 raise TypeError(
999 "count must be a positive integer (got {!r})".format(count))
1000 if count <= 0:
1001 raise ValueError(
1002 "count must be a positive integer (got {!r})".format(count))
1003 if not isinstance(offset, int):
1004 raise TypeError(
1005 "offset must be a non-negative integer (got {!r})".format(
1006 offset))
1007 if offset < 0:
1008 raise ValueError(
1009 "offset must be a non-negative integer (got {!r})".format(
1010 offset))
1011
1012 async def _connect_sock(self, exceptions, addr_info, local_addr_infos=None):
1013 """Create, bind and connect one socket."""
1014 my_exceptions = []
1015 exceptions.append(my_exceptions)
1016 family, type_, proto, _, address = addr_info
1017 sock = None
1018 try:
1019 try:
1020 sock = socket.socket(family=family, type=type_, proto=proto)
1021 sock.setblocking(False)
1022 if local_addr_infos is not None:
1023 for lfamily, _, _, _, laddr in local_addr_infos:
1024 # skip local addresses of different family
1025 if lfamily != family:
1026 continue
1027 try:
1028 sock.bind(laddr)
1029 break
1030 except OSError as exc:
1031 msg = (
1032 f'error while attempting to bind on '
1033 f'address {laddr!r}: {str(exc).lower()}'
1034 )
1035 exc = OSError(exc.errno, msg)
1036 my_exceptions.append(exc)
1037 else: # all bind attempts failed
1038 if my_exceptions:
1039 raise my_exceptions.pop()
1040 else:
1041 raise OSError(f"no matching local address with {family=} found")
1042 await self.sock_connect(sock, address)
1043 return sock
1044 except OSError as exc:
1045 my_exceptions.append(exc)
1046 raise
1047 except:
1048 if sock is not None:
1049 try:
1050 sock.close()
1051 except OSError:
1052 # An error when closing a newly created socket is
1053 # not important, but it can overwrite more important
1054 # non-OSError error. So ignore it.
1055 pass
1056 raise
1057 finally:
1058 exceptions = my_exceptions = None
1059
1060 async def create_connection(
1061 self, protocol_factory, host=None, port=None,
1062 *, ssl=None, family=0,
1063 proto=0, flags=0, sock=None,
1064 local_addr=None, server_hostname=None,
1065 ssl_handshake_timeout=None,
1066 ssl_shutdown_timeout=None,
1067 happy_eyeballs_delay=None, interleave=None,
1068 all_errors=False):
1069 """Connect to a TCP server.
1070
1071 Create a streaming transport connection to a given internet host and
1072 port: socket family AF_INET or socket.AF_INET6 depending on host (or
1073 family if specified), socket type SOCK_STREAM. protocol_factory must be
1074 a callable returning a protocol instance.
1075
1076 This method is a coroutine which will try to establish the connection
1077 in the background. When successful, the coroutine returns a
1078 (transport, protocol) pair.
1079 """
1080 if server_hostname is not None and not ssl:
1081 raise ValueError('server_hostname is only meaningful with ssl')
1082
1083 if server_hostname is None and ssl:
1084 # Use host as default for server_hostname. It is an error
1085 # if host is empty or not set, e.g. when an
1086 # already-connected socket was passed or when only a port
1087 # is given. To avoid this error, you can pass
1088 # server_hostname='' -- this will bypass the hostname
1089 # check. (This also means that if host is a numeric
1090 # IP/IPv6 address, we will attempt to verify that exact
1091 # address; this will probably fail, but it is possible to
1092 # create a certificate for a specific IP address, so we
1093 # don't judge it here.)
1094 if not host:
1095 raise ValueError('You must set server_hostname '
1096 'when using ssl without a host')
1097 server_hostname = host
1098
1099 if ssl_handshake_timeout is not None and not ssl:
1100 raise ValueError(
1101 'ssl_handshake_timeout is only meaningful with ssl')
1102
1103 if ssl_shutdown_timeout is not None and not ssl:
1104 raise ValueError(
1105 'ssl_shutdown_timeout is only meaningful with ssl')
1106
1107 if sock is not None:
1108 _check_ssl_socket(sock)
1109
1110 if happy_eyeballs_delay is not None and interleave is None:
1111 # If using happy eyeballs, default to interleave addresses by family
1112 interleave = 1
1113
1114 if host is not None or port is not None:
1115 if sock is not None:
1116 raise ValueError(
1117 'host/port and sock can not be specified at the same time')
1118
1119 infos = await self._ensure_resolved(
1120 (host, port), family=family,
1121 type=socket.SOCK_STREAM, proto=proto, flags=flags, loop=self)
1122 if not infos:
1123 raise OSError('getaddrinfo() returned empty list')
1124
1125 if local_addr is not None:
1126 laddr_infos = await self._ensure_resolved(
1127 local_addr, family=family,
1128 type=socket.SOCK_STREAM, proto=proto,
1129 flags=flags, loop=self)
1130 if not laddr_infos:
1131 raise OSError('getaddrinfo() returned empty list')
1132 else:
1133 laddr_infos = None
1134
1135 if interleave:
1136 infos = _interleave_addrinfos(infos, interleave)
1137
1138 exceptions = []
1139 if happy_eyeballs_delay is None:
1140 # not using happy eyeballs
1141 for addrinfo in infos:
1142 try:
1143 sock = await self._connect_sock(
1144 exceptions, addrinfo, laddr_infos)
1145 break
1146 except OSError:
1147 continue
1148 else: # using happy eyeballs
1149 sock = (await staggered.staggered_race(
1150 (
1151 # can't use functools.partial as it keeps a reference
1152 # to exceptions
1153 lambda addrinfo=addrinfo: self._connect_sock(
1154 exceptions, addrinfo, laddr_infos
1155 )
1156 for addrinfo in infos
1157 ),
1158 happy_eyeballs_delay,
1159 loop=self,
1160 ))[0] # can't use sock, _, _ as it keeks a reference to exceptions
1161
1162 if sock is None:
1163 exceptions = [exc for sub in exceptions for exc in sub]
1164 try:
1165 if all_errors:
1166 raise ExceptionGroup("create_connection failed", exceptions)
1167 if len(exceptions) == 1:
1168 raise exceptions[0]
1169 elif exceptions:
1170 # If they all have the same str(), raise one.
1171 model = str(exceptions[0])
1172 if all(str(exc) == model for exc in exceptions):
1173 raise exceptions[0]
1174 # Raise a combined exception so the user can see all
1175 # the various error messages.
1176 raise OSError('Multiple exceptions: {}'.format(
1177 ', '.join(str(exc) for exc in exceptions)))
1178 else:
1179 # No exceptions were collected, raise a timeout error
1180 raise TimeoutError('create_connection failed')
1181 finally:
1182 exceptions = None
1183
1184 else:
1185 if sock is None:
1186 raise ValueError(
1187 'host and port was not specified and no sock specified')
1188 if sock.type != socket.SOCK_STREAM:
1189 # We allow AF_INET, AF_INET6, AF_UNIX as long as they
1190 # are SOCK_STREAM.
1191 # We support passing AF_UNIX sockets even though we have
1192 # a dedicated API for that: create_unix_connection.
1193 # Disallowing AF_UNIX in this method, breaks backwards
1194 # compatibility.
1195 raise ValueError(
1196 f'A Stream Socket was expected, got {sock!r}')
1197
1198 transport, protocol = await self._create_connection_transport(
1199 sock, protocol_factory, ssl, server_hostname,
1200 ssl_handshake_timeout=ssl_handshake_timeout,
1201 ssl_shutdown_timeout=ssl_shutdown_timeout)
1202 if self._debug:
1203 # Get the socket from the transport because SSL transport closes
1204 # the old socket and creates a new SSL socket
1205 sock = transport.get_extra_info('socket')
1206 logger.debug("%r connected to %s:%r: (%r, %r)",
1207 sock, host, port, transport, protocol)
1208 return transport, protocol
1209
1210 async def _create_connection_transport(
1211 self, sock, protocol_factory, ssl,
1212 server_hostname, server_side=False,
1213 ssl_handshake_timeout=None,
1214 ssl_shutdown_timeout=None):
1215
1216 sock.setblocking(False)
1217
1218 protocol = protocol_factory()
1219 waiter = self.create_future()
1220 if ssl:
1221 sslcontext = None if isinstance(ssl, bool) else ssl
1222 transport = self._make_ssl_transport(
1223 sock, protocol, sslcontext, waiter,
1224 server_side=server_side, server_hostname=server_hostname,
1225 ssl_handshake_timeout=ssl_handshake_timeout,
1226 ssl_shutdown_timeout=ssl_shutdown_timeout)
1227 else:
1228 transport = self._make_socket_transport(sock, protocol, waiter)
1229
1230 try:
1231 await waiter
1232 except:
1233 transport.close()
1234 raise
1235
1236 return transport, protocol
1237
1238 async def sendfile(self, transport, file, offset=0, count=None,
1239 *, fallback=True):
1240 """Send a file to transport.
1241
1242 Return the total number of bytes which were sent.
1243
1244 The method uses high-performance os.sendfile if available.
1245
1246 file must be a regular file object opened in binary mode.
1247
1248 offset tells from where to start reading the file. If specified,
1249 count is the total number of bytes to transmit as opposed to
1250 sending the file until EOF is reached. File position is updated on
1251 return or also in case of error in which case file.tell()
1252 can be used to figure out the number of bytes
1253 which were sent.
1254
1255 fallback set to True makes asyncio to manually read and send
1256 the file when the platform does not support the sendfile syscall
1257 (e.g. Windows or SSL socket on Unix).
1258
1259 Raise SendfileNotAvailableError if the system does not support
1260 sendfile syscall and fallback is False.
1261 """
1262 if transport.is_closing():
1263 raise RuntimeError("Transport is closing")
1264 mode = getattr(transport, '_sendfile_compatible',
1265 constants._SendfileMode.UNSUPPORTED)
1266 if mode is constants._SendfileMode.UNSUPPORTED:
1267 raise RuntimeError(
1268 f"sendfile is not supported for transport {transport!r}")
1269 if mode is constants._SendfileMode.TRY_NATIVE:
1270 try:
1271 return await self._sendfile_native(transport, file,
1272 offset, count)
1273 except exceptions.SendfileNotAvailableError as exc:
1274 if not fallback:
1275 raise
1276
1277 if not fallback:
1278 raise RuntimeError(
1279 f"fallback is disabled and native sendfile is not "
1280 f"supported for transport {transport!r}")
1281
1282 return await self._sendfile_fallback(transport, file,
1283 offset, count)
1284
1285 async def _sendfile_native(self, transp, file, offset, count):
1286 raise exceptions.SendfileNotAvailableError(
1287 "sendfile syscall is not supported")
1288
1289 async def _sendfile_fallback(self, transp, file, offset, count):
1290 if offset:
1291 file.seek(offset)
1292 blocksize = min(count, 16384) if count else 16384
1293 buf = bytearray(blocksize)
1294 total_sent = 0
1295 proto = _SendfileFallbackProtocol(transp)
1296 try:
1297 while True:
1298 if count:
1299 blocksize = min(count - total_sent, blocksize)
1300 if blocksize <= 0:
1301 return total_sent
1302 view = memoryview(buf)[:blocksize]
1303 read = await self.run_in_executor(None, file.readinto, view)
1304 if not read:
1305 return total_sent # EOF
1306 transp.write(view[:read])
1307 await proto.drain()
1308 total_sent += read
1309 finally:
1310 if total_sent > 0 and hasattr(file, 'seek'):
1311 file.seek(offset + total_sent)
1312 await proto.restore()
1313
1314 async def start_tls(self, transport, protocol, sslcontext, *,
1315 server_side=False,
1316 server_hostname=None,
1317 ssl_handshake_timeout=None,
1318 ssl_shutdown_timeout=None):
1319 """Upgrade transport to TLS.
1320
1321 Return a new transport that *protocol* should start using
1322 immediately.
1323 """
1324 if ssl is None:
1325 raise RuntimeError('Python ssl module is not available')
1326
1327 if not isinstance(sslcontext, ssl.SSLContext):
1328 raise TypeError(
1329 f'sslcontext is expected to be an instance of ssl.SSLContext, '
1330 f'got {sslcontext!r}')
1331
1332 if not getattr(transport, '_start_tls_compatible', False):
1333 raise TypeError(
1334 f'transport {transport!r} is not supported by start_tls()')
1335
1336 waiter = self.create_future()
1337 ssl_protocol = sslproto.SSLProtocol(
1338 self, protocol, sslcontext, waiter,
1339 server_side, server_hostname,
1340 ssl_handshake_timeout=ssl_handshake_timeout,
1341 ssl_shutdown_timeout=ssl_shutdown_timeout,
1342 call_connection_made=False)
1343
1344 # Pause early so that "ssl_protocol.data_received()" doesn't
1345 # have a chance to get called before "ssl_protocol.connection_made()".
1346 transport.pause_reading()
1347
1348 # gh-142352: move buffered StreamReader data to SSLProtocol
1349 if server_side:
1350 from .streams import StreamReaderProtocol
1351 if isinstance(protocol, StreamReaderProtocol):
1352 stream_reader = getattr(protocol, '_stream_reader', None)
1353 if stream_reader is not None:
1354 buffer = stream_reader._buffer
1355 if buffer:
1356 ssl_protocol._incoming.write(buffer)
1357 buffer.clear()
1358
1359 transport.set_protocol(ssl_protocol)
1360 conmade_cb = self.call_soon(ssl_protocol.connection_made, transport)
1361 resume_cb = self.call_soon(transport.resume_reading)
1362
1363 try:
1364 await waiter
1365 except BaseException:
1366 transport.close()
1367 conmade_cb.cancel()
1368 resume_cb.cancel()
1369 raise
1370
1371 return ssl_protocol._app_transport
1372
1373 async def create_datagram_endpoint(self, protocol_factory,
1374 local_addr=None, remote_addr=None, *,
1375 family=0, proto=0, flags=0,
1376 reuse_port=None,
1377 allow_broadcast=None, sock=None):
1378 """Create datagram connection."""
1379 if sock is not None:
1380 if sock.type == socket.SOCK_STREAM:
1381 raise ValueError(
1382 f'A datagram socket was expected, got {sock!r}')
1383 if (local_addr or remote_addr or
1384 family or proto or flags or
1385 reuse_port or allow_broadcast):
1386 # show the problematic kwargs in exception msg
1387 opts = dict(local_addr=local_addr, remote_addr=remote_addr,
1388 family=family, proto=proto, flags=flags,
1389 reuse_port=reuse_port,
1390 allow_broadcast=allow_broadcast)
1391 problems = ', '.join(f'{k}={v}' for k, v in opts.items() if v)
1392 raise ValueError(
1393 f'socket modifier keyword arguments can not be used '
1394 f'when sock is specified. ({problems})')
1395 sock.setblocking(False)
1396 r_addr = None
1397 else:
1398 if not (local_addr or remote_addr):
1399 if family == 0:
1400 raise ValueError('unexpected address family')
1401 addr_pairs_info = (((family, proto), (None, None)),)
1402 elif hasattr(socket, 'AF_UNIX') and family == socket.AF_UNIX:
1403 for addr in (local_addr, remote_addr):
1404 if addr is not None and not isinstance(addr, str):
1405 raise TypeError('string is expected')
1406
1407 if local_addr and local_addr[0] not in (0, '\x00'):
1408 try:
1409 if stat.S_ISSOCK(os.stat(local_addr).st_mode):
1410 os.remove(local_addr)
1411 except FileNotFoundError:
1412 pass
1413 except OSError as err:
1414 # Directory may have permissions only to create socket.
1415 logger.error('Unable to check or remove stale UNIX '
1416 'socket %r: %r',
1417 local_addr, err)
1418
1419 addr_pairs_info = (((family, proto),
1420 (local_addr, remote_addr)), )
1421 else:
1422 # join address by (family, protocol)
1423 addr_infos = {} # Using order preserving dict
1424 for idx, addr in ((0, local_addr), (1, remote_addr)):
1425 if addr is not None:
1426 if not (isinstance(addr, tuple) and len(addr) == 2):
1427 raise TypeError('2-tuple is expected')
1428
1429 infos = await self._ensure_resolved(
1430 addr, family=family, type=socket.SOCK_DGRAM,
1431 proto=proto, flags=flags, loop=self)
1432 if not infos:
1433 raise OSError('getaddrinfo() returned empty list')
1434
1435 for fam, _, pro, _, address in infos:
1436 key = (fam, pro)
1437 if key not in addr_infos:
1438 addr_infos[key] = [None, None]
1439 addr_infos[key][idx] = address
1440
1441 # each addr has to have info for each (family, proto) pair
1442 addr_pairs_info = [
1443 (key, addr_pair) for key, addr_pair in addr_infos.items()
1444 if not ((local_addr and addr_pair[0] is None) or
1445 (remote_addr and addr_pair[1] is None))]
1446
1447 if not addr_pairs_info:
1448 raise ValueError('can not get address information')
1449
1450 exceptions = []
1451
1452 for ((family, proto),
1453 (local_address, remote_address)) in addr_pairs_info:
1454 sock = None
1455 r_addr = None
1456 try:
1457 sock = socket.socket(
1458 family=family, type=socket.SOCK_DGRAM, proto=proto)
1459 if reuse_port:
1460 _set_reuseport(sock)
1461 if allow_broadcast:
1462 sock.setsockopt(
1463 socket.SOL_SOCKET, socket.SO_BROADCAST, 1)
1464 sock.setblocking(False)
1465
1466 if local_addr:
1467 sock.bind(local_address)
1468 if remote_addr:
1469 if not allow_broadcast:
1470 await self.sock_connect(sock, remote_address)
1471 r_addr = remote_address
1472 except OSError as exc:
1473 if sock is not None:
1474 sock.close()
1475 exceptions.append(exc)
1476 except:
1477 if sock is not None:
1478 sock.close()
1479 raise
1480 else:
1481 break
1482 else:
1483 raise exceptions[0]
1484
1485 protocol = protocol_factory()
1486 waiter = self.create_future()
1487 transport = self._make_datagram_transport(
1488 sock, protocol, r_addr, waiter)
1489 if self._debug:
1490 if local_addr:
1491 logger.info("Datagram endpoint local_addr=%r remote_addr=%r "
1492 "created: (%r, %r)",
1493 local_addr, remote_addr, transport, protocol)
1494 else:
1495 logger.debug("Datagram endpoint remote_addr=%r created: "
1496 "(%r, %r)",
1497 remote_addr, transport, protocol)
1498
1499 try:
1500 await waiter
1501 except:
1502 transport.close()
1503 raise
1504
1505 return transport, protocol
1506
1507 async def _ensure_resolved(self, address, *,
1508 family=0, type=socket.SOCK_STREAM,
1509 proto=0, flags=0, loop):
1510 host, port = address[:2]
1511 info = _ipaddr_info(host, port, family, type, proto, *address[2:])
1512 if info is not None:
1513 # "host" is already a resolved IP.
1514 return [info]
1515 else:
1516 return await loop.getaddrinfo(host, port, family=family, type=type,
1517 proto=proto, flags=flags)
1518
1519 async def _create_server_getaddrinfo(self, host, port, family, flags):
1520 infos = await self._ensure_resolved((host, port), family=family,
1521 type=socket.SOCK_STREAM,
1522 flags=flags, loop=self)
1523 if not infos:
1524 raise OSError(f'getaddrinfo({host!r}) returned empty list')
1525 return infos
1526
1527 async def create_server(
1528 self, protocol_factory, host=None, port=None,
1529 *,
1530 family=socket.AF_UNSPEC,
1531 flags=socket.AI_PASSIVE,
1532 sock=None,
1533 backlog=100,
1534 ssl=None,
1535 reuse_address=None,
1536 reuse_port=None,
1537 keep_alive=None,
1538 ssl_handshake_timeout=None,
1539 ssl_shutdown_timeout=None,
1540 start_serving=True):
1541 """Create a TCP server.
1542
1543 The host parameter can be a string, in that case the TCP server is
1544 bound to host and port.
1545
1546 The host parameter can also be a sequence of strings and in that case
1547 the TCP server is bound to all hosts of the sequence. If a host
1548 appears multiple times (possibly indirectly e.g. when hostnames
1549 resolve to the same IP address), the server is only bound once to that
1550 host.
1551
1552 Return a Server object which can be used to stop the service.
1553
1554 This method is a coroutine.
1555 """
1556 if isinstance(ssl, bool):
1557 raise TypeError('ssl argument must be an SSLContext or None')
1558
1559 if ssl_handshake_timeout is not None and ssl is None:
1560 raise ValueError(
1561 'ssl_handshake_timeout is only meaningful with ssl')
1562
1563 if ssl_shutdown_timeout is not None and ssl is None:
1564 raise ValueError(
1565 'ssl_shutdown_timeout is only meaningful with ssl')
1566
1567 if sock is not None:
1568 _check_ssl_socket(sock)
1569
1570 if host is not None or port is not None:
1571 if sock is not None:
1572 raise ValueError(
1573 'host/port and sock can not be specified at the same time')
1574
1575 if reuse_address is None:
1576 reuse_address = os.name == "posix" and sys.platform != "cygwin"
1577 sockets = []
1578 if host == '':
1579 hosts = [None]
1580 elif (isinstance(host, str) or
1581 not isinstance(host, collections.abc.Iterable)):
1582 hosts = [host]
1583 else:
1584 hosts = host
1585
1586 fs = [self._create_server_getaddrinfo(host, port, family=family,
1587 flags=flags)
1588 for host in hosts]
1589 infos = await tasks.gather(*fs)
1590 infos = set(itertools.chain.from_iterable(infos))
1591
1592 completed = False
1593 try:
1594 for res in infos:
1595 af, socktype, proto, canonname, sa = res
1596 try:
1597 sock = socket.socket(af, socktype, proto)
1598 except socket.error:
1599 # Assume it's a bad family/type/protocol combination.
1600 if self._debug:
1601 logger.warning('create_server() failed to create '
1602 'socket.socket(%r, %r, %r)',
1603 af, socktype, proto, exc_info=True)
1604 continue
1605 sockets.append(sock)
1606 if reuse_address:
1607 sock.setsockopt(
1608 socket.SOL_SOCKET, socket.SO_REUSEADDR, True)
1609 # Since Linux 6.12.9, SO_REUSEPORT is not allowed
1610 # on other address families than AF_INET/AF_INET6.
1611 if reuse_port and af in (socket.AF_INET, socket.AF_INET6):
1612 _set_reuseport(sock)
1613 if keep_alive:
1614 sock.setsockopt(
1615 socket.SOL_SOCKET, socket.SO_KEEPALIVE, True)
1616 # Disable IPv4/IPv6 dual stack support (enabled by
1617 # default on Linux) which makes a single socket
1618 # listen on both address families.
1619 if (_HAS_IPv6 and
1620 af == socket.AF_INET6 and
1621 hasattr(socket, 'IPPROTO_IPV6')):
1622 sock.setsockopt(socket.IPPROTO_IPV6,
1623 socket.IPV6_V6ONLY,
1624 True)
1625 try:
1626 sock.bind(sa)
1627 except OSError as err:
1628 msg = ('error while attempting '
1629 'to bind on address %r: %s'
1630 % (sa, str(err).lower()))
1631 if err.errno == errno.EADDRNOTAVAIL:
1632 # Assume the family is not enabled (bpo-30945)
1633 sockets.pop()
1634 sock.close()
1635 if self._debug:
1636 logger.warning(msg)
1637 continue
1638 raise OSError(err.errno, msg) from None
1639
1640 if not sockets:
1641 raise OSError('could not bind on any address out of %r'
1642 % ([info[4] for info in infos],))
1643
1644 completed = True
1645 finally:
1646 if not completed:
1647 for sock in sockets:
1648 sock.close()
1649 else:
1650 if sock is None:
1651 raise ValueError('Neither host/port nor sock were specified')
1652 if sock.type != socket.SOCK_STREAM:
1653 raise ValueError(f'A Stream Socket was expected, got {sock!r}')
1654 sockets = [sock]
1655
1656 for sock in sockets:
1657 sock.setblocking(False)
1658
1659 server = Server(self, sockets, protocol_factory,
1660 ssl, backlog, ssl_handshake_timeout,
1661 ssl_shutdown_timeout)
1662 if start_serving:
1663 server._start_serving()
1664 # Skip one loop iteration so that all 'loop.add_reader'
1665 # go through.
1666 await tasks.sleep(0)
1667
1668 if self._debug:
1669 logger.info("%r is serving", server)
1670 return server
1671
1672 async def connect_accepted_socket(
1673 self, protocol_factory, sock,
1674 *, ssl=None,
1675 ssl_handshake_timeout=None,
1676 ssl_shutdown_timeout=None):
1677 if sock.type != socket.SOCK_STREAM:
1678 raise ValueError(f'A Stream Socket was expected, got {sock!r}')
1679
1680 if ssl_handshake_timeout is not None and not ssl:
1681 raise ValueError(
1682 'ssl_handshake_timeout is only meaningful with ssl')
1683
1684 if ssl_shutdown_timeout is not None and not ssl:
1685 raise ValueError(
1686 'ssl_shutdown_timeout is only meaningful with ssl')
1687
1688 _check_ssl_socket(sock)
1689
1690 transport, protocol = await self._create_connection_transport(
1691 sock, protocol_factory, ssl, '', server_side=True,
1692 ssl_handshake_timeout=ssl_handshake_timeout,
1693 ssl_shutdown_timeout=ssl_shutdown_timeout)
1694 if self._debug:
1695 # Get the socket from the transport because SSL transport closes
1696 # the old socket and creates a new SSL socket
1697 sock = transport.get_extra_info('socket')
1698 logger.debug("%r handled: (%r, %r)", sock, transport, protocol)
1699 return transport, protocol
1700
1701 async def connect_read_pipe(self, protocol_factory, pipe):
1702 protocol = protocol_factory()
1703 waiter = self.create_future()
1704 transport = self._make_read_pipe_transport(pipe, protocol, waiter)
1705
1706 try:
1707 await waiter
1708 except:
1709 transport.close()
1710 raise
1711
1712 if self._debug:
1713 logger.debug('Read pipe %r connected: (%r, %r)',
1714 pipe.fileno(), transport, protocol)
1715 return transport, protocol
1716
1717 async def connect_write_pipe(self, protocol_factory, pipe):
1718 protocol = protocol_factory()
1719 waiter = self.create_future()
1720 transport = self._make_write_pipe_transport(pipe, protocol, waiter)
1721
1722 try:
1723 await waiter
1724 except:
1725 transport.close()
1726 raise
1727
1728 if self._debug:
1729 logger.debug('Write pipe %r connected: (%r, %r)',
1730 pipe.fileno(), transport, protocol)
1731 return transport, protocol
1732
1733 def _log_subprocess(self, msg, stdin, stdout, stderr):
1734 info = [msg]
1735 if stdin is not None:
1736 info.append(f'stdin={_format_pipe(stdin)}')
1737 if stdout is not None and stderr == subprocess.STDOUT:
1738 info.append(f'stdout=stderr={_format_pipe(stdout)}')
1739 else:
1740 if stdout is not None:
1741 info.append(f'stdout={_format_pipe(stdout)}')
1742 if stderr is not None:
1743 info.append(f'stderr={_format_pipe(stderr)}')
1744 logger.debug(' '.join(info))
1745
1746 async def subprocess_shell(self, protocol_factory, cmd, *,
1747 stdin=subprocess.PIPE,
1748 stdout=subprocess.PIPE,
1749 stderr=subprocess.PIPE,
1750 universal_newlines=False,
1751 shell=True, bufsize=0,
1752 encoding=None, errors=None, text=None,
1753 **kwargs):
1754 if not isinstance(cmd, (bytes, str)):
1755 raise ValueError("cmd must be a string")
1756 if universal_newlines:
1757 raise ValueError("universal_newlines must be False")
1758 if not shell:
1759 raise ValueError("shell must be True")
1760 if bufsize != 0:
1761 raise ValueError("bufsize must be 0")
1762 if text:
1763 raise ValueError("text must be False")
1764 if encoding is not None:
1765 raise ValueError("encoding must be None")
1766 if errors is not None:
1767 raise ValueError("errors must be None")
1768
1769 protocol = protocol_factory()
1770 debug_log = None
1771 if self._debug:
1772 # don't log parameters: they may contain sensitive information
1773 # (password) and may be too long
1774 debug_log = 'run shell command %r' % cmd
1775 self._log_subprocess(debug_log, stdin, stdout, stderr)
1776 transport = await self._make_subprocess_transport(
1777 protocol, cmd, True, stdin, stdout, stderr, bufsize, **kwargs)
1778 if self._debug and debug_log is not None:
1779 logger.info('%s: %r', debug_log, transport)
1780 return transport, protocol
1781
1782 async def subprocess_exec(self, protocol_factory, program, *args,
1783 stdin=subprocess.PIPE, stdout=subprocess.PIPE,
1784 stderr=subprocess.PIPE, universal_newlines=False,
1785 shell=False, bufsize=0,
1786 encoding=None, errors=None, text=None,
1787 **kwargs):
1788 if universal_newlines:
1789 raise ValueError("universal_newlines must be False")
1790 if shell:
1791 raise ValueError("shell must be False")
1792 if bufsize != 0:
1793 raise ValueError("bufsize must be 0")
1794 if text:
1795 raise ValueError("text must be False")
1796 if encoding is not None:
1797 raise ValueError("encoding must be None")
1798 if errors is not None:
1799 raise ValueError("errors must be None")
1800
1801 popen_args = (program,) + args
1802 protocol = protocol_factory()
1803 debug_log = None
1804 if self._debug:
1805 # don't log parameters: they may contain sensitive information
1806 # (password) and may be too long
1807 debug_log = f'execute program {program!r}'
1808 self._log_subprocess(debug_log, stdin, stdout, stderr)
1809 transport = await self._make_subprocess_transport(
1810 protocol, popen_args, False, stdin, stdout, stderr,
1811 bufsize, **kwargs)
1812 if self._debug and debug_log is not None:
1813 logger.info('%s: %r', debug_log, transport)
1814 return transport, protocol
1815
1816 def get_exception_handler(self):
1817 """Return an exception handler, or None if the default one is in use.
1818 """
1819 return self._exception_handler
1820
1821 def set_exception_handler(self, handler):
1822 """Set handler as the new event loop exception handler.
1823
1824 If handler is None, the default exception handler will
1825 be set.
1826
1827 If handler is a callable object, it should have a
1828 signature matching '(loop, context)', where 'loop'
1829 will be a reference to the active event loop, 'context'
1830 will be a dict object (see `call_exception_handler()`
1831 documentation for details about context).
1832 """
1833 if handler is not None and not callable(handler):
1834 raise TypeError(f'A callable object or None is expected, '
1835 f'got {handler!r}')
1836 self._exception_handler = handler
1837
1838 def default_exception_handler(self, context):
1839 """Default exception handler.
1840
1841 This is called when an exception occurs and no exception
1842 handler is set, and can be called by a custom exception
1843 handler that wants to defer to the default behavior.
1844
1845 This default handler logs the error message and other
1846 context-dependent information. In debug mode, a truncated
1847 stack trace is also appended showing where the given object
1848 (e.g. a handle or future or task) was created, if any.
1849
1850 The context parameter has the same meaning as in
1851 `call_exception_handler()`.
1852 """
1853 message = context.get('message')
1854 if not message:
1855 message = 'Unhandled exception in event loop'
1856
1857 exception = context.get('exception')
1858 if exception is not None:
1859 exc_info = (type(exception), exception, exception.__traceback__)
1860 else:
1861 exc_info = False
1862
1863 if ('source_traceback' not in context and
1864 self._current_handle is not None and
1865 self._current_handle._source_traceback):
1866 context['handle_traceback'] = \
1867 self._current_handle._source_traceback
1868
1869 log_lines = [message]
1870 for key in sorted(context):
1871 if key in {'message', 'exception'}:
1872 continue
1873 value = context[key]
1874 if key == 'source_traceback':
1875 tb = ''.join(traceback.format_list(value))
1876 value = 'Object created at (most recent call last):\n'
1877 value += tb.rstrip()
1878 elif key == 'handle_traceback':
1879 tb = ''.join(traceback.format_list(value))
1880 value = 'Handle created at (most recent call last):\n'
1881 value += tb.rstrip()
1882 else:
1883 value = repr(value)
1884 log_lines.append(f'{key}: {value}')
1885
1886 logger.error('\n'.join(log_lines), exc_info=exc_info)
1887
1888 def call_exception_handler(self, context):
1889 """Call the current event loop's exception handler.
1890
1891 The context argument is a dict containing the following keys:
1892
1893 - 'message': Error message;
1894 - 'exception' (optional): Exception object;
1895 - 'future' (optional): Future instance;
1896 - 'task' (optional): Task instance;
1897 - 'handle' (optional): Handle instance;
1898 - 'protocol' (optional): Protocol instance;
1899 - 'transport' (optional): Transport instance;
1900 - 'socket' (optional): Socket instance;
1901 - 'source_traceback' (optional): Traceback of the source;
1902 - 'handle_traceback' (optional): Traceback of the handle;
1903 - 'asyncgen' (optional): Asynchronous generator that caused
1904 the exception.
1905
1906 New keys maybe introduced in the future.
1907
1908 Note: do not overload this method in an event loop subclass.
1909 For custom exception handling, use the
1910 `set_exception_handler()` method.
1911 """
1912 if self._exception_handler is None:
1913 try:
1914 self.default_exception_handler(context)
1915 except (SystemExit, KeyboardInterrupt):
1916 raise
1917 except BaseException:
1918 # Second protection layer for unexpected errors
1919 # in the default implementation, as well as for subclassed
1920 # event loops with overloaded "default_exception_handler".
1921 logger.error('Exception in default exception handler',
1922 exc_info=True)
1923 else:
1924 try:
1925 ctx = None
1926 thing = context.get("task")
1927 if thing is None:
1928 # Even though Futures don't have a context,
1929 # Task is a subclass of Future,
1930 # and sometimes the 'future' key holds a Task.
1931 thing = context.get("future")
1932 if thing is None:
1933 # Handles also have a context.
1934 thing = context.get("handle")
1935 if thing is not None and hasattr(thing, "get_context"):
1936 ctx = thing.get_context()
1937 if ctx is not None and hasattr(ctx, "run"):
1938 ctx.run(self._exception_handler, self, context)
1939 else:
1940 self._exception_handler(self, context)
1941 except (SystemExit, KeyboardInterrupt):
1942 raise
1943 except BaseException as exc:
1944 # Exception in the user set custom exception handler.
1945 try:
1946 # Let's try default handler.
1947 self.default_exception_handler({
1948 'message': 'Unhandled error in exception handler',
1949 'exception': exc,
1950 'context': context,
1951 })
1952 except (SystemExit, KeyboardInterrupt):
1953 raise
1954 except BaseException:
1955 # Guard 'default_exception_handler' in case it is
1956 # overloaded.
1957 logger.error('Exception in default exception handler '
1958 'while handling an unexpected error '
1959 'in custom exception handler',
1960 exc_info=True)
1961
1962 def _add_callback(self, handle):
1963 """Add a Handle to _ready."""
1964 if not handle._cancelled:
1965 self._ready.append(handle)
1966
1967 def _add_callback_signalsafe(self, handle):
1968 """Like _add_callback() but called from a signal handler."""
1969 self._add_callback(handle)
1970 self._write_to_self()
1971
1972 def _timer_handle_cancelled(self, handle):
1973 """Notification that a TimerHandle has been cancelled."""
1974 if handle._scheduled:
1975 self._timer_cancelled_count += 1
1976
1977 def _run_once(self):
1978 """Run one full iteration of the event loop.
1979
1980 This calls all currently ready callbacks, polls for I/O,
1981 schedules the resulting callbacks, and finally schedules
1982 'call_later' callbacks.
1983 """
1984
1985 sched_count = len(self._scheduled)
1986 if (sched_count > _MIN_SCHEDULED_TIMER_HANDLES and
1987 self._timer_cancelled_count / sched_count >
1988 _MIN_CANCELLED_TIMER_HANDLES_FRACTION):
1989 # Remove delayed calls that were cancelled if their number
1990 # is too high
1991 new_scheduled = []
1992 for handle in self._scheduled:
1993 if handle._cancelled:
1994 handle._scheduled = False
1995 else:
1996 new_scheduled.append(handle)
1997
1998 heapq.heapify(new_scheduled)
1999 self._scheduled = new_scheduled
2000 self._timer_cancelled_count = 0
2001 else:
2002 # Remove delayed calls that were cancelled from head of queue.
2003 while self._scheduled and self._scheduled[0]._cancelled:
2004 self._timer_cancelled_count -= 1
2005 handle = heapq.heappop(self._scheduled)
2006 handle._scheduled = False
2007
2008 timeout = None
2009 if self._ready or self._stopping:
2010 timeout = 0
2011 elif self._scheduled:
2012 # Compute the desired timeout.
2013 timeout = self._scheduled[0]._when - self.time()
2014 if timeout > MAXIMUM_SELECT_TIMEOUT:
2015 timeout = MAXIMUM_SELECT_TIMEOUT
2016 elif timeout < 0:
2017 timeout = 0
2018
2019 event_list = self._selector.select(timeout)
2020 self._process_events(event_list)
2021 # Needed to break cycles when an exception occurs.
2022 event_list = None
2023
2024 # Handle 'later' callbacks that are ready.
2025 end_time = self.time() + self._clock_resolution
2026 while self._scheduled:
2027 handle = self._scheduled[0]
2028 if handle._when >= end_time:
2029 break
2030 handle = heapq.heappop(self._scheduled)
2031 handle._scheduled = False
2032 self._ready.append(handle)
2033
2034 # This is the only place where callbacks are actually *called*.
2035 # All other places just add them to ready.
2036 # Note: We run all currently scheduled callbacks, but not any
2037 # callbacks scheduled by callbacks run this time around --
2038 # they will be run the next time (after another I/O poll).
2039 # Use an idiom that is thread-safe without using locks.
2040 ntodo = len(self._ready)
2041 for i in range(ntodo):
2042 handle = self._ready.popleft()
2043 if handle._cancelled:
2044 continue
2045 if self._debug:
2046 try:
2047 self._current_handle = handle
2048 t0 = self.time()
2049 handle._run()
2050 dt = self.time() - t0
2051 if dt >= self.slow_callback_duration:
2052 logger.warning('Executing %s took %.3f seconds',
2053 _format_handle(handle), dt)
2054 finally:
2055 self._current_handle = None
2056 else:
2057 handle._run()
2058 handle = None # Needed to break cycles when an exception occurs.
2059
2060 def _set_coroutine_origin_tracking(self, enabled):
2061 if bool(enabled) == bool(self._coroutine_origin_tracking_enabled):
2062 return
2063
2064 if enabled:
2065 self._coroutine_origin_tracking_saved_depth = (
2066 sys.get_coroutine_origin_tracking_depth())
2067 sys.set_coroutine_origin_tracking_depth(
2068 constants.DEBUG_STACK_DEPTH)
2069 else:
2070 sys.set_coroutine_origin_tracking_depth(
2071 self._coroutine_origin_tracking_saved_depth)
2072
2073 self._coroutine_origin_tracking_enabled = enabled
2074
2075 def get_debug(self):
2076 return self._debug
2077
2078 def set_debug(self, enabled):
2079 self._debug = enabled
2080
2081 if self.is_running():
2082 self.call_soon_threadsafe(self._set_coroutine_origin_tracking, enabled)