Debian Codegraph @

Lib/asyncio/staggered.py

raw
1 """Support for running coroutines in parallel with staggered start times."""
2
3 __all__ = 'staggered_race',
4
5 import contextlib
6
7 from . import events
8 from . import exceptions as exceptions_mod
9 from . import locks
10 from . import tasks
11
12
13 async def staggered_race(coro_fns, delay, *, loop=None):
14 """Run coroutines with staggered start times and take the first to finish.
15
16 This method takes an iterable of coroutine functions. The first one is
17 started immediately. From then on, whenever the immediately preceding one
18 fails (raises an exception), or when *delay* seconds has passed, the next
19 coroutine is started. This continues until one of the coroutines complete
20 successfully, in which case all others are cancelled, or until all
21 coroutines fail.
22
23 The coroutines provided should be well-behaved in the following way:
24
25 * They should only ``return`` if completed successfully.
26
27 * They should always raise an exception if they did not complete
28 successfully. In particular, if they handle cancellation, they should
29 probably reraise, like this::
30
31 try:
32 # do work
33 except asyncio.CancelledError:
34 # undo partially completed work
35 raise
36
37 Args:
38 coro_fns: an iterable of coroutine functions, i.e. callables that
39 return a coroutine object when called. Use ``functools.partial`` or
40 lambdas to pass arguments.
41
42 delay: amount of time, in seconds, between starting coroutines. If
43 ``None``, the coroutines will run sequentially.
44
45 loop: the event loop to use.
46
47 Returns:
48 tuple *(winner_result, winner_index, exceptions)* where
49
50 - *winner_result*: the result of the winning coroutine, or ``None``
51 if no coroutines won.
52
53 - *winner_index*: the index of the winning coroutine in
54 ``coro_fns``, or ``None`` if no coroutines won. If the winning
55 coroutine may return None on success, *winner_index* can be used
56 to definitively determine whether any coroutine won.
57
58 - *exceptions*: list of exceptions returned by the coroutines.
59 ``len(exceptions)`` is equal to the number of coroutines actually
60 started, and the order is the same as in ``coro_fns``. The winning
61 coroutine's entry is ``None``.
62
63 """
64 # TODO: when we have aiter() and anext(), allow async iterables in coro_fns.
65 loop = loop or events.get_running_loop()
66 enum_coro_fns = enumerate(coro_fns)
67 winner_result = None
68 winner_index = None
69 unhandled_exceptions = []
70 exceptions = []
71 running_tasks = set()
72 on_completed_fut = None
73
74 def task_done(task):
75 running_tasks.discard(task)
76 if (
77 on_completed_fut is not None
78 and not on_completed_fut.done()
79 and not running_tasks
80 ):
81 on_completed_fut.set_result(None)
82
83 if task.cancelled():
84 return
85
86 exc = task.exception()
87 if exc is None:
88 return
89 unhandled_exceptions.append(exc)
90
91 async def run_one_coro(ok_to_start, previous_failed) -> None:
92 # in eager tasks this waits for the calling task to append this task
93 # to running_tasks, in regular tasks this wait is a no-op that does
94 # not yield a future. See gh-124309.
95 await ok_to_start.wait()
96 # Wait for the previous task to finish, or for delay seconds
97 if previous_failed is not None:
98 with contextlib.suppress(exceptions_mod.TimeoutError):
99 # Use asyncio.wait_for() instead of asyncio.wait() here, so
100 # that if we get cancelled at this point, Event.wait() is also
101 # cancelled, otherwise there will be a "Task destroyed but it is
102 # pending" later.
103 await tasks.wait_for(previous_failed.wait(), delay)
104 # Get the next coroutine to run
105 try:
106 this_index, coro_fn = next(enum_coro_fns)
107 except StopIteration:
108 return
109 # Start task that will run the next coroutine
110 this_failed = locks.Event()
111 next_ok_to_start = locks.Event()
112 next_task = loop.create_task(run_one_coro(next_ok_to_start, this_failed))
113 running_tasks.add(next_task)
114 next_task.add_done_callback(task_done)
115 # next_task has been appended to running_tasks so next_task is ok to
116 # start.
117 next_ok_to_start.set()
118 # Prepare place to put this coroutine's exceptions if not won
119 exceptions.append(None)
120 assert len(exceptions) == this_index + 1
121
122 try:
123 result = await coro_fn()
124 except (SystemExit, KeyboardInterrupt):
125 raise
126 except BaseException as e:
127 exceptions[this_index] = e
128 this_failed.set() # Kickstart the next coroutine
129 else:
130 # Store winner's results
131 nonlocal winner_index, winner_result
132 assert winner_index is None
133 winner_index = this_index
134 winner_result = result
135 # Cancel all other tasks. We take care to not cancel the current
136 # task as well. If we do so, then since there is no `await` after
137 # here and CancelledError are usually thrown at one, we will
138 # encounter a curious corner case where the current task will end
139 # up as done() == True, cancelled() == False, exception() ==
140 # asyncio.CancelledError. This behavior is specified in
141 # https://bugs.python.org/issue30048
142 current_task = tasks.current_task(loop)
143 for t in running_tasks:
144 if t is not current_task:
145 t.cancel()
146
147 propagate_cancellation_error = None
148 try:
149 ok_to_start = locks.Event()
150 first_task = loop.create_task(run_one_coro(ok_to_start, None))
151 running_tasks.add(first_task)
152 first_task.add_done_callback(task_done)
153 # first_task has been appended to running_tasks so first_task is ok to start.
154 ok_to_start.set()
155 propagate_cancellation_error = None
156 # Make sure no tasks are left running if we leave this function
157 while running_tasks:
158 on_completed_fut = loop.create_future()
159 try:
160 await on_completed_fut
161 except exceptions_mod.CancelledError as ex:
162 propagate_cancellation_error = ex
163 for task in running_tasks:
164 task.cancel(*ex.args)
165 on_completed_fut = None
166 if __debug__ and unhandled_exceptions:
167 # If run_one_coro raises an unhandled exception, it's probably a
168 # programming error, and I want to see it.
169 raise ExceptionGroup("staggered race failed", unhandled_exceptions)
170 if propagate_cancellation_error is not None:
171 raise propagate_cancellation_error
172 return winner_result, winner_index, exceptions
173 finally:
174 del exceptions, propagate_cancellation_error, unhandled_exceptions