Debian Codegraph @

Lib/concurrent/futures/_base.py

raw
1 # Copyright 2009 Brian Quinlan. All Rights Reserved.
2 # Licensed to PSF under a Contributor Agreement.
3
4 __author__ = 'Brian Quinlan ([email protected])'
5
6 import collections
7 import logging
8 import threading
9 import time
10 import types
11 import weakref
12 from itertools import islice
13
14 FIRST_COMPLETED = 'FIRST_COMPLETED'
15 FIRST_EXCEPTION = 'FIRST_EXCEPTION'
16 ALL_COMPLETED = 'ALL_COMPLETED'
17 _AS_COMPLETED = '_AS_COMPLETED'
18
19 # Possible future states (for internal use by the futures package).
20 PENDING = 'PENDING'
21 RUNNING = 'RUNNING'
22 # The future was cancelled by the user...
23 CANCELLED = 'CANCELLED'
24 # ...and _Waiter.add_cancelled() was called by a worker.
25 CANCELLED_AND_NOTIFIED = 'CANCELLED_AND_NOTIFIED'
26 FINISHED = 'FINISHED'
27
28 _STATE_TO_DESCRIPTION_MAP = {
29 PENDING: "pending",
30 RUNNING: "running",
31 CANCELLED: "cancelled",
32 CANCELLED_AND_NOTIFIED: "cancelled",
33 FINISHED: "finished"
34 }
35
36 # Logger for internal use by the futures package.
37 LOGGER = logging.getLogger("concurrent.futures")
38
39 class Error(Exception):
40 """Base class for all future-related exceptions."""
41 pass
42
43 class CancelledError(Error):
44 """The Future was cancelled."""
45 pass
46
47 TimeoutError = TimeoutError # make local alias for the standard exception
48
49 class InvalidStateError(Error):
50 """The operation is not allowed in this state."""
51 pass
52
53 class _Waiter(object):
54 """Provides the event that wait() and as_completed() block on."""
55 def __init__(self):
56 self.event = threading.Event()
57 self.finished_futures = []
58
59 def add_result(self, future):
60 self.finished_futures.append(future)
61
62 def add_exception(self, future):
63 self.finished_futures.append(future)
64
65 def add_cancelled(self, future):
66 self.finished_futures.append(future)
67
68 class _AsCompletedWaiter(_Waiter):
69 """Used by as_completed()."""
70
71 def __init__(self):
72 super(_AsCompletedWaiter, self).__init__()
73 self.lock = threading.Lock()
74
75 def add_result(self, future):
76 with self.lock:
77 super(_AsCompletedWaiter, self).add_result(future)
78 self.event.set()
79
80 def add_exception(self, future):
81 with self.lock:
82 super(_AsCompletedWaiter, self).add_exception(future)
83 self.event.set()
84
85 def add_cancelled(self, future):
86 with self.lock:
87 super(_AsCompletedWaiter, self).add_cancelled(future)
88 self.event.set()
89
90 class _FirstCompletedWaiter(_Waiter):
91 """Used by wait(return_when=FIRST_COMPLETED)."""
92
93 def add_result(self, future):
94 super().add_result(future)
95 self.event.set()
96
97 def add_exception(self, future):
98 super().add_exception(future)
99 self.event.set()
100
101 def add_cancelled(self, future):
102 super().add_cancelled(future)
103 self.event.set()
104
105 class _AllCompletedWaiter(_Waiter):
106 """Used by wait(return_when=FIRST_EXCEPTION and ALL_COMPLETED)."""
107
108 def __init__(self, num_pending_calls, stop_on_exception):
109 self.num_pending_calls = num_pending_calls
110 self.stop_on_exception = stop_on_exception
111 self.lock = threading.Lock()
112 super().__init__()
113
114 def _decrement_pending_calls(self):
115 with self.lock:
116 self.num_pending_calls -= 1
117 if not self.num_pending_calls:
118 self.event.set()
119
120 def add_result(self, future):
121 super().add_result(future)
122 self._decrement_pending_calls()
123
124 def add_exception(self, future):
125 super().add_exception(future)
126 if self.stop_on_exception:
127 self.event.set()
128 else:
129 self._decrement_pending_calls()
130
131 def add_cancelled(self, future):
132 super().add_cancelled(future)
133 self._decrement_pending_calls()
134
135 class _AcquireFutures(object):
136 """A context manager that does an ordered acquire of Future conditions."""
137
138 def __init__(self, futures):
139 self.futures = sorted(futures, key=id)
140
141 def __enter__(self):
142 for future in self.futures:
143 future._condition.acquire()
144
145 def __exit__(self, *args):
146 for future in self.futures:
147 future._condition.release()
148
149 def _create_and_install_waiters(fs, return_when):
150 if return_when == _AS_COMPLETED:
151 waiter = _AsCompletedWaiter()
152 elif return_when == FIRST_COMPLETED:
153 waiter = _FirstCompletedWaiter()
154 else:
155 pending_count = sum(
156 f._state not in [CANCELLED_AND_NOTIFIED, FINISHED] for f in fs)
157
158 if return_when == FIRST_EXCEPTION:
159 waiter = _AllCompletedWaiter(pending_count, stop_on_exception=True)
160 elif return_when == ALL_COMPLETED:
161 waiter = _AllCompletedWaiter(pending_count, stop_on_exception=False)
162 else:
163 raise ValueError("Invalid return condition: %r" % return_when)
164
165 for f in fs:
166 f._waiters.append(waiter)
167
168 return waiter
169
170
171 def _yield_finished_futures(fs, waiter, ref_collect):
172 """
173 Iterate on the list *fs*, yielding finished futures one by one in
174 reverse order.
175 Before yielding a future, *waiter* is removed from its waiters
176 and the future is removed from each set in the collection of sets
177 *ref_collect*.
178
179 The aim of this function is to avoid keeping stale references after
180 the future is yielded and before the iterator resumes.
181 """
182 while fs:
183 f = fs[-1]
184 for futures_set in ref_collect:
185 futures_set.remove(f)
186 with f._condition:
187 f._waiters.remove(waiter)
188 del f
189 # Careful not to keep a reference to the popped value
190 yield fs.pop()
191
192
193 def as_completed(fs, timeout=None):
194 """An iterator over the given futures that yields each as it completes.
195
196 Args:
197 fs: The sequence of Futures (possibly created by different
198 Executors) to iterate over.
199 timeout: The maximum number of seconds to wait. If None, then
200 there is no limit on the wait time.
201
202 Returns:
203 An iterator that yields the given Futures as they complete
204 (finished or cancelled). If any given Futures are duplicated,
205 they will be returned once.
206
207 Raises:
208 TimeoutError: If the entire result iterator could not be generated
209 before the given timeout.
210 """
211 if timeout is not None:
212 end_time = timeout + time.monotonic()
213
214 fs = set(fs)
215 total_futures = len(fs)
216 with _AcquireFutures(fs):
217 finished = set(
218 f for f in fs
219 if f._state in [CANCELLED_AND_NOTIFIED, FINISHED])
220 pending = fs - finished
221 waiter = _create_and_install_waiters(fs, _AS_COMPLETED)
222 finished = list(finished)
223 try:
224 yield from _yield_finished_futures(finished, waiter,
225 ref_collect=(fs,))
226
227 while pending:
228 if timeout is None:
229 wait_timeout = None
230 else:
231 wait_timeout = end_time - time.monotonic()
232 if wait_timeout < 0:
233 raise TimeoutError(
234 '%d (of %d) futures unfinished' % (
235 len(pending), total_futures))
236
237 waiter.event.wait(wait_timeout)
238
239 with waiter.lock:
240 finished = waiter.finished_futures
241 waiter.finished_futures = []
242 waiter.event.clear()
243
244 # reverse to keep finishing order
245 finished.reverse()
246 yield from _yield_finished_futures(finished, waiter,
247 ref_collect=(fs, pending))
248
249 finally:
250 # Remove waiter from unfinished futures
251 for f in fs:
252 with f._condition:
253 f._waiters.remove(waiter)
254
255 DoneAndNotDoneFutures = collections.namedtuple(
256 'DoneAndNotDoneFutures', 'done not_done')
257 def wait(fs, timeout=None, return_when=ALL_COMPLETED):
258 """Wait for the futures in the given sequence to complete.
259
260 Args:
261 fs: The sequence of Futures (possibly created by different
262 Executors) to wait upon.
263 timeout: The maximum number of seconds to wait. If None, then
264 there is no limit on the wait time.
265 return_when: Indicates when this function should return.
266 The options are:
267
268 FIRST_COMPLETED - Return when any future finishes or is
269 cancelled.
270 FIRST_EXCEPTION - Return when any future finishes by raising an
271 exception. If no future raises an exception
272 then it is equivalent to ALL_COMPLETED.
273 ALL_COMPLETED - Return when all futures finish or are
274 cancelled.
275
276 Returns:
277 A named 2-tuple of sets. The first set, named 'done', contains the
278 futures that completed (is finished or cancelled) before the wait
279 completed. The second set, named 'not_done', contains uncompleted
280 futures. Duplicate futures given to *fs* are removed and will be
281 returned only once.
282 """
283 fs = set(fs)
284 with _AcquireFutures(fs):
285 done = {f for f in fs
286 if f._state in [CANCELLED_AND_NOTIFIED, FINISHED]}
287 not_done = fs - done
288 if (return_when == FIRST_COMPLETED) and done:
289 return DoneAndNotDoneFutures(done, not_done)
290 elif (return_when == FIRST_EXCEPTION) and done:
291 if any(f for f in done
292 if not f.cancelled() and f.exception() is not None):
293 return DoneAndNotDoneFutures(done, not_done)
294
295 if len(done) == len(fs):
296 return DoneAndNotDoneFutures(done, not_done)
297
298 waiter = _create_and_install_waiters(fs, return_when)
299
300 waiter.event.wait(timeout)
301 for f in fs:
302 with f._condition:
303 f._waiters.remove(waiter)
304
305 done.update(waiter.finished_futures)
306 return DoneAndNotDoneFutures(done, fs - done)
307
308
309 def _result_or_cancel(fut, timeout=None):
310 try:
311 try:
312 return fut.result(timeout)
313 finally:
314 fut.cancel()
315 finally:
316 # Break a reference cycle with the exception in self._exception
317 del fut
318
319
320 class Future(object):
321 """Represents the result of an asynchronous computation."""
322
323 def __init__(self):
324 """Initializes the future. Should not be called by clients."""
325 self._condition = threading.Condition()
326 self._state = PENDING
327 self._result = None
328 self._exception = None
329 self._waiters = []
330 self._done_callbacks = []
331
332 def _invoke_callbacks(self):
333 for callback in self._done_callbacks:
334 try:
335 callback(self)
336 except Exception:
337 LOGGER.exception('exception calling callback for %r', self)
338
339 def __repr__(self):
340 with self._condition:
341 if self._state == FINISHED:
342 if self._exception:
343 return '<%s at %#x state=%s raised %s>' % (
344 self.__class__.__name__,
345 id(self),
346 _STATE_TO_DESCRIPTION_MAP[self._state],
347 self._exception.__class__.__name__)
348 else:
349 return '<%s at %#x state=%s returned %s>' % (
350 self.__class__.__name__,
351 id(self),
352 _STATE_TO_DESCRIPTION_MAP[self._state],
353 self._result.__class__.__name__)
354 return '<%s at %#x state=%s>' % (
355 self.__class__.__name__,
356 id(self),
357 _STATE_TO_DESCRIPTION_MAP[self._state])
358
359 def cancel(self):
360 """Cancel the future if possible.
361
362 Returns True if the future was cancelled, False otherwise. A future
363 cannot be cancelled if it is running or has already completed.
364 """
365 with self._condition:
366 if self._state in [RUNNING, FINISHED]:
367 return False
368
369 if self._state in [CANCELLED, CANCELLED_AND_NOTIFIED]:
370 return True
371
372 self._state = CANCELLED
373 self._condition.notify_all()
374
375 self._invoke_callbacks()
376 return True
377
378 def cancelled(self):
379 """Return True if the future was cancelled."""
380 with self._condition:
381 return self._state in [CANCELLED, CANCELLED_AND_NOTIFIED]
382
383 def running(self):
384 """Return True if the future is currently executing."""
385 with self._condition:
386 return self._state == RUNNING
387
388 def done(self):
389 """Return True if the future was cancelled or finished executing."""
390 with self._condition:
391 return self._state in [CANCELLED, CANCELLED_AND_NOTIFIED, FINISHED]
392
393 def __get_result(self):
394 if self._exception is not None:
395 try:
396 raise self._exception
397 finally:
398 # Break a reference cycle with the exception in self._exception
399 self = None
400 else:
401 return self._result
402
403 def add_done_callback(self, fn):
404 """Attaches a callable that will be called when the future finishes.
405
406 Args:
407 fn: A callable that will be called with this future as its only
408 argument when the future completes or is cancelled. The
409 callable will always be called by a thread in the same
410 process in which it was added. If the future has already
411 completed or been cancelled then the callable will be
412 called immediately. These callables are called in the
413 order that they were added.
414 """
415 with self._condition:
416 if self._state not in [CANCELLED, CANCELLED_AND_NOTIFIED, FINISHED]:
417 self._done_callbacks.append(fn)
418 return
419 try:
420 fn(self)
421 except Exception:
422 LOGGER.exception('exception calling callback for %r', self)
423
424 def result(self, timeout=None):
425 """Return the result of the call that the future represents.
426
427 Args:
428 timeout: The number of seconds to wait for the result if the
429 future isn't done. If None, then there is no limit on the
430 wait time.
431
432 Returns:
433 The result of the call that the future represents.
434
435 Raises:
436 CancelledError: If the future was cancelled.
437 TimeoutError: If the future didn't finish executing before the
438 given timeout.
439 Exception: If the call raised then that exception will be
440 raised.
441 """
442 try:
443 with self._condition:
444 if self._state in [CANCELLED, CANCELLED_AND_NOTIFIED]:
445 raise CancelledError()
446 elif self._state == FINISHED:
447 return self.__get_result()
448
449 self._condition.wait(timeout)
450
451 if self._state in [CANCELLED, CANCELLED_AND_NOTIFIED]:
452 raise CancelledError()
453 elif self._state == FINISHED:
454 return self.__get_result()
455 else:
456 raise TimeoutError()
457 finally:
458 # Break a reference cycle with the exception in self._exception
459 self = None
460
461 def exception(self, timeout=None):
462 """Return the exception raised by the call that the future represents.
463
464 Args:
465 timeout: The number of seconds to wait for the exception if the
466 future isn't done. If None, then there is no limit on the
467 wait time.
468
469 Returns:
470 The exception raised by the call that the future represents or
471 None if the call completed without raising.
472
473 Raises:
474 CancelledError: If the future was cancelled.
475 TimeoutError: If the future didn't finish executing before the
476 given timeout.
477 """
478
479 with self._condition:
480 if self._state in [CANCELLED, CANCELLED_AND_NOTIFIED]:
481 raise CancelledError()
482 elif self._state == FINISHED:
483 return self._exception
484
485 self._condition.wait(timeout)
486
487 if self._state in [CANCELLED, CANCELLED_AND_NOTIFIED]:
488 raise CancelledError()
489 elif self._state == FINISHED:
490 return self._exception
491 else:
492 raise TimeoutError()
493
494 # The following methods should only be used by Executors and in tests.
495 def set_running_or_notify_cancel(self):
496 """Mark the future as running or process any cancel notifications.
497
498 Should only be used by Executor implementations and unit tests.
499
500 If the future has been cancelled (cancel() was called and returned
501 True) then any threads waiting on the future completing (though
502 calls to as_completed() or wait()) are notified and False is
503 returned.
504
505 If the future was not cancelled then it is put in the running state
506 (future calls to running() will return True) and True is returned.
507
508 This method should be called by Executor implementations before
509 executing the work associated with this future. If this method
510 returns False then the work should not be executed.
511
512 Returns:
513 False if the Future was cancelled, True otherwise.
514
515 Raises:
516 RuntimeError: if this method was already called or if
517 set_result() or set_exception() was called.
518 """
519 with self._condition:
520 if self._state == CANCELLED:
521 self._state = CANCELLED_AND_NOTIFIED
522 for waiter in self._waiters:
523 waiter.add_cancelled(self)
524 # self._condition.notify_all() is not necessary because
525 # self.cancel() triggers a notification.
526 return False
527 elif self._state == PENDING:
528 self._state = RUNNING
529 return True
530 else:
531 LOGGER.critical('Future %s in unexpected state: %s',
532 id(self),
533 self._state)
534 raise RuntimeError('Future in unexpected state')
535
536 def set_result(self, result):
537 """Sets the return value of work associated with the future.
538
539 Should only be used by Executor implementations and unit tests.
540 """
541 with self._condition:
542 if self._state in {CANCELLED, CANCELLED_AND_NOTIFIED, FINISHED}:
543 raise InvalidStateError('{}: {!r}'.format(self._state, self))
544 self._result = result
545 self._state = FINISHED
546 for waiter in self._waiters:
547 waiter.add_result(self)
548 self._condition.notify_all()
549 self._invoke_callbacks()
550
551 def set_exception(self, exception):
552 """Sets the result of the future as being the given exception.
553
554 Should only be used by Executor implementations and unit tests.
555 """
556 with self._condition:
557 if self._state in {CANCELLED, CANCELLED_AND_NOTIFIED, FINISHED}:
558 raise InvalidStateError('{}: {!r}'.format(self._state, self))
559 self._exception = exception
560 self._state = FINISHED
561 for waiter in self._waiters:
562 waiter.add_exception(self)
563 self._condition.notify_all()
564 self._invoke_callbacks()
565
566 __class_getitem__ = classmethod(types.GenericAlias)
567
568 class Executor(object):
569 """This is an abstract base class for concrete asynchronous executors."""
570
571 def submit(self, fn, /, *args, **kwargs):
572 """Submits a callable to be executed with the given arguments.
573
574 Schedules the callable to be executed as fn(*args, **kwargs) and
575 returns a Future instance representing the execution of the
576 callable.
577
578 Returns:
579 A Future representing the given call.
580 """
581 raise NotImplementedError()
582
583 def map(self, fn, *iterables, timeout=None, chunksize=1, buffersize=None):
584 """Returns an iterator equivalent to map(fn, iter).
585
586 Args:
587 fn: A callable that will take as many arguments as there are
588 passed iterables.
589 timeout: The maximum number of seconds to wait. If None, then
590 there is no limit on the wait time.
591 chunksize: The size of the chunks the iterable will be broken
592 into before being passed to a child process. This argument
593 is only used by ProcessPoolExecutor; it is ignored by
594 ThreadPoolExecutor.
595 buffersize: The number of submitted tasks whose results have not
596 yet been yielded. If the buffer is full, iteration over the
597 iterables pauses until a result is yielded from the buffer.
598 If None, all input elements are eagerly collected, and
599 a task is submitted for each.
600
601 Returns:
602 An iterator equivalent to: map(func, *iterables) but the calls
603 may be evaluated out-of-order.
604
605 Raises:
606 TimeoutError: If the entire result iterator could not be
607 generated before the given timeout.
608 Exception: If fn(*args) raises for any values.
609 """
610 if buffersize is not None and not isinstance(buffersize, int):
611 raise TypeError("buffersize must be an integer or None")
612 if buffersize is not None and buffersize < 1:
613 raise ValueError("buffersize must be None or > 0")
614
615 if timeout is not None:
616 end_time = timeout + time.monotonic()
617
618 zipped_iterables = zip(*iterables)
619 if buffersize:
620 fs = collections.deque(
621 self.submit(fn, *args) for args in islice(zipped_iterables, buffersize)
622 )
623 else:
624 fs = [self.submit(fn, *args) for args in zipped_iterables]
625
626 # Use a weak reference to ensure that the executor can be garbage
627 # collected independently of the result_iterator closure.
628 executor_weakref = weakref.ref(self)
629
630 # Yield must be hidden in closure so that the futures are submitted
631 # before the first iterator value is required.
632 def result_iterator():
633 try:
634 # reverse to keep finishing order
635 fs.reverse()
636 while fs:
637 if (
638 buffersize
639 and (executor := executor_weakref())
640 and (args := next(zipped_iterables, None))
641 ):
642 fs.appendleft(executor.submit(fn, *args))
643 # Careful not to keep a reference to the popped future
644 if timeout is None:
645 yield _result_or_cancel(fs.pop())
646 else:
647 yield _result_or_cancel(fs.pop(), end_time - time.monotonic())
648 finally:
649 for future in fs:
650 future.cancel()
651 return result_iterator()
652
653 def shutdown(self, wait=True, *, cancel_futures=False):
654 """Clean-up the resources associated with the Executor.
655
656 It is safe to call this method several times. Otherwise, no other
657 methods can be called after this one.
658
659 Args:
660 wait: If True then shutdown will not return until all running
661 futures have finished executing and the resources used by
662 the executor have been reclaimed.
663 cancel_futures: If True then shutdown will cancel all pending
664 futures. Futures that are completed or running will not be
665 cancelled.
666 """
667 pass
668
669 def __enter__(self):
670 return self
671
672 def __exit__(self, exc_type, exc_val, exc_tb):
673 self.shutdown(wait=True)
674 return False
675
676
677 class BrokenExecutor(RuntimeError):
678 """
679 Raised when a executor has become non-functional after a severe failure.
680 """