Debian Codegraph @

Lib/concurrent/futures/_base.py

raw
1 # Copyright 2009 Brian Quinlan. All Rights Reserved.
2 # Licensed to PSF under a Contributor Agreement.
3
4 __author__ = 'Brian Quinlan ([email protected])'
5
6 import collections
7 import logging
8 import threading
9 import time
10 import types
11 import weakref
12 from itertools import islice
13
14 FIRST_COMPLETED = 'FIRST_COMPLETED'
15 FIRST_EXCEPTION = 'FIRST_EXCEPTION'
16 ALL_COMPLETED = 'ALL_COMPLETED'
17 _AS_COMPLETED = '_AS_COMPLETED'
18
19 # Possible future states (for internal use by the futures package).
20 PENDING = 'PENDING'
21 RUNNING = 'RUNNING'
22 # The future was cancelled by the user...
23 CANCELLED = 'CANCELLED'
24 # ...and _Waiter.add_cancelled() was called by a worker.
25 CANCELLED_AND_NOTIFIED = 'CANCELLED_AND_NOTIFIED'
26 FINISHED = 'FINISHED'
27
28 _STATE_TO_DESCRIPTION_MAP = {
29 PENDING: "pending",
30 RUNNING: "running",
31 CANCELLED: "cancelled",
32 CANCELLED_AND_NOTIFIED: "cancelled",
33 FINISHED: "finished"
34 }
35
36 # Logger for internal use by the futures package.
37 LOGGER = logging.getLogger("concurrent.futures")
38
39 class Error(Exception):
40 """Base class for all future-related exceptions."""
41 pass
42
43 class CancelledError(Error):
44 """The Future was cancelled."""
45 pass
46
47 TimeoutError = TimeoutError # make local alias for the standard exception
48
49 class InvalidStateError(Error):
50 """The operation is not allowed in this state."""
51 pass
52
53 class _Waiter(object):
54 """Provides the event that wait() and as_completed() block on."""
55 def __init__(self):
56 self.event = threading.Event()
57 self.finished_futures = []
58
59 def add_result(self, future):
60 self.finished_futures.append(future)
61
62 def add_exception(self, future):
63 self.finished_futures.append(future)
64
65 def add_cancelled(self, future):
66 self.finished_futures.append(future)
67
68 class _AsCompletedWaiter(_Waiter):
69 """Used by as_completed()."""
70
71 def __init__(self):
72 super(_AsCompletedWaiter, self).__init__()
73 self.lock = threading.Lock()
74
75 def add_result(self, future):
76 with self.lock:
77 super(_AsCompletedWaiter, self).add_result(future)
78 self.event.set()
79
80 def add_exception(self, future):
81 with self.lock:
82 super(_AsCompletedWaiter, self).add_exception(future)
83 self.event.set()
84
85 def add_cancelled(self, future):
86 with self.lock:
87 super(_AsCompletedWaiter, self).add_cancelled(future)
88 self.event.set()
89
90 class _FirstCompletedWaiter(_Waiter):
91 """Used by wait(return_when=FIRST_COMPLETED)."""
92
93 def add_result(self, future):
94 super().add_result(future)
95 self.event.set()
96
97 def add_exception(self, future):
98 super().add_exception(future)
99 self.event.set()
100
101 def add_cancelled(self, future):
102 super().add_cancelled(future)
103 self.event.set()
104
105 class _AllCompletedWaiter(_Waiter):
106 """Used by wait(return_when=FIRST_EXCEPTION and ALL_COMPLETED)."""
107
108 def __init__(self, num_pending_calls, stop_on_exception):
109 self.num_pending_calls = num_pending_calls
110 self.stop_on_exception = stop_on_exception
111 self.lock = threading.Lock()
112 super().__init__()
113
114 def _decrement_pending_calls(self):
115 with self.lock:
116 self.num_pending_calls -= 1
117 if not self.num_pending_calls:
118 self.event.set()
119
120 def add_result(self, future):
121 super().add_result(future)
122 self._decrement_pending_calls()
123
124 def add_exception(self, future):
125 super().add_exception(future)
126 if self.stop_on_exception:
127 self.event.set()
128 else:
129 self._decrement_pending_calls()
130
131 def add_cancelled(self, future):
132 super().add_cancelled(future)
133 self._decrement_pending_calls()
134
135 class _AcquireFutures(object):
136 """A context manager that does an ordered acquire of Future conditions."""
137
138 def __init__(self, futures):
139 self.futures = sorted(futures, key=id)
140
141 def __enter__(self):
142 for future in self.futures:
143 future._condition.acquire()
144
145 def __exit__(self, *args):
146 for future in self.futures:
147 future._condition.release()
148
149 def _create_and_install_waiters(fs, return_when):
150 if return_when == _AS_COMPLETED:
151 waiter = _AsCompletedWaiter()
152 elif return_when == FIRST_COMPLETED:
153 waiter = _FirstCompletedWaiter()
154 else:
155 pending_count = sum(
156 f._state not in [CANCELLED_AND_NOTIFIED, FINISHED] for f in fs)
157
158 if return_when == FIRST_EXCEPTION:
159 waiter = _AllCompletedWaiter(pending_count, stop_on_exception=True)
160 elif return_when == ALL_COMPLETED:
161 waiter = _AllCompletedWaiter(pending_count, stop_on_exception=False)
162 else:
163 raise ValueError("Invalid return condition: %r" % return_when)
164
165 for f in fs:
166 f._waiters.append(waiter)
167
168 return waiter
169
170
171 def _yield_finished_futures(fs, waiter, ref_collect):
172 """
173 Iterate on the list *fs*, yielding finished futures one by one in
174 reverse order.
175 Before yielding a future, *waiter* is removed from its waiters
176 and the future is removed from each set in the collection of sets
177 *ref_collect*.
178
179 The aim of this function is to avoid keeping stale references after
180 the future is yielded and before the iterator resumes.
181 """
182 while fs:
183 f = fs[-1]
184 for futures_set in ref_collect:
185 futures_set.remove(f)
186 with f._condition:
187 f._waiters.remove(waiter)
188 del f
189 # Careful not to keep a reference to the popped value
190 yield fs.pop()
191
192
193 def as_completed(fs, timeout=None):
194 """An iterator over the given futures that yields each as it completes.
195
196 Args:
197 fs: The sequence of Futures (possibly created by different Executors) to
198 iterate over.
199 timeout: The maximum number of seconds to wait. If None, then there
200 is no limit on the wait time.
201
202 Returns:
203 An iterator that yields the given Futures as they complete (finished or
204 cancelled). If any given Futures are duplicated, they will be returned
205 once.
206
207 Raises:
208 TimeoutError: If the entire result iterator could not be generated
209 before the given timeout.
210 """
211 if timeout is not None:
212 end_time = timeout + time.monotonic()
213
214 fs = set(fs)
215 total_futures = len(fs)
216 with _AcquireFutures(fs):
217 finished = set(
218 f for f in fs
219 if f._state in [CANCELLED_AND_NOTIFIED, FINISHED])
220 pending = fs - finished
221 waiter = _create_and_install_waiters(fs, _AS_COMPLETED)
222 finished = list(finished)
223 try:
224 yield from _yield_finished_futures(finished, waiter,
225 ref_collect=(fs,))
226
227 while pending:
228 if timeout is None:
229 wait_timeout = None
230 else:
231 wait_timeout = end_time - time.monotonic()
232 if wait_timeout < 0:
233 raise TimeoutError(
234 '%d (of %d) futures unfinished' % (
235 len(pending), total_futures))
236
237 waiter.event.wait(wait_timeout)
238
239 with waiter.lock:
240 finished = waiter.finished_futures
241 waiter.finished_futures = []
242 waiter.event.clear()
243
244 # reverse to keep finishing order
245 finished.reverse()
246 yield from _yield_finished_futures(finished, waiter,
247 ref_collect=(fs, pending))
248
249 finally:
250 # Remove waiter from unfinished futures
251 for f in fs:
252 with f._condition:
253 f._waiters.remove(waiter)
254
255 DoneAndNotDoneFutures = collections.namedtuple(
256 'DoneAndNotDoneFutures', 'done not_done')
257 def wait(fs, timeout=None, return_when=ALL_COMPLETED):
258 """Wait for the futures in the given sequence to complete.
259
260 Args:
261 fs: The sequence of Futures (possibly created by different Executors) to
262 wait upon.
263 timeout: The maximum number of seconds to wait. If None, then there
264 is no limit on the wait time.
265 return_when: Indicates when this function should return. The options
266 are:
267
268 FIRST_COMPLETED - Return when any future finishes or is
269 cancelled.
270 FIRST_EXCEPTION - Return when any future finishes by raising an
271 exception. If no future raises an exception
272 then it is equivalent to ALL_COMPLETED.
273 ALL_COMPLETED - Return when all futures finish or are cancelled.
274
275 Returns:
276 A named 2-tuple of sets. The first set, named 'done', contains the
277 futures that completed (is finished or cancelled) before the wait
278 completed. The second set, named 'not_done', contains uncompleted
279 futures. Duplicate futures given to *fs* are removed and will be
280 returned only once.
281 """
282 fs = set(fs)
283 with _AcquireFutures(fs):
284 done = {f for f in fs
285 if f._state in [CANCELLED_AND_NOTIFIED, FINISHED]}
286 not_done = fs - done
287 if (return_when == FIRST_COMPLETED) and done:
288 return DoneAndNotDoneFutures(done, not_done)
289 elif (return_when == FIRST_EXCEPTION) and done:
290 if any(f for f in done
291 if not f.cancelled() and f.exception() is not None):
292 return DoneAndNotDoneFutures(done, not_done)
293
294 if len(done) == len(fs):
295 return DoneAndNotDoneFutures(done, not_done)
296
297 waiter = _create_and_install_waiters(fs, return_when)
298
299 waiter.event.wait(timeout)
300 for f in fs:
301 with f._condition:
302 f._waiters.remove(waiter)
303
304 done.update(waiter.finished_futures)
305 return DoneAndNotDoneFutures(done, fs - done)
306
307
308 def _result_or_cancel(fut, timeout=None):
309 try:
310 try:
311 return fut.result(timeout)
312 finally:
313 fut.cancel()
314 finally:
315 # Break a reference cycle with the exception in self._exception
316 del fut
317
318
319 class Future(object):
320 """Represents the result of an asynchronous computation."""
321
322 def __init__(self):
323 """Initializes the future. Should not be called by clients."""
324 self._condition = threading.Condition()
325 self._state = PENDING
326 self._result = None
327 self._exception = None
328 self._waiters = []
329 self._done_callbacks = []
330
331 def _invoke_callbacks(self):
332 for callback in self._done_callbacks:
333 try:
334 callback(self)
335 except Exception:
336 LOGGER.exception('exception calling callback for %r', self)
337
338 def __repr__(self):
339 with self._condition:
340 if self._state == FINISHED:
341 if self._exception:
342 return '<%s at %#x state=%s raised %s>' % (
343 self.__class__.__name__,
344 id(self),
345 _STATE_TO_DESCRIPTION_MAP[self._state],
346 self._exception.__class__.__name__)
347 else:
348 return '<%s at %#x state=%s returned %s>' % (
349 self.__class__.__name__,
350 id(self),
351 _STATE_TO_DESCRIPTION_MAP[self._state],
352 self._result.__class__.__name__)
353 return '<%s at %#x state=%s>' % (
354 self.__class__.__name__,
355 id(self),
356 _STATE_TO_DESCRIPTION_MAP[self._state])
357
358 def cancel(self):
359 """Cancel the future if possible.
360
361 Returns True if the future was cancelled, False otherwise. A future
362 cannot be cancelled if it is running or has already completed.
363 """
364 with self._condition:
365 if self._state in [RUNNING, FINISHED]:
366 return False
367
368 if self._state in [CANCELLED, CANCELLED_AND_NOTIFIED]:
369 return True
370
371 self._state = CANCELLED
372 self._condition.notify_all()
373
374 self._invoke_callbacks()
375 return True
376
377 def cancelled(self):
378 """Return True if the future was cancelled."""
379 with self._condition:
380 return self._state in [CANCELLED, CANCELLED_AND_NOTIFIED]
381
382 def running(self):
383 """Return True if the future is currently executing."""
384 with self._condition:
385 return self._state == RUNNING
386
387 def done(self):
388 """Return True if the future was cancelled or finished executing."""
389 with self._condition:
390 return self._state in [CANCELLED, CANCELLED_AND_NOTIFIED, FINISHED]
391
392 def __get_result(self):
393 if self._exception is not None:
394 try:
395 raise self._exception
396 finally:
397 # Break a reference cycle with the exception in self._exception
398 self = None
399 else:
400 return self._result
401
402 def add_done_callback(self, fn):
403 """Attaches a callable that will be called when the future finishes.
404
405 Args:
406 fn: A callable that will be called with this future as its only
407 argument when the future completes or is cancelled. The callable
408 will always be called by a thread in the same process in which
409 it was added. If the future has already completed or been
410 cancelled then the callable will be called immediately. These
411 callables are called in the order that they were added.
412 """
413 with self._condition:
414 if self._state not in [CANCELLED, CANCELLED_AND_NOTIFIED, FINISHED]:
415 self._done_callbacks.append(fn)
416 return
417 try:
418 fn(self)
419 except Exception:
420 LOGGER.exception('exception calling callback for %r', self)
421
422 def result(self, timeout=None):
423 """Return the result of the call that the future represents.
424
425 Args:
426 timeout: The number of seconds to wait for the result if the future
427 isn't done. If None, then there is no limit on the wait time.
428
429 Returns:
430 The result of the call that the future represents.
431
432 Raises:
433 CancelledError: If the future was cancelled.
434 TimeoutError: If the future didn't finish executing before the given
435 timeout.
436 Exception: If the call raised then that exception will be raised.
437 """
438 try:
439 with self._condition:
440 if self._state in [CANCELLED, CANCELLED_AND_NOTIFIED]:
441 raise CancelledError()
442 elif self._state == FINISHED:
443 return self.__get_result()
444
445 self._condition.wait(timeout)
446
447 if self._state in [CANCELLED, CANCELLED_AND_NOTIFIED]:
448 raise CancelledError()
449 elif self._state == FINISHED:
450 return self.__get_result()
451 else:
452 raise TimeoutError()
453 finally:
454 # Break a reference cycle with the exception in self._exception
455 self = None
456
457 def exception(self, timeout=None):
458 """Return the exception raised by the call that the future represents.
459
460 Args:
461 timeout: The number of seconds to wait for the exception if the
462 future isn't done. If None, then there is no limit on the wait
463 time.
464
465 Returns:
466 The exception raised by the call that the future represents or None
467 if the call completed without raising.
468
469 Raises:
470 CancelledError: If the future was cancelled.
471 TimeoutError: If the future didn't finish executing before the given
472 timeout.
473 """
474
475 with self._condition:
476 if self._state in [CANCELLED, CANCELLED_AND_NOTIFIED]:
477 raise CancelledError()
478 elif self._state == FINISHED:
479 return self._exception
480
481 self._condition.wait(timeout)
482
483 if self._state in [CANCELLED, CANCELLED_AND_NOTIFIED]:
484 raise CancelledError()
485 elif self._state == FINISHED:
486 return self._exception
487 else:
488 raise TimeoutError()
489
490 # The following methods should only be used by Executors and in tests.
491 def set_running_or_notify_cancel(self):
492 """Mark the future as running or process any cancel notifications.
493
494 Should only be used by Executor implementations and unit tests.
495
496 If the future has been cancelled (cancel() was called and returned
497 True) then any threads waiting on the future completing (though calls
498 to as_completed() or wait()) are notified and False is returned.
499
500 If the future was not cancelled then it is put in the running state
501 (future calls to running() will return True) and True is returned.
502
503 This method should be called by Executor implementations before
504 executing the work associated with this future. If this method returns
505 False then the work should not be executed.
506
507 Returns:
508 False if the Future was cancelled, True otherwise.
509
510 Raises:
511 RuntimeError: if this method was already called or if set_result()
512 or set_exception() was called.
513 """
514 with self._condition:
515 if self._state == CANCELLED:
516 self._state = CANCELLED_AND_NOTIFIED
517 for waiter in self._waiters:
518 waiter.add_cancelled(self)
519 # self._condition.notify_all() is not necessary because
520 # self.cancel() triggers a notification.
521 return False
522 elif self._state == PENDING:
523 self._state = RUNNING
524 return True
525 else:
526 LOGGER.critical('Future %s in unexpected state: %s',
527 id(self),
528 self._state)
529 raise RuntimeError('Future in unexpected state')
530
531 def set_result(self, result):
532 """Sets the return value of work associated with the future.
533
534 Should only be used by Executor implementations and unit tests.
535 """
536 with self._condition:
537 if self._state in {CANCELLED, CANCELLED_AND_NOTIFIED, FINISHED}:
538 raise InvalidStateError('{}: {!r}'.format(self._state, self))
539 self._result = result
540 self._state = FINISHED
541 for waiter in self._waiters:
542 waiter.add_result(self)
543 self._condition.notify_all()
544 self._invoke_callbacks()
545
546 def set_exception(self, exception):
547 """Sets the result of the future as being the given exception.
548
549 Should only be used by Executor implementations and unit tests.
550 """
551 with self._condition:
552 if self._state in {CANCELLED, CANCELLED_AND_NOTIFIED, FINISHED}:
553 raise InvalidStateError('{}: {!r}'.format(self._state, self))
554 self._exception = exception
555 self._state = FINISHED
556 for waiter in self._waiters:
557 waiter.add_exception(self)
558 self._condition.notify_all()
559 self._invoke_callbacks()
560
561 __class_getitem__ = classmethod(types.GenericAlias)
562
563 class Executor(object):
564 """This is an abstract base class for concrete asynchronous executors."""
565
566 def submit(self, fn, /, *args, **kwargs):
567 """Submits a callable to be executed with the given arguments.
568
569 Schedules the callable to be executed as fn(*args, **kwargs) and returns
570 a Future instance representing the execution of the callable.
571
572 Returns:
573 A Future representing the given call.
574 """
575 raise NotImplementedError()
576
577 def map(self, fn, *iterables, timeout=None, chunksize=1, buffersize=None):
578 """Returns an iterator equivalent to map(fn, iter).
579
580 Args:
581 fn: A callable that will take as many arguments as there are
582 passed iterables.
583 timeout: The maximum number of seconds to wait. If None, then there
584 is no limit on the wait time.
585 chunksize: The size of the chunks the iterable will be broken into
586 before being passed to a child process. This argument is only
587 used by ProcessPoolExecutor; it is ignored by
588 ThreadPoolExecutor.
589 buffersize: The number of submitted tasks whose results have not
590 yet been yielded. If the buffer is full, iteration over the
591 iterables pauses until a result is yielded from the buffer.
592 If None, all input elements are eagerly collected, and a task is
593 submitted for each.
594
595 Returns:
596 An iterator equivalent to: map(func, *iterables) but the calls may
597 be evaluated out-of-order.
598
599 Raises:
600 TimeoutError: If the entire result iterator could not be generated
601 before the given timeout.
602 Exception: If fn(*args) raises for any values.
603 """
604 if buffersize is not None and not isinstance(buffersize, int):
605 raise TypeError("buffersize must be an integer or None")
606 if buffersize is not None and buffersize < 1:
607 raise ValueError("buffersize must be None or > 0")
608
609 if timeout is not None:
610 end_time = timeout + time.monotonic()
611
612 zipped_iterables = zip(*iterables)
613 if buffersize:
614 fs = collections.deque(
615 self.submit(fn, *args) for args in islice(zipped_iterables, buffersize)
616 )
617 else:
618 fs = [self.submit(fn, *args) for args in zipped_iterables]
619
620 # Use a weak reference to ensure that the executor can be garbage
621 # collected independently of the result_iterator closure.
622 executor_weakref = weakref.ref(self)
623
624 # Yield must be hidden in closure so that the futures are submitted
625 # before the first iterator value is required.
626 def result_iterator():
627 try:
628 # reverse to keep finishing order
629 fs.reverse()
630 while fs:
631 if (
632 buffersize
633 and (executor := executor_weakref())
634 and (args := next(zipped_iterables, None))
635 ):
636 fs.appendleft(executor.submit(fn, *args))
637 # Careful not to keep a reference to the popped future
638 if timeout is None:
639 yield _result_or_cancel(fs.pop())
640 else:
641 yield _result_or_cancel(fs.pop(), end_time - time.monotonic())
642 finally:
643 for future in fs:
644 future.cancel()
645 return result_iterator()
646
647 def shutdown(self, wait=True, *, cancel_futures=False):
648 """Clean-up the resources associated with the Executor.
649
650 It is safe to call this method several times. Otherwise, no other
651 methods can be called after this one.
652
653 Args:
654 wait: If True then shutdown will not return until all running
655 futures have finished executing and the resources used by the
656 executor have been reclaimed.
657 cancel_futures: If True then shutdown will cancel all pending
658 futures. Futures that are completed or running will not be
659 cancelled.
660 """
661 pass
662
663 def __enter__(self):
664 return self
665
666 def __exit__(self, exc_type, exc_val, exc_tb):
667 self.shutdown(wait=True)
668 return False
669
670
671 class BrokenExecutor(RuntimeError):
672 """
673 Raised when a executor has become non-functional after a severe failure.
674 """