Debian Codegraph @

Lib/concurrent/futures/_base.py

raw
1 # Copyright 2009 Brian Quinlan. All Rights Reserved.
2 # Licensed to PSF under a Contributor Agreement.
3
4 __author__ = 'Brian Quinlan ([email protected])'
5
6 import collections
7 import logging
8 import threading
9 import time
10 import types
11
12 FIRST_COMPLETED = 'FIRST_COMPLETED'
13 FIRST_EXCEPTION = 'FIRST_EXCEPTION'
14 ALL_COMPLETED = 'ALL_COMPLETED'
15 _AS_COMPLETED = '_AS_COMPLETED'
16
17 # Possible future states (for internal use by the futures package).
18 PENDING = 'PENDING'
19 RUNNING = 'RUNNING'
20 # The future was cancelled by the user...
21 CANCELLED = 'CANCELLED'
22 # ...and _Waiter.add_cancelled() was called by a worker.
23 CANCELLED_AND_NOTIFIED = 'CANCELLED_AND_NOTIFIED'
24 FINISHED = 'FINISHED'
25
26 _FUTURE_STATES = [
27 PENDING,
28 RUNNING,
29 CANCELLED,
30 CANCELLED_AND_NOTIFIED,
31 FINISHED
32 ]
33
34 _STATE_TO_DESCRIPTION_MAP = {
35 PENDING: "pending",
36 RUNNING: "running",
37 CANCELLED: "cancelled",
38 CANCELLED_AND_NOTIFIED: "cancelled",
39 FINISHED: "finished"
40 }
41
42 # Logger for internal use by the futures package.
43 LOGGER = logging.getLogger("concurrent.futures")
44
45 class Error(Exception):
46 """Base class for all future-related exceptions."""
47 pass
48
49 class CancelledError(Error):
50 """The Future was cancelled."""
51 pass
52
53 TimeoutError = TimeoutError # make local alias for the standard exception
54
55 class InvalidStateError(Error):
56 """The operation is not allowed in this state."""
57 pass
58
59 class _Waiter(object):
60 """Provides the event that wait() and as_completed() block on."""
61 def __init__(self):
62 self.event = threading.Event()
63 self.finished_futures = []
64
65 def add_result(self, future):
66 self.finished_futures.append(future)
67
68 def add_exception(self, future):
69 self.finished_futures.append(future)
70
71 def add_cancelled(self, future):
72 self.finished_futures.append(future)
73
74 class _AsCompletedWaiter(_Waiter):
75 """Used by as_completed()."""
76
77 def __init__(self):
78 super(_AsCompletedWaiter, self).__init__()
79 self.lock = threading.Lock()
80
81 def add_result(self, future):
82 with self.lock:
83 super(_AsCompletedWaiter, self).add_result(future)
84 self.event.set()
85
86 def add_exception(self, future):
87 with self.lock:
88 super(_AsCompletedWaiter, self).add_exception(future)
89 self.event.set()
90
91 def add_cancelled(self, future):
92 with self.lock:
93 super(_AsCompletedWaiter, self).add_cancelled(future)
94 self.event.set()
95
96 class _FirstCompletedWaiter(_Waiter):
97 """Used by wait(return_when=FIRST_COMPLETED)."""
98
99 def add_result(self, future):
100 super().add_result(future)
101 self.event.set()
102
103 def add_exception(self, future):
104 super().add_exception(future)
105 self.event.set()
106
107 def add_cancelled(self, future):
108 super().add_cancelled(future)
109 self.event.set()
110
111 class _AllCompletedWaiter(_Waiter):
112 """Used by wait(return_when=FIRST_EXCEPTION and ALL_COMPLETED)."""
113
114 def __init__(self, num_pending_calls, stop_on_exception):
115 self.num_pending_calls = num_pending_calls
116 self.stop_on_exception = stop_on_exception
117 self.lock = threading.Lock()
118 super().__init__()
119
120 def _decrement_pending_calls(self):
121 with self.lock:
122 self.num_pending_calls -= 1
123 if not self.num_pending_calls:
124 self.event.set()
125
126 def add_result(self, future):
127 super().add_result(future)
128 self._decrement_pending_calls()
129
130 def add_exception(self, future):
131 super().add_exception(future)
132 if self.stop_on_exception:
133 self.event.set()
134 else:
135 self._decrement_pending_calls()
136
137 def add_cancelled(self, future):
138 super().add_cancelled(future)
139 self._decrement_pending_calls()
140
141 class _AcquireFutures(object):
142 """A context manager that does an ordered acquire of Future conditions."""
143
144 def __init__(self, futures):
145 self.futures = sorted(futures, key=id)
146
147 def __enter__(self):
148 for future in self.futures:
149 future._condition.acquire()
150
151 def __exit__(self, *args):
152 for future in self.futures:
153 future._condition.release()
154
155 def _create_and_install_waiters(fs, return_when):
156 if return_when == _AS_COMPLETED:
157 waiter = _AsCompletedWaiter()
158 elif return_when == FIRST_COMPLETED:
159 waiter = _FirstCompletedWaiter()
160 else:
161 pending_count = sum(
162 f._state not in [CANCELLED_AND_NOTIFIED, FINISHED] for f in fs)
163
164 if return_when == FIRST_EXCEPTION:
165 waiter = _AllCompletedWaiter(pending_count, stop_on_exception=True)
166 elif return_when == ALL_COMPLETED:
167 waiter = _AllCompletedWaiter(pending_count, stop_on_exception=False)
168 else:
169 raise ValueError("Invalid return condition: %r" % return_when)
170
171 for f in fs:
172 f._waiters.append(waiter)
173
174 return waiter
175
176
177 def _yield_finished_futures(fs, waiter, ref_collect):
178 """
179 Iterate on the list *fs*, yielding finished futures one by one in
180 reverse order.
181 Before yielding a future, *waiter* is removed from its waiters
182 and the future is removed from each set in the collection of sets
183 *ref_collect*.
184
185 The aim of this function is to avoid keeping stale references after
186 the future is yielded and before the iterator resumes.
187 """
188 while fs:
189 f = fs[-1]
190 for futures_set in ref_collect:
191 futures_set.remove(f)
192 with f._condition:
193 f._waiters.remove(waiter)
194 del f
195 # Careful not to keep a reference to the popped value
196 yield fs.pop()
197
198
199 def as_completed(fs, timeout=None):
200 """An iterator over the given futures that yields each as it completes.
201
202 Args:
203 fs: The sequence of Futures (possibly created by different
204 Executors) to iterate over.
205 timeout: The maximum number of seconds to wait. If None, then
206 there is no limit on the wait time.
207
208 Returns:
209 An iterator that yields the given Futures as they complete
210 (finished or cancelled). If any given Futures are duplicated,
211 they will be returned once.
212
213 Raises:
214 TimeoutError: If the entire result iterator could not be generated
215 before the given timeout.
216 """
217 if timeout is not None:
218 end_time = timeout + time.monotonic()
219
220 fs = set(fs)
221 total_futures = len(fs)
222 with _AcquireFutures(fs):
223 finished = set(
224 f for f in fs
225 if f._state in [CANCELLED_AND_NOTIFIED, FINISHED])
226 pending = fs - finished
227 waiter = _create_and_install_waiters(fs, _AS_COMPLETED)
228 finished = list(finished)
229 try:
230 yield from _yield_finished_futures(finished, waiter,
231 ref_collect=(fs,))
232
233 while pending:
234 if timeout is None:
235 wait_timeout = None
236 else:
237 wait_timeout = end_time - time.monotonic()
238 if wait_timeout < 0:
239 raise TimeoutError(
240 '%d (of %d) futures unfinished' % (
241 len(pending), total_futures))
242
243 waiter.event.wait(wait_timeout)
244
245 with waiter.lock:
246 finished = waiter.finished_futures
247 waiter.finished_futures = []
248 waiter.event.clear()
249
250 # reverse to keep finishing order
251 finished.reverse()
252 yield from _yield_finished_futures(finished, waiter,
253 ref_collect=(fs, pending))
254
255 finally:
256 # Remove waiter from unfinished futures
257 for f in fs:
258 with f._condition:
259 f._waiters.remove(waiter)
260
261 DoneAndNotDoneFutures = collections.namedtuple(
262 'DoneAndNotDoneFutures', 'done not_done')
263 def wait(fs, timeout=None, return_when=ALL_COMPLETED):
264 """Wait for the futures in the given sequence to complete.
265
266 Args:
267 fs: The sequence of Futures (possibly created by different
268 Executors) to wait upon.
269 timeout: The maximum number of seconds to wait. If None, then
270 there is no limit on the wait time.
271 return_when: Indicates when this function should return.
272 The options are:
273
274 FIRST_COMPLETED - Return when any future finishes or is
275 cancelled.
276 FIRST_EXCEPTION - Return when any future finishes by raising an
277 exception. If no future raises an exception
278 then it is equivalent to ALL_COMPLETED.
279 ALL_COMPLETED - Return when all futures finish or are
280 cancelled.
281
282 Returns:
283 A named 2-tuple of sets. The first set, named 'done', contains the
284 futures that completed (is finished or cancelled) before the wait
285 completed. The second set, named 'not_done', contains uncompleted
286 futures. Duplicate futures given to *fs* are removed and will be
287 returned only once.
288 """
289 fs = set(fs)
290 with _AcquireFutures(fs):
291 done = {f for f in fs
292 if f._state in [CANCELLED_AND_NOTIFIED, FINISHED]}
293 not_done = fs - done
294 if (return_when == FIRST_COMPLETED) and done:
295 return DoneAndNotDoneFutures(done, not_done)
296 elif (return_when == FIRST_EXCEPTION) and done:
297 if any(f for f in done
298 if not f.cancelled() and f.exception() is not None):
299 return DoneAndNotDoneFutures(done, not_done)
300
301 if len(done) == len(fs):
302 return DoneAndNotDoneFutures(done, not_done)
303
304 waiter = _create_and_install_waiters(fs, return_when)
305
306 waiter.event.wait(timeout)
307 for f in fs:
308 with f._condition:
309 f._waiters.remove(waiter)
310
311 done.update(waiter.finished_futures)
312 return DoneAndNotDoneFutures(done, fs - done)
313
314
315 def _result_or_cancel(fut, timeout=None):
316 try:
317 try:
318 return fut.result(timeout)
319 finally:
320 fut.cancel()
321 finally:
322 # Break a reference cycle with the exception in self._exception
323 del fut
324
325
326 class Future(object):
327 """Represents the result of an asynchronous computation."""
328
329 def __init__(self):
330 """Initializes the future. Should not be called by clients."""
331 self._condition = threading.Condition()
332 self._state = PENDING
333 self._result = None
334 self._exception = None
335 self._waiters = []
336 self._done_callbacks = []
337
338 def _invoke_callbacks(self):
339 for callback in self._done_callbacks:
340 try:
341 callback(self)
342 except Exception:
343 LOGGER.exception('exception calling callback for %r', self)
344
345 def __repr__(self):
346 with self._condition:
347 if self._state == FINISHED:
348 if self._exception:
349 return '<%s at %#x state=%s raised %s>' % (
350 self.__class__.__name__,
351 id(self),
352 _STATE_TO_DESCRIPTION_MAP[self._state],
353 self._exception.__class__.__name__)
354 else:
355 return '<%s at %#x state=%s returned %s>' % (
356 self.__class__.__name__,
357 id(self),
358 _STATE_TO_DESCRIPTION_MAP[self._state],
359 self._result.__class__.__name__)
360 return '<%s at %#x state=%s>' % (
361 self.__class__.__name__,
362 id(self),
363 _STATE_TO_DESCRIPTION_MAP[self._state])
364
365 def cancel(self):
366 """Cancel the future if possible.
367
368 Returns True if the future was cancelled, False otherwise. A future
369 cannot be cancelled if it is running or has already completed.
370 """
371 with self._condition:
372 if self._state in [RUNNING, FINISHED]:
373 return False
374
375 if self._state in [CANCELLED, CANCELLED_AND_NOTIFIED]:
376 return True
377
378 self._state = CANCELLED
379 self._condition.notify_all()
380
381 self._invoke_callbacks()
382 return True
383
384 def cancelled(self):
385 """Return True if the future was cancelled."""
386 with self._condition:
387 return self._state in [CANCELLED, CANCELLED_AND_NOTIFIED]
388
389 def running(self):
390 """Return True if the future is currently executing."""
391 with self._condition:
392 return self._state == RUNNING
393
394 def done(self):
395 """Return True if the future was cancelled or finished executing."""
396 with self._condition:
397 return self._state in [CANCELLED, CANCELLED_AND_NOTIFIED, FINISHED]
398
399 def __get_result(self):
400 if self._exception is not None:
401 try:
402 raise self._exception
403 finally:
404 # Break a reference cycle with the exception in self._exception
405 self = None
406 else:
407 return self._result
408
409 def add_done_callback(self, fn):
410 """Attaches a callable that will be called when the future finishes.
411
412 Args:
413 fn: A callable that will be called with this future as its only
414 argument when the future completes or is cancelled. The
415 callable will always be called by a thread in the same
416 process in which it was added. If the future has already
417 completed or been cancelled then the callable will be
418 called immediately. These callables are called in the
419 order that they were added.
420 """
421 with self._condition:
422 if self._state not in [CANCELLED, CANCELLED_AND_NOTIFIED, FINISHED]:
423 self._done_callbacks.append(fn)
424 return
425 try:
426 fn(self)
427 except Exception:
428 LOGGER.exception('exception calling callback for %r', self)
429
430 def result(self, timeout=None):
431 """Return the result of the call that the future represents.
432
433 Args:
434 timeout: The number of seconds to wait for the result if the
435 future isn't done. If None, then there is no limit on the
436 wait time.
437
438 Returns:
439 The result of the call that the future represents.
440
441 Raises:
442 CancelledError: If the future was cancelled.
443 TimeoutError: If the future didn't finish executing before the
444 given timeout.
445 Exception: If the call raised then that exception will be
446 raised.
447 """
448 try:
449 with self._condition:
450 if self._state in [CANCELLED, CANCELLED_AND_NOTIFIED]:
451 raise CancelledError()
452 elif self._state == FINISHED:
453 return self.__get_result()
454
455 self._condition.wait(timeout)
456
457 if self._state in [CANCELLED, CANCELLED_AND_NOTIFIED]:
458 raise CancelledError()
459 elif self._state == FINISHED:
460 return self.__get_result()
461 else:
462 raise TimeoutError()
463 finally:
464 # Break a reference cycle with the exception in self._exception
465 self = None
466
467 def exception(self, timeout=None):
468 """Return the exception raised by the call that the future represents.
469
470 Args:
471 timeout: The number of seconds to wait for the exception if the
472 future isn't done. If None, then there is no limit on the
473 wait time.
474
475 Returns:
476 The exception raised by the call that the future represents or
477 None if the call completed without raising.
478
479 Raises:
480 CancelledError: If the future was cancelled.
481 TimeoutError: If the future didn't finish executing before the
482 given timeout.
483 """
484
485 with self._condition:
486 if self._state in [CANCELLED, CANCELLED_AND_NOTIFIED]:
487 raise CancelledError()
488 elif self._state == FINISHED:
489 return self._exception
490
491 self._condition.wait(timeout)
492
493 if self._state in [CANCELLED, CANCELLED_AND_NOTIFIED]:
494 raise CancelledError()
495 elif self._state == FINISHED:
496 return self._exception
497 else:
498 raise TimeoutError()
499
500 # The following methods should only be used by Executors and in tests.
501 def set_running_or_notify_cancel(self):
502 """Mark the future as running or process any cancel notifications.
503
504 Should only be used by Executor implementations and unit tests.
505
506 If the future has been cancelled (cancel() was called and returned
507 True) then any threads waiting on the future completing (though
508 calls to as_completed() or wait()) are notified and False is
509 returned.
510
511 If the future was not cancelled then it is put in the running state
512 (future calls to running() will return True) and True is returned.
513
514 This method should be called by Executor implementations before
515 executing the work associated with this future. If this method
516 returns False then the work should not be executed.
517
518 Returns:
519 False if the Future was cancelled, True otherwise.
520
521 Raises:
522 RuntimeError: if this method was already called or if
523 set_result() or set_exception() was called.
524 """
525 with self._condition:
526 if self._state == CANCELLED:
527 self._state = CANCELLED_AND_NOTIFIED
528 for waiter in self._waiters:
529 waiter.add_cancelled(self)
530 # self._condition.notify_all() is not necessary because
531 # self.cancel() triggers a notification.
532 return False
533 elif self._state == PENDING:
534 self._state = RUNNING
535 return True
536 else:
537 LOGGER.critical('Future %s in unexpected state: %s',
538 id(self),
539 self._state)
540 raise RuntimeError('Future in unexpected state')
541
542 def set_result(self, result):
543 """Sets the return value of work associated with the future.
544
545 Should only be used by Executor implementations and unit tests.
546 """
547 with self._condition:
548 if self._state in {CANCELLED, CANCELLED_AND_NOTIFIED, FINISHED}:
549 raise InvalidStateError('{}: {!r}'.format(self._state, self))
550 self._result = result
551 self._state = FINISHED
552 for waiter in self._waiters:
553 waiter.add_result(self)
554 self._condition.notify_all()
555 self._invoke_callbacks()
556
557 def set_exception(self, exception):
558 """Sets the result of the future as being the given exception.
559
560 Should only be used by Executor implementations and unit tests.
561 """
562 with self._condition:
563 if self._state in {CANCELLED, CANCELLED_AND_NOTIFIED, FINISHED}:
564 raise InvalidStateError('{}: {!r}'.format(self._state, self))
565 self._exception = exception
566 self._state = FINISHED
567 for waiter in self._waiters:
568 waiter.add_exception(self)
569 self._condition.notify_all()
570 self._invoke_callbacks()
571
572 __class_getitem__ = classmethod(types.GenericAlias)
573
574 class Executor(object):
575 """This is an abstract base class for concrete asynchronous executors."""
576
577 def submit(self, fn, /, *args, **kwargs):
578 """Submits a callable to be executed with the given arguments.
579
580 Schedules the callable to be executed as fn(*args, **kwargs) and
581 returns a Future instance representing the execution of the
582 callable.
583
584 Returns:
585 A Future representing the given call.
586 """
587 raise NotImplementedError()
588
589 def map(self, fn, *iterables, timeout=None, chunksize=1):
590 """Returns an iterator equivalent to map(fn, iter).
591
592 Args:
593 fn: A callable that will take as many arguments as there are
594 passed iterables.
595 timeout: The maximum number of seconds to wait. If None, then
596 there is no limit on the wait time.
597 chunksize: The size of the chunks the iterable will be broken
598 into before being passed to a child process. This argument
599 is only used by ProcessPoolExecutor; it is ignored by
600 ThreadPoolExecutor.
601
602 Returns:
603 An iterator equivalent to: map(func, *iterables) but the calls
604 may be evaluated out-of-order.
605
606 Raises:
607 TimeoutError: If the entire result iterator could not be
608 generated before the given timeout.
609 Exception: If fn(*args) raises for any values.
610 """
611 if timeout is not None:
612 end_time = timeout + time.monotonic()
613
614 fs = [self.submit(fn, *args) for args in zip(*iterables)]
615
616 # Yield must be hidden in closure so that the futures are submitted
617 # before the first iterator value is required.
618 def result_iterator():
619 try:
620 # reverse to keep finishing order
621 fs.reverse()
622 while fs:
623 # Careful not to keep a reference to the popped future
624 if timeout is None:
625 yield _result_or_cancel(fs.pop())
626 else:
627 yield _result_or_cancel(fs.pop(), end_time - time.monotonic())
628 finally:
629 for future in fs:
630 future.cancel()
631 return result_iterator()
632
633 def shutdown(self, wait=True, *, cancel_futures=False):
634 """Clean-up the resources associated with the Executor.
635
636 It is safe to call this method several times. Otherwise, no other
637 methods can be called after this one.
638
639 Args:
640 wait: If True then shutdown will not return until all running
641 futures have finished executing and the resources used by
642 the executor have been reclaimed.
643 cancel_futures: If True then shutdown will cancel all pending
644 futures. Futures that are completed or running will not be
645 cancelled.
646 """
647 pass
648
649 def __enter__(self):
650 return self
651
652 def __exit__(self, exc_type, exc_val, exc_tb):
653 self.shutdown(wait=True)
654 return False
655
656
657 class BrokenExecutor(RuntimeError):
658 """
659 Raised when a executor has become non-functional after a severe failure.
660 """