| 1 |
# Copyright 2009 Brian Quinlan. All Rights Reserved. |
| 2 |
# Licensed to PSF under a Contributor Agreement. |
| 3 |
|
| 4 |
"""Implements ThreadPoolExecutor.""" |
| 5 |
|
| 6 |
__author__ = 'Brian Quinlan ([email protected])' |
| 7 |
|
| 8 |
from concurrent.futures import _base |
| 9 |
import itertools |
| 10 |
import queue |
| 11 |
import threading |
| 12 |
import types |
| 13 |
import weakref |
| 14 |
import os |
| 15 |
|
| 16 |
|
| 17 |
_threads_queues = weakref.WeakKeyDictionary() |
| 18 |
_shutdown = False |
| 19 |
# Lock that ensures that new workers are not created while the interpreter is |
| 20 |
# shutting down. Must be held while mutating _threads_queues and _shutdown. |
| 21 |
_global_shutdown_lock = threading.Lock() |
| 22 |
|
| 23 |
def _python_exit(): |
| 24 |
global _shutdown |
| 25 |
with _global_shutdown_lock: |
| 26 |
_shutdown = True |
| 27 |
items = list(_threads_queues.items()) |
| 28 |
for t, q in items: |
| 29 |
q.put(None) |
| 30 |
for t, q in items: |
| 31 |
t.join() |
| 32 |
|
| 33 |
# Register for `_python_exit()` to be called just before joining all |
| 34 |
# non-daemon threads. This is used instead of `atexit.register()` for |
| 35 |
# compatibility with subinterpreters, which no longer support daemon threads. |
| 36 |
# See bpo-39812 for context. |
| 37 |
threading._register_atexit(_python_exit) |
| 38 |
|
| 39 |
# At fork, reinitialize the `_global_shutdown_lock` lock in the child process |
| 40 |
if hasattr(os, 'register_at_fork'): |
| 41 |
os.register_at_fork(before=_global_shutdown_lock.acquire, |
| 42 |
after_in_child=_global_shutdown_lock._at_fork_reinit, |
| 43 |
after_in_parent=_global_shutdown_lock.release) |
| 44 |
os.register_at_fork(after_in_child=_threads_queues.clear) |
| 45 |
|
| 46 |
|
| 47 |
class WorkerContext: |
| 48 |
|
| 49 |
@classmethod |
| 50 |
def prepare(cls, initializer, initargs): |
| 51 |
if initializer is not None: |
| 52 |
if not callable(initializer): |
| 53 |
raise TypeError("initializer must be a callable") |
| 54 |
def create_context(): |
| 55 |
return cls(initializer, initargs) |
| 56 |
def resolve_task(fn, args, kwargs): |
| 57 |
return (fn, args, kwargs) |
| 58 |
return create_context, resolve_task |
| 59 |
|
| 60 |
def __init__(self, initializer, initargs): |
| 61 |
self.initializer = initializer |
| 62 |
self.initargs = initargs |
| 63 |
|
| 64 |
def initialize(self): |
| 65 |
if self.initializer is not None: |
| 66 |
self.initializer(*self.initargs) |
| 67 |
|
| 68 |
def finalize(self): |
| 69 |
pass |
| 70 |
|
| 71 |
def run(self, task): |
| 72 |
fn, args, kwargs = task |
| 73 |
return fn(*args, **kwargs) |
| 74 |
|
| 75 |
|
| 76 |
class _WorkItem: |
| 77 |
def __init__(self, future, task): |
| 78 |
self.future = future |
| 79 |
self.task = task |
| 80 |
|
| 81 |
def run(self, ctx): |
| 82 |
if not self.future.set_running_or_notify_cancel(): |
| 83 |
return |
| 84 |
|
| 85 |
try: |
| 86 |
result = ctx.run(self.task) |
| 87 |
except BaseException as exc: |
| 88 |
self.future.set_exception(exc) |
| 89 |
# Break a reference cycle with the exception 'exc' |
| 90 |
self = None |
| 91 |
else: |
| 92 |
self.future.set_result(result) |
| 93 |
|
| 94 |
__class_getitem__ = classmethod(types.GenericAlias) |
| 95 |
|
| 96 |
|
| 97 |
def _worker(executor_reference, ctx, work_queue): |
| 98 |
try: |
| 99 |
ctx.initialize() |
| 100 |
except BaseException: |
| 101 |
_base.LOGGER.critical('Exception in initializer:', exc_info=True) |
| 102 |
executor = executor_reference() |
| 103 |
if executor is not None: |
| 104 |
executor._initializer_failed() |
| 105 |
return |
| 106 |
try: |
| 107 |
while True: |
| 108 |
try: |
| 109 |
work_item = work_queue.get_nowait() |
| 110 |
except queue.Empty: |
| 111 |
# attempt to increment idle count if queue is empty |
| 112 |
executor = executor_reference() |
| 113 |
if executor is not None: |
| 114 |
executor._idle_semaphore.release() |
| 115 |
del executor |
| 116 |
work_item = work_queue.get(block=True) |
| 117 |
|
| 118 |
if work_item is not None: |
| 119 |
work_item.run(ctx) |
| 120 |
# Delete references to object. See GH-60488 |
| 121 |
del work_item |
| 122 |
continue |
| 123 |
|
| 124 |
executor = executor_reference() |
| 125 |
# Exit if: |
| 126 |
# - The interpreter is shutting down OR |
| 127 |
# - The executor that owns the worker has been collected OR |
| 128 |
# - The executor that owns the worker has been shutdown. |
| 129 |
if _shutdown or executor is None or executor._shutdown: |
| 130 |
# Flag the executor as shutting down as early as possible if it |
| 131 |
# is not gc-ed yet. |
| 132 |
if executor is not None: |
| 133 |
executor._shutdown = True |
| 134 |
# Notice other workers |
| 135 |
work_queue.put(None) |
| 136 |
return |
| 137 |
del executor |
| 138 |
except BaseException: |
| 139 |
_base.LOGGER.critical('Exception in worker', exc_info=True) |
| 140 |
finally: |
| 141 |
ctx.finalize() |
| 142 |
|
| 143 |
|
| 144 |
class BrokenThreadPool(_base.BrokenExecutor): |
| 145 |
""" |
| 146 |
Raised when a worker thread in a ThreadPoolExecutor failed initializing. |
| 147 |
""" |
| 148 |
|
| 149 |
|
| 150 |
class ThreadPoolExecutor(_base.Executor): |
| 151 |
|
| 152 |
BROKEN = BrokenThreadPool |
| 153 |
|
| 154 |
# Used to assign unique thread names when thread_name_prefix is not supplied. |
| 155 |
_counter = itertools.count().__next__ |
| 156 |
|
| 157 |
@classmethod |
| 158 |
def prepare_context(cls, initializer, initargs): |
| 159 |
return WorkerContext.prepare(initializer, initargs) |
| 160 |
|
| 161 |
def __init__(self, max_workers=None, thread_name_prefix='', |
| 162 |
initializer=None, initargs=(), **ctxkwargs): |
| 163 |
"""Initializes a new ThreadPoolExecutor instance. |
| 164 |
|
| 165 |
Args: |
| 166 |
max_workers: The maximum number of threads that can be used to |
| 167 |
execute the given calls. |
| 168 |
thread_name_prefix: An optional name prefix to give our threads. |
| 169 |
initializer: A callable used to initialize worker threads. |
| 170 |
initargs: A tuple of arguments to pass to the initializer. |
| 171 |
ctxkwargs: Additional arguments to cls.prepare_context(). |
| 172 |
""" |
| 173 |
if max_workers is None: |
| 174 |
# ThreadPoolExecutor is often used to: |
| 175 |
# * CPU bound task which releases GIL |
| 176 |
# * I/O bound task (which releases GIL, of course) |
| 177 |
# |
| 178 |
# We use process_cpu_count + 4 for both types of tasks. |
| 179 |
# But we limit it to 32 to avoid consuming surprisingly large resource |
| 180 |
# on many core machine. |
| 181 |
max_workers = min(32, (os.process_cpu_count() or 1) + 4) |
| 182 |
if max_workers <= 0: |
| 183 |
raise ValueError("max_workers must be greater than 0") |
| 184 |
|
| 185 |
(self._create_worker_context, |
| 186 |
self._resolve_work_item_task, |
| 187 |
) = type(self).prepare_context(initializer, initargs, **ctxkwargs) |
| 188 |
|
| 189 |
self._max_workers = max_workers |
| 190 |
self._work_queue = queue.SimpleQueue() |
| 191 |
self._idle_semaphore = threading.Semaphore(0) |
| 192 |
self._threads = set() |
| 193 |
self._broken = False |
| 194 |
self._shutdown = False |
| 195 |
self._shutdown_lock = threading.Lock() |
| 196 |
self._thread_name_prefix = (thread_name_prefix or |
| 197 |
("ThreadPoolExecutor-%d" % self._counter())) |
| 198 |
|
| 199 |
def submit(self, fn, /, *args, **kwargs): |
| 200 |
with self._shutdown_lock, _global_shutdown_lock: |
| 201 |
if self._broken: |
| 202 |
raise self.BROKEN(self._broken) |
| 203 |
|
| 204 |
if self._shutdown: |
| 205 |
raise RuntimeError('cannot schedule new futures after shutdown') |
| 206 |
if _shutdown: |
| 207 |
raise RuntimeError('cannot schedule new futures after ' |
| 208 |
'interpreter shutdown') |
| 209 |
|
| 210 |
f = _base.Future() |
| 211 |
task = self._resolve_work_item_task(fn, args, kwargs) |
| 212 |
w = _WorkItem(f, task) |
| 213 |
|
| 214 |
self._work_queue.put(w) |
| 215 |
self._adjust_thread_count() |
| 216 |
return f |
| 217 |
submit.__doc__ = _base.Executor.submit.__doc__ |
| 218 |
|
| 219 |
def _adjust_thread_count(self): |
| 220 |
# if idle threads are available, don't spin new threads |
| 221 |
if self._idle_semaphore.acquire(timeout=0): |
| 222 |
return |
| 223 |
|
| 224 |
# When the executor gets lost, the weakref callback will wake up |
| 225 |
# the worker threads. |
| 226 |
def weakref_cb(_, q=self._work_queue): |
| 227 |
q.put(None) |
| 228 |
|
| 229 |
num_threads = len(self._threads) |
| 230 |
if num_threads < self._max_workers: |
| 231 |
thread_name = '%s_%d' % (self._thread_name_prefix or self, |
| 232 |
num_threads) |
| 233 |
t = threading.Thread(name=thread_name, target=_worker, |
| 234 |
args=(weakref.ref(self, weakref_cb), |
| 235 |
self._create_worker_context(), |
| 236 |
self._work_queue)) |
| 237 |
t.start() |
| 238 |
self._threads.add(t) |
| 239 |
_threads_queues[t] = self._work_queue |
| 240 |
|
| 241 |
def _initializer_failed(self): |
| 242 |
with self._shutdown_lock: |
| 243 |
self._broken = ('A thread initializer failed, the thread pool ' |
| 244 |
'is not usable anymore') |
| 245 |
# Drain work queue and mark pending futures failed |
| 246 |
while True: |
| 247 |
try: |
| 248 |
work_item = self._work_queue.get_nowait() |
| 249 |
except queue.Empty: |
| 250 |
break |
| 251 |
if work_item is not None: |
| 252 |
work_item.future.set_exception(self.BROKEN(self._broken)) |
| 253 |
|
| 254 |
def shutdown(self, wait=True, *, cancel_futures=False): |
| 255 |
with self._shutdown_lock: |
| 256 |
self._shutdown = True |
| 257 |
if cancel_futures: |
| 258 |
# Drain all work items from the queue, and then cancel their |
| 259 |
# associated futures. |
| 260 |
while True: |
| 261 |
try: |
| 262 |
work_item = self._work_queue.get_nowait() |
| 263 |
except queue.Empty: |
| 264 |
break |
| 265 |
if work_item is not None: |
| 266 |
work_item.future.cancel() |
| 267 |
|
| 268 |
# Send a wake-up to prevent threads calling |
| 269 |
# _work_queue.get(block=True) from permanently blocking. |
| 270 |
self._work_queue.put(None) |
| 271 |
if wait: |
| 272 |
for t in self._threads: |
| 273 |
t.join() |
| 274 |
shutdown.__doc__ = _base.Executor.shutdown.__doc__ |