| 1 |
# Copyright 2009 Brian Quinlan. All Rights Reserved. |
| 2 |
# Licensed to PSF under a Contributor Agreement. |
| 3 |
|
| 4 |
"""Implements ProcessPoolExecutor. |
| 5 |
|
| 6 |
The following diagram and text describe the data-flow through the system: |
| 7 |
|
| 8 |
|======================= In-process =====================|== Out-of-process ==| |
| 9 |
|
| 10 |
+----------+ +----------+ +--------+ +-----------+ +---------+ |
| 11 |
| | => | Work Ids | | | | Call Q | | Process | |
| 12 |
| | +----------+ | | +-----------+ | Pool | |
| 13 |
| | | ... | | | | ... | +---------+ |
| 14 |
| | | 6 | => | | => | 5, call() | => | | |
| 15 |
| | | 7 | | | | ... | | | |
| 16 |
| Process | | ... | | Local | +-----------+ | Process | |
| 17 |
| Pool | +----------+ | Worker | | #1..n | |
| 18 |
| Executor | | Thread | | | |
| 19 |
| | +----------- + | | +-----------+ | | |
| 20 |
| | <=> | Work Items | <=> | | <= | Result Q | <= | | |
| 21 |
| | +------------+ | | +-----------+ | | |
| 22 |
| | | 6: call() | | | | ... | | | |
| 23 |
| | | future | | | | 4, result | | | |
| 24 |
| | | ... | | | | 3, except | | | |
| 25 |
+----------+ +------------+ +--------+ +-----------+ +---------+ |
| 26 |
|
| 27 |
Executor.submit() called: |
| 28 |
- creates a uniquely numbered _WorkItem and adds it to the "Work Items" dict |
| 29 |
- adds the id of the _WorkItem to the "Work Ids" queue |
| 30 |
|
| 31 |
Local worker thread: |
| 32 |
- reads work ids from the "Work Ids" queue and looks up the corresponding |
| 33 |
WorkItem from the "Work Items" dict: if the work item has been cancelled then |
| 34 |
it is simply removed from the dict, otherwise it is repackaged as a |
| 35 |
_CallItem and put in the "Call Q". New _CallItems are put in the "Call Q" |
| 36 |
until "Call Q" is full. NOTE: the size of the "Call Q" is kept small because |
| 37 |
calls placed in the "Call Q" can no longer be cancelled with Future.cancel(). |
| 38 |
- reads _ResultItems from "Result Q", updates the future stored in the |
| 39 |
"Work Items" dict and deletes the dict entry |
| 40 |
|
| 41 |
Process #1..n: |
| 42 |
- reads _CallItems from "Call Q", executes the calls, and puts the resulting |
| 43 |
_ResultItems in "Result Q" |
| 44 |
""" |
| 45 |
|
| 46 |
__author__ = 'Brian Quinlan ([email protected])' |
| 47 |
|
| 48 |
import os |
| 49 |
from concurrent.futures import _base |
| 50 |
import queue |
| 51 |
import multiprocessing as mp |
| 52 |
# This import is required to load the multiprocessing.connection submodule |
| 53 |
# so that it can be accessed later as `mp.connection` |
| 54 |
import multiprocessing.connection |
| 55 |
from multiprocessing.queues import Queue |
| 56 |
import threading |
| 57 |
import weakref |
| 58 |
from functools import partial |
| 59 |
import itertools |
| 60 |
import sys |
| 61 |
from traceback import format_exception |
| 62 |
|
| 63 |
|
| 64 |
_threads_wakeups = weakref.WeakKeyDictionary() |
| 65 |
_global_shutdown = False |
| 66 |
|
| 67 |
|
| 68 |
class _ThreadWakeup: |
| 69 |
def __init__(self): |
| 70 |
self._closed = False |
| 71 |
self._lock = threading.Lock() |
| 72 |
self._reader, self._writer = mp.Pipe(duplex=False) |
| 73 |
|
| 74 |
def close(self): |
| 75 |
# Please note that we do not take the self._lock when |
| 76 |
# calling clear() (to avoid deadlocking) so this method can |
| 77 |
# only be called safely from the same thread as all calls to |
| 78 |
# clear() even if you hold the lock. Otherwise we |
| 79 |
# might try to read from the closed pipe. |
| 80 |
with self._lock: |
| 81 |
if not self._closed: |
| 82 |
self._closed = True |
| 83 |
self._writer.close() |
| 84 |
self._reader.close() |
| 85 |
|
| 86 |
def wakeup(self): |
| 87 |
with self._lock: |
| 88 |
if not self._closed: |
| 89 |
self._writer.send_bytes(b"") |
| 90 |
|
| 91 |
def clear(self): |
| 92 |
if self._closed: |
| 93 |
raise RuntimeError('operation on closed _ThreadWakeup') |
| 94 |
while self._reader.poll(): |
| 95 |
self._reader.recv_bytes() |
| 96 |
|
| 97 |
|
| 98 |
def _python_exit(): |
| 99 |
global _global_shutdown |
| 100 |
_global_shutdown = True |
| 101 |
items = list(_threads_wakeups.items()) |
| 102 |
for _, thread_wakeup in items: |
| 103 |
# call not protected by ProcessPoolExecutor._shutdown_lock |
| 104 |
thread_wakeup.wakeup() |
| 105 |
for t, _ in items: |
| 106 |
t.join() |
| 107 |
|
| 108 |
# Register for `_python_exit()` to be called just before joining all |
| 109 |
# non-daemon threads. This is used instead of `atexit.register()` for |
| 110 |
# compatibility with subinterpreters, which no longer support daemon threads. |
| 111 |
# See bpo-39812 for context. |
| 112 |
threading._register_atexit(_python_exit) |
| 113 |
|
| 114 |
# Controls how many more calls than processes will be queued in the call queue. |
| 115 |
# A smaller number will mean that processes spend more time idle waiting for |
| 116 |
# work while a larger number will make Future.cancel() succeed less frequently |
| 117 |
# (Futures in the call queue cannot be cancelled). |
| 118 |
EXTRA_QUEUED_CALLS = 1 |
| 119 |
|
| 120 |
|
| 121 |
# On Windows, WaitForMultipleObjects is used to wait for processes to finish. |
| 122 |
# It can wait on, at most, 63 objects. There is an overhead of two objects: |
| 123 |
# - the result queue reader |
| 124 |
# - the thread wakeup reader |
| 125 |
_MAX_WINDOWS_WORKERS = 63 - 2 |
| 126 |
|
| 127 |
# Hack to embed stringification of remote traceback in local traceback |
| 128 |
|
| 129 |
class _RemoteTraceback(Exception): |
| 130 |
def __init__(self, tb): |
| 131 |
self.tb = tb |
| 132 |
def __str__(self): |
| 133 |
return self.tb |
| 134 |
|
| 135 |
class _ExceptionWithTraceback: |
| 136 |
def __init__(self, exc, tb): |
| 137 |
tb = ''.join(format_exception(type(exc), exc, tb)) |
| 138 |
self.exc = exc |
| 139 |
# Traceback object needs to be garbage-collected as its frames |
| 140 |
# contain references to all the objects in the exception scope |
| 141 |
self.exc.__traceback__ = None |
| 142 |
self.tb = '\n"""\n%s"""' % tb |
| 143 |
def __reduce__(self): |
| 144 |
return _rebuild_exc, (self.exc, self.tb) |
| 145 |
|
| 146 |
def _rebuild_exc(exc, tb): |
| 147 |
exc.__cause__ = _RemoteTraceback(tb) |
| 148 |
return exc |
| 149 |
|
| 150 |
class _WorkItem(object): |
| 151 |
def __init__(self, future, fn, args, kwargs): |
| 152 |
self.future = future |
| 153 |
self.fn = fn |
| 154 |
self.args = args |
| 155 |
self.kwargs = kwargs |
| 156 |
|
| 157 |
class _ResultItem(object): |
| 158 |
def __init__(self, work_id, exception=None, result=None, exit_pid=None): |
| 159 |
self.work_id = work_id |
| 160 |
self.exception = exception |
| 161 |
self.result = result |
| 162 |
self.exit_pid = exit_pid |
| 163 |
|
| 164 |
class _CallItem(object): |
| 165 |
def __init__(self, work_id, fn, args, kwargs): |
| 166 |
self.work_id = work_id |
| 167 |
self.fn = fn |
| 168 |
self.args = args |
| 169 |
self.kwargs = kwargs |
| 170 |
|
| 171 |
|
| 172 |
class _SafeQueue(Queue): |
| 173 |
"""Safe Queue set exception to the future object linked to a job""" |
| 174 |
def __init__(self, max_size=0, *, ctx, pending_work_items, thread_wakeup): |
| 175 |
self.pending_work_items = pending_work_items |
| 176 |
self.thread_wakeup = thread_wakeup |
| 177 |
super().__init__(max_size, ctx=ctx) |
| 178 |
|
| 179 |
def _on_queue_feeder_error(self, e, obj): |
| 180 |
if isinstance(obj, _CallItem): |
| 181 |
tb = format_exception(type(e), e, e.__traceback__) |
| 182 |
e.__cause__ = _RemoteTraceback('\n"""\n{}"""'.format(''.join(tb))) |
| 183 |
work_item = self.pending_work_items.pop(obj.work_id, None) |
| 184 |
self.thread_wakeup.wakeup() |
| 185 |
# work_item can be None if another process terminated. In this |
| 186 |
# case, the executor_manager_thread fails all work_items |
| 187 |
# with BrokenProcessPool |
| 188 |
if work_item is not None: |
| 189 |
work_item.future.set_exception(e) |
| 190 |
else: |
| 191 |
super()._on_queue_feeder_error(e, obj) |
| 192 |
|
| 193 |
|
| 194 |
def _process_chunk(fn, chunk): |
| 195 |
""" Processes a chunk of an iterable passed to map. |
| 196 |
|
| 197 |
Runs the function passed to map() on a chunk of the |
| 198 |
iterable passed to map. |
| 199 |
|
| 200 |
This function is run in a separate process. |
| 201 |
|
| 202 |
""" |
| 203 |
return [fn(*args) for args in chunk] |
| 204 |
|
| 205 |
|
| 206 |
def _sendback_result(result_queue, work_id, result=None, exception=None, |
| 207 |
exit_pid=None): |
| 208 |
"""Safely send back the given result or exception""" |
| 209 |
try: |
| 210 |
result_queue.put(_ResultItem(work_id, result=result, |
| 211 |
exception=exception, exit_pid=exit_pid)) |
| 212 |
except BaseException as e: |
| 213 |
exc = _ExceptionWithTraceback(e, e.__traceback__) |
| 214 |
result_queue.put(_ResultItem(work_id, exception=exc, |
| 215 |
exit_pid=exit_pid)) |
| 216 |
|
| 217 |
|
| 218 |
def _process_worker(call_queue, result_queue, initializer, initargs, max_tasks=None): |
| 219 |
"""Evaluates calls from call_queue and places the results in result_queue. |
| 220 |
|
| 221 |
This worker is run in a separate process. |
| 222 |
|
| 223 |
Args: |
| 224 |
call_queue: A ctx.Queue of _CallItems that will be read and |
| 225 |
evaluated by the worker. |
| 226 |
result_queue: A ctx.Queue of _ResultItems that will written |
| 227 |
to by the worker. |
| 228 |
initializer: A callable initializer, or None |
| 229 |
initargs: A tuple of args for the initializer |
| 230 |
""" |
| 231 |
if initializer is not None: |
| 232 |
try: |
| 233 |
initializer(*initargs) |
| 234 |
except BaseException: |
| 235 |
_base.LOGGER.critical('Exception in initializer:', exc_info=True) |
| 236 |
# The parent will notice that the process stopped and |
| 237 |
# mark the pool broken |
| 238 |
return |
| 239 |
num_tasks = 0 |
| 240 |
exit_pid = None |
| 241 |
while True: |
| 242 |
call_item = call_queue.get(block=True) |
| 243 |
if call_item is None: |
| 244 |
# Wake up queue management thread |
| 245 |
result_queue.put(os.getpid()) |
| 246 |
return |
| 247 |
|
| 248 |
if max_tasks is not None: |
| 249 |
num_tasks += 1 |
| 250 |
if num_tasks >= max_tasks: |
| 251 |
exit_pid = os.getpid() |
| 252 |
|
| 253 |
try: |
| 254 |
r = call_item.fn(*call_item.args, **call_item.kwargs) |
| 255 |
except BaseException as e: |
| 256 |
exc = _ExceptionWithTraceback(e, e.__traceback__) |
| 257 |
_sendback_result(result_queue, call_item.work_id, exception=exc, |
| 258 |
exit_pid=exit_pid) |
| 259 |
else: |
| 260 |
_sendback_result(result_queue, call_item.work_id, result=r, |
| 261 |
exit_pid=exit_pid) |
| 262 |
del r |
| 263 |
|
| 264 |
# Liberate the resource as soon as possible, to avoid holding onto |
| 265 |
# open files or shared memory that is not needed anymore |
| 266 |
del call_item |
| 267 |
|
| 268 |
if exit_pid is not None: |
| 269 |
return |
| 270 |
|
| 271 |
|
| 272 |
class _ExecutorManagerThread(threading.Thread): |
| 273 |
"""Manages the communication between this process and the worker processes. |
| 274 |
|
| 275 |
The manager is run in a local thread. |
| 276 |
|
| 277 |
Args: |
| 278 |
executor: A reference to the ProcessPoolExecutor that owns |
| 279 |
this thread. A weakref will be own by the manager as well as |
| 280 |
references to internal objects used to introspect the state of |
| 281 |
the executor. |
| 282 |
""" |
| 283 |
|
| 284 |
def __init__(self, executor): |
| 285 |
# Store references to necessary internals of the executor. |
| 286 |
|
| 287 |
# A _ThreadWakeup to allow waking up the queue_manager_thread from the |
| 288 |
# main Thread and avoid deadlocks caused by permanently locked queues. |
| 289 |
self.thread_wakeup = executor._executor_manager_thread_wakeup |
| 290 |
self.shutdown_lock = executor._shutdown_lock |
| 291 |
|
| 292 |
# A weakref.ref to the ProcessPoolExecutor that owns this thread. Used |
| 293 |
# to determine if the ProcessPoolExecutor has been garbage collected |
| 294 |
# and that the manager can exit. |
| 295 |
# When the executor gets garbage collected, the weakref callback |
| 296 |
# will wake up the queue management thread so that it can terminate |
| 297 |
# if there is no pending work item. |
| 298 |
def weakref_cb(_, |
| 299 |
thread_wakeup=self.thread_wakeup, |
| 300 |
mp_util_debug=mp.util.debug): |
| 301 |
mp_util_debug('Executor collected: triggering callback for' |
| 302 |
' QueueManager wakeup') |
| 303 |
thread_wakeup.wakeup() |
| 304 |
|
| 305 |
self.executor_reference = weakref.ref(executor, weakref_cb) |
| 306 |
|
| 307 |
# A list of the ctx.Process instances used as workers. |
| 308 |
self.processes = executor._processes |
| 309 |
|
| 310 |
# A ctx.Queue that will be filled with _CallItems derived from |
| 311 |
# _WorkItems for processing by the process workers. |
| 312 |
self.call_queue = executor._call_queue |
| 313 |
|
| 314 |
# A ctx.SimpleQueue of _ResultItems generated by the process workers. |
| 315 |
self.result_queue = executor._result_queue |
| 316 |
|
| 317 |
# A queue.Queue of work ids e.g. Queue([5, 6, ...]). |
| 318 |
self.work_ids_queue = executor._work_ids |
| 319 |
|
| 320 |
# Maximum number of tasks a worker process can execute before |
| 321 |
# exiting safely |
| 322 |
self.max_tasks_per_child = executor._max_tasks_per_child |
| 323 |
|
| 324 |
# A dict mapping work ids to _WorkItems e.g. |
| 325 |
# {5: <_WorkItem...>, 6: <_WorkItem...>, ...} |
| 326 |
self.pending_work_items = executor._pending_work_items |
| 327 |
|
| 328 |
super().__init__() |
| 329 |
|
| 330 |
def run(self): |
| 331 |
# Main loop for the executor manager thread. |
| 332 |
|
| 333 |
while True: |
| 334 |
# gh-109047: During Python finalization, self.call_queue.put() |
| 335 |
# creation of a thread can fail with RuntimeError. |
| 336 |
try: |
| 337 |
self.add_call_item_to_queue() |
| 338 |
except BaseException as exc: |
| 339 |
cause = format_exception(exc) |
| 340 |
self.terminate_broken(cause) |
| 341 |
return |
| 342 |
|
| 343 |
result_item, is_broken, cause = self.wait_result_broken_or_wakeup() |
| 344 |
|
| 345 |
if is_broken: |
| 346 |
self.terminate_broken(cause) |
| 347 |
return |
| 348 |
if result_item is not None: |
| 349 |
self.process_result_item(result_item) |
| 350 |
|
| 351 |
process_exited = result_item.exit_pid is not None |
| 352 |
if process_exited: |
| 353 |
p = self.processes.pop(result_item.exit_pid) |
| 354 |
p.join() |
| 355 |
|
| 356 |
# Delete reference to result_item to avoid keeping references |
| 357 |
# while waiting on new results. |
| 358 |
del result_item |
| 359 |
|
| 360 |
if executor := self.executor_reference(): |
| 361 |
if process_exited: |
| 362 |
with self.shutdown_lock: |
| 363 |
executor._adjust_process_count() |
| 364 |
else: |
| 365 |
executor._idle_worker_semaphore.release() |
| 366 |
del executor |
| 367 |
|
| 368 |
if self.is_shutting_down(): |
| 369 |
self.flag_executor_shutting_down() |
| 370 |
|
| 371 |
# When only canceled futures remain in pending_work_items, our |
| 372 |
# next call to wait_result_broken_or_wakeup would hang forever. |
| 373 |
# This makes sure we have some running futures or none at all. |
| 374 |
self.add_call_item_to_queue() |
| 375 |
|
| 376 |
# Since no new work items can be added, it is safe to shutdown |
| 377 |
# this thread if there are no pending work items. |
| 378 |
if not self.pending_work_items: |
| 379 |
self.join_executor_internals() |
| 380 |
return |
| 381 |
|
| 382 |
def add_call_item_to_queue(self): |
| 383 |
# Fills call_queue with _WorkItems from pending_work_items. |
| 384 |
# This function never blocks. |
| 385 |
while True: |
| 386 |
if self.call_queue.full(): |
| 387 |
return |
| 388 |
try: |
| 389 |
work_id = self.work_ids_queue.get(block=False) |
| 390 |
except queue.Empty: |
| 391 |
return |
| 392 |
else: |
| 393 |
work_item = self.pending_work_items[work_id] |
| 394 |
|
| 395 |
if work_item.future.set_running_or_notify_cancel(): |
| 396 |
self.call_queue.put(_CallItem(work_id, |
| 397 |
work_item.fn, |
| 398 |
work_item.args, |
| 399 |
work_item.kwargs), |
| 400 |
block=True) |
| 401 |
else: |
| 402 |
del self.pending_work_items[work_id] |
| 403 |
continue |
| 404 |
|
| 405 |
def wait_result_broken_or_wakeup(self): |
| 406 |
# Wait for a result to be ready in the result_queue while checking |
| 407 |
# that all worker processes are still running, or for a wake up |
| 408 |
# signal send. The wake up signals come either from new tasks being |
| 409 |
# submitted, from the executor being shutdown/gc-ed, or from the |
| 410 |
# shutdown of the python interpreter. |
| 411 |
result_reader = self.result_queue._reader |
| 412 |
assert not self.thread_wakeup._closed |
| 413 |
wakeup_reader = self.thread_wakeup._reader |
| 414 |
readers = [result_reader, wakeup_reader] |
| 415 |
worker_sentinels = [p.sentinel for p in list(self.processes.values())] |
| 416 |
ready = mp.connection.wait(readers + worker_sentinels) |
| 417 |
|
| 418 |
cause = None |
| 419 |
is_broken = True |
| 420 |
result_item = None |
| 421 |
if result_reader in ready: |
| 422 |
try: |
| 423 |
result_item = result_reader.recv() |
| 424 |
is_broken = False |
| 425 |
except BaseException as exc: |
| 426 |
cause = format_exception(exc) |
| 427 |
|
| 428 |
elif wakeup_reader in ready: |
| 429 |
is_broken = False |
| 430 |
|
| 431 |
self.thread_wakeup.clear() |
| 432 |
|
| 433 |
return result_item, is_broken, cause |
| 434 |
|
| 435 |
def process_result_item(self, result_item): |
| 436 |
# Process the received a result_item. This can be either the PID of a |
| 437 |
# worker that exited gracefully or a _ResultItem |
| 438 |
|
| 439 |
# Received a _ResultItem so mark the future as completed. |
| 440 |
work_item = self.pending_work_items.pop(result_item.work_id, None) |
| 441 |
# work_item can be None if another process terminated (see above) |
| 442 |
if work_item is not None: |
| 443 |
if result_item.exception is not None: |
| 444 |
work_item.future.set_exception(result_item.exception) |
| 445 |
else: |
| 446 |
work_item.future.set_result(result_item.result) |
| 447 |
|
| 448 |
def is_shutting_down(self): |
| 449 |
# Check whether we should start shutting down the executor. |
| 450 |
executor = self.executor_reference() |
| 451 |
# No more work items can be added if: |
| 452 |
# - The interpreter is shutting down OR |
| 453 |
# - The executor that owns this worker has been collected OR |
| 454 |
# - The executor that owns this worker has been shutdown. |
| 455 |
return (_global_shutdown or executor is None |
| 456 |
or executor._shutdown_thread) |
| 457 |
|
| 458 |
def _terminate_broken(self, cause): |
| 459 |
# Terminate the executor because it is in a broken state. The cause |
| 460 |
# argument can be used to display more information on the error that |
| 461 |
# lead the executor into becoming broken. |
| 462 |
|
| 463 |
# Mark the process pool broken so that submits fail right now. |
| 464 |
executor = self.executor_reference() |
| 465 |
if executor is not None: |
| 466 |
executor._broken = ('A child process terminated ' |
| 467 |
'abruptly, the process pool is not ' |
| 468 |
'usable anymore') |
| 469 |
executor._shutdown_thread = True |
| 470 |
executor = None |
| 471 |
|
| 472 |
# All pending tasks are to be marked failed with the following |
| 473 |
# BrokenProcessPool error |
| 474 |
bpe = BrokenProcessPool("A process in the process pool was " |
| 475 |
"terminated abruptly while the future was " |
| 476 |
"running or pending.") |
| 477 |
if cause is not None: |
| 478 |
bpe.__cause__ = _RemoteTraceback( |
| 479 |
f"\n'''\n{''.join(cause)}'''") |
| 480 |
|
| 481 |
# Mark pending tasks as failed. |
| 482 |
for work_id, work_item in self.pending_work_items.items(): |
| 483 |
try: |
| 484 |
work_item.future.set_exception(bpe) |
| 485 |
except _base.InvalidStateError: |
| 486 |
# set_exception() fails if the future is cancelled: ignore it. |
| 487 |
# Trying to check if the future is cancelled before calling |
| 488 |
# set_exception() would leave a race condition if the future is |
| 489 |
# cancelled between the check and set_exception(). |
| 490 |
pass |
| 491 |
# Delete references to object. See issue16284 |
| 492 |
del work_item |
| 493 |
self.pending_work_items.clear() |
| 494 |
|
| 495 |
# Terminate remaining workers forcibly: the queues or their |
| 496 |
# locks may be in a dirty state and block forever. |
| 497 |
for p in self.processes.values(): |
| 498 |
p.terminate() |
| 499 |
|
| 500 |
self.call_queue._terminate_broken() |
| 501 |
|
| 502 |
# clean up resources |
| 503 |
self._join_executor_internals(broken=True) |
| 504 |
|
| 505 |
def terminate_broken(self, cause): |
| 506 |
with self.shutdown_lock: |
| 507 |
self._terminate_broken(cause) |
| 508 |
|
| 509 |
def flag_executor_shutting_down(self): |
| 510 |
# Flag the executor as shutting down and cancel remaining tasks if |
| 511 |
# requested as early as possible if it is not gc-ed yet. |
| 512 |
executor = self.executor_reference() |
| 513 |
if executor is not None: |
| 514 |
executor._shutdown_thread = True |
| 515 |
# Cancel pending work items if requested. |
| 516 |
if executor._cancel_pending_futures: |
| 517 |
# Cancel all pending futures and update pending_work_items |
| 518 |
# to only have futures that are currently running. |
| 519 |
new_pending_work_items = {} |
| 520 |
for work_id, work_item in self.pending_work_items.items(): |
| 521 |
if not work_item.future.cancel(): |
| 522 |
new_pending_work_items[work_id] = work_item |
| 523 |
self.pending_work_items = new_pending_work_items |
| 524 |
# Drain work_ids_queue since we no longer need to |
| 525 |
# add items to the call queue. |
| 526 |
while True: |
| 527 |
try: |
| 528 |
self.work_ids_queue.get_nowait() |
| 529 |
except queue.Empty: |
| 530 |
break |
| 531 |
# Make sure we do this only once to not waste time looping |
| 532 |
# on running processes over and over. |
| 533 |
executor._cancel_pending_futures = False |
| 534 |
|
| 535 |
def shutdown_workers(self): |
| 536 |
n_children_to_stop = self.get_n_children_alive() |
| 537 |
n_sentinels_sent = 0 |
| 538 |
# Send the right number of sentinels, to make sure all children are |
| 539 |
# properly terminated. |
| 540 |
while (n_sentinels_sent < n_children_to_stop |
| 541 |
and self.get_n_children_alive() > 0): |
| 542 |
for i in range(n_children_to_stop - n_sentinels_sent): |
| 543 |
try: |
| 544 |
self.call_queue.put_nowait(None) |
| 545 |
n_sentinels_sent += 1 |
| 546 |
except queue.Full: |
| 547 |
break |
| 548 |
|
| 549 |
def join_executor_internals(self): |
| 550 |
with self.shutdown_lock: |
| 551 |
self._join_executor_internals() |
| 552 |
|
| 553 |
def _join_executor_internals(self, broken=False): |
| 554 |
# If broken, call_queue was closed and so can no longer be used. |
| 555 |
if not broken: |
| 556 |
self.shutdown_workers() |
| 557 |
|
| 558 |
# Release the queue's resources as soon as possible. |
| 559 |
self.call_queue.close() |
| 560 |
self.call_queue.join_thread() |
| 561 |
self.thread_wakeup.close() |
| 562 |
|
| 563 |
# If .join() is not called on the created processes then |
| 564 |
# some ctx.Queue methods may deadlock on Mac OS X. |
| 565 |
for p in self.processes.values(): |
| 566 |
if broken: |
| 567 |
p.terminate() |
| 568 |
p.join() |
| 569 |
|
| 570 |
def get_n_children_alive(self): |
| 571 |
# This is an upper bound on the number of children alive. |
| 572 |
return sum(p.is_alive() for p in self.processes.values()) |
| 573 |
|
| 574 |
|
| 575 |
_system_limits_checked = False |
| 576 |
_system_limited = None |
| 577 |
|
| 578 |
|
| 579 |
def _check_system_limits(): |
| 580 |
global _system_limits_checked, _system_limited |
| 581 |
if _system_limits_checked: |
| 582 |
if _system_limited: |
| 583 |
raise NotImplementedError(_system_limited) |
| 584 |
_system_limits_checked = True |
| 585 |
try: |
| 586 |
import multiprocessing.synchronize # noqa: F401 |
| 587 |
except ImportError: |
| 588 |
_system_limited = ( |
| 589 |
"This Python build lacks multiprocessing.synchronize, usually due " |
| 590 |
"to named semaphores being unavailable on this platform." |
| 591 |
) |
| 592 |
raise NotImplementedError(_system_limited) |
| 593 |
try: |
| 594 |
nsems_max = os.sysconf("SC_SEM_NSEMS_MAX") |
| 595 |
except (AttributeError, ValueError): |
| 596 |
# sysconf not available or setting not available |
| 597 |
return |
| 598 |
if nsems_max == -1: |
| 599 |
# indetermined limit, assume that limit is determined |
| 600 |
# by available memory only |
| 601 |
return |
| 602 |
if nsems_max >= 256: |
| 603 |
# minimum number of semaphores available |
| 604 |
# according to POSIX |
| 605 |
return |
| 606 |
_system_limited = ("system provides too few semaphores (%d" |
| 607 |
" available, 256 necessary)" % nsems_max) |
| 608 |
raise NotImplementedError(_system_limited) |
| 609 |
|
| 610 |
|
| 611 |
def _chain_from_iterable_of_lists(iterable): |
| 612 |
""" |
| 613 |
Specialized implementation of itertools.chain.from_iterable. |
| 614 |
Each item in *iterable* should be a list. This function is |
| 615 |
careful not to keep references to yielded objects. |
| 616 |
""" |
| 617 |
for element in iterable: |
| 618 |
element.reverse() |
| 619 |
while element: |
| 620 |
yield element.pop() |
| 621 |
|
| 622 |
|
| 623 |
class BrokenProcessPool(_base.BrokenExecutor): |
| 624 |
""" |
| 625 |
Raised when a process in a ProcessPoolExecutor terminated abruptly |
| 626 |
while a future was in the running state. |
| 627 |
""" |
| 628 |
|
| 629 |
_TERMINATE = "terminate" |
| 630 |
_KILL = "kill" |
| 631 |
|
| 632 |
_SHUTDOWN_CALLBACK_OPERATION = { |
| 633 |
_TERMINATE, |
| 634 |
_KILL |
| 635 |
} |
| 636 |
|
| 637 |
|
| 638 |
class ProcessPoolExecutor(_base.Executor): |
| 639 |
def __init__(self, max_workers=None, mp_context=None, |
| 640 |
initializer=None, initargs=(), *, max_tasks_per_child=None): |
| 641 |
"""Initializes a new ProcessPoolExecutor instance. |
| 642 |
|
| 643 |
Args: |
| 644 |
max_workers: The maximum number of processes that can be used to |
| 645 |
execute the given calls. If None or not given then as many |
| 646 |
worker processes will be created as the machine has |
| 647 |
processors. |
| 648 |
mp_context: A multiprocessing context to launch the workers |
| 649 |
created using the multiprocessing.get_context('start method') |
| 650 |
API. This object should provide SimpleQueue, Queue and |
| 651 |
Process. |
| 652 |
initializer: A callable used to initialize worker processes. |
| 653 |
initargs: A tuple of arguments to pass to the initializer. |
| 654 |
max_tasks_per_child: The maximum number of tasks a worker |
| 655 |
process can complete before it will exit and be replaced |
| 656 |
with a fresh worker process. The default of None means |
| 657 |
worker process will live as long as the executor. Requires |
| 658 |
a non-'fork' mp_context start method. When given, we |
| 659 |
default to using 'spawn' if no mp_context is supplied. |
| 660 |
""" |
| 661 |
_check_system_limits() |
| 662 |
|
| 663 |
if max_workers is None: |
| 664 |
self._max_workers = os.process_cpu_count() or 1 |
| 665 |
if sys.platform == 'win32': |
| 666 |
self._max_workers = min(_MAX_WINDOWS_WORKERS, |
| 667 |
self._max_workers) |
| 668 |
else: |
| 669 |
if max_workers <= 0: |
| 670 |
raise ValueError("max_workers must be greater than 0") |
| 671 |
elif (sys.platform == 'win32' and |
| 672 |
max_workers > _MAX_WINDOWS_WORKERS): |
| 673 |
raise ValueError( |
| 674 |
f"max_workers must be <= {_MAX_WINDOWS_WORKERS}") |
| 675 |
|
| 676 |
self._max_workers = max_workers |
| 677 |
|
| 678 |
if mp_context is None: |
| 679 |
if max_tasks_per_child is not None: |
| 680 |
mp_context = mp.get_context("spawn") |
| 681 |
else: |
| 682 |
mp_context = mp.get_context() |
| 683 |
self._mp_context = mp_context |
| 684 |
|
| 685 |
# https://github.com/python/cpython/issues/90622 |
| 686 |
self._safe_to_dynamically_spawn_children = ( |
| 687 |
self._mp_context.get_start_method(allow_none=False) != "fork") |
| 688 |
|
| 689 |
if initializer is not None and not callable(initializer): |
| 690 |
raise TypeError("initializer must be a callable") |
| 691 |
self._initializer = initializer |
| 692 |
self._initargs = initargs |
| 693 |
|
| 694 |
if max_tasks_per_child is not None: |
| 695 |
if not isinstance(max_tasks_per_child, int): |
| 696 |
raise TypeError("max_tasks_per_child must be an integer") |
| 697 |
elif max_tasks_per_child <= 0: |
| 698 |
raise ValueError("max_tasks_per_child must be >= 1") |
| 699 |
if self._mp_context.get_start_method(allow_none=False) == "fork": |
| 700 |
# https://github.com/python/cpython/issues/90622 |
| 701 |
raise ValueError("max_tasks_per_child is incompatible with" |
| 702 |
" the 'fork' multiprocessing start method;" |
| 703 |
" supply a different mp_context.") |
| 704 |
self._max_tasks_per_child = max_tasks_per_child |
| 705 |
|
| 706 |
# Management thread |
| 707 |
self._executor_manager_thread = None |
| 708 |
|
| 709 |
# Map of pids to processes |
| 710 |
self._processes = {} |
| 711 |
|
| 712 |
# Shutdown is a two-step process. |
| 713 |
self._shutdown_thread = False |
| 714 |
self._shutdown_lock = threading.Lock() |
| 715 |
self._idle_worker_semaphore = threading.Semaphore(0) |
| 716 |
self._broken = False |
| 717 |
self._queue_count = 0 |
| 718 |
self._pending_work_items = {} |
| 719 |
self._cancel_pending_futures = False |
| 720 |
|
| 721 |
# _ThreadWakeup is a communication channel used to interrupt the wait |
| 722 |
# of the main loop of executor_manager_thread from another thread (e.g. |
| 723 |
# when calling executor.submit or executor.shutdown). We do not use the |
| 724 |
# _result_queue to send wakeup signals to the executor_manager_thread |
| 725 |
# as it could result in a deadlock if a worker process dies with the |
| 726 |
# _result_queue write lock still acquired. |
| 727 |
# |
| 728 |
# Care must be taken to only call clear and close from the |
| 729 |
# executor_manager_thread, since _ThreadWakeup.clear() is not protected |
| 730 |
# by a lock. |
| 731 |
self._executor_manager_thread_wakeup = _ThreadWakeup() |
| 732 |
|
| 733 |
# Create communication channels for the executor |
| 734 |
# Make the call queue slightly larger than the number of processes to |
| 735 |
# prevent the worker processes from idling. But don't make it too big |
| 736 |
# because futures in the call queue cannot be cancelled. |
| 737 |
queue_size = self._max_workers + EXTRA_QUEUED_CALLS |
| 738 |
self._call_queue = _SafeQueue( |
| 739 |
max_size=queue_size, ctx=self._mp_context, |
| 740 |
pending_work_items=self._pending_work_items, |
| 741 |
thread_wakeup=self._executor_manager_thread_wakeup) |
| 742 |
# Killed worker processes can produce spurious "broken pipe" |
| 743 |
# tracebacks in the queue's own worker thread. But we detect killed |
| 744 |
# processes anyway, so silence the tracebacks. |
| 745 |
self._call_queue._ignore_epipe = True |
| 746 |
self._result_queue = mp_context.SimpleQueue() |
| 747 |
self._work_ids = queue.Queue() |
| 748 |
|
| 749 |
def _start_executor_manager_thread(self): |
| 750 |
if self._executor_manager_thread is None: |
| 751 |
# Start the processes so that their sentinels are known. |
| 752 |
if not self._safe_to_dynamically_spawn_children: # ie, using fork. |
| 753 |
self._launch_processes() |
| 754 |
self._executor_manager_thread = _ExecutorManagerThread(self) |
| 755 |
self._executor_manager_thread.start() |
| 756 |
_threads_wakeups[self._executor_manager_thread] = \ |
| 757 |
self._executor_manager_thread_wakeup |
| 758 |
|
| 759 |
def _adjust_process_count(self): |
| 760 |
# gh-132969: avoid error when state is reset and executor is still running, |
| 761 |
# which will happen when shutdown(wait=False) is called. |
| 762 |
if self._processes is None: |
| 763 |
return |
| 764 |
|
| 765 |
# if there's an idle process, we don't need to spawn a new one. |
| 766 |
if self._idle_worker_semaphore.acquire(blocking=False): |
| 767 |
return |
| 768 |
|
| 769 |
process_count = len(self._processes) |
| 770 |
if process_count < self._max_workers: |
| 771 |
# Assertion disabled as this codepath is also used to replace a |
| 772 |
# worker that unexpectedly dies, even when using the 'fork' start |
| 773 |
# method. That means there is still a potential deadlock bug. If a |
| 774 |
# 'fork' mp_context worker dies, we'll be forking a new one when |
| 775 |
# we know a thread is running (self._executor_manager_thread). |
| 776 |
#assert self._safe_to_dynamically_spawn_children or not self._executor_manager_thread, 'https://github.com/python/cpython/issues/90622' |
| 777 |
self._spawn_process() |
| 778 |
|
| 779 |
def _launch_processes(self): |
| 780 |
# https://github.com/python/cpython/issues/90622 |
| 781 |
assert not self._executor_manager_thread, ( |
| 782 |
'Processes cannot be fork()ed after the thread has started, ' |
| 783 |
'deadlock in the child processes could result.') |
| 784 |
for _ in range(len(self._processes), self._max_workers): |
| 785 |
self._spawn_process() |
| 786 |
|
| 787 |
def _spawn_process(self): |
| 788 |
p = self._mp_context.Process( |
| 789 |
target=_process_worker, |
| 790 |
args=(self._call_queue, |
| 791 |
self._result_queue, |
| 792 |
self._initializer, |
| 793 |
self._initargs, |
| 794 |
self._max_tasks_per_child)) |
| 795 |
p.start() |
| 796 |
self._processes[p.pid] = p |
| 797 |
|
| 798 |
def submit(self, fn, /, *args, **kwargs): |
| 799 |
with self._shutdown_lock: |
| 800 |
if self._broken: |
| 801 |
raise BrokenProcessPool(self._broken) |
| 802 |
if self._shutdown_thread: |
| 803 |
raise RuntimeError('cannot schedule new futures after shutdown') |
| 804 |
if _global_shutdown: |
| 805 |
raise RuntimeError('cannot schedule new futures after ' |
| 806 |
'interpreter shutdown') |
| 807 |
|
| 808 |
f = _base.Future() |
| 809 |
w = _WorkItem(f, fn, args, kwargs) |
| 810 |
|
| 811 |
self._pending_work_items[self._queue_count] = w |
| 812 |
self._work_ids.put(self._queue_count) |
| 813 |
self._queue_count += 1 |
| 814 |
# Wake up queue management thread |
| 815 |
self._executor_manager_thread_wakeup.wakeup() |
| 816 |
|
| 817 |
if self._safe_to_dynamically_spawn_children: |
| 818 |
self._adjust_process_count() |
| 819 |
self._start_executor_manager_thread() |
| 820 |
return f |
| 821 |
submit.__doc__ = _base.Executor.submit.__doc__ |
| 822 |
|
| 823 |
def map(self, fn, *iterables, timeout=None, chunksize=1, buffersize=None): |
| 824 |
"""Returns an iterator equivalent to map(fn, iter). |
| 825 |
|
| 826 |
Args: |
| 827 |
fn: A callable that will take as many arguments as there are |
| 828 |
passed iterables. |
| 829 |
timeout: The maximum number of seconds to wait. If None, then |
| 830 |
there is no limit on the wait time. |
| 831 |
chunksize: If greater than one, the iterables will be chopped |
| 832 |
into chunks of size chunksize and submitted to the process |
| 833 |
pool. If set to one, the items in the list will be sent |
| 834 |
one at a time. |
| 835 |
buffersize: The number of submitted tasks whose results have not |
| 836 |
yet been yielded. If the buffer is full, iteration over the |
| 837 |
iterables pauses until a result is yielded from the buffer. |
| 838 |
If None, all input elements are eagerly collected, and |
| 839 |
a task is submitted for each. |
| 840 |
|
| 841 |
Returns: |
| 842 |
An iterator equivalent to: map(func, *iterables) but the calls |
| 843 |
may be evaluated out-of-order. |
| 844 |
|
| 845 |
Raises: |
| 846 |
TimeoutError: If the entire result iterator could not be |
| 847 |
generated before the given timeout. |
| 848 |
Exception: If fn(*args) raises for any values. |
| 849 |
""" |
| 850 |
if chunksize < 1: |
| 851 |
raise ValueError("chunksize must be >= 1.") |
| 852 |
|
| 853 |
results = super().map(partial(_process_chunk, fn), |
| 854 |
itertools.batched(zip(*iterables), chunksize), |
| 855 |
timeout=timeout, |
| 856 |
buffersize=buffersize) |
| 857 |
return _chain_from_iterable_of_lists(results) |
| 858 |
|
| 859 |
def shutdown(self, wait=True, *, cancel_futures=False): |
| 860 |
with self._shutdown_lock: |
| 861 |
self._cancel_pending_futures = cancel_futures |
| 862 |
self._shutdown_thread = True |
| 863 |
if self._executor_manager_thread_wakeup is not None: |
| 864 |
# Wake up queue management thread |
| 865 |
self._executor_manager_thread_wakeup.wakeup() |
| 866 |
|
| 867 |
if self._executor_manager_thread is not None and wait: |
| 868 |
self._executor_manager_thread.join() |
| 869 |
# To reduce the risk of opening too many files, remove references to |
| 870 |
# objects that use file descriptors. |
| 871 |
self._executor_manager_thread = None |
| 872 |
self._call_queue = None |
| 873 |
if self._result_queue is not None and wait: |
| 874 |
self._result_queue.close() |
| 875 |
self._result_queue = None |
| 876 |
self._processes = None |
| 877 |
self._executor_manager_thread_wakeup = None |
| 878 |
|
| 879 |
shutdown.__doc__ = _base.Executor.shutdown.__doc__ |
| 880 |
|
| 881 |
def _force_shutdown(self, operation): |
| 882 |
"""Attempts to terminate or kill the executor's workers based off the |
| 883 |
given operation. Iterates through all of the current processes and |
| 884 |
performs the relevant task if the process is still alive. |
| 885 |
|
| 886 |
After terminating workers, the pool will be in a broken state |
| 887 |
and no longer usable (for instance, new tasks should not be |
| 888 |
submitted). |
| 889 |
""" |
| 890 |
if operation not in _SHUTDOWN_CALLBACK_OPERATION: |
| 891 |
raise ValueError(f"Unsupported operation: {operation!r}") |
| 892 |
|
| 893 |
processes = {} |
| 894 |
if self._processes: |
| 895 |
processes = self._processes.copy() |
| 896 |
|
| 897 |
# shutdown will invalidate ._processes, so we copy it right before |
| 898 |
# calling. If we waited here, we would deadlock if a process decides not |
| 899 |
# to exit. |
| 900 |
self.shutdown(wait=False, cancel_futures=True) |
| 901 |
|
| 902 |
if not processes: |
| 903 |
return |
| 904 |
|
| 905 |
for proc in processes.values(): |
| 906 |
try: |
| 907 |
if not proc.is_alive(): |
| 908 |
continue |
| 909 |
except ValueError: |
| 910 |
# The process is already exited/closed out. |
| 911 |
continue |
| 912 |
|
| 913 |
try: |
| 914 |
if operation == _TERMINATE: |
| 915 |
proc.terminate() |
| 916 |
elif operation == _KILL: |
| 917 |
proc.kill() |
| 918 |
except ProcessLookupError: |
| 919 |
# The process just ended before our signal |
| 920 |
continue |
| 921 |
|
| 922 |
def terminate_workers(self): |
| 923 |
"""Attempts to terminate the executor's workers. |
| 924 |
Iterates through all of the current worker processes and terminates |
| 925 |
each one that is still alive. |
| 926 |
|
| 927 |
After terminating workers, the pool will be in a broken state |
| 928 |
and no longer usable (for instance, new tasks should not be |
| 929 |
submitted). |
| 930 |
""" |
| 931 |
return self._force_shutdown(operation=_TERMINATE) |
| 932 |
|
| 933 |
def kill_workers(self): |
| 934 |
"""Attempts to kill the executor's workers. |
| 935 |
Iterates through all of the current worker processes and kills |
| 936 |
each one that is still alive. |
| 937 |
|
| 938 |
After killing workers, the pool will be in a broken state |
| 939 |
and no longer usable (for instance, new tasks should not be |
| 940 |
submitted). |
| 941 |
""" |
| 942 |
return self._force_shutdown(operation=_KILL) |