| 1 |
import os |
| 2 |
import queue |
| 3 |
import sys |
| 4 |
import threading |
| 5 |
import time |
| 6 |
import unittest |
| 7 |
import unittest.mock |
| 8 |
from concurrent import futures |
| 9 |
from concurrent.futures.process import BrokenProcessPool |
| 10 |
|
| 11 |
from test import support |
| 12 |
from test.support import hashlib_helper |
| 13 |
from test.test_importlib.metadata.fixtures import parameterize |
| 14 |
|
| 15 |
from .executor import ExecutorTest, mul |
| 16 |
from .util import ( |
| 17 |
ProcessPoolForkMixin, ProcessPoolForkserverMixin, ProcessPoolSpawnMixin, |
| 18 |
create_executor_tests, setup_module) |
| 19 |
|
| 20 |
|
| 21 |
class EventfulGCObj(): |
| 22 |
def __init__(self, mgr): |
| 23 |
self.event = mgr.Event() |
| 24 |
|
| 25 |
def __del__(self): |
| 26 |
self.event.set() |
| 27 |
|
| 28 |
TERMINATE_WORKERS = futures.ProcessPoolExecutor.terminate_workers.__name__ |
| 29 |
KILL_WORKERS = futures.ProcessPoolExecutor.kill_workers.__name__ |
| 30 |
FORCE_SHUTDOWN_PARAMS = [ |
| 31 |
dict(function_name=TERMINATE_WORKERS), |
| 32 |
dict(function_name=KILL_WORKERS), |
| 33 |
] |
| 34 |
|
| 35 |
def _put_wait_put(queue, event): |
| 36 |
""" Used as part of test_terminate_workers """ |
| 37 |
queue.put('started') |
| 38 |
event.wait() |
| 39 |
|
| 40 |
# We should never get here since the event will not get set |
| 41 |
queue.put('finished') |
| 42 |
|
| 43 |
|
| 44 |
class ProcessPoolExecutorTest(ExecutorTest): |
| 45 |
|
| 46 |
@unittest.skipUnless(sys.platform=='win32', 'Windows-only process limit') |
| 47 |
def test_max_workers_too_large(self): |
| 48 |
with self.assertRaisesRegex(ValueError, |
| 49 |
"max_workers must be <= 61"): |
| 50 |
futures.ProcessPoolExecutor(max_workers=62) |
| 51 |
|
| 52 |
def test_killed_child(self): |
| 53 |
# When a child process is abruptly terminated, the whole pool gets |
| 54 |
# "broken". |
| 55 |
futures = [self.executor.submit(time.sleep, 3)] |
| 56 |
# Get one of the processes, and terminate (kill) it |
| 57 |
p = next(iter(self.executor._processes.values())) |
| 58 |
p.terminate() |
| 59 |
for fut in futures: |
| 60 |
self.assertRaises(BrokenProcessPool, fut.result) |
| 61 |
# Submitting other jobs fails as well. |
| 62 |
self.assertRaises(BrokenProcessPool, self.executor.submit, pow, 2, 8) |
| 63 |
|
| 64 |
def test_map_chunksize(self): |
| 65 |
def bad_map(): |
| 66 |
list(self.executor.map(pow, range(40), range(40), chunksize=-1)) |
| 67 |
|
| 68 |
ref = list(map(pow, range(40), range(40))) |
| 69 |
self.assertEqual( |
| 70 |
list(self.executor.map(pow, range(40), range(40), chunksize=6)), |
| 71 |
ref) |
| 72 |
self.assertEqual( |
| 73 |
list(self.executor.map(pow, range(40), range(40), chunksize=50)), |
| 74 |
ref) |
| 75 |
self.assertEqual( |
| 76 |
list(self.executor.map(pow, range(40), range(40), chunksize=40)), |
| 77 |
ref) |
| 78 |
self.assertRaises(ValueError, bad_map) |
| 79 |
|
| 80 |
@classmethod |
| 81 |
def _test_traceback(cls): |
| 82 |
raise RuntimeError(123) # some comment |
| 83 |
|
| 84 |
def test_traceback(self): |
| 85 |
# We want ensure that the traceback from the child process is |
| 86 |
# contained in the traceback raised in the main process. |
| 87 |
future = self.executor.submit(self._test_traceback) |
| 88 |
with self.assertRaises(Exception) as cm: |
| 89 |
future.result() |
| 90 |
|
| 91 |
exc = cm.exception |
| 92 |
self.assertIs(type(exc), RuntimeError) |
| 93 |
self.assertEqual(exc.args, (123,)) |
| 94 |
cause = exc.__cause__ |
| 95 |
self.assertIs(type(cause), futures.process._RemoteTraceback) |
| 96 |
self.assertIn('raise RuntimeError(123) # some comment', cause.tb) |
| 97 |
|
| 98 |
with support.captured_stderr() as f1: |
| 99 |
try: |
| 100 |
raise exc |
| 101 |
except RuntimeError: |
| 102 |
sys.excepthook(*sys.exc_info()) |
| 103 |
self.assertIn('raise RuntimeError(123) # some comment', |
| 104 |
f1.getvalue()) |
| 105 |
|
| 106 |
@hashlib_helper.requires_hashdigest('md5') |
| 107 |
def test_ressources_gced_in_workers(self): |
| 108 |
# Ensure that argument for a job are correctly gc-ed after the job |
| 109 |
# is finished |
| 110 |
mgr = self.get_context().Manager() |
| 111 |
obj = EventfulGCObj(mgr) |
| 112 |
future = self.executor.submit(id, obj) |
| 113 |
future.result() |
| 114 |
|
| 115 |
self.assertTrue(obj.event.wait(timeout=1)) |
| 116 |
|
| 117 |
# explicitly destroy the object to ensure that EventfulGCObj.__del__() |
| 118 |
# is called while manager is still running. |
| 119 |
support.gc_collect() |
| 120 |
obj = None |
| 121 |
support.gc_collect() |
| 122 |
|
| 123 |
mgr.shutdown() |
| 124 |
mgr.join() |
| 125 |
|
| 126 |
def test_saturation(self): |
| 127 |
executor = self.executor |
| 128 |
mp_context = self.get_context() |
| 129 |
sem = mp_context.Semaphore(0) |
| 130 |
job_count = 15 * executor._max_workers |
| 131 |
for _ in range(job_count): |
| 132 |
executor.submit(sem.acquire) |
| 133 |
self.assertEqual(len(executor._processes), executor._max_workers) |
| 134 |
for _ in range(job_count): |
| 135 |
sem.release() |
| 136 |
|
| 137 |
@support.requires_gil_enabled("gh-117344: test is flaky without the GIL") |
| 138 |
def test_idle_process_reuse_one(self): |
| 139 |
executor = self.executor |
| 140 |
assert executor._max_workers >= 4 |
| 141 |
if self.get_context().get_start_method(allow_none=False) == "fork": |
| 142 |
raise unittest.SkipTest("Incompatible with the fork start method.") |
| 143 |
executor.submit(mul, 21, 2).result() |
| 144 |
executor.submit(mul, 6, 7).result() |
| 145 |
executor.submit(mul, 3, 14).result() |
| 146 |
self.assertEqual(len(executor._processes), 1) |
| 147 |
|
| 148 |
def test_idle_process_reuse_multiple(self): |
| 149 |
executor = self.executor |
| 150 |
assert executor._max_workers <= 5 |
| 151 |
if self.get_context().get_start_method(allow_none=False) == "fork": |
| 152 |
raise unittest.SkipTest("Incompatible with the fork start method.") |
| 153 |
executor.submit(mul, 12, 7).result() |
| 154 |
executor.submit(mul, 33, 25) |
| 155 |
executor.submit(mul, 25, 26).result() |
| 156 |
executor.submit(mul, 18, 29) |
| 157 |
executor.submit(mul, 1, 2).result() |
| 158 |
executor.submit(mul, 0, 9) |
| 159 |
self.assertLessEqual(len(executor._processes), 3) |
| 160 |
executor.shutdown() |
| 161 |
|
| 162 |
def test_max_tasks_per_child(self): |
| 163 |
context = self.get_context() |
| 164 |
if context.get_start_method(allow_none=False) == "fork": |
| 165 |
with self.assertRaises(ValueError): |
| 166 |
self.executor_type(1, mp_context=context, max_tasks_per_child=3) |
| 167 |
return |
| 168 |
# not using self.executor as we need to control construction. |
| 169 |
# arguably this could go in another class w/o that mixin. |
| 170 |
executor = self.executor_type( |
| 171 |
1, mp_context=context, max_tasks_per_child=3) |
| 172 |
f1 = executor.submit(os.getpid) |
| 173 |
original_pid = f1.result() |
| 174 |
# The worker pid remains the same as the worker could be reused |
| 175 |
f2 = executor.submit(os.getpid) |
| 176 |
self.assertEqual(f2.result(), original_pid) |
| 177 |
self.assertEqual(len(executor._processes), 1) |
| 178 |
f3 = executor.submit(os.getpid) |
| 179 |
self.assertEqual(f3.result(), original_pid) |
| 180 |
|
| 181 |
# A new worker is spawned, with a statistically different pid, |
| 182 |
# while the previous was reaped. |
| 183 |
f4 = executor.submit(os.getpid) |
| 184 |
new_pid = f4.result() |
| 185 |
self.assertNotEqual(original_pid, new_pid) |
| 186 |
self.assertEqual(len(executor._processes), 1) |
| 187 |
|
| 188 |
executor.shutdown() |
| 189 |
|
| 190 |
def test_max_tasks_per_child_defaults_to_spawn_context(self): |
| 191 |
# not using self.executor as we need to control construction. |
| 192 |
# arguably this could go in another class w/o that mixin. |
| 193 |
executor = self.executor_type(1, max_tasks_per_child=3) |
| 194 |
self.assertEqual(executor._mp_context.get_start_method(), "spawn") |
| 195 |
|
| 196 |
def test_max_tasks_early_shutdown(self): |
| 197 |
context = self.get_context() |
| 198 |
if context.get_start_method(allow_none=False) == "fork": |
| 199 |
raise unittest.SkipTest("Incompatible with the fork start method.") |
| 200 |
# not using self.executor as we need to control construction. |
| 201 |
# arguably this could go in another class w/o that mixin. |
| 202 |
executor = self.executor_type( |
| 203 |
3, mp_context=context, max_tasks_per_child=1) |
| 204 |
futures = [] |
| 205 |
for i in range(6): |
| 206 |
futures.append(executor.submit(mul, i, i)) |
| 207 |
executor.shutdown() |
| 208 |
for i, future in enumerate(futures): |
| 209 |
self.assertEqual(future.result(), mul(i, i)) |
| 210 |
|
| 211 |
def test_python_finalization_error(self): |
| 212 |
# gh-109047: Catch RuntimeError on thread creation |
| 213 |
# during Python finalization. |
| 214 |
|
| 215 |
context = self.get_context() |
| 216 |
|
| 217 |
# gh-109047: Mock the threading.start_joinable_thread() function to inject |
| 218 |
# RuntimeError: simulate the error raised during Python finalization. |
| 219 |
# Block the second creation: create _ExecutorManagerThread, but block |
| 220 |
# QueueFeederThread. |
| 221 |
orig_start_new_thread = threading._start_joinable_thread |
| 222 |
nthread = 0 |
| 223 |
def mock_start_new_thread(func, *args, **kwargs): |
| 224 |
nonlocal nthread |
| 225 |
if nthread >= 1: |
| 226 |
raise RuntimeError("can't create new thread at " |
| 227 |
"interpreter shutdown") |
| 228 |
nthread += 1 |
| 229 |
return orig_start_new_thread(func, *args, **kwargs) |
| 230 |
|
| 231 |
with support.swap_attr(threading, '_start_joinable_thread', |
| 232 |
mock_start_new_thread): |
| 233 |
executor = self.executor_type(max_workers=2, mp_context=context) |
| 234 |
with executor: |
| 235 |
with self.assertRaises(BrokenProcessPool): |
| 236 |
list(executor.map(mul, [(2, 3)] * 10)) |
| 237 |
executor.shutdown() |
| 238 |
|
| 239 |
def test_terminate_workers(self): |
| 240 |
mock_fn = unittest.mock.Mock() |
| 241 |
with self.executor_type(max_workers=1) as executor: |
| 242 |
executor._force_shutdown = mock_fn |
| 243 |
executor.terminate_workers() |
| 244 |
|
| 245 |
mock_fn.assert_called_once_with(operation=futures.process._TERMINATE) |
| 246 |
|
| 247 |
def test_kill_workers(self): |
| 248 |
mock_fn = unittest.mock.Mock() |
| 249 |
with self.executor_type(max_workers=1) as executor: |
| 250 |
executor._force_shutdown = mock_fn |
| 251 |
executor.kill_workers() |
| 252 |
|
| 253 |
mock_fn.assert_called_once_with(operation=futures.process._KILL) |
| 254 |
|
| 255 |
def test_force_shutdown_workers_invalid_op(self): |
| 256 |
with self.executor_type(max_workers=1) as executor: |
| 257 |
self.assertRaises(ValueError, |
| 258 |
executor._force_shutdown, |
| 259 |
operation='invalid operation'), |
| 260 |
|
| 261 |
@parameterize(*FORCE_SHUTDOWN_PARAMS) |
| 262 |
def test_force_shutdown_workers(self, function_name): |
| 263 |
manager = self.get_context().Manager() |
| 264 |
q = manager.Queue() |
| 265 |
e = manager.Event() |
| 266 |
|
| 267 |
with self.executor_type(max_workers=1) as executor: |
| 268 |
executor.submit(_put_wait_put, q, e) |
| 269 |
|
| 270 |
# We should get started, but not finished since we'll terminate the |
| 271 |
# workers just after and never set the event. |
| 272 |
self.assertEqual(q.get(timeout=support.SHORT_TIMEOUT), 'started') |
| 273 |
|
| 274 |
worker_process = list(executor._processes.values())[0] |
| 275 |
|
| 276 |
Mock = unittest.mock.Mock |
| 277 |
worker_process.terminate = Mock(wraps=worker_process.terminate) |
| 278 |
worker_process.kill = Mock(wraps=worker_process.kill) |
| 279 |
|
| 280 |
getattr(executor, function_name)() |
| 281 |
worker_process.join() |
| 282 |
|
| 283 |
if function_name == TERMINATE_WORKERS: |
| 284 |
worker_process.terminate.assert_called() |
| 285 |
elif function_name == KILL_WORKERS: |
| 286 |
worker_process.kill.assert_called() |
| 287 |
else: |
| 288 |
self.fail(f"Unknown operation: {function_name}") |
| 289 |
|
| 290 |
self.assertRaises(queue.Empty, q.get, timeout=0.01) |
| 291 |
|
| 292 |
@parameterize(*FORCE_SHUTDOWN_PARAMS) |
| 293 |
def test_force_shutdown_workers_dead_workers(self, function_name): |
| 294 |
with self.executor_type(max_workers=1) as executor: |
| 295 |
future = executor.submit(os._exit, 1) |
| 296 |
self.assertRaises(BrokenProcessPool, future.result) |
| 297 |
|
| 298 |
# even though the pool is broken, this shouldn't raise |
| 299 |
getattr(executor, function_name)() |
| 300 |
|
| 301 |
@parameterize(*FORCE_SHUTDOWN_PARAMS) |
| 302 |
def test_force_shutdown_workers_not_started_yet(self, function_name): |
| 303 |
ctx = self.get_context() |
| 304 |
with unittest.mock.patch.object(ctx, 'Process') as mock_process: |
| 305 |
with self.executor_type(max_workers=1, mp_context=ctx) as executor: |
| 306 |
# The worker has not been started yet, terminate/kill_workers |
| 307 |
# should basically no-op |
| 308 |
getattr(executor, function_name)() |
| 309 |
|
| 310 |
mock_process.return_value.kill.assert_not_called() |
| 311 |
mock_process.return_value.terminate.assert_not_called() |
| 312 |
|
| 313 |
@parameterize(*FORCE_SHUTDOWN_PARAMS) |
| 314 |
def test_force_shutdown_workers_stops_pool(self, function_name): |
| 315 |
with self.executor_type(max_workers=1) as executor: |
| 316 |
task = executor.submit(time.sleep, 0) |
| 317 |
self.assertIsNone(task.result()) |
| 318 |
|
| 319 |
worker_process = list(executor._processes.values())[0] |
| 320 |
getattr(executor, function_name)() |
| 321 |
|
| 322 |
self.assertRaises(RuntimeError, executor.submit, time.sleep, 0) |
| 323 |
|
| 324 |
# A signal sent, is not a signal reacted to. |
| 325 |
# So wait a moment here for the process to die. |
| 326 |
# If we don't, every once in a while we may get an ENV CHANGE |
| 327 |
# error since the process would be alive immediately after the |
| 328 |
# test run.. and die a moment later. |
| 329 |
worker_process.join(support.SHORT_TIMEOUT) |
| 330 |
|
| 331 |
# Oddly enough, even though join completes, sometimes it takes a |
| 332 |
# moment for the process to actually be marked as dead. |
| 333 |
# ... that seems a bit buggy. |
| 334 |
# We need it dead before ending the test to ensure it doesn't |
| 335 |
# get marked as an ENV CHANGE due to living child process. |
| 336 |
for _ in support.sleeping_retry(support.SHORT_TIMEOUT): |
| 337 |
if not worker_process.is_alive(): |
| 338 |
break |
| 339 |
|
| 340 |
|
| 341 |
create_executor_tests(globals(), ProcessPoolExecutorTest, |
| 342 |
executor_mixins=(ProcessPoolForkMixin, |
| 343 |
ProcessPoolForkserverMixin, |
| 344 |
ProcessPoolSpawnMixin)) |
| 345 |
|
| 346 |
|
| 347 |
def setUpModule(): |
| 348 |
setup_module() |
| 349 |
|
| 350 |
|
| 351 |
if __name__ == "__main__": |
| 352 |
unittest.main() |