| 1 |
import contextlib |
| 2 |
import multiprocessing as mp |
| 3 |
import multiprocessing.process |
| 4 |
import multiprocessing.util |
| 5 |
import os |
| 6 |
import threading |
| 7 |
import unittest |
| 8 |
from concurrent import futures |
| 9 |
from test import support |
| 10 |
|
| 11 |
from .executor import ExecutorTest, mul |
| 12 |
from .util import BaseTestCase, ThreadPoolMixin, setup_module |
| 13 |
|
| 14 |
|
| 15 |
class ThreadPoolExecutorTest(ThreadPoolMixin, ExecutorTest, BaseTestCase): |
| 16 |
def test_map_submits_without_iteration(self): |
| 17 |
"""Tests verifying issue 11777.""" |
| 18 |
finished = [] |
| 19 |
def record_finished(n): |
| 20 |
finished.append(n) |
| 21 |
|
| 22 |
self.executor.map(record_finished, range(10)) |
| 23 |
self.executor.shutdown(wait=True) |
| 24 |
self.assertCountEqual(finished, range(10)) |
| 25 |
|
| 26 |
def test_default_workers(self): |
| 27 |
executor = self.executor_type() |
| 28 |
expected = min(32, (os.process_cpu_count() or 1) + 4) |
| 29 |
self.assertEqual(executor._max_workers, expected) |
| 30 |
|
| 31 |
def test_saturation(self): |
| 32 |
executor = self.executor_type(4) |
| 33 |
def acquire_lock(lock): |
| 34 |
lock.acquire() |
| 35 |
|
| 36 |
sem = threading.Semaphore(0) |
| 37 |
for i in range(15 * executor._max_workers): |
| 38 |
executor.submit(acquire_lock, sem) |
| 39 |
self.assertEqual(len(executor._threads), executor._max_workers) |
| 40 |
for i in range(15 * executor._max_workers): |
| 41 |
sem.release() |
| 42 |
executor.shutdown(wait=True) |
| 43 |
|
| 44 |
@support.requires_gil_enabled("gh-117344: test is flaky without the GIL") |
| 45 |
def test_idle_thread_reuse(self): |
| 46 |
executor = self.executor_type() |
| 47 |
executor.submit(mul, 21, 2).result() |
| 48 |
executor.submit(mul, 6, 7).result() |
| 49 |
executor.submit(mul, 3, 14).result() |
| 50 |
self.assertEqual(len(executor._threads), 1) |
| 51 |
executor.shutdown(wait=True) |
| 52 |
|
| 53 |
@support.requires_fork() |
| 54 |
@unittest.skipUnless(hasattr(os, 'register_at_fork'), 'need os.register_at_fork') |
| 55 |
@support.requires_resource('cpu') |
| 56 |
def test_hang_global_shutdown_lock(self): |
| 57 |
# bpo-45021: _global_shutdown_lock should be reinitialized in the child |
| 58 |
# process, otherwise it will never exit |
| 59 |
def submit(pool): |
| 60 |
pool.submit(submit, pool) |
| 61 |
|
| 62 |
with futures.ThreadPoolExecutor(1) as pool: |
| 63 |
pool.submit(submit, pool) |
| 64 |
|
| 65 |
for _ in range(50): |
| 66 |
with futures.ProcessPoolExecutor(1, mp_context=mp.get_context('fork')) as workers: |
| 67 |
workers.submit(tuple) |
| 68 |
|
| 69 |
@support.requires_fork() |
| 70 |
@unittest.skipUnless(hasattr(os, 'register_at_fork'), 'need os.register_at_fork') |
| 71 |
def test_process_fork_from_a_threadpool(self): |
| 72 |
# bpo-43944: clear concurrent.futures.thread._threads_queues after fork, |
| 73 |
# otherwise child process will try to join parent thread |
| 74 |
def fork_process_and_return_exitcode(): |
| 75 |
# Ignore the warning about fork with threads. |
| 76 |
with self.assertWarnsRegex(DeprecationWarning, |
| 77 |
r"use of fork\(\) may lead to deadlocks in the child"): |
| 78 |
p = mp.get_context('fork').Process(target=lambda: 1) |
| 79 |
p.start() |
| 80 |
p.join() |
| 81 |
return p.exitcode |
| 82 |
|
| 83 |
with futures.ThreadPoolExecutor(1) as pool: |
| 84 |
process_exitcode = pool.submit(fork_process_and_return_exitcode).result() |
| 85 |
|
| 86 |
self.assertEqual(process_exitcode, 0) |
| 87 |
|
| 88 |
def test_executor_map_current_future_cancel(self): |
| 89 |
stop_event = threading.Event() |
| 90 |
log = [] |
| 91 |
|
| 92 |
def log_n_wait(ident): |
| 93 |
log.append(f"{ident=} started") |
| 94 |
try: |
| 95 |
stop_event.wait() |
| 96 |
finally: |
| 97 |
log.append(f"{ident=} stopped") |
| 98 |
|
| 99 |
with self.executor_type(max_workers=1) as pool: |
| 100 |
# submit work to saturate the pool |
| 101 |
fut = pool.submit(log_n_wait, ident="first") |
| 102 |
try: |
| 103 |
with contextlib.closing( |
| 104 |
pool.map(log_n_wait, ["second", "third"], timeout=0) |
| 105 |
) as gen: |
| 106 |
with self.assertRaises(TimeoutError): |
| 107 |
next(gen) |
| 108 |
finally: |
| 109 |
stop_event.set() |
| 110 |
fut.result() |
| 111 |
# ident='second' is cancelled as a result of raising a TimeoutError |
| 112 |
# ident='third' is cancelled because it remained in the collection of futures |
| 113 |
self.assertListEqual(log, ["ident='first' started", "ident='first' stopped"]) |
| 114 |
|
| 115 |
|
| 116 |
def setUpModule(): |
| 117 |
setup_module() |
| 118 |
|
| 119 |
|
| 120 |
if __name__ == "__main__": |
| 121 |
unittest.main() |