Debian Codegraph @

Lib/test/test_concurrent_futures/test_deadlock.py

raw
1 import contextlib
2 import queue
3 import signal
4 import sys
5 import time
6 import unittest
7 import unittest.mock
8 from pickle import PicklingError
9 from concurrent import futures
10 from concurrent.futures.process import BrokenProcessPool, _ThreadWakeup
11
12 from test import support
13
14 from .util import (
15 create_executor_tests, setup_module,
16 ProcessPoolForkMixin, ProcessPoolForkserverMixin, ProcessPoolSpawnMixin)
17
18
19 def _crash(delay=None):
20 """Induces a segfault."""
21 if delay:
22 time.sleep(delay)
23 import faulthandler
24 faulthandler.disable()
25 faulthandler._sigsegv()
26
27
28 def _crash_with_data(data):
29 """Induces a segfault with dummy data in input."""
30 _crash()
31
32
33 def _exit():
34 """Induces a sys exit with exitcode 1."""
35 sys.exit(1)
36
37
38 def _raise_error(Err):
39 """Function that raises an Exception in process."""
40 raise Err()
41
42
43 def _raise_error_ignore_stderr(Err):
44 """Function that raises an Exception in process and ignores stderr."""
45 import io
46 sys.stderr = io.StringIO()
47 raise Err()
48
49
50 def _return_instance(cls):
51 """Function that returns a instance of cls."""
52 return cls()
53
54
55 class CrashAtPickle(object):
56 """Bad object that triggers a segfault at pickling time."""
57 def __reduce__(self):
58 _crash()
59
60
61 class CrashAtUnpickle(object):
62 """Bad object that triggers a segfault at unpickling time."""
63 def __reduce__(self):
64 return _crash, ()
65
66
67 class ExitAtPickle(object):
68 """Bad object that triggers a process exit at pickling time."""
69 def __reduce__(self):
70 _exit()
71
72
73 class ExitAtUnpickle(object):
74 """Bad object that triggers a process exit at unpickling time."""
75 def __reduce__(self):
76 return _exit, ()
77
78
79 class ErrorAtPickle(object):
80 """Bad object that triggers an error at pickling time."""
81 def __reduce__(self):
82 from pickle import PicklingError
83 raise PicklingError("Error in pickle")
84
85
86 class ErrorAtUnpickle(object):
87 """Bad object that triggers an error at unpickling time."""
88 def __reduce__(self):
89 from pickle import UnpicklingError
90 return _raise_error_ignore_stderr, (UnpicklingError, )
91
92
93 class ExecutorDeadlockTest:
94 TIMEOUT = support.LONG_TIMEOUT
95
96 def _fail_on_deadlock(self, executor):
97 # If we did not recover before TIMEOUT seconds, consider that the
98 # executor is in a deadlock state and forcefully clean all its
99 # composants.
100 import faulthandler
101 from tempfile import TemporaryFile
102 with TemporaryFile(mode="w+") as f:
103 faulthandler.dump_traceback(file=f)
104 f.seek(0)
105 tb = f.read()
106 for p in executor._processes.values():
107 p.terminate()
108 # This should be safe to call executor.shutdown here as all possible
109 # deadlocks should have been broken.
110 executor.shutdown(wait=True)
111 print(f"\nTraceback:\n {tb}", file=sys.__stderr__)
112 self.fail(f"Executor deadlock:\n\n{tb}")
113
114 def _check_error(self, error, func, *args, ignore_stderr=False):
115 # test for deadlock caused by crashes or exiting in a pool
116 self.executor.shutdown(wait=True)
117
118 executor = self.executor_type(
119 max_workers=2, mp_context=self.get_context())
120 res = executor.submit(func, *args)
121
122 if ignore_stderr:
123 cm = support.captured_stderr()
124 else:
125 cm = contextlib.nullcontext()
126
127 try:
128 with self.assertRaises(error):
129 with cm:
130 res.result(timeout=self.TIMEOUT)
131 except futures.TimeoutError:
132 # If we did not recover before TIMEOUT seconds,
133 # consider that the executor is in a deadlock state
134 self._fail_on_deadlock(executor)
135 executor.shutdown(wait=True)
136
137 def test_error_at_task_pickle(self):
138 # Check problem occurring while pickling a task in
139 # the task_handler thread
140 self._check_error(PicklingError, id, ErrorAtPickle())
141
142 def test_exit_at_task_unpickle(self):
143 # Check problem occurring while unpickling a task on workers
144 self._check_error(BrokenProcessPool, id, ExitAtUnpickle())
145
146 def test_error_at_task_unpickle(self):
147 # gh-109832: Restore stderr overridden by _raise_error_ignore_stderr()
148 self.addCleanup(setattr, sys, 'stderr', sys.stderr)
149
150 # Check problem occurring while unpickling a task on workers
151 self._check_error(BrokenProcessPool, id, ErrorAtUnpickle())
152
153 @support.skip_if_sanitizer("UBSan: explicit SIGSEV not allowed", ub=True)
154 def test_crash_at_task_unpickle(self):
155 # Check problem occurring while unpickling a task on workers
156 self._check_error(BrokenProcessPool, id, CrashAtUnpickle())
157
158 @support.skip_if_sanitizer("UBSan: explicit SIGSEV not allowed", ub=True)
159 def test_crash_during_func_exec_on_worker(self):
160 # Check problem occurring during func execution on workers
161 self._check_error(BrokenProcessPool, _crash)
162
163 def test_exit_during_func_exec_on_worker(self):
164 # Check problem occurring during func execution on workers
165 self._check_error(SystemExit, _exit)
166
167 def test_error_during_func_exec_on_worker(self):
168 # Check problem occurring during func execution on workers
169 self._check_error(RuntimeError, _raise_error, RuntimeError)
170
171 @support.skip_if_sanitizer("UBSan: explicit SIGSEV not allowed", ub=True)
172 def test_crash_during_result_pickle_on_worker(self):
173 # Check problem occurring while pickling a task result
174 # on workers
175 self._check_error(BrokenProcessPool, _return_instance, CrashAtPickle)
176
177 def test_exit_during_result_pickle_on_worker(self):
178 # Check problem occurring while pickling a task result
179 # on workers
180 self._check_error(SystemExit, _return_instance, ExitAtPickle)
181
182 def test_error_during_result_pickle_on_worker(self):
183 # Check problem occurring while pickling a task result
184 # on workers
185 self._check_error(PicklingError, _return_instance, ErrorAtPickle)
186
187 def test_error_during_result_unpickle_in_result_handler(self):
188 # gh-109832: Restore stderr overridden by _raise_error_ignore_stderr()
189 self.addCleanup(setattr, sys, 'stderr', sys.stderr)
190
191 # Check problem occurring while unpickling a task in
192 # the result_handler thread
193 self._check_error(BrokenProcessPool,
194 _return_instance, ErrorAtUnpickle,
195 ignore_stderr=True)
196
197 def test_exit_during_result_unpickle_in_result_handler(self):
198 # Check problem occurring while unpickling a task in
199 # the result_handler thread
200 self._check_error(BrokenProcessPool, _return_instance, ExitAtUnpickle)
201
202 @support.skip_if_sanitizer("UBSan: explicit SIGSEV not allowed", ub=True)
203 def test_shutdown_deadlock(self):
204 # Test that the pool calling shutdown do not cause deadlock
205 # if a worker fails after the shutdown call.
206 self.executor.shutdown(wait=True)
207 with self.executor_type(max_workers=2,
208 mp_context=self.get_context()) as executor:
209 self.executor = executor # Allow clean up in fail_on_deadlock
210 f = executor.submit(_crash, delay=.1)
211 executor.shutdown(wait=True)
212 with self.assertRaises(BrokenProcessPool):
213 f.result()
214
215 def test_shutdown_deadlock_pickle(self):
216 # Test that the pool calling shutdown with wait=False does not cause
217 # a deadlock if a task fails at pickle after the shutdown call.
218 # Reported in bpo-39104.
219 self.executor.shutdown(wait=True)
220 with self.executor_type(max_workers=2,
221 mp_context=self.get_context()) as executor:
222 self.executor = executor # Allow clean up in fail_on_deadlock
223
224 # Start the executor and get the executor_manager_thread to collect
225 # the threads and avoid dangling thread that should be cleaned up
226 # asynchronously.
227 executor.submit(id, 42).result()
228 executor_manager = executor._executor_manager_thread
229
230 # Submit a task that fails at pickle and shutdown the executor
231 # without waiting
232 f = executor.submit(id, ErrorAtPickle())
233 executor.shutdown(wait=False)
234 with self.assertRaises(PicklingError):
235 f.result()
236
237 # Make sure the executor is eventually shutdown and do not leave
238 # dangling threads
239 executor_manager.join()
240
241 @support.skip_if_sanitizer("UBSan: explicit SIGSEV not allowed", ub=True)
242 def test_crash_big_data(self):
243 # Test that there is a clean exception instead of a deadlock when a
244 # child process crashes while some data is being written into the
245 # queue.
246 # https://github.com/python/cpython/issues/94777
247 self.executor.shutdown(wait=True)
248 data = "a" * support.PIPE_MAX_SIZE
249 with self.executor_type(max_workers=2,
250 mp_context=self.get_context()) as executor:
251 self.executor = executor # Allow clean up in fail_on_deadlock
252 with self.assertRaises(BrokenProcessPool):
253 list(executor.map(_crash_with_data, [data] * 10))
254
255 executor.shutdown(wait=True)
256
257 def test_gh105829_should_not_deadlock_if_wakeup_pipe_full(self):
258 # Issue #105829: The _ExecutorManagerThread wakeup pipe could
259 # fill up and block. See: https://github.com/python/cpython/issues/105829
260
261 # Lots of cargo culting while writing this test, apologies if
262 # something is really stupid...
263
264 self.executor.shutdown(wait=True)
265
266 if not hasattr(signal, 'alarm'):
267 raise unittest.SkipTest(
268 "Tested platform does not support the alarm signal")
269
270 def timeout(_signum, _frame):
271 import faulthandler
272 faulthandler.dump_traceback()
273
274 raise RuntimeError("timed out while submitting jobs?")
275
276 thread_run = futures.process._ExecutorManagerThread.run
277 def mock_run(self):
278 # Delay thread startup so the wakeup pipe can fill up and block
279 time.sleep(3)
280 thread_run(self)
281
282 class MockWakeup(_ThreadWakeup):
283 """Mock wakeup object to force the wakeup to block"""
284 def __init__(self):
285 super().__init__()
286 self._dummy_queue = queue.Queue(maxsize=1)
287
288 def wakeup(self):
289 self._dummy_queue.put(None, block=True)
290 super().wakeup()
291
292 def clear(self):
293 super().clear()
294 try:
295 while True:
296 self._dummy_queue.get_nowait()
297 except queue.Empty:
298 pass
299
300 with (unittest.mock.patch.object(futures.process._ExecutorManagerThread,
301 'run', mock_run),
302 unittest.mock.patch('concurrent.futures.process._ThreadWakeup',
303 MockWakeup)):
304 with self.executor_type(max_workers=2,
305 mp_context=self.get_context()) as executor:
306 self.executor = executor # Allow clean up in fail_on_deadlock
307
308 job_num = 100
309 job_data = range(job_num)
310
311 # Need to use sigalarm for timeout detection because
312 # Executor.submit is not guarded by any timeout (both
313 # self._work_ids.put(self._queue_count) and
314 # self._executor_manager_thread_wakeup.wakeup() might
315 # timeout, maybe more?). In this specific case it was
316 # the wakeup call that deadlocked on a blocking pipe.
317 old_handler = signal.signal(signal.SIGALRM, timeout)
318 try:
319 signal.alarm(int(self.TIMEOUT))
320 self.assertEqual(job_num, len(list(executor.map(int, job_data))))
321 finally:
322 signal.alarm(0)
323 signal.signal(signal.SIGALRM, old_handler)
324
325
326 create_executor_tests(globals(), ExecutorDeadlockTest,
327 executor_mixins=(ProcessPoolForkMixin,
328 ProcessPoolForkserverMixin,
329 ProcessPoolSpawnMixin))
330
331 def setUpModule():
332 setup_module()
333
334
335 if __name__ == "__main__":
336 unittest.main()