Debian Codegraph @

Lib/test/support/threading_helper.py

raw
1 import _thread
2 import contextlib
3 import functools
4 import sys
5 import threading
6 import time
7 import unittest
8
9 from test import support
10
11
12 #=======================================================================
13 # Threading support to prevent reporting refleaks when running regrtest.py -R
14
15 # NOTE: we use thread._count() rather than threading.enumerate() (or the
16 # moral equivalent thereof) because a threading.Thread object is still alive
17 # until its __bootstrap() method has returned, even after it has been
18 # unregistered from the threading module.
19 # thread._count(), on the other hand, only gets decremented *after* the
20 # __bootstrap() method has returned, which gives us reliable reference counts
21 # at the end of a test run.
22
23
24 def threading_setup():
25 return _thread._count(), len(threading._dangling)
26
27
28 def threading_cleanup(*original_values):
29 orig_count, orig_ndangling = original_values
30
31 timeout = 1.0
32 for _ in support.sleeping_retry(timeout, error=False):
33 # Copy the thread list to get a consistent output. threading._dangling
34 # is a WeakSet, its value changes when it's read.
35 dangling_threads = list(threading._dangling)
36 count = _thread._count()
37
38 if count <= orig_count:
39 return
40
41 # Timeout!
42 support.environment_altered = True
43 support.print_warning(
44 f"threading_cleanup() failed to clean up threads "
45 f"in {timeout:.1f} seconds\n"
46 f" before: thread count={orig_count}, dangling={orig_ndangling}\n"
47 f" after: thread count={count}, dangling={len(dangling_threads)}")
48 for thread in dangling_threads:
49 support.print_warning(f"Dangling thread: {thread!r}")
50
51 # The warning happens when a test spawns threads and some of these threads
52 # are still running after the test completes. To fix this warning, join
53 # threads explicitly to wait until they complete.
54 #
55 # To make the warning more likely, reduce the timeout.
56
57
58 def reap_threads(func):
59 """Use this function when threads are being used. This will
60 ensure that the threads are cleaned up even when the test fails.
61 """
62 @functools.wraps(func)
63 def decorator(*args):
64 key = threading_setup()
65 try:
66 return func(*args)
67 finally:
68 threading_cleanup(*key)
69 return decorator
70
71
72 @contextlib.contextmanager
73 def wait_threads_exit(timeout=None):
74 """
75 bpo-31234: Context manager to wait until all threads created in the with
76 statement exit.
77
78 Use _thread.count() to check if threads exited. Indirectly, wait until
79 threads exit the internal t_bootstrap() C function of the _thread module.
80
81 threading_setup() and threading_cleanup() are designed to emit a warning
82 if a test leaves running threads in the background. This context manager
83 is designed to cleanup threads started by the _thread.start_new_thread()
84 which doesn't allow to wait for thread exit, whereas thread.Thread has a
85 join() method.
86 """
87 if timeout is None:
88 timeout = support.SHORT_TIMEOUT
89 old_count = _thread._count()
90 try:
91 yield
92 finally:
93 start_time = time.monotonic()
94 for _ in support.sleeping_retry(timeout, error=False):
95 support.gc_collect()
96 count = _thread._count()
97 if count <= old_count:
98 break
99 else:
100 dt = time.monotonic() - start_time
101 msg = (f"wait_threads() failed to cleanup {count - old_count} "
102 f"threads after {dt:.1f} seconds "
103 f"(count: {count}, old count: {old_count})")
104 raise AssertionError(msg)
105
106
107 def join_thread(thread, timeout=None):
108 """Join a thread. Raise an AssertionError if the thread is still alive
109 after timeout seconds.
110 """
111 if timeout is None:
112 timeout = support.SHORT_TIMEOUT
113 thread.join(timeout)
114 if thread.is_alive():
115 msg = f"failed to join the thread in {timeout:.1f} seconds"
116 raise AssertionError(msg)
117
118
119 @contextlib.contextmanager
120 def start_threads(threads, unlock=None):
121 try:
122 import faulthandler
123 except ImportError:
124 # It isn't supported on subinterpreters yet.
125 faulthandler = None
126 threads = list(threads)
127 started = []
128 try:
129 try:
130 for t in threads:
131 t.start()
132 started.append(t)
133 except:
134 if support.verbose:
135 print("Can't start %d threads, only %d threads started" %
136 (len(threads), len(started)))
137 raise
138 yield
139 finally:
140 try:
141 if unlock:
142 unlock()
143 endtime = time.monotonic()
144 for timeout in range(1, 16):
145 endtime += 60
146 for t in started:
147 t.join(max(endtime - time.monotonic(), 0.01))
148 started = [t for t in started if t.is_alive()]
149 if not started:
150 break
151 if support.verbose:
152 print('Unable to join %d threads during a period of '
153 '%d minutes' % (len(started), timeout))
154 finally:
155 started = [t for t in started if t.is_alive()]
156 if started:
157 if faulthandler is not None:
158 faulthandler.dump_traceback(sys.stdout)
159 raise AssertionError('Unable to join %d threads' % len(started))
160
161
162 class catch_threading_exception:
163 """
164 Context manager catching threading.Thread exception using
165 threading.excepthook.
166
167 Attributes set when an exception is caught:
168
169 * exc_type
170 * exc_value
171 * exc_traceback
172 * thread
173
174 See threading.excepthook() documentation for these attributes.
175
176 These attributes are deleted at the context manager exit.
177
178 Usage:
179
180 with threading_helper.catch_threading_exception() as cm:
181 # code spawning a thread which raises an exception
182 ...
183
184 # check the thread exception, use cm attributes:
185 # exc_type, exc_value, exc_traceback, thread
186 ...
187
188 # exc_type, exc_value, exc_traceback, thread attributes of cm no longer
189 # exists at this point
190 # (to avoid reference cycles)
191 """
192
193 def __init__(self):
194 self.exc_type = None
195 self.exc_value = None
196 self.exc_traceback = None
197 self.thread = None
198 self._old_hook = None
199
200 def _hook(self, args):
201 self.exc_type = args.exc_type
202 self.exc_value = args.exc_value
203 self.exc_traceback = args.exc_traceback
204 self.thread = args.thread
205
206 def __enter__(self):
207 self._old_hook = threading.excepthook
208 threading.excepthook = self._hook
209 return self
210
211 def __exit__(self, *exc_info):
212 threading.excepthook = self._old_hook
213 del self.exc_type
214 del self.exc_value
215 del self.exc_traceback
216 del self.thread
217
218
219 def _can_start_thread() -> bool:
220 """Detect whether Python can start new threads.
221
222 Some WebAssembly platforms do not provide a working pthread
223 implementation. Thread support is stubbed and any attempt
224 to create a new thread fails.
225
226 - wasm32-wasi does not have threading.
227 - wasm32-emscripten can be compiled with or without pthread
228 support (-s USE_PTHREADS / __EMSCRIPTEN_PTHREADS__).
229 """
230 if sys.platform == "emscripten":
231 return sys._emscripten_info.pthreads
232 elif sys.platform == "wasi":
233 return False
234 else:
235 # assume all other platforms have working thread support.
236 return True
237
238 can_start_thread = _can_start_thread()
239
240 def requires_working_threading(*, module=False):
241 """Skip tests or modules that require working threading.
242
243 Can be used as a function/class decorator or to skip an entire module.
244 """
245 msg = "requires threading support"
246 if module:
247 if not can_start_thread:
248 raise unittest.SkipTest(msg)
249 else:
250 return unittest.skipUnless(can_start_thread, msg)
251
252
253 def run_concurrently(worker_func, nthreads=None, args=(), kwargs={}):
254 """
255 Run the worker function(s) concurrently in multiple threads.
256
257 If `worker_func` is a single callable, it is used for all threads.
258 If it is a list of callables, each callable is used for one thread.
259 """
260 from collections.abc import Iterable
261
262 if nthreads is None:
263 nthreads = len(worker_func)
264 if not isinstance(worker_func, Iterable):
265 worker_func = [worker_func] * nthreads
266 assert len(worker_func) == nthreads
267
268 barrier = threading.Barrier(nthreads)
269
270 def wrapper_func(func, *args, **kwargs):
271 # Wait for all threads to reach this point before proceeding.
272 barrier.wait()
273 func(*args, **kwargs)
274
275 with catch_threading_exception() as cm:
276 workers = [
277 threading.Thread(target=wrapper_func, args=(func, *args), kwargs=kwargs)
278 for func in worker_func
279 ]
280 with start_threads(workers):
281 pass
282
283 # If a worker thread raises an exception, re-raise it.
284 if cm.exc_value is not None:
285 raise cm.exc_value