| 1 |
import _thread |
| 2 |
import contextlib |
| 3 |
import functools |
| 4 |
import sys |
| 5 |
import threading |
| 6 |
import time |
| 7 |
import unittest |
| 8 |
|
| 9 |
from test import support |
| 10 |
|
| 11 |
|
| 12 |
#======================================================================= |
| 13 |
# Threading support to prevent reporting refleaks when running regrtest.py -R |
| 14 |
|
| 15 |
# NOTE: we use thread._count() rather than threading.enumerate() (or the |
| 16 |
# moral equivalent thereof) because a threading.Thread object is still alive |
| 17 |
# until its __bootstrap() method has returned, even after it has been |
| 18 |
# unregistered from the threading module. |
| 19 |
# thread._count(), on the other hand, only gets decremented *after* the |
| 20 |
# __bootstrap() method has returned, which gives us reliable reference counts |
| 21 |
# at the end of a test run. |
| 22 |
|
| 23 |
|
| 24 |
def threading_setup(): |
| 25 |
return _thread._count(), len(threading._dangling) |
| 26 |
|
| 27 |
|
| 28 |
def threading_cleanup(*original_values): |
| 29 |
orig_count, orig_ndangling = original_values |
| 30 |
|
| 31 |
timeout = 1.0 |
| 32 |
for _ in support.sleeping_retry(timeout, error=False): |
| 33 |
# Copy the thread list to get a consistent output. threading._dangling |
| 34 |
# is a WeakSet, its value changes when it's read. |
| 35 |
dangling_threads = list(threading._dangling) |
| 36 |
count = _thread._count() |
| 37 |
|
| 38 |
if count <= orig_count: |
| 39 |
return |
| 40 |
|
| 41 |
# Timeout! |
| 42 |
support.environment_altered = True |
| 43 |
support.print_warning( |
| 44 |
f"threading_cleanup() failed to clean up threads " |
| 45 |
f"in {timeout:.1f} seconds\n" |
| 46 |
f" before: thread count={orig_count}, dangling={orig_ndangling}\n" |
| 47 |
f" after: thread count={count}, dangling={len(dangling_threads)}") |
| 48 |
for thread in dangling_threads: |
| 49 |
support.print_warning(f"Dangling thread: {thread!r}") |
| 50 |
|
| 51 |
# The warning happens when a test spawns threads and some of these threads |
| 52 |
# are still running after the test completes. To fix this warning, join |
| 53 |
# threads explicitly to wait until they complete. |
| 54 |
# |
| 55 |
# To make the warning more likely, reduce the timeout. |
| 56 |
|
| 57 |
|
| 58 |
def reap_threads(func): |
| 59 |
"""Use this function when threads are being used. This will |
| 60 |
ensure that the threads are cleaned up even when the test fails. |
| 61 |
""" |
| 62 |
@functools.wraps(func) |
| 63 |
def decorator(*args): |
| 64 |
key = threading_setup() |
| 65 |
try: |
| 66 |
return func(*args) |
| 67 |
finally: |
| 68 |
threading_cleanup(*key) |
| 69 |
return decorator |
| 70 |
|
| 71 |
|
| 72 |
@contextlib.contextmanager |
| 73 |
def wait_threads_exit(timeout=None): |
| 74 |
""" |
| 75 |
bpo-31234: Context manager to wait until all threads created in the with |
| 76 |
statement exit. |
| 77 |
|
| 78 |
Use _thread.count() to check if threads exited. Indirectly, wait until |
| 79 |
threads exit the internal t_bootstrap() C function of the _thread module. |
| 80 |
|
| 81 |
threading_setup() and threading_cleanup() are designed to emit a warning |
| 82 |
if a test leaves running threads in the background. This context manager |
| 83 |
is designed to cleanup threads started by the _thread.start_new_thread() |
| 84 |
which doesn't allow to wait for thread exit, whereas thread.Thread has a |
| 85 |
join() method. |
| 86 |
""" |
| 87 |
if timeout is None: |
| 88 |
timeout = support.SHORT_TIMEOUT |
| 89 |
old_count = _thread._count() |
| 90 |
try: |
| 91 |
yield |
| 92 |
finally: |
| 93 |
start_time = time.monotonic() |
| 94 |
for _ in support.sleeping_retry(timeout, error=False): |
| 95 |
support.gc_collect() |
| 96 |
count = _thread._count() |
| 97 |
if count <= old_count: |
| 98 |
break |
| 99 |
else: |
| 100 |
dt = time.monotonic() - start_time |
| 101 |
msg = (f"wait_threads() failed to cleanup {count - old_count} " |
| 102 |
f"threads after {dt:.1f} seconds " |
| 103 |
f"(count: {count}, old count: {old_count})") |
| 104 |
raise AssertionError(msg) |
| 105 |
|
| 106 |
|
| 107 |
def join_thread(thread, timeout=None): |
| 108 |
"""Join a thread. Raise an AssertionError if the thread is still alive |
| 109 |
after timeout seconds. |
| 110 |
""" |
| 111 |
if timeout is None: |
| 112 |
timeout = support.SHORT_TIMEOUT |
| 113 |
thread.join(timeout) |
| 114 |
if thread.is_alive(): |
| 115 |
msg = f"failed to join the thread in {timeout:.1f} seconds" |
| 116 |
raise AssertionError(msg) |
| 117 |
|
| 118 |
|
| 119 |
@contextlib.contextmanager |
| 120 |
def start_threads(threads, unlock=None): |
| 121 |
try: |
| 122 |
import faulthandler |
| 123 |
except ImportError: |
| 124 |
# It isn't supported on subinterpreters yet. |
| 125 |
faulthandler = None |
| 126 |
threads = list(threads) |
| 127 |
started = [] |
| 128 |
try: |
| 129 |
try: |
| 130 |
for t in threads: |
| 131 |
t.start() |
| 132 |
started.append(t) |
| 133 |
except: |
| 134 |
if support.verbose: |
| 135 |
print("Can't start %d threads, only %d threads started" % |
| 136 |
(len(threads), len(started))) |
| 137 |
raise |
| 138 |
yield |
| 139 |
finally: |
| 140 |
try: |
| 141 |
if unlock: |
| 142 |
unlock() |
| 143 |
endtime = time.monotonic() |
| 144 |
for timeout in range(1, 16): |
| 145 |
endtime += 60 |
| 146 |
for t in started: |
| 147 |
t.join(max(endtime - time.monotonic(), 0.01)) |
| 148 |
started = [t for t in started if t.is_alive()] |
| 149 |
if not started: |
| 150 |
break |
| 151 |
if support.verbose: |
| 152 |
print('Unable to join %d threads during a period of ' |
| 153 |
'%d minutes' % (len(started), timeout)) |
| 154 |
finally: |
| 155 |
started = [t for t in started if t.is_alive()] |
| 156 |
if started: |
| 157 |
if faulthandler is not None: |
| 158 |
faulthandler.dump_traceback(sys.stdout) |
| 159 |
raise AssertionError('Unable to join %d threads' % len(started)) |
| 160 |
|
| 161 |
|
| 162 |
class catch_threading_exception: |
| 163 |
""" |
| 164 |
Context manager catching threading.Thread exception using |
| 165 |
threading.excepthook. |
| 166 |
|
| 167 |
Attributes set when an exception is caught: |
| 168 |
|
| 169 |
* exc_type |
| 170 |
* exc_value |
| 171 |
* exc_traceback |
| 172 |
* thread |
| 173 |
|
| 174 |
See threading.excepthook() documentation for these attributes. |
| 175 |
|
| 176 |
These attributes are deleted at the context manager exit. |
| 177 |
|
| 178 |
Usage: |
| 179 |
|
| 180 |
with threading_helper.catch_threading_exception() as cm: |
| 181 |
# code spawning a thread which raises an exception |
| 182 |
... |
| 183 |
|
| 184 |
# check the thread exception, use cm attributes: |
| 185 |
# exc_type, exc_value, exc_traceback, thread |
| 186 |
... |
| 187 |
|
| 188 |
# exc_type, exc_value, exc_traceback, thread attributes of cm no longer |
| 189 |
# exists at this point |
| 190 |
# (to avoid reference cycles) |
| 191 |
""" |
| 192 |
|
| 193 |
def __init__(self): |
| 194 |
self.exc_type = None |
| 195 |
self.exc_value = None |
| 196 |
self.exc_traceback = None |
| 197 |
self.thread = None |
| 198 |
self._old_hook = None |
| 199 |
|
| 200 |
def _hook(self, args): |
| 201 |
self.exc_type = args.exc_type |
| 202 |
self.exc_value = args.exc_value |
| 203 |
self.exc_traceback = args.exc_traceback |
| 204 |
self.thread = args.thread |
| 205 |
|
| 206 |
def __enter__(self): |
| 207 |
self._old_hook = threading.excepthook |
| 208 |
threading.excepthook = self._hook |
| 209 |
return self |
| 210 |
|
| 211 |
def __exit__(self, *exc_info): |
| 212 |
threading.excepthook = self._old_hook |
| 213 |
del self.exc_type |
| 214 |
del self.exc_value |
| 215 |
del self.exc_traceback |
| 216 |
del self.thread |
| 217 |
|
| 218 |
|
| 219 |
def _can_start_thread() -> bool: |
| 220 |
"""Detect whether Python can start new threads. |
| 221 |
|
| 222 |
Some WebAssembly platforms do not provide a working pthread |
| 223 |
implementation. Thread support is stubbed and any attempt |
| 224 |
to create a new thread fails. |
| 225 |
|
| 226 |
- wasm32-wasi does not have threading. |
| 227 |
- wasm32-emscripten can be compiled with or without pthread |
| 228 |
support (-s USE_PTHREADS / __EMSCRIPTEN_PTHREADS__). |
| 229 |
""" |
| 230 |
if sys.platform == "emscripten": |
| 231 |
return sys._emscripten_info.pthreads |
| 232 |
elif sys.platform == "wasi": |
| 233 |
return False |
| 234 |
else: |
| 235 |
# assume all other platforms have working thread support. |
| 236 |
return True |
| 237 |
|
| 238 |
can_start_thread = _can_start_thread() |
| 239 |
|
| 240 |
def requires_working_threading(*, module=False): |
| 241 |
"""Skip tests or modules that require working threading. |
| 242 |
|
| 243 |
Can be used as a function/class decorator or to skip an entire module. |
| 244 |
""" |
| 245 |
msg = "requires threading support" |
| 246 |
if module: |
| 247 |
if not can_start_thread: |
| 248 |
raise unittest.SkipTest(msg) |
| 249 |
else: |
| 250 |
return unittest.skipUnless(can_start_thread, msg) |
| 251 |
|
| 252 |
|
| 253 |
def run_concurrently(worker_func, nthreads=None, args=(), kwargs={}): |
| 254 |
""" |
| 255 |
Run the worker function(s) concurrently in multiple threads. |
| 256 |
|
| 257 |
If `worker_func` is a single callable, it is used for all threads. |
| 258 |
If it is a list of callables, each callable is used for one thread. |
| 259 |
""" |
| 260 |
from collections.abc import Iterable |
| 261 |
|
| 262 |
if nthreads is None: |
| 263 |
nthreads = len(worker_func) |
| 264 |
if not isinstance(worker_func, Iterable): |
| 265 |
worker_func = [worker_func] * nthreads |
| 266 |
assert len(worker_func) == nthreads |
| 267 |
|
| 268 |
barrier = threading.Barrier(nthreads) |
| 269 |
|
| 270 |
def wrapper_func(func, *args, **kwargs): |
| 271 |
# Wait for all threads to reach this point before proceeding. |
| 272 |
barrier.wait() |
| 273 |
func(*args, **kwargs) |
| 274 |
|
| 275 |
with catch_threading_exception() as cm: |
| 276 |
workers = [ |
| 277 |
threading.Thread(target=wrapper_func, args=(func, *args), kwargs=kwargs) |
| 278 |
for func in worker_func |
| 279 |
] |
| 280 |
with start_threads(workers): |
| 281 |
pass |
| 282 |
|
| 283 |
# If a worker thread raises an exception, re-raise it. |
| 284 |
if cm.exc_value is not None: |
| 285 |
raise cm.exc_value |