| 1 |
import itertools |
| 2 |
import threading |
| 3 |
import time |
| 4 |
import weakref |
| 5 |
from concurrent import futures |
| 6 |
from operator import add |
| 7 |
from test import support |
| 8 |
from test.support import Py_GIL_DISABLED |
| 9 |
|
| 10 |
|
| 11 |
def mul(x, y): |
| 12 |
return x * y |
| 13 |
|
| 14 |
def capture(*args, **kwargs): |
| 15 |
return args, kwargs |
| 16 |
|
| 17 |
|
| 18 |
class MyObject(object): |
| 19 |
def my_method(self): |
| 20 |
pass |
| 21 |
|
| 22 |
|
| 23 |
def make_dummy_object(_): |
| 24 |
return MyObject() |
| 25 |
|
| 26 |
|
| 27 |
# Used in test_swallows_falsey_exceptions |
| 28 |
def raiser(exception, msg='std'): |
| 29 |
raise exception(msg) |
| 30 |
|
| 31 |
|
| 32 |
class FalseyBoolException(Exception): |
| 33 |
def __bool__(self): |
| 34 |
return False |
| 35 |
|
| 36 |
|
| 37 |
class FalseyLenException(Exception): |
| 38 |
def __len__(self): |
| 39 |
return 0 |
| 40 |
|
| 41 |
|
| 42 |
class ExecutorTest: |
| 43 |
|
| 44 |
# Executor.shutdown() and context manager usage is tested by |
| 45 |
# ExecutorShutdownTest. |
| 46 |
def test_submit(self): |
| 47 |
future = self.executor.submit(pow, 2, 8) |
| 48 |
self.assertEqual(256, future.result()) |
| 49 |
|
| 50 |
def test_submit_keyword(self): |
| 51 |
future = self.executor.submit(mul, 2, y=8) |
| 52 |
self.assertEqual(16, future.result()) |
| 53 |
future = self.executor.submit(capture, 1, self=2, fn=3) |
| 54 |
self.assertEqual(future.result(), ((1,), {'self': 2, 'fn': 3})) |
| 55 |
with self.assertRaises(TypeError): |
| 56 |
self.executor.submit(fn=capture, arg=1) |
| 57 |
with self.assertRaises(TypeError): |
| 58 |
self.executor.submit(arg=1) |
| 59 |
|
| 60 |
def test_map(self): |
| 61 |
self.assertEqual( |
| 62 |
list(self.executor.map(pow, range(10), range(10))), |
| 63 |
list(map(pow, range(10), range(10)))) |
| 64 |
|
| 65 |
self.assertEqual( |
| 66 |
list(self.executor.map(pow, range(10), range(10), chunksize=3)), |
| 67 |
list(map(pow, range(10), range(10)))) |
| 68 |
|
| 69 |
def test_map_exception(self): |
| 70 |
i = self.executor.map(divmod, [1, 1, 1, 1], [2, 3, 0, 5]) |
| 71 |
self.assertEqual(i.__next__(), (0, 1)) |
| 72 |
self.assertEqual(i.__next__(), (0, 1)) |
| 73 |
with self.assertRaises(ZeroDivisionError): |
| 74 |
i.__next__() |
| 75 |
|
| 76 |
@support.requires_resource('walltime') |
| 77 |
def test_map_timeout(self): |
| 78 |
results = [] |
| 79 |
try: |
| 80 |
for i in self.executor.map(time.sleep, |
| 81 |
[0, 0, 6], |
| 82 |
timeout=5): |
| 83 |
results.append(i) |
| 84 |
except futures.TimeoutError: |
| 85 |
pass |
| 86 |
else: |
| 87 |
self.fail('expected TimeoutError') |
| 88 |
|
| 89 |
# gh-110097: On heavily loaded systems, the launch of the worker may |
| 90 |
# take longer than the specified timeout. |
| 91 |
self.assertIn(results, ([None, None], [None], [])) |
| 92 |
|
| 93 |
def test_map_buffersize_type_validation(self): |
| 94 |
for buffersize in ("foo", 2.0): |
| 95 |
with self.subTest(buffersize=buffersize): |
| 96 |
with self.assertRaisesRegex( |
| 97 |
TypeError, |
| 98 |
"buffersize must be an integer or None", |
| 99 |
): |
| 100 |
self.executor.map(str, range(4), buffersize=buffersize) |
| 101 |
|
| 102 |
def test_map_buffersize_value_validation(self): |
| 103 |
for buffersize in (0, -1): |
| 104 |
with self.subTest(buffersize=buffersize): |
| 105 |
with self.assertRaisesRegex( |
| 106 |
ValueError, |
| 107 |
"buffersize must be None or > 0", |
| 108 |
): |
| 109 |
self.executor.map(str, range(4), buffersize=buffersize) |
| 110 |
|
| 111 |
def test_map_buffersize(self): |
| 112 |
ints = range(4) |
| 113 |
for buffersize in (1, 2, len(ints), len(ints) * 2): |
| 114 |
with self.subTest(buffersize=buffersize): |
| 115 |
res = self.executor.map(str, ints, buffersize=buffersize) |
| 116 |
self.assertListEqual(list(res), ["0", "1", "2", "3"]) |
| 117 |
|
| 118 |
def test_map_buffersize_on_multiple_iterables(self): |
| 119 |
ints = range(4) |
| 120 |
for buffersize in (1, 2, len(ints), len(ints) * 2): |
| 121 |
with self.subTest(buffersize=buffersize): |
| 122 |
res = self.executor.map(add, ints, ints, buffersize=buffersize) |
| 123 |
self.assertListEqual(list(res), [0, 2, 4, 6]) |
| 124 |
|
| 125 |
def test_map_buffersize_on_infinite_iterable(self): |
| 126 |
res = self.executor.map(str, itertools.count(), buffersize=2) |
| 127 |
self.assertEqual(next(res, None), "0") |
| 128 |
self.assertEqual(next(res, None), "1") |
| 129 |
self.assertEqual(next(res, None), "2") |
| 130 |
|
| 131 |
def test_map_buffersize_on_multiple_infinite_iterables(self): |
| 132 |
res = self.executor.map( |
| 133 |
add, |
| 134 |
itertools.count(), |
| 135 |
itertools.count(), |
| 136 |
buffersize=2 |
| 137 |
) |
| 138 |
self.assertEqual(next(res, None), 0) |
| 139 |
self.assertEqual(next(res, None), 2) |
| 140 |
self.assertEqual(next(res, None), 4) |
| 141 |
|
| 142 |
def test_map_buffersize_on_empty_iterable(self): |
| 143 |
res = self.executor.map(str, [], buffersize=2) |
| 144 |
self.assertIsNone(next(res, None)) |
| 145 |
|
| 146 |
def test_map_buffersize_without_iterable(self): |
| 147 |
res = self.executor.map(str, buffersize=2) |
| 148 |
self.assertIsNone(next(res, None)) |
| 149 |
|
| 150 |
def test_map_buffersize_when_buffer_is_full(self): |
| 151 |
ints = iter(range(4)) |
| 152 |
buffersize = 2 |
| 153 |
self.executor.map(str, ints, buffersize=buffersize) |
| 154 |
self.executor.shutdown(wait=True) # wait for tasks to complete |
| 155 |
self.assertEqual( |
| 156 |
next(ints), |
| 157 |
buffersize, |
| 158 |
msg="should have fetched only `buffersize` elements from `ints`.", |
| 159 |
) |
| 160 |
|
| 161 |
def test_shutdown_race_issue12456(self): |
| 162 |
# Issue #12456: race condition at shutdown where trying to post a |
| 163 |
# sentinel in the call queue blocks (the queue is full while processes |
| 164 |
# have exited). |
| 165 |
self.executor.map(str, [2] * (self.worker_count + 1)) |
| 166 |
self.executor.shutdown() |
| 167 |
|
| 168 |
@support.cpython_only |
| 169 |
def test_no_stale_references(self): |
| 170 |
# Issue #16284: check that the executors don't unnecessarily hang onto |
| 171 |
# references. |
| 172 |
my_object = MyObject() |
| 173 |
my_object_collected = threading.Event() |
| 174 |
def set_event(): |
| 175 |
if Py_GIL_DISABLED: |
| 176 |
# gh-117688 Avoid deadlock by setting the event in a |
| 177 |
# background thread. The current thread may be in the middle |
| 178 |
# of the my_object_collected.wait() call, which holds locks |
| 179 |
# needed by my_object_collected.set(). |
| 180 |
threading.Thread(target=my_object_collected.set).start() |
| 181 |
else: |
| 182 |
my_object_collected.set() |
| 183 |
my_object_callback = weakref.ref(my_object, lambda obj: set_event()) |
| 184 |
# Deliberately discarding the future. |
| 185 |
self.executor.submit(my_object.my_method) |
| 186 |
del my_object |
| 187 |
|
| 188 |
if Py_GIL_DISABLED: |
| 189 |
# Due to biased reference counting, my_object might only be |
| 190 |
# deallocated while the thread that created it runs -- if the |
| 191 |
# thread is paused waiting on an event, it may not merge the |
| 192 |
# refcount of the queued object. For that reason, we alternate |
| 193 |
# between running the GC and waiting for the event. |
| 194 |
wait_time = 0 |
| 195 |
collected = False |
| 196 |
while not collected and wait_time <= support.SHORT_TIMEOUT: |
| 197 |
support.gc_collect() |
| 198 |
collected = my_object_collected.wait(timeout=1.0) |
| 199 |
wait_time += 1.0 |
| 200 |
else: |
| 201 |
collected = my_object_collected.wait(timeout=support.SHORT_TIMEOUT) |
| 202 |
self.assertTrue(collected, |
| 203 |
"Stale reference not collected within timeout.") |
| 204 |
|
| 205 |
def test_max_workers_negative(self): |
| 206 |
for number in (0, -1): |
| 207 |
with self.assertRaisesRegex(ValueError, |
| 208 |
"max_workers must be greater " |
| 209 |
"than 0"): |
| 210 |
self.executor_type(max_workers=number) |
| 211 |
|
| 212 |
def test_free_reference(self): |
| 213 |
# Issue #14406: Result iterator should not keep an internal |
| 214 |
# reference to result objects. |
| 215 |
for obj in self.executor.map(make_dummy_object, range(10)): |
| 216 |
wr = weakref.ref(obj) |
| 217 |
del obj |
| 218 |
support.gc_collect() # For PyPy or other GCs. |
| 219 |
|
| 220 |
for _ in support.sleeping_retry(support.SHORT_TIMEOUT): |
| 221 |
if wr() is None: |
| 222 |
break |
| 223 |
|
| 224 |
def test_swallows_falsey_exceptions(self): |
| 225 |
# see gh-132063: Prevent exceptions that evaluate as falsey |
| 226 |
# from being ignored. |
| 227 |
# Recall: `x` is falsey if `len(x)` returns 0 or `bool(x)` returns False. |
| 228 |
|
| 229 |
msg = 'boolbool' |
| 230 |
with self.assertRaisesRegex(FalseyBoolException, msg): |
| 231 |
self.executor.submit(raiser, FalseyBoolException, msg).result() |
| 232 |
|
| 233 |
msg = 'lenlen' |
| 234 |
with self.assertRaisesRegex(FalseyLenException, msg): |
| 235 |
self.executor.submit(raiser, FalseyLenException, msg).result() |