Debian Codegraph @

Lib/test/test_concurrent_futures/executor.py

raw
1 import itertools
2 import threading
3 import time
4 import weakref
5 from concurrent import futures
6 from operator import add
7 from test import support
8 from test.support import Py_GIL_DISABLED
9
10
11 def mul(x, y):
12 return x * y
13
14 def capture(*args, **kwargs):
15 return args, kwargs
16
17
18 class MyObject(object):
19 def my_method(self):
20 pass
21
22
23 def make_dummy_object(_):
24 return MyObject()
25
26
27 # Used in test_swallows_falsey_exceptions
28 def raiser(exception, msg='std'):
29 raise exception(msg)
30
31
32 class FalseyBoolException(Exception):
33 def __bool__(self):
34 return False
35
36
37 class FalseyLenException(Exception):
38 def __len__(self):
39 return 0
40
41
42 class ExecutorTest:
43
44 # Executor.shutdown() and context manager usage is tested by
45 # ExecutorShutdownTest.
46 def test_submit(self):
47 future = self.executor.submit(pow, 2, 8)
48 self.assertEqual(256, future.result())
49
50 def test_submit_keyword(self):
51 future = self.executor.submit(mul, 2, y=8)
52 self.assertEqual(16, future.result())
53 future = self.executor.submit(capture, 1, self=2, fn=3)
54 self.assertEqual(future.result(), ((1,), {'self': 2, 'fn': 3}))
55 with self.assertRaises(TypeError):
56 self.executor.submit(fn=capture, arg=1)
57 with self.assertRaises(TypeError):
58 self.executor.submit(arg=1)
59
60 def test_map(self):
61 self.assertEqual(
62 list(self.executor.map(pow, range(10), range(10))),
63 list(map(pow, range(10), range(10))))
64
65 self.assertEqual(
66 list(self.executor.map(pow, range(10), range(10), chunksize=3)),
67 list(map(pow, range(10), range(10))))
68
69 def test_map_exception(self):
70 i = self.executor.map(divmod, [1, 1, 1, 1], [2, 3, 0, 5])
71 self.assertEqual(i.__next__(), (0, 1))
72 self.assertEqual(i.__next__(), (0, 1))
73 with self.assertRaises(ZeroDivisionError):
74 i.__next__()
75
76 @support.requires_resource('walltime')
77 def test_map_timeout(self):
78 results = []
79 try:
80 for i in self.executor.map(time.sleep,
81 [0, 0, 6],
82 timeout=5):
83 results.append(i)
84 except futures.TimeoutError:
85 pass
86 else:
87 self.fail('expected TimeoutError')
88
89 # gh-110097: On heavily loaded systems, the launch of the worker may
90 # take longer than the specified timeout.
91 self.assertIn(results, ([None, None], [None], []))
92
93 def test_map_buffersize_type_validation(self):
94 for buffersize in ("foo", 2.0):
95 with self.subTest(buffersize=buffersize):
96 with self.assertRaisesRegex(
97 TypeError,
98 "buffersize must be an integer or None",
99 ):
100 self.executor.map(str, range(4), buffersize=buffersize)
101
102 def test_map_buffersize_value_validation(self):
103 for buffersize in (0, -1):
104 with self.subTest(buffersize=buffersize):
105 with self.assertRaisesRegex(
106 ValueError,
107 "buffersize must be None or > 0",
108 ):
109 self.executor.map(str, range(4), buffersize=buffersize)
110
111 def test_map_buffersize(self):
112 ints = range(4)
113 for buffersize in (1, 2, len(ints), len(ints) * 2):
114 with self.subTest(buffersize=buffersize):
115 res = self.executor.map(str, ints, buffersize=buffersize)
116 self.assertListEqual(list(res), ["0", "1", "2", "3"])
117
118 def test_map_buffersize_on_multiple_iterables(self):
119 ints = range(4)
120 for buffersize in (1, 2, len(ints), len(ints) * 2):
121 with self.subTest(buffersize=buffersize):
122 res = self.executor.map(add, ints, ints, buffersize=buffersize)
123 self.assertListEqual(list(res), [0, 2, 4, 6])
124
125 def test_map_buffersize_on_infinite_iterable(self):
126 res = self.executor.map(str, itertools.count(), buffersize=2)
127 self.assertEqual(next(res, None), "0")
128 self.assertEqual(next(res, None), "1")
129 self.assertEqual(next(res, None), "2")
130
131 def test_map_buffersize_on_multiple_infinite_iterables(self):
132 res = self.executor.map(
133 add,
134 itertools.count(),
135 itertools.count(),
136 buffersize=2
137 )
138 self.assertEqual(next(res, None), 0)
139 self.assertEqual(next(res, None), 2)
140 self.assertEqual(next(res, None), 4)
141
142 def test_map_buffersize_on_empty_iterable(self):
143 res = self.executor.map(str, [], buffersize=2)
144 self.assertIsNone(next(res, None))
145
146 def test_map_buffersize_without_iterable(self):
147 res = self.executor.map(str, buffersize=2)
148 self.assertIsNone(next(res, None))
149
150 def test_map_buffersize_when_buffer_is_full(self):
151 ints = iter(range(4))
152 buffersize = 2
153 self.executor.map(str, ints, buffersize=buffersize)
154 self.executor.shutdown(wait=True) # wait for tasks to complete
155 self.assertEqual(
156 next(ints),
157 buffersize,
158 msg="should have fetched only `buffersize` elements from `ints`.",
159 )
160
161 def test_shutdown_race_issue12456(self):
162 # Issue #12456: race condition at shutdown where trying to post a
163 # sentinel in the call queue blocks (the queue is full while processes
164 # have exited).
165 self.executor.map(str, [2] * (self.worker_count + 1))
166 self.executor.shutdown()
167
168 @support.cpython_only
169 def test_no_stale_references(self):
170 # Issue #16284: check that the executors don't unnecessarily hang onto
171 # references.
172 my_object = MyObject()
173 my_object_collected = threading.Event()
174 def set_event():
175 if Py_GIL_DISABLED:
176 # gh-117688 Avoid deadlock by setting the event in a
177 # background thread. The current thread may be in the middle
178 # of the my_object_collected.wait() call, which holds locks
179 # needed by my_object_collected.set().
180 threading.Thread(target=my_object_collected.set).start()
181 else:
182 my_object_collected.set()
183 my_object_callback = weakref.ref(my_object, lambda obj: set_event())
184 # Deliberately discarding the future.
185 self.executor.submit(my_object.my_method)
186 del my_object
187
188 if Py_GIL_DISABLED:
189 # Due to biased reference counting, my_object might only be
190 # deallocated while the thread that created it runs -- if the
191 # thread is paused waiting on an event, it may not merge the
192 # refcount of the queued object. For that reason, we alternate
193 # between running the GC and waiting for the event.
194 wait_time = 0
195 collected = False
196 while not collected and wait_time <= support.SHORT_TIMEOUT:
197 support.gc_collect()
198 collected = my_object_collected.wait(timeout=1.0)
199 wait_time += 1.0
200 else:
201 collected = my_object_collected.wait(timeout=support.SHORT_TIMEOUT)
202 self.assertTrue(collected,
203 "Stale reference not collected within timeout.")
204
205 def test_max_workers_negative(self):
206 for number in (0, -1):
207 with self.assertRaisesRegex(ValueError,
208 "max_workers must be greater "
209 "than 0"):
210 self.executor_type(max_workers=number)
211
212 def test_free_reference(self):
213 # Issue #14406: Result iterator should not keep an internal
214 # reference to result objects.
215 for obj in self.executor.map(make_dummy_object, range(10)):
216 wr = weakref.ref(obj)
217 del obj
218 support.gc_collect() # For PyPy or other GCs.
219
220 for _ in support.sleeping_retry(support.SHORT_TIMEOUT):
221 if wr() is None:
222 break
223
224 def test_swallows_falsey_exceptions(self):
225 # see gh-132063: Prevent exceptions that evaluate as falsey
226 # from being ignored.
227 # Recall: `x` is falsey if `len(x)` returns 0 or `bool(x)` returns False.
228
229 msg = 'boolbool'
230 with self.assertRaisesRegex(FalseyBoolException, msg):
231 self.executor.submit(raiser, FalseyBoolException, msg).result()
232
233 msg = 'lenlen'
234 with self.assertRaisesRegex(FalseyLenException, msg):
235 self.executor.submit(raiser, FalseyLenException, msg).result()