Debian Codegraph @

Lib/test/test_concurrent_futures/test_interpreter_pool.py

raw
1 import _thread
2 import asyncio
3 import contextlib
4 import io
5 import os
6 import subprocess
7 import sys
8 import textwrap
9 import time
10 import unittest
11 from concurrent.futures.interpreter import BrokenInterpreterPool
12 from concurrent import interpreters
13 from concurrent.interpreters import _queues as queues
14 import _interpreters
15 from test import support
16 from test.support import os_helper
17 from test.support import script_helper
18 import test.test_asyncio.utils as testasyncio_utils
19
20 from .executor import ExecutorTest, mul
21 from .util import BaseTestCase, InterpreterPoolMixin, setup_module
22
23
24 WINDOWS = sys.platform.startswith('win')
25
26
27 @contextlib.contextmanager
28 def nonblocking(fd):
29 blocking = os.get_blocking(fd)
30 if blocking:
31 os.set_blocking(fd, False)
32 try:
33 yield
34 finally:
35 if blocking:
36 os.set_blocking(fd, blocking)
37
38
39 def read_file_with_timeout(fd, nbytes, timeout):
40 with nonblocking(fd):
41 end = time.time() + timeout
42 try:
43 return os.read(fd, nbytes)
44 except BlockingIOError:
45 pass
46 while time.time() < end:
47 try:
48 return os.read(fd, nbytes)
49 except BlockingIOError:
50 continue
51 else:
52 raise TimeoutError('nothing to read')
53
54
55 if not WINDOWS:
56 import select
57 def read_file_with_timeout(fd, nbytes, timeout):
58 r, _, _ = select.select([fd], [], [], timeout)
59 if fd not in r:
60 raise TimeoutError('nothing to read')
61 return os.read(fd, nbytes)
62
63
64 def noop():
65 pass
66
67
68 def write_msg(fd, msg):
69 import os
70 os.write(fd, msg + b'\0')
71
72
73 def read_msg(fd, timeout=10.0):
74 msg = b''
75 ch = read_file_with_timeout(fd, 1, timeout)
76 while ch != b'\0':
77 msg += ch
78 ch = os.read(fd, 1)
79 return msg
80
81
82 def get_current_name():
83 return __name__
84
85
86 def fail(exctype, msg=None):
87 raise exctype(msg)
88
89
90 def get_current_interpid(*extra):
91 interpid, _ = _interpreters.get_current()
92 return (interpid, *extra)
93
94
95 class InterpretersMixin(InterpreterPoolMixin):
96
97 def pipe(self):
98 r, w = os.pipe()
99 self.addCleanup(lambda: os.close(r))
100 self.addCleanup(lambda: os.close(w))
101 return r, w
102
103
104 class PickleShenanigans:
105 """Succeeds with pickle.dumps(), but fails with pickle.loads()"""
106 def __init__(self, value):
107 if value == 1:
108 raise RuntimeError("gotcha")
109
110 def __reduce__(self):
111 return (self.__class__, (1,))
112
113
114 class InterpreterPoolExecutorTest(
115 InterpretersMixin, ExecutorTest, BaseTestCase):
116
117 @unittest.expectedFailure
118 def test_init_script(self):
119 msg1 = b'step: init'
120 msg2 = b'step: run'
121 r, w = self.pipe()
122 initscript = f"""
123 import os
124 msg = {msg2!r}
125 os.write({w}, {msg1!r} + b'\\0')
126 """
127 script = f"""
128 os.write({w}, msg + b'\\0')
129 """
130 os.write(w, b'\0')
131
132 executor = self.executor_type(initializer=initscript)
133 before_init = os.read(r, 100)
134 fut = executor.submit(script)
135 after_init = read_msg(r)
136 fut.result()
137 after_run = read_msg(r)
138
139 self.assertEqual(before_init, b'\0')
140 self.assertEqual(after_init, msg1)
141 self.assertEqual(after_run, msg2)
142
143 @unittest.expectedFailure
144 def test_init_script_args(self):
145 with self.assertRaises(ValueError):
146 self.executor_type(initializer='pass', initargs=('spam',))
147
148 def test_init_func(self):
149 msg = b'step: init'
150 r, w = self.pipe()
151 os.write(w, b'\0')
152
153 executor = self.executor_type(
154 initializer=write_msg, initargs=(w, msg))
155 before = os.read(r, 100)
156 executor.submit(mul, 10, 10)
157 after = read_msg(r)
158
159 self.assertEqual(before, b'\0')
160 self.assertEqual(after, msg)
161
162 def test_init_with___main___global(self):
163 # See https://github.com/python/cpython/pull/133957#issuecomment-2927415311.
164 text = """if True:
165 from concurrent.futures import InterpreterPoolExecutor
166
167 INITIALIZER_STATUS = 'uninitialized'
168
169 def init(x):
170 global INITIALIZER_STATUS
171 INITIALIZER_STATUS = x
172 INITIALIZER_STATUS
173
174 def get_init_status():
175 return INITIALIZER_STATUS
176
177 if __name__ == "__main__":
178 exe = InterpreterPoolExecutor(initializer=init,
179 initargs=('initialized',))
180 fut = exe.submit(get_init_status)
181 print(fut.result()) # 'initialized'
182 exe.shutdown(wait=True)
183 print(INITIALIZER_STATUS) # 'uninitialized'
184 """
185 with os_helper.temp_dir() as tempdir:
186 filename = script_helper.make_script(tempdir, 'my-script', text)
187 res = script_helper.assert_python_ok(filename)
188 stdout = res.out.decode('utf-8').strip()
189 self.assertEqual(stdout.splitlines(), [
190 'initialized',
191 'uninitialized',
192 ])
193
194 def test_init_closure(self):
195 count = 0
196 def init1():
197 assert count == 0, count
198 def init2():
199 nonlocal count
200 count += 1
201
202 with contextlib.redirect_stderr(io.StringIO()) as stderr:
203 with self.executor_type(initializer=init1) as executor:
204 fut = executor.submit(lambda: None)
205 self.assertIn('NotShareableError', stderr.getvalue())
206 with self.assertRaises(BrokenInterpreterPool):
207 fut.result()
208
209 with contextlib.redirect_stderr(io.StringIO()) as stderr:
210 with self.executor_type(initializer=init2) as executor:
211 fut = executor.submit(lambda: None)
212 self.assertIn('NotShareableError', stderr.getvalue())
213 with self.assertRaises(BrokenInterpreterPool):
214 fut.result()
215
216 def test_init_instance_method(self):
217 class Spam:
218 def initializer(self):
219 raise NotImplementedError
220 spam = Spam()
221
222 with contextlib.redirect_stderr(io.StringIO()) as stderr:
223 with self.executor_type(initializer=spam.initializer) as executor:
224 fut = executor.submit(lambda: None)
225 self.assertIn('NotShareableError', stderr.getvalue())
226 with self.assertRaises(BrokenInterpreterPool):
227 fut.result()
228
229 @unittest.expectedFailure
230 def test_init_exception_in_script(self):
231 executor = self.executor_type(initializer='raise Exception("spam")')
232 with executor:
233 with contextlib.redirect_stderr(io.StringIO()) as stderr:
234 fut = executor.submit('pass')
235 with self.assertRaises(BrokenInterpreterPool):
236 fut.result()
237 stderr = stderr.getvalue()
238 self.assertIn('ExecutionFailed: Exception: spam', stderr)
239 self.assertIn('Uncaught in the interpreter:', stderr)
240 self.assertIn('The above exception was the direct cause of the following exception:',
241 stderr)
242
243 def test_init_exception_in_func(self):
244 executor = self.executor_type(initializer=fail,
245 initargs=(Exception, 'spam'))
246 with executor:
247 with contextlib.redirect_stderr(io.StringIO()) as stderr:
248 fut = executor.submit(noop)
249 with self.assertRaises(BrokenInterpreterPool):
250 fut.result()
251 stderr = stderr.getvalue()
252 self.assertIn('ExecutionFailed: Exception: spam', stderr)
253 self.assertIn('Uncaught in the interpreter:', stderr)
254
255 @unittest.expectedFailure
256 def test_submit_script(self):
257 msg = b'spam'
258 r, w = self.pipe()
259 script = f"""
260 import os
261 os.write({w}, __name__.encode('utf-8') + b'\\0')
262 """
263 executor = self.executor_type()
264
265 fut = executor.submit(script)
266 res = fut.result()
267 after = read_msg(r)
268
269 self.assertEqual(after, b'__main__')
270 self.assertIs(res, None)
271
272 def test_submit_closure(self):
273 spam = True
274 def task1():
275 return spam
276 def task2():
277 nonlocal spam
278 spam += 1
279 return spam
280
281 executor = self.executor_type()
282
283 fut = executor.submit(task1)
284 with self.assertRaises(_interpreters.NotShareableError):
285 fut.result()
286
287 fut = executor.submit(task2)
288 with self.assertRaises(_interpreters.NotShareableError):
289 fut.result()
290
291 def test_submit_local_instance(self):
292 class Spam:
293 def __init__(self):
294 self.value = True
295
296 executor = self.executor_type()
297 fut = executor.submit(Spam)
298 with self.assertRaises(_interpreters.NotShareableError):
299 fut.result()
300
301 def test_submit_instance_method(self):
302 class Spam:
303 def run(self):
304 return True
305 spam = Spam()
306
307 executor = self.executor_type()
308 fut = executor.submit(spam.run)
309 with self.assertRaises(_interpreters.NotShareableError):
310 fut.result()
311
312 def test_submit_func_globals(self):
313 executor = self.executor_type()
314 fut = executor.submit(get_current_name)
315 name = fut.result()
316
317 self.assertEqual(name, __name__)
318 self.assertNotEqual(name, '__main__')
319
320 @unittest.expectedFailure
321 def test_submit_exception_in_script(self):
322 # Scripts are not supported currently.
323 fut = self.executor.submit('raise Exception("spam")')
324 with self.assertRaises(Exception) as captured:
325 fut.result()
326 self.assertIs(type(captured.exception), Exception)
327 self.assertEqual(str(captured.exception), 'spam')
328 cause = captured.exception.__cause__
329 self.assertIs(type(cause), interpreters.ExecutionFailed)
330 for attr in ('__name__', '__qualname__', '__module__'):
331 self.assertEqual(getattr(cause.excinfo.type, attr),
332 getattr(Exception, attr))
333 self.assertEqual(cause.excinfo.msg, 'spam')
334
335 def test_submit_exception_in_func(self):
336 fut = self.executor.submit(fail, Exception, 'spam')
337 with self.assertRaises(Exception) as captured:
338 fut.result()
339 self.assertIs(type(captured.exception), Exception)
340 self.assertEqual(str(captured.exception), 'spam')
341 cause = captured.exception.__cause__
342 self.assertIs(type(cause), interpreters.ExecutionFailed)
343 for attr in ('__name__', '__qualname__', '__module__'):
344 self.assertEqual(getattr(cause.excinfo.type, attr),
345 getattr(Exception, attr))
346 self.assertEqual(cause.excinfo.msg, 'spam')
347
348 def test_saturation(self):
349 blocker = queues.create()
350 executor = self.executor_type(4)
351
352 for i in range(15 * executor._max_workers):
353 executor.submit(blocker.get)
354 self.assertEqual(len(executor._threads), executor._max_workers)
355 for i in range(15 * executor._max_workers):
356 blocker.put_nowait(None)
357 executor.shutdown(wait=True)
358
359 def test_blocking(self):
360 # There is no guarantee that a worker will be created for every
361 # submitted task. That's because there's a race between:
362 #
363 # * a new worker thread, created when task A was just submitted,
364 # becoming non-idle when it picks up task A
365 # * after task B is added to the queue, a new worker thread
366 # is started only if there are no idle workers
367 # (the check in ThreadPoolExecutor._adjust_thread_count())
368 #
369 # That means we must not block waiting for *all* tasks to report
370 # "ready" before we unblock the known-ready workers.
371 ready = queues.create()
372 blocker = queues.create()
373
374 def run(taskid, ready, blocker):
375 # There can't be any globals here.
376 ready.put_nowait(taskid)
377 blocker.get() # blocking
378
379 numtasks = 10
380 futures = []
381 with self.executor_type() as executor:
382 # Request the jobs.
383 for i in range(numtasks):
384 fut = executor.submit(run, i, ready, blocker)
385 futures.append(fut)
386 pending = numtasks
387 while pending > 0:
388 # Wait for any to be ready.
389 done = 0
390 for _ in range(pending):
391 try:
392 ready.get(timeout=1) # blocking
393 except interpreters.QueueEmpty:
394 pass
395 else:
396 done += 1
397 pending -= done
398 # Unblock the workers.
399 for _ in range(done):
400 blocker.put_nowait(None)
401
402 def test_blocking_with_limited_workers(self):
403 # This is essentially the same as test_blocking,
404 # but we explicitly force a limited number of workers,
405 # instead of it happening implicitly sometimes due to a race.
406 ready = queues.create()
407 blocker = queues.create()
408
409 def run(taskid, ready, blocker):
410 # There can't be any globals here.
411 ready.put_nowait(taskid)
412 blocker.get() # blocking
413
414 numtasks = 10
415 futures = []
416 with self.executor_type(4) as executor:
417 # Request the jobs.
418 for i in range(numtasks):
419 fut = executor.submit(run, i, ready, blocker)
420 futures.append(fut)
421 pending = numtasks
422 while pending > 0:
423 # Wait for any to be ready.
424 done = 0
425 for _ in range(pending):
426 try:
427 ready.get(timeout=1) # blocking
428 except interpreters.QueueEmpty:
429 pass
430 else:
431 done += 1
432 pending -= done
433 # Unblock the workers.
434 for _ in range(done):
435 blocker.put_nowait(None)
436
437 @support.requires_gil_enabled("gh-117344: test is flaky without the GIL")
438 def test_idle_thread_reuse(self):
439 executor = self.executor_type()
440 executor.submit(mul, 21, 2).result()
441 executor.submit(mul, 6, 7).result()
442 executor.submit(mul, 3, 14).result()
443 self.assertEqual(len(executor._threads), 1)
444 executor.shutdown(wait=True)
445
446 def test_pickle_errors_propagate(self):
447 # GH-125864: Pickle errors happen before the script tries to execute,
448 # so the queue used to wait infinitely.
449 fut = self.executor.submit(PickleShenanigans(0))
450 expected = interpreters.NotShareableError
451 with self.assertRaisesRegex(expected, 'args not shareable') as cm:
452 fut.result()
453 self.assertRegex(str(cm.exception.__cause__), 'unpickled')
454
455 def test_no_stale_references(self):
456 # Weak references don't cross between interpreters.
457 raise unittest.SkipTest('not applicable')
458
459 def test_free_reference(self):
460 # Weak references don't cross between interpreters.
461 raise unittest.SkipTest('not applicable')
462
463 @support.requires_subprocess()
464 def test_import_interpreter_pool_executor(self):
465 # Test the import behavior normally if _interpreters is unavailable.
466 code = textwrap.dedent("""
467 import sys
468 # Set it to None to emulate the case when _interpreter is unavailable.
469 sys.modules['_interpreters'] = None
470 from concurrent import futures
471
472 try:
473 futures.InterpreterPoolExecutor
474 except AttributeError:
475 pass
476 else:
477 print('AttributeError not raised!', file=sys.stderr)
478 sys.exit(1)
479
480 try:
481 from concurrent.futures import InterpreterPoolExecutor
482 except ImportError:
483 pass
484 else:
485 print('ImportError not raised!', file=sys.stderr)
486 sys.exit(1)
487
488 from concurrent.futures import *
489
490 if 'InterpreterPoolExecutor' in globals():
491 print('InterpreterPoolExecutor should not be imported!',
492 file=sys.stderr)
493 sys.exit(1)
494 """)
495
496 cmd = [sys.executable, '-c', code]
497 p = subprocess.run(cmd, capture_output=True)
498 self.assertEqual(p.returncode, 0, p.stderr.decode())
499 self.assertEqual(p.stdout.decode(), '')
500 self.assertEqual(p.stderr.decode(), '')
501
502 def test_thread_name_prefix(self):
503 self.assertStartsWith(self.executor._thread_name_prefix,
504 "InterpreterPoolExecutor-")
505
506 @unittest.skipUnless(hasattr(_thread, '_get_name'), "missing _thread._get_name")
507 def test_thread_name_prefix_with_thread_get_name(self):
508 def get_thread_name():
509 import _thread
510 return _thread._get_name()
511
512 # Some platforms (Linux) are using 16 bytes to store the thread name,
513 # so only compare the first 15 bytes (without the trailing \n).
514 self.assertStartsWith(self.executor.submit(get_thread_name).result(),
515 "InterpreterPoolExecutor-"[:15])
516
517 class AsyncioTest(InterpretersMixin, testasyncio_utils.TestCase):
518
519 @classmethod
520 def setUpClass(cls):
521 # Most uses of asyncio will implicitly call set_event_loop_policy()
522 # with the default policy if a policy hasn't been set already.
523 # If that happens in a test, like here, we'll end up with a failure
524 # when --fail-env-changed is used. That's why the other tests that
525 # use asyncio are careful to set the policy back to None and why
526 # we're careful to do so here. We also validate that no other
527 # tests left a policy in place, just in case.
528 policy = support.maybe_get_event_loop_policy()
529 assert policy is None, policy
530 cls.addClassCleanup(lambda: asyncio.events._set_event_loop_policy(None))
531
532 def setUp(self):
533 super().setUp()
534 self.loop = asyncio.new_event_loop()
535 self.set_event_loop(self.loop)
536
537 self.executor = self.executor_type()
538 self.addCleanup(lambda: self.executor.shutdown())
539
540 def tearDown(self):
541 if not self.loop.is_closed():
542 testasyncio_utils.run_briefly(self.loop)
543
544 self.doCleanups()
545 support.gc_collect()
546 super().tearDown()
547
548 def test_run_in_executor(self):
549 unexpected, _ = _interpreters.get_current()
550
551 func = get_current_interpid
552 fut = self.loop.run_in_executor(self.executor, func, 'yo')
553 interpid, res = self.loop.run_until_complete(fut)
554
555 self.assertEqual(res, 'yo')
556 self.assertNotEqual(interpid, unexpected)
557
558 def test_run_in_executor_cancel(self):
559 executor = self.executor_type()
560
561 called = False
562
563 def patched_call_soon(*args):
564 nonlocal called
565 called = True
566
567 func = time.sleep
568 fut = self.loop.run_in_executor(self.executor, func, 0.05)
569 fut.cancel()
570 self.loop.run_until_complete(
571 self.loop.shutdown_default_executor())
572 self.loop.close()
573 self.loop.call_soon = patched_call_soon
574 self.loop.call_soon_threadsafe = patched_call_soon
575 time.sleep(0.4)
576 self.assertFalse(called)
577
578 def test_default_executor(self):
579 unexpected, _ = _interpreters.get_current()
580
581 self.loop.set_default_executor(self.executor)
582 fut = self.loop.run_in_executor(None, get_current_interpid)
583 interpid, = self.loop.run_until_complete(fut)
584
585 self.assertNotEqual(interpid, unexpected)
586
587
588 def setUpModule():
589 setup_module()
590
591
592 if __name__ == "__main__":
593 unittest.main()