Debian Codegraph @

Lib/test/_test_multiprocessing.py

raw
1 #
2 # Unit tests for the multiprocessing package
3 #
4
5 import unittest
6 import unittest.mock
7 import queue as pyqueue
8 import textwrap
9 import time
10 import io
11 import itertools
12 import sys
13 import os
14 import gc
15 import importlib
16 import errno
17 import functools
18 import signal
19 import array
20 import collections.abc
21 import socket
22 import random
23 import logging
24 import shutil
25 import subprocess
26 import struct
27 import tempfile
28 import operator
29 import pickle
30 import weakref
31 import warnings
32 import test.support
33 import test.support.script_helper
34 from test import support
35 from test.support import hashlib_helper
36 from test.support import import_helper
37 from test.support import os_helper
38 from test.support import script_helper
39 from test.support import socket_helper
40 from test.support import threading_helper
41 from test.support import warnings_helper
42 from test.support import subTests
43
44 # Skip tests if _multiprocessing wasn't built.
45 _multiprocessing = import_helper.import_module('_multiprocessing')
46 # Skip tests if sem_open implementation is broken.
47 support.skip_if_broken_multiprocessing_synchronize()
48 import threading
49
50 import multiprocessing.connection
51 import multiprocessing.dummy
52 import multiprocessing.heap
53 import multiprocessing.managers
54 import multiprocessing.pool
55 import multiprocessing.queues
56 from multiprocessing.connection import wait
57
58 from multiprocessing import util
59
60 try:
61 from multiprocessing import reduction
62 HAS_REDUCTION = reduction.HAVE_SEND_HANDLE
63 except ImportError:
64 HAS_REDUCTION = False
65
66 try:
67 from multiprocessing.sharedctypes import Value, copy
68 HAS_SHAREDCTYPES = True
69 except ImportError:
70 HAS_SHAREDCTYPES = False
71
72 try:
73 from multiprocessing import shared_memory
74 HAS_SHMEM = True
75 except ImportError:
76 HAS_SHMEM = False
77
78 try:
79 import msvcrt
80 except ImportError:
81 msvcrt = None
82
83
84 if support.HAVE_ASAN_FORK_BUG:
85 # gh-89363: Skip multiprocessing tests if Python is built with ASAN to
86 # work around a libasan race condition: dead lock in pthread_create().
87 raise unittest.SkipTest("libasan has a pthread_create() dead lock related to thread+fork")
88
89
90 # gh-110666: Tolerate a difference of 100 ms when comparing timings
91 # (clock resolution)
92 CLOCK_RES = 0.100
93
94
95 def latin(s):
96 return s.encode('latin')
97
98
99 def close_queue(queue):
100 if isinstance(queue, multiprocessing.queues.Queue):
101 queue.close()
102 queue.join_thread()
103
104
105 def join_process(process):
106 # Since multiprocessing.Process has the same API than threading.Thread
107 # (join() and is_alive(), the support function can be reused
108 threading_helper.join_thread(process)
109
110
111 if os.name == "posix":
112 from multiprocessing import resource_tracker
113
114 def _resource_unlink(name, rtype):
115 resource_tracker._CLEANUP_FUNCS[rtype](name)
116
117
118 #
119 # Constants
120 #
121
122 LOG_LEVEL = util.SUBWARNING
123 #LOG_LEVEL = logging.DEBUG
124
125 DELTA = 0.1
126 CHECK_TIMINGS = False # making true makes tests take a lot longer
127 # and can sometimes cause some non-serious
128 # failures because some calls block a bit
129 # longer than expected
130 if CHECK_TIMINGS:
131 TIMEOUT1, TIMEOUT2, TIMEOUT3 = 0.82, 0.35, 1.4
132 else:
133 TIMEOUT1, TIMEOUT2, TIMEOUT3 = 0.1, 0.1, 0.1
134
135 # BaseManager.shutdown_timeout
136 SHUTDOWN_TIMEOUT = support.SHORT_TIMEOUT
137
138 WAIT_ACTIVE_CHILDREN_TIMEOUT = 5.0
139
140 HAVE_GETVALUE = not getattr(_multiprocessing,
141 'HAVE_BROKEN_SEM_GETVALUE', False)
142
143 WIN32 = (sys.platform == "win32")
144
145 def wait_for_handle(handle, timeout):
146 if timeout is not None and timeout < 0.0:
147 timeout = None
148 return wait([handle], timeout)
149
150 try:
151 MAXFD = os.sysconf("SC_OPEN_MAX")
152 except:
153 MAXFD = 256
154
155 # To speed up tests when using the forkserver, we can preload these:
156 PRELOAD = ['__main__', 'test.test_multiprocessing_forkserver']
157
158 #
159 # Some tests require ctypes
160 #
161
162 try:
163 from ctypes import Structure, c_int, c_double, c_longlong
164 except ImportError:
165 Structure = object
166 c_int = c_double = c_longlong = None
167
168
169 def check_enough_semaphores():
170 """Check that the system supports enough semaphores to run the test."""
171 # minimum number of semaphores available according to POSIX
172 nsems_min = 256
173 try:
174 nsems = os.sysconf("SC_SEM_NSEMS_MAX")
175 except (AttributeError, ValueError):
176 # sysconf not available or setting not available
177 return
178 if nsems == -1 or nsems >= nsems_min:
179 return
180 raise unittest.SkipTest("The OS doesn't support enough semaphores "
181 "to run the test (required: %d)." % nsems_min)
182
183
184 def only_run_in_spawn_testsuite(reason):
185 """Returns a decorator: raises SkipTest when SM != spawn at test time.
186
187 This can be useful to save overall Python test suite execution time.
188 "spawn" is the universal mode available on all platforms so this limits the
189 decorated test to only execute within test_multiprocessing_spawn.
190
191 This would not be necessary if we refactored our test suite to split things
192 into other test files when they are not start method specific to be rerun
193 under all start methods.
194 """
195
196 def decorator(test_item):
197
198 @functools.wraps(test_item)
199 def spawn_check_wrapper(*args, **kwargs):
200 if (start_method := multiprocessing.get_start_method()) != "spawn":
201 raise unittest.SkipTest(f"{start_method=}, not 'spawn'; {reason}")
202 return test_item(*args, **kwargs)
203
204 return spawn_check_wrapper
205
206 return decorator
207
208
209 def only_run_in_forkserver_testsuite(reason):
210 """Returns a decorator: raises SkipTest unless fork is supported
211 and the current start method is forkserver.
212
213 Combines @support.requires_fork() with the single-run semantics of
214 only_run_in_spawn_testsuite(), but uses the forkserver testsuite as
215 the single-run target. Appropriate for tests that exercise
216 os.fork() directly (raw fork or mp.set_start_method("fork") in a
217 subprocess) and don't vary by start method, since forkserver is
218 only available on platforms that support fork.
219 """
220
221 def decorator(test_item):
222
223 @functools.wraps(test_item)
224 def forkserver_check_wrapper(*args, **kwargs):
225 if not support.has_fork_support:
226 raise unittest.SkipTest("requires working os.fork()")
227 if (start_method := multiprocessing.get_start_method()) != "forkserver":
228 raise unittest.SkipTest(
229 f"{start_method=}, not 'forkserver'; {reason}")
230 return test_item(*args, **kwargs)
231
232 return forkserver_check_wrapper
233
234 return decorator
235
236
237 class TestInternalDecorators(unittest.TestCase):
238 """Logic within a test suite that could errantly skip tests? Test it!"""
239
240 @support.requires_fork()
241 def test_only_run_in_spawn_testsuite(self):
242 if multiprocessing.get_start_method() != "spawn":
243 raise unittest.SkipTest("only run in test_multiprocessing_spawn.")
244
245 try:
246 @only_run_in_spawn_testsuite("testing this decorator")
247 def return_four_if_spawn():
248 return 4
249 except Exception as err:
250 self.fail(f"expected decorated `def` not to raise; caught {err}")
251
252 orig_start_method = multiprocessing.get_start_method(allow_none=True)
253 try:
254 multiprocessing.set_start_method("spawn", force=True)
255 self.assertEqual(return_four_if_spawn(), 4)
256 multiprocessing.set_start_method("fork", force=True)
257 with self.assertRaises(unittest.SkipTest) as ctx:
258 return_four_if_spawn()
259 self.assertIn("testing this decorator", str(ctx.exception))
260 self.assertIn("start_method=", str(ctx.exception))
261 finally:
262 multiprocessing.set_start_method(orig_start_method, force=True)
263
264 @support.requires_fork()
265 def test_only_run_in_forkserver_testsuite(self):
266 if multiprocessing.get_start_method() != "forkserver":
267 raise unittest.SkipTest("only run in test_multiprocessing_forkserver.")
268
269 try:
270 @only_run_in_forkserver_testsuite("testing this decorator")
271 def return_four_if_forkserver():
272 return 4
273 except Exception as err:
274 self.fail(f"expected decorated `def` not to raise; caught {err}")
275
276 orig_start_method = multiprocessing.get_start_method(allow_none=True)
277 try:
278 multiprocessing.set_start_method("forkserver", force=True)
279 self.assertEqual(return_four_if_forkserver(), 4)
280 multiprocessing.set_start_method("spawn", force=True)
281 with self.assertRaises(unittest.SkipTest) as ctx:
282 return_four_if_forkserver()
283 self.assertIn("testing this decorator", str(ctx.exception))
284 self.assertIn("start_method=", str(ctx.exception))
285 finally:
286 multiprocessing.set_start_method(orig_start_method, force=True)
287
288
289 #
290 # Creates a wrapper for a function which records the time it takes to finish
291 #
292
293 class TimingWrapper(object):
294
295 def __init__(self, func):
296 self.func = func
297 self.elapsed = None
298
299 def __call__(self, *args, **kwds):
300 t = time.monotonic()
301 try:
302 return self.func(*args, **kwds)
303 finally:
304 self.elapsed = time.monotonic() - t
305
306 #
307 # Base class for test cases
308 #
309
310 class BaseTestCase(object):
311
312 ALLOWED_TYPES = ('processes', 'manager', 'threads')
313 # If not empty, limit which start method suites run this class.
314 START_METHODS: set[str] = set()
315 start_method = None # set by install_tests_in_module_dict()
316
317 def assertTimingAlmostEqual(self, a, b):
318 if CHECK_TIMINGS:
319 self.assertAlmostEqual(a, b, 1)
320
321 def assertReturnsIfImplemented(self, value, func, *args):
322 try:
323 res = func(*args)
324 except NotImplementedError:
325 pass
326 else:
327 return self.assertEqual(value, res)
328
329 # For the sanity of Windows users, rather than crashing or freezing in
330 # multiple ways.
331 def __reduce__(self, *args):
332 raise NotImplementedError("shouldn't try to pickle a test case")
333
334 __reduce_ex__ = __reduce__
335
336 #
337 # Return the value of a semaphore
338 #
339
340 def get_value(self):
341 try:
342 return self.get_value()
343 except AttributeError:
344 try:
345 return self._Semaphore__value
346 except AttributeError:
347 try:
348 return self._value
349 except AttributeError:
350 raise NotImplementedError
351
352 #
353 # Testcases
354 #
355
356 class DummyCallable:
357 def __call__(self, q, c):
358 assert isinstance(c, DummyCallable)
359 q.put(5)
360
361
362 class _TestProcess(BaseTestCase):
363
364 ALLOWED_TYPES = ('processes', 'threads')
365
366 def test_current(self):
367 if self.TYPE == 'threads':
368 self.skipTest('test not appropriate for {}'.format(self.TYPE))
369
370 current = self.current_process()
371 authkey = current.authkey
372
373 self.assertTrue(current.is_alive())
374 self.assertFalse(current.daemon)
375 self.assertIsInstance(authkey, bytes)
376 self.assertTrue(len(authkey) > 0)
377 self.assertEqual(current.ident, os.getpid())
378 self.assertEqual(current.exitcode, None)
379
380 def test_set_executable(self):
381 if self.TYPE == 'threads':
382 self.skipTest(f'test not appropriate for {self.TYPE}')
383 paths = [
384 sys.executable, # str
385 os.fsencode(sys.executable), # bytes
386 os_helper.FakePath(sys.executable), # os.PathLike
387 os_helper.FakePath(os.fsencode(sys.executable)), # os.PathLike bytes
388 ]
389 for path in paths:
390 self.set_executable(path)
391 p = self.Process()
392 p.start()
393 p.join()
394 self.assertEqual(p.exitcode, 0)
395
396 @support.requires_resource('cpu')
397 def test_args_argument(self):
398 # bpo-45735: Using list or tuple as *args* in constructor could
399 # achieve the same effect.
400 args_cases = (1, "str", [1], (1,))
401 args_types = (list, tuple)
402
403 test_cases = itertools.product(args_cases, args_types)
404
405 for args, args_type in test_cases:
406 with self.subTest(args=args, args_type=args_type):
407 q = self.Queue(1)
408 # pass a tuple or list as args
409 p = self.Process(target=self._test_args, args=args_type((q, args)))
410 p.daemon = True
411 p.start()
412 child_args = q.get()
413 self.assertEqual(child_args, args)
414 p.join()
415 close_queue(q)
416
417 @classmethod
418 def _test_args(cls, q, arg):
419 q.put(arg)
420
421 def test_daemon_argument(self):
422 if self.TYPE == "threads":
423 self.skipTest('test not appropriate for {}'.format(self.TYPE))
424
425 # By default uses the current process's daemon flag.
426 proc0 = self.Process(target=self._test)
427 self.assertEqual(proc0.daemon, self.current_process().daemon)
428 proc1 = self.Process(target=self._test, daemon=True)
429 self.assertTrue(proc1.daemon)
430 proc2 = self.Process(target=self._test, daemon=False)
431 self.assertFalse(proc2.daemon)
432
433 @classmethod
434 def _test(cls, q, *args, **kwds):
435 current = cls.current_process()
436 q.put(args)
437 q.put(kwds)
438 q.put(current.name)
439 if cls.TYPE != 'threads':
440 q.put(bytes(current.authkey))
441 q.put(current.pid)
442
443 def test_parent_process_attributes(self):
444 if self.TYPE == "threads":
445 self.skipTest('test not appropriate for {}'.format(self.TYPE))
446
447 self.assertIsNone(self.parent_process())
448
449 rconn, wconn = self.Pipe(duplex=False)
450 p = self.Process(target=self._test_send_parent_process, args=(wconn,))
451 p.start()
452 p.join()
453 parent_pid, parent_name = rconn.recv()
454 self.assertEqual(parent_pid, self.current_process().pid)
455 self.assertEqual(parent_pid, os.getpid())
456 self.assertEqual(parent_name, self.current_process().name)
457
458 @classmethod
459 def _test_send_parent_process(cls, wconn):
460 from multiprocessing.process import parent_process
461 wconn.send([parent_process().pid, parent_process().name])
462
463 def test_parent_process(self):
464 if self.TYPE == "threads":
465 self.skipTest('test not appropriate for {}'.format(self.TYPE))
466
467 # Launch a child process. Make it launch a grandchild process. Kill the
468 # child process and make sure that the grandchild notices the death of
469 # its parent (a.k.a the child process).
470 rconn, wconn = self.Pipe(duplex=False)
471 p = self.Process(
472 target=self._test_create_grandchild_process, args=(wconn, ))
473 p.start()
474
475 if not rconn.poll(timeout=support.LONG_TIMEOUT):
476 raise AssertionError("Could not communicate with child process")
477 parent_process_status = rconn.recv()
478 self.assertEqual(parent_process_status, "alive")
479
480 p.terminate()
481 p.join()
482
483 if not rconn.poll(timeout=support.LONG_TIMEOUT):
484 raise AssertionError("Could not communicate with child process")
485 parent_process_status = rconn.recv()
486 self.assertEqual(parent_process_status, "not alive")
487
488 @classmethod
489 def _test_create_grandchild_process(cls, wconn):
490 p = cls.Process(target=cls._test_report_parent_status, args=(wconn, ))
491 p.start()
492 time.sleep(300)
493
494 @classmethod
495 def _test_report_parent_status(cls, wconn):
496 from multiprocessing.process import parent_process
497 wconn.send("alive" if parent_process().is_alive() else "not alive")
498 parent_process().join(timeout=support.SHORT_TIMEOUT)
499 wconn.send("alive" if parent_process().is_alive() else "not alive")
500
501 def test_process(self):
502 q = self.Queue(1)
503 e = self.Event()
504 args = (q, 1, 2)
505 kwargs = {'hello':23, 'bye':2.54}
506 name = 'SomeProcess'
507 p = self.Process(
508 target=self._test, args=args, kwargs=kwargs, name=name
509 )
510 p.daemon = True
511 current = self.current_process()
512
513 if self.TYPE != 'threads':
514 self.assertEqual(p.authkey, current.authkey)
515 self.assertEqual(p.is_alive(), False)
516 self.assertEqual(p.daemon, True)
517 self.assertNotIn(p, self.active_children())
518 self.assertIs(type(self.active_children()), list)
519 self.assertEqual(p.exitcode, None)
520
521 p.start()
522
523 self.assertEqual(p.exitcode, None)
524 self.assertEqual(p.is_alive(), True)
525 self.assertIn(p, self.active_children())
526
527 self.assertEqual(q.get(), args[1:])
528 self.assertEqual(q.get(), kwargs)
529 self.assertEqual(q.get(), p.name)
530 if self.TYPE != 'threads':
531 self.assertEqual(q.get(), current.authkey)
532 self.assertEqual(q.get(), p.pid)
533
534 p.join()
535
536 self.assertEqual(p.exitcode, 0)
537 self.assertEqual(p.is_alive(), False)
538 self.assertNotIn(p, self.active_children())
539 close_queue(q)
540
541 @unittest.skipUnless(threading._HAVE_THREAD_NATIVE_ID, "needs native_id")
542 def test_process_mainthread_native_id(self):
543 if self.TYPE == 'threads':
544 self.skipTest('test not appropriate for {}'.format(self.TYPE))
545
546 current_mainthread_native_id = threading.main_thread().native_id
547
548 q = self.Queue(1)
549 p = self.Process(target=self._test_process_mainthread_native_id, args=(q,))
550 p.start()
551
552 child_mainthread_native_id = q.get()
553 p.join()
554 close_queue(q)
555
556 self.assertNotEqual(current_mainthread_native_id, child_mainthread_native_id)
557
558 @classmethod
559 def _test_process_mainthread_native_id(cls, q):
560 mainthread_native_id = threading.main_thread().native_id
561 q.put(mainthread_native_id)
562
563 @classmethod
564 def _sleep_some(cls):
565 time.sleep(100)
566
567 @classmethod
568 def _sleep_some_event(cls, event):
569 event.set()
570 time.sleep(100)
571
572 @classmethod
573 def _sleep_no_int_handler(cls, event):
574 signal.signal(signal.SIGINT, signal.SIG_DFL)
575 cls._sleep_some_event(event)
576
577 @classmethod
578 def _test_sleep(cls, delay):
579 time.sleep(delay)
580
581 def _kill_process(self, meth, target=None):
582 if self.TYPE == 'threads':
583 self.skipTest('test not appropriate for {}'.format(self.TYPE))
584
585 event = self.Event()
586 if not target:
587 target = self._sleep_some_event
588 p = self.Process(target=target, args=(event,))
589 p.daemon = True
590 p.start()
591
592 self.assertEqual(p.is_alive(), True)
593 self.assertIn(p, self.active_children())
594 self.assertEqual(p.exitcode, None)
595
596 join = TimingWrapper(p.join)
597
598 self.assertEqual(join(0), None)
599 self.assertTimingAlmostEqual(join.elapsed, 0.0)
600 self.assertEqual(p.is_alive(), True)
601
602 self.assertEqual(join(-1), None)
603 self.assertTimingAlmostEqual(join.elapsed, 0.0)
604 self.assertEqual(p.is_alive(), True)
605
606 timeout = support.SHORT_TIMEOUT
607 if not event.wait(timeout):
608 p.terminate()
609 p.join()
610 self.fail(f"event not signaled in {timeout} seconds")
611
612 meth(p)
613
614 if hasattr(signal, 'alarm'):
615 # On the Gentoo buildbot waitpid() often seems to block forever.
616 # We use alarm() to interrupt it if it blocks for too long.
617 def handler(*args):
618 raise RuntimeError('join took too long: %s' % p)
619 old_handler = signal.signal(signal.SIGALRM, handler)
620 try:
621 signal.alarm(10)
622 self.assertEqual(join(), None)
623 finally:
624 signal.alarm(0)
625 signal.signal(signal.SIGALRM, old_handler)
626 else:
627 self.assertEqual(join(), None)
628
629 self.assertTimingAlmostEqual(join.elapsed, 0.0)
630
631 self.assertEqual(p.is_alive(), False)
632 self.assertNotIn(p, self.active_children())
633
634 p.join()
635
636 return p.exitcode
637
638 @unittest.skipIf(os.name == 'nt', "POSIX only")
639 def test_interrupt(self):
640 exitcode = self._kill_process(multiprocessing.Process.interrupt)
641 self.assertEqual(exitcode, 1)
642 # exit code 1 is hard-coded for uncaught exceptions
643 # (KeyboardInterrupt in this case)
644 # in multiprocessing.BaseProcess._bootstrap
645
646 @unittest.skipIf(os.name == 'nt', "POSIX only")
647 def test_interrupt_no_handler(self):
648 exitcode = self._kill_process(multiprocessing.Process.interrupt, target=self._sleep_no_int_handler)
649 self.assertEqual(exitcode, -signal.SIGINT)
650
651 def test_terminate(self):
652 exitcode = self._kill_process(multiprocessing.Process.terminate)
653 self.assertEqual(exitcode, -signal.SIGTERM)
654
655 def test_kill(self):
656 exitcode = self._kill_process(multiprocessing.Process.kill)
657 if os.name != 'nt':
658 self.assertEqual(exitcode, -signal.SIGKILL)
659 else:
660 self.assertEqual(exitcode, -signal.SIGTERM)
661
662 def test_cpu_count(self):
663 try:
664 cpus = multiprocessing.cpu_count()
665 except NotImplementedError:
666 cpus = 1
667 self.assertIsInstance(cpus, int)
668 self.assertGreaterEqual(cpus, 1)
669
670 def test_active_children(self):
671 self.assertEqual(type(self.active_children()), list)
672
673 event = self.Event()
674 p = self.Process(target=event.wait, args=())
675 self.assertNotIn(p, self.active_children())
676
677 try:
678 p.daemon = True
679 p.start()
680 self.assertIn(p, self.active_children())
681 finally:
682 event.set()
683
684 p.join()
685 self.assertNotIn(p, self.active_children())
686
687 @classmethod
688 def _test_recursion(cls, wconn, id):
689 wconn.send(id)
690 if len(id) < 2:
691 for i in range(2):
692 p = cls.Process(
693 target=cls._test_recursion, args=(wconn, id+[i])
694 )
695 p.start()
696 p.join()
697
698 def test_recursion(self):
699 rconn, wconn = self.Pipe(duplex=False)
700 self._test_recursion(wconn, [])
701
702 time.sleep(DELTA)
703 result = []
704 while rconn.poll():
705 result.append(rconn.recv())
706
707 expected = [
708 [],
709 [0],
710 [0, 0],
711 [0, 1],
712 [1],
713 [1, 0],
714 [1, 1]
715 ]
716 self.assertEqual(result, expected)
717
718 @classmethod
719 def _test_sentinel(cls, event):
720 event.wait(10.0)
721
722 def test_sentinel(self):
723 if self.TYPE == "threads":
724 self.skipTest('test not appropriate for {}'.format(self.TYPE))
725 event = self.Event()
726 p = self.Process(target=self._test_sentinel, args=(event,))
727 with self.assertRaises(ValueError):
728 p.sentinel
729 p.start()
730 self.addCleanup(p.join)
731 sentinel = p.sentinel
732 self.assertIsInstance(sentinel, int)
733 self.assertFalse(wait_for_handle(sentinel, timeout=0.0))
734 event.set()
735 p.join()
736 self.assertTrue(wait_for_handle(sentinel, timeout=1))
737
738 @classmethod
739 def _test_close(cls, rc=0, q=None):
740 if q is not None:
741 q.get()
742 sys.exit(rc)
743
744 def test_close(self):
745 if self.TYPE == "threads":
746 self.skipTest('test not appropriate for {}'.format(self.TYPE))
747 q = self.Queue()
748 p = self.Process(target=self._test_close, kwargs={'q': q})
749 p.daemon = True
750 p.start()
751 self.assertEqual(p.is_alive(), True)
752 # Child is still alive, cannot close
753 with self.assertRaises(ValueError):
754 p.close()
755
756 q.put(None)
757 p.join()
758 self.assertEqual(p.is_alive(), False)
759 self.assertEqual(p.exitcode, 0)
760 p.close()
761 with self.assertRaises(ValueError):
762 p.is_alive()
763 with self.assertRaises(ValueError):
764 p.join()
765 with self.assertRaises(ValueError):
766 p.terminate()
767 p.close()
768
769 wr = weakref.ref(p)
770 del p
771 gc.collect()
772 self.assertIs(wr(), None)
773
774 close_queue(q)
775
776 @support.requires_resource('walltime')
777 def test_many_processes(self):
778 if self.TYPE == 'threads':
779 self.skipTest('test not appropriate for {}'.format(self.TYPE))
780
781 sm = multiprocessing.get_start_method()
782 N = 5 if sm == 'spawn' else 100
783
784 # Try to overwhelm the forkserver loop with events
785 procs = [self.Process(target=self._test_sleep, args=(0.01,))
786 for i in range(N)]
787 for p in procs:
788 p.start()
789 for p in procs:
790 join_process(p)
791 for p in procs:
792 self.assertEqual(p.exitcode, 0)
793
794 procs = [self.Process(target=self._sleep_some)
795 for i in range(N)]
796 for p in procs:
797 p.start()
798 time.sleep(0.001) # let the children start...
799 for p in procs:
800 p.terminate()
801 for p in procs:
802 join_process(p)
803 if os.name != 'nt':
804 exitcodes = [-signal.SIGTERM]
805 if sys.platform == 'darwin':
806 # bpo-31510: On macOS, killing a freshly started process with
807 # SIGTERM sometimes kills the process with SIGKILL.
808 exitcodes.append(-signal.SIGKILL)
809 for p in procs:
810 self.assertIn(p.exitcode, exitcodes)
811
812 def test_lose_target_ref(self):
813 c = DummyCallable()
814 wr = weakref.ref(c)
815 q = self.Queue()
816 p = self.Process(target=c, args=(q, c))
817 del c
818 p.start()
819 p.join()
820 gc.collect() # For PyPy or other GCs.
821 self.assertIs(wr(), None)
822 self.assertEqual(q.get(), 5)
823 close_queue(q)
824
825 @classmethod
826 def _test_child_fd_inflation(self, evt, q):
827 q.put(os_helper.fd_count())
828 evt.wait()
829
830 def test_child_fd_inflation(self):
831 # Number of fds in child processes should not grow with the
832 # number of running children.
833 if self.TYPE == 'threads':
834 self.skipTest('test not appropriate for {}'.format(self.TYPE))
835
836 sm = multiprocessing.get_start_method()
837 if sm == 'fork':
838 # The fork method by design inherits all fds from the parent,
839 # trying to go against it is a lost battle
840 self.skipTest('test not appropriate for {}'.format(sm))
841
842 N = 5
843 evt = self.Event()
844 q = self.Queue()
845
846 procs = [self.Process(target=self._test_child_fd_inflation, args=(evt, q))
847 for i in range(N)]
848 for p in procs:
849 p.start()
850
851 try:
852 fd_counts = [q.get() for i in range(N)]
853 self.assertEqual(len(set(fd_counts)), 1, fd_counts)
854
855 finally:
856 evt.set()
857 for p in procs:
858 p.join()
859 close_queue(q)
860
861 @classmethod
862 def _test_wait_for_threads(self, evt):
863 def func1():
864 time.sleep(0.5)
865 evt.set()
866
867 def func2():
868 time.sleep(20)
869 evt.clear()
870
871 threading.Thread(target=func1).start()
872 threading.Thread(target=func2, daemon=True).start()
873
874 def test_wait_for_threads(self):
875 # A child process should wait for non-daemonic threads to end
876 # before exiting
877 if self.TYPE == 'threads':
878 self.skipTest('test not appropriate for {}'.format(self.TYPE))
879
880 evt = self.Event()
881 proc = self.Process(target=self._test_wait_for_threads, args=(evt,))
882 proc.start()
883 proc.join()
884 self.assertTrue(evt.is_set())
885
886 @classmethod
887 def _test_error_on_stdio_flush(self, evt, break_std_streams={}):
888 for stream_name, action in break_std_streams.items():
889 if action == 'close':
890 stream = io.StringIO()
891 stream.close()
892 else:
893 assert action == 'remove'
894 stream = None
895 setattr(sys, stream_name, None)
896 evt.set()
897
898 def test_error_on_stdio_flush_1(self):
899 # Check that Process works with broken standard streams
900 streams = [io.StringIO(), None]
901 streams[0].close()
902 for stream_name in ('stdout', 'stderr'):
903 for stream in streams:
904 old_stream = getattr(sys, stream_name)
905 setattr(sys, stream_name, stream)
906 try:
907 evt = self.Event()
908 proc = self.Process(target=self._test_error_on_stdio_flush,
909 args=(evt,))
910 proc.start()
911 proc.join()
912 self.assertTrue(evt.is_set())
913 self.assertEqual(proc.exitcode, 0)
914 finally:
915 setattr(sys, stream_name, old_stream)
916
917 def test_error_on_stdio_flush_2(self):
918 # Same as test_error_on_stdio_flush_1(), but standard streams are
919 # broken by the child process
920 for stream_name in ('stdout', 'stderr'):
921 for action in ('close', 'remove'):
922 old_stream = getattr(sys, stream_name)
923 try:
924 evt = self.Event()
925 proc = self.Process(target=self._test_error_on_stdio_flush,
926 args=(evt, {stream_name: action}))
927 proc.start()
928 proc.join()
929 self.assertTrue(evt.is_set())
930 self.assertEqual(proc.exitcode, 0)
931 finally:
932 setattr(sys, stream_name, old_stream)
933
934 @staticmethod
935 def _sleep_and_set_event(evt, delay=0.0):
936 time.sleep(delay)
937 evt.set()
938
939 def check_forkserver_death(self, signum):
940 # bpo-31308: if the forkserver process has died, we should still
941 # be able to create and run new Process instances (the forkserver
942 # is implicitly restarted).
943 if self.TYPE == 'threads':
944 self.skipTest('test not appropriate for {}'.format(self.TYPE))
945 sm = multiprocessing.get_start_method()
946 if sm != 'forkserver':
947 # The fork method by design inherits all fds from the parent,
948 # trying to go against it is a lost battle
949 self.skipTest('test not appropriate for {}'.format(sm))
950
951 from multiprocessing.forkserver import _forkserver
952 _forkserver.ensure_running()
953
954 # First process sleeps 500 ms
955 delay = 0.5
956
957 evt = self.Event()
958 proc = self.Process(target=self._sleep_and_set_event, args=(evt, delay))
959 proc.start()
960
961 pid = _forkserver._forkserver_pid
962 os.kill(pid, signum)
963 # give time to the fork server to die and time to proc to complete
964 time.sleep(delay * 2.0)
965
966 evt2 = self.Event()
967 proc2 = self.Process(target=self._sleep_and_set_event, args=(evt2,))
968 proc2.start()
969 proc2.join()
970 self.assertTrue(evt2.is_set())
971 self.assertEqual(proc2.exitcode, 0)
972
973 proc.join()
974 self.assertTrue(evt.is_set())
975 self.assertIn(proc.exitcode, (0, 255))
976
977 def test_forkserver_sigint(self):
978 # Catchable signal
979 self.check_forkserver_death(signal.SIGINT)
980
981 def test_forkserver_sigkill(self):
982 # Uncatchable signal
983 if os.name != 'nt':
984 self.check_forkserver_death(signal.SIGKILL)
985
986 def test_forkserver_auth_is_enabled(self):
987 if self.TYPE == "threads":
988 self.skipTest(f"test not appropriate for {self.TYPE}")
989 if multiprocessing.get_start_method() != "forkserver":
990 self.skipTest("forkserver start method specific")
991
992 forkserver = multiprocessing.forkserver._forkserver
993 forkserver.ensure_running()
994 self.assertTrue(forkserver._forkserver_pid)
995 authkey = forkserver._forkserver_authkey
996 self.assertTrue(authkey)
997 self.assertGreater(len(authkey), 15)
998 addr = forkserver._forkserver_address
999 self.assertTrue(addr)
1000
1001 # Demonstrate that a raw auth handshake, as Client performs, does not
1002 # raise an error.
1003 client = multiprocessing.connection.Client(addr, authkey=authkey)
1004 client.close()
1005
1006 # That worked, now launch a quick process.
1007 proc = self.Process(target=sys.exit)
1008 proc.start()
1009 proc.join()
1010 self.assertEqual(proc.exitcode, 0)
1011
1012 def test_forkserver_without_auth_fails(self):
1013 if self.TYPE == "threads":
1014 self.skipTest(f"test not appropriate for {self.TYPE}")
1015 if multiprocessing.get_start_method() != "forkserver":
1016 self.skipTest("forkserver start method specific")
1017
1018 forkserver = multiprocessing.forkserver._forkserver
1019 forkserver.ensure_running()
1020 self.assertTrue(forkserver._forkserver_pid)
1021 authkey_len = len(forkserver._forkserver_authkey)
1022 with unittest.mock.patch.object(
1023 forkserver, '_forkserver_authkey', None):
1024 # With an incorrect authkey we should get an auth rejection
1025 # rather than the above protocol error.
1026 forkserver._forkserver_authkey = b'T' * authkey_len
1027 proc = self.Process(target=sys.exit)
1028 with self.assertRaises(multiprocessing.AuthenticationError):
1029 proc.start()
1030 del proc
1031
1032 # authkey restored, launching processes should work again.
1033 proc = self.Process(target=sys.exit)
1034 proc.start()
1035 proc.join()
1036
1037 #
1038 #
1039 #
1040
1041 class _UpperCaser(multiprocessing.Process):
1042
1043 def __init__(self):
1044 multiprocessing.Process.__init__(self)
1045 self.child_conn, self.parent_conn = multiprocessing.Pipe()
1046
1047 def run(self):
1048 self.parent_conn.close()
1049 for s in iter(self.child_conn.recv, None):
1050 self.child_conn.send(s.upper())
1051 self.child_conn.close()
1052
1053 def submit(self, s):
1054 assert type(s) is str
1055 self.parent_conn.send(s)
1056 return self.parent_conn.recv()
1057
1058 def stop(self):
1059 self.parent_conn.send(None)
1060 self.parent_conn.close()
1061 self.child_conn.close()
1062
1063 class _TestSubclassingProcess(BaseTestCase):
1064
1065 ALLOWED_TYPES = ('processes',)
1066
1067 def test_subclassing(self):
1068 uppercaser = _UpperCaser()
1069 uppercaser.daemon = True
1070 uppercaser.start()
1071 self.assertEqual(uppercaser.submit('hello'), 'HELLO')
1072 self.assertEqual(uppercaser.submit('world'), 'WORLD')
1073 uppercaser.stop()
1074 uppercaser.join()
1075
1076 def test_stderr_flush(self):
1077 # sys.stderr is flushed at process shutdown (issue #13812)
1078 if self.TYPE == "threads":
1079 self.skipTest('test not appropriate for {}'.format(self.TYPE))
1080
1081 testfn = os_helper.TESTFN
1082 self.addCleanup(os_helper.unlink, testfn)
1083 proc = self.Process(target=self._test_stderr_flush, args=(testfn,))
1084 proc.start()
1085 proc.join()
1086 with open(testfn, encoding="utf-8") as f:
1087 err = f.read()
1088 # The whole traceback was printed
1089 self.assertIn("ZeroDivisionError", err)
1090 self.assertIn("test_multiprocessing.py", err)
1091 self.assertIn("1/0 # MARKER", err)
1092
1093 @classmethod
1094 def _test_stderr_flush(cls, testfn):
1095 fd = os.open(testfn, os.O_WRONLY | os.O_CREAT | os.O_EXCL)
1096 sys.stderr = open(fd, 'w', encoding="utf-8", closefd=False)
1097 1/0 # MARKER
1098
1099
1100 @classmethod
1101 def _test_sys_exit(cls, reason, testfn):
1102 fd = os.open(testfn, os.O_WRONLY | os.O_CREAT | os.O_EXCL)
1103 sys.stderr = open(fd, 'w', encoding="utf-8", closefd=False)
1104 sys.exit(reason)
1105
1106 def test_sys_exit(self):
1107 # See Issue 13854
1108 if self.TYPE == 'threads':
1109 self.skipTest('test not appropriate for {}'.format(self.TYPE))
1110
1111 testfn = os_helper.TESTFN
1112 self.addCleanup(os_helper.unlink, testfn)
1113
1114 for reason in (
1115 [1, 2, 3],
1116 'ignore this',
1117 ):
1118 p = self.Process(target=self._test_sys_exit, args=(reason, testfn))
1119 p.daemon = True
1120 p.start()
1121 join_process(p)
1122 self.assertEqual(p.exitcode, 1)
1123
1124 with open(testfn, encoding="utf-8") as f:
1125 content = f.read()
1126 self.assertEqual(content.rstrip(), str(reason))
1127
1128 os.unlink(testfn)
1129
1130 cases = [
1131 ((True,), 1),
1132 ((False,), 0),
1133 ((8,), 8),
1134 ((None,), 0),
1135 ((), 0),
1136 ]
1137
1138 for args, expected in cases:
1139 with self.subTest(args=args):
1140 p = self.Process(target=sys.exit, args=args)
1141 p.daemon = True
1142 p.start()
1143 join_process(p)
1144 self.assertEqual(p.exitcode, expected)
1145
1146 #
1147 #
1148 #
1149
1150 def queue_empty(q):
1151 if hasattr(q, 'empty'):
1152 return q.empty()
1153 else:
1154 return q.qsize() == 0
1155
1156 def queue_full(q, maxsize):
1157 if hasattr(q, 'full'):
1158 return q.full()
1159 else:
1160 return q.qsize() == maxsize
1161
1162
1163 class _TestQueue(BaseTestCase):
1164
1165
1166 @classmethod
1167 def _test_put(cls, queue, child_can_start, parent_can_continue):
1168 child_can_start.wait()
1169 for i in range(6):
1170 queue.get()
1171 parent_can_continue.set()
1172
1173 def test_put(self):
1174 MAXSIZE = 6
1175 queue = self.Queue(maxsize=MAXSIZE)
1176 child_can_start = self.Event()
1177 parent_can_continue = self.Event()
1178
1179 proc = self.Process(
1180 target=self._test_put,
1181 args=(queue, child_can_start, parent_can_continue)
1182 )
1183 proc.daemon = True
1184 proc.start()
1185
1186 self.assertEqual(queue_empty(queue), True)
1187 self.assertEqual(queue_full(queue, MAXSIZE), False)
1188
1189 queue.put(1)
1190 queue.put(2, True)
1191 queue.put(3, True, None)
1192 queue.put(4, False)
1193 queue.put(5, False, None)
1194 queue.put_nowait(6)
1195
1196 # the values may be in buffer but not yet in pipe so sleep a bit
1197 time.sleep(DELTA)
1198
1199 self.assertEqual(queue_empty(queue), False)
1200 self.assertEqual(queue_full(queue, MAXSIZE), True)
1201
1202 put = TimingWrapper(queue.put)
1203 put_nowait = TimingWrapper(queue.put_nowait)
1204
1205 self.assertRaises(pyqueue.Full, put, 7, False)
1206 self.assertTimingAlmostEqual(put.elapsed, 0)
1207
1208 self.assertRaises(pyqueue.Full, put, 7, False, None)
1209 self.assertTimingAlmostEqual(put.elapsed, 0)
1210
1211 self.assertRaises(pyqueue.Full, put_nowait, 7)
1212 self.assertTimingAlmostEqual(put_nowait.elapsed, 0)
1213
1214 self.assertRaises(pyqueue.Full, put, 7, True, TIMEOUT1)
1215 self.assertTimingAlmostEqual(put.elapsed, TIMEOUT1)
1216
1217 self.assertRaises(pyqueue.Full, put, 7, False, TIMEOUT2)
1218 self.assertTimingAlmostEqual(put.elapsed, 0)
1219
1220 self.assertRaises(pyqueue.Full, put, 7, True, timeout=TIMEOUT3)
1221 self.assertTimingAlmostEqual(put.elapsed, TIMEOUT3)
1222
1223 child_can_start.set()
1224 parent_can_continue.wait()
1225
1226 self.assertEqual(queue_empty(queue), True)
1227 self.assertEqual(queue_full(queue, MAXSIZE), False)
1228
1229 proc.join()
1230 close_queue(queue)
1231
1232 @classmethod
1233 def _test_get(cls, queue, child_can_start, parent_can_continue):
1234 child_can_start.wait()
1235 queue.put(1)
1236 queue.put(2)
1237 queue.put(3)
1238 queue.put(4)
1239 queue.put(5)
1240 parent_can_continue.set()
1241
1242 def test_get(self):
1243 queue = self.Queue()
1244 child_can_start = self.Event()
1245 parent_can_continue = self.Event()
1246
1247 proc = self.Process(
1248 target=self._test_get,
1249 args=(queue, child_can_start, parent_can_continue)
1250 )
1251 proc.daemon = True
1252 proc.start()
1253
1254 self.assertEqual(queue_empty(queue), True)
1255
1256 child_can_start.set()
1257 parent_can_continue.wait()
1258
1259 for _ in support.sleeping_retry(support.SHORT_TIMEOUT):
1260 if not queue_empty(queue):
1261 break
1262 self.assertEqual(queue_empty(queue), False)
1263
1264 self.assertEqual(queue.get_nowait(), 1)
1265 self.assertEqual(queue.get(True, None), 2)
1266 self.assertEqual(queue.get(True), 3)
1267 self.assertEqual(queue.get(timeout=1), 4)
1268 self.assertEqual(queue.get(), 5)
1269
1270 self.assertEqual(queue_empty(queue), True)
1271
1272 get = TimingWrapper(queue.get)
1273 get_nowait = TimingWrapper(queue.get_nowait)
1274
1275 self.assertRaises(pyqueue.Empty, get, False)
1276 self.assertTimingAlmostEqual(get.elapsed, 0)
1277
1278 self.assertRaises(pyqueue.Empty, get, False, None)
1279 self.assertTimingAlmostEqual(get.elapsed, 0)
1280
1281 self.assertRaises(pyqueue.Empty, get_nowait)
1282 self.assertTimingAlmostEqual(get_nowait.elapsed, 0)
1283
1284 self.assertRaises(pyqueue.Empty, get, True, TIMEOUT1)
1285 self.assertTimingAlmostEqual(get.elapsed, TIMEOUT1)
1286
1287 self.assertRaises(pyqueue.Empty, get, False, TIMEOUT2)
1288 self.assertTimingAlmostEqual(get.elapsed, 0)
1289
1290 self.assertRaises(pyqueue.Empty, get, timeout=TIMEOUT3)
1291 self.assertTimingAlmostEqual(get.elapsed, TIMEOUT3)
1292
1293 proc.join()
1294 close_queue(queue)
1295
1296 @classmethod
1297 def _test_fork(cls, queue):
1298 for i in range(10, 20):
1299 queue.put(i)
1300 # note that at this point the items may only be buffered, so the
1301 # process cannot shutdown until the feeder thread has finished
1302 # pushing items onto the pipe.
1303
1304 def test_fork(self):
1305 # Old versions of Queue would fail to create a new feeder
1306 # thread for a forked process if the original process had its
1307 # own feeder thread. This test checks that this no longer
1308 # happens.
1309
1310 queue = self.Queue()
1311
1312 # put items on queue so that main process starts a feeder thread
1313 for i in range(10):
1314 queue.put(i)
1315
1316 # wait to make sure thread starts before we fork a new process
1317 time.sleep(DELTA)
1318
1319 # fork process
1320 p = self.Process(target=self._test_fork, args=(queue,))
1321 p.daemon = True
1322 p.start()
1323
1324 # check that all expected items are in the queue
1325 for i in range(20):
1326 self.assertEqual(queue.get(), i)
1327 self.assertRaises(pyqueue.Empty, queue.get, False)
1328
1329 p.join()
1330 close_queue(queue)
1331
1332 def test_qsize(self):
1333 q = self.Queue()
1334 try:
1335 self.assertEqual(q.qsize(), 0)
1336 except NotImplementedError:
1337 self.skipTest('qsize method not implemented')
1338 q.put(1)
1339 self.assertEqual(q.qsize(), 1)
1340 q.put(5)
1341 self.assertEqual(q.qsize(), 2)
1342 q.get()
1343 self.assertEqual(q.qsize(), 1)
1344 q.get()
1345 self.assertEqual(q.qsize(), 0)
1346 close_queue(q)
1347
1348 @classmethod
1349 def _test_task_done(cls, q):
1350 for obj in iter(q.get, None):
1351 time.sleep(DELTA)
1352 q.task_done()
1353
1354 def test_task_done(self):
1355 queue = self.JoinableQueue()
1356
1357 workers = [self.Process(target=self._test_task_done, args=(queue,))
1358 for i in range(4)]
1359
1360 for p in workers:
1361 p.daemon = True
1362 p.start()
1363
1364 for i in range(10):
1365 queue.put(i)
1366
1367 queue.join()
1368
1369 for p in workers:
1370 queue.put(None)
1371
1372 for p in workers:
1373 p.join()
1374 close_queue(queue)
1375
1376 def test_no_import_lock_contention(self):
1377 with os_helper.temp_cwd():
1378 module_name = 'imported_by_an_imported_module'
1379 with open(module_name + '.py', 'w', encoding="utf-8") as f:
1380 f.write("""if 1:
1381 import multiprocessing
1382
1383 q = multiprocessing.Queue()
1384 q.put('knock knock')
1385 q.get(timeout=3)
1386 q.close()
1387 del q
1388 """)
1389
1390 with import_helper.DirsOnSysPath(os.getcwd()):
1391 try:
1392 __import__(module_name)
1393 except pyqueue.Empty:
1394 self.fail("Probable regression on import lock contention;"
1395 " see Issue #22853")
1396
1397 def test_timeout(self):
1398 q = multiprocessing.Queue()
1399 start = time.monotonic()
1400 self.assertRaises(pyqueue.Empty, q.get, True, 0.200)
1401 delta = time.monotonic() - start
1402 # bpo-30317: Tolerate a delta of 100 ms because of the bad clock
1403 # resolution on Windows (usually 15.6 ms). x86 Windows7 3.x once
1404 # failed because the delta was only 135.8 ms.
1405 self.assertGreaterEqual(delta, 0.100)
1406 close_queue(q)
1407
1408 def test_queue_feeder_donot_stop_onexc(self):
1409 # bpo-30414: verify feeder handles exceptions correctly
1410 if self.TYPE != 'processes':
1411 self.skipTest('test not appropriate for {}'.format(self.TYPE))
1412
1413 class NotSerializable(object):
1414 def __reduce__(self):
1415 raise AttributeError
1416 with test.support.captured_stderr():
1417 q = self.Queue()
1418 q.put(NotSerializable())
1419 q.put(True)
1420 self.assertTrue(q.get(timeout=support.SHORT_TIMEOUT))
1421 close_queue(q)
1422
1423 with test.support.captured_stderr():
1424 # bpo-33078: verify that the queue size is correctly handled
1425 # on errors.
1426 q = self.Queue(maxsize=1)
1427 q.put(NotSerializable())
1428 q.put(True)
1429 try:
1430 self.assertEqual(q.qsize(), 1)
1431 except NotImplementedError:
1432 # qsize is not available on all platform as it
1433 # relies on sem_getvalue
1434 pass
1435 self.assertTrue(q.get(timeout=support.SHORT_TIMEOUT))
1436 # Check that the size of the queue is correct
1437 self.assertTrue(q.empty())
1438 close_queue(q)
1439
1440 def test_queue_feeder_on_queue_feeder_error(self):
1441 # bpo-30006: verify feeder handles exceptions using the
1442 # _on_queue_feeder_error hook.
1443 if self.TYPE != 'processes':
1444 self.skipTest('test not appropriate for {}'.format(self.TYPE))
1445
1446 class NotSerializable(object):
1447 """Mock unserializable object"""
1448 def __init__(self):
1449 self.reduce_was_called = False
1450 self.on_queue_feeder_error_was_called = False
1451
1452 def __reduce__(self):
1453 self.reduce_was_called = True
1454 raise AttributeError
1455
1456 class SafeQueue(multiprocessing.queues.Queue):
1457 """Queue with overloaded _on_queue_feeder_error hook"""
1458 @staticmethod
1459 def _on_queue_feeder_error(e, obj):
1460 if (isinstance(e, AttributeError) and
1461 isinstance(obj, NotSerializable)):
1462 obj.on_queue_feeder_error_was_called = True
1463
1464 not_serializable_obj = NotSerializable()
1465 # The captured_stderr reduces the noise in the test report
1466 with test.support.captured_stderr():
1467 q = SafeQueue(ctx=multiprocessing.get_context())
1468 q.put(not_serializable_obj)
1469
1470 # Verify that q is still functioning correctly
1471 q.put(True)
1472 self.assertTrue(q.get(timeout=support.SHORT_TIMEOUT))
1473
1474 # Assert that the serialization and the hook have been called correctly
1475 self.assertTrue(not_serializable_obj.reduce_was_called)
1476 self.assertTrue(not_serializable_obj.on_queue_feeder_error_was_called)
1477
1478 def test_closed_queue_empty_exceptions(self):
1479 # Assert that checking the emptiness of an unused closed queue
1480 # does not raise an OSError. The rationale is that q.close() is
1481 # a no-op upon construction and becomes effective once the queue
1482 # has been used (e.g., by calling q.put()).
1483 for q in multiprocessing.Queue(), multiprocessing.JoinableQueue():
1484 q.close() # this is a no-op since the feeder thread is None
1485 q.join_thread() # this is also a no-op
1486 self.assertTrue(q.empty())
1487
1488 for q in multiprocessing.Queue(), multiprocessing.JoinableQueue():
1489 q.put('foo') # make sure that the queue is 'used'
1490 q.close() # close the feeder thread
1491 q.join_thread() # make sure to join the feeder thread
1492 with self.assertRaisesRegex(OSError, 'is closed'):
1493 q.empty()
1494
1495 def test_closed_queue_put_get_exceptions(self):
1496 for q in multiprocessing.Queue(), multiprocessing.JoinableQueue():
1497 q.close()
1498 with self.assertRaisesRegex(ValueError, 'is closed'):
1499 q.put('foo')
1500 with self.assertRaisesRegex(ValueError, 'is closed'):
1501 q.get()
1502 #
1503 #
1504 #
1505
1506 class _TestLock(BaseTestCase):
1507
1508 @staticmethod
1509 def _acquire(lock, l=None):
1510 lock.acquire()
1511 if l is not None:
1512 l.append(repr(lock))
1513
1514 @staticmethod
1515 def _acquire_event(lock, event):
1516 lock.acquire()
1517 event.set()
1518 time.sleep(1.0)
1519
1520 def test_repr_lock(self):
1521 if self.TYPE != 'processes':
1522 self.skipTest('test not appropriate for {}'.format(self.TYPE))
1523
1524 lock = self.Lock()
1525 self.assertEqual(f'<Lock(owner=None)>', repr(lock))
1526
1527 lock.acquire()
1528 self.assertEqual(f'<Lock(owner=MainProcess)>', repr(lock))
1529 lock.release()
1530
1531 tname = 'T1'
1532 l = []
1533 t = threading.Thread(target=self._acquire,
1534 args=(lock, l),
1535 name=tname)
1536 t.start()
1537 time.sleep(0.1)
1538 self.assertEqual(f'<Lock(owner=MainProcess|{tname})>', l[0])
1539 lock.release()
1540
1541 t = threading.Thread(target=self._acquire,
1542 args=(lock,),
1543 name=tname)
1544 t.start()
1545 time.sleep(0.1)
1546 self.assertEqual('<Lock(owner=SomeOtherThread)>', repr(lock))
1547 lock.release()
1548
1549 pname = 'P1'
1550 l = multiprocessing.Manager().list()
1551 p = self.Process(target=self._acquire,
1552 args=(lock, l),
1553 name=pname)
1554 p.start()
1555 p.join()
1556 self.assertEqual(f'<Lock(owner={pname})>', l[0])
1557
1558 lock = self.Lock()
1559 event = self.Event()
1560 p = self.Process(target=self._acquire_event,
1561 args=(lock, event),
1562 name='P2')
1563 p.start()
1564 event.wait()
1565 self.assertEqual(f'<Lock(owner=SomeOtherProcess)>', repr(lock))
1566 p.terminate()
1567
1568 def test_lock(self):
1569 lock = self.Lock()
1570 self.assertEqual(lock.acquire(), True)
1571 self.assertTrue(lock.locked())
1572 self.assertEqual(lock.acquire(False), False)
1573 self.assertEqual(lock.release(), None)
1574 self.assertFalse(lock.locked())
1575 self.assertRaises((ValueError, threading.ThreadError), lock.release)
1576
1577 @classmethod
1578 def _test_lock_locked_2processes(cls, lock, event, res):
1579 lock.acquire()
1580 res.value = lock.locked()
1581 event.set()
1582
1583 @unittest.skipUnless(HAS_SHAREDCTYPES, 'needs sharedctypes')
1584 def test_lock_locked_2processes(self):
1585 if self.TYPE != 'processes':
1586 self.skipTest('test not appropriate for {}'.format(self.TYPE))
1587
1588 lock = self.Lock()
1589 event = self.Event()
1590 res = self.Value('b', 0)
1591 p = self.Process(target=self._test_lock_locked_2processes,
1592 args=(lock, event, res))
1593 p.start()
1594 event.wait()
1595 self.assertTrue(lock.locked())
1596 self.assertTrue(res.value)
1597 p.join()
1598
1599 @staticmethod
1600 def _acquire_release(lock, timeout, l=None, n=1):
1601 for _ in range(n):
1602 lock.acquire()
1603 if l is not None:
1604 l.append(repr(lock))
1605 time.sleep(timeout)
1606 for _ in range(n):
1607 lock.release()
1608
1609 def test_repr_rlock(self):
1610 if self.TYPE != 'processes':
1611 self.skipTest('test not appropriate for {}'.format(self.TYPE))
1612
1613 lock = self.RLock()
1614 self.assertEqual('<RLock(None, 0)>', repr(lock))
1615
1616 n = 3
1617 for _ in range(n):
1618 lock.acquire()
1619 self.assertEqual(f'<RLock(MainProcess, {n})>', repr(lock))
1620 for _ in range(n):
1621 lock.release()
1622
1623 t, l = [], []
1624 for i in range(n):
1625 t.append(threading.Thread(target=self._acquire_release,
1626 args=(lock, 0.1, l, i+1),
1627 name=f'T{i+1}'))
1628 t[-1].start()
1629 for t_ in t:
1630 t_.join()
1631 for i in range(n):
1632 self.assertIn(f'<RLock(MainProcess|T{i+1}, {i+1})>', l)
1633
1634 rlock = self.RLock()
1635 t = threading.Thread(target=rlock.acquire)
1636 t.start()
1637 t.join()
1638 self.assertEqual('<RLock(SomeOtherThread, nonzero)>', repr(rlock))
1639
1640 pname = 'P1'
1641 l = multiprocessing.Manager().list()
1642 p = self.Process(target=self._acquire_release,
1643 args=(lock, 0.1, l),
1644 name=pname)
1645 p.start()
1646 p.join()
1647 self.assertEqual(f'<RLock({pname}, 1)>', l[0])
1648
1649 rlock = self.RLock()
1650 p = self.Process(target=self._acquire, args=(rlock,))
1651 p.start()
1652 p.join()
1653 self.assertEqual('<RLock(SomeOtherProcess, nonzero)>', repr(rlock))
1654
1655 def test_rlock(self):
1656 lock = self.RLock()
1657 self.assertEqual(lock.acquire(), True)
1658 self.assertTrue(lock.locked())
1659 self.assertEqual(lock.acquire(), True)
1660 self.assertEqual(lock.acquire(), True)
1661 self.assertEqual(lock.release(), None)
1662 self.assertTrue(lock.locked())
1663 self.assertEqual(lock.release(), None)
1664 self.assertEqual(lock.release(), None)
1665 self.assertFalse(lock.locked())
1666 self.assertRaises((AssertionError, RuntimeError), lock.release)
1667
1668 @unittest.skipUnless(HAS_SHAREDCTYPES, 'needs sharedctypes')
1669 def test_rlock_locked_2processes(self):
1670 if self.TYPE != 'processes':
1671 self.skipTest('test not appropriate for {}'.format(self.TYPE))
1672
1673 rlock = self.RLock()
1674 event = self.Event()
1675 res = self.Value('b', 0)
1676 # target is the same as for the test_lock_locked_2processes test.
1677 p = self.Process(target=self._test_lock_locked_2processes,
1678 args=(rlock, event, res))
1679 p.start()
1680 event.wait()
1681 self.assertTrue(rlock.locked())
1682 self.assertTrue(res.value)
1683 p.join()
1684
1685 def test_lock_context(self):
1686 with self.Lock() as locked:
1687 self.assertTrue(locked)
1688
1689 def test_rlock_context(self):
1690 with self.RLock() as locked:
1691 self.assertTrue(locked)
1692
1693
1694 class _TestSemaphore(BaseTestCase):
1695
1696 def _test_semaphore(self, sem):
1697 self.assertReturnsIfImplemented(2, get_value, sem)
1698 self.assertEqual(sem.acquire(), True)
1699 self.assertReturnsIfImplemented(1, get_value, sem)
1700 self.assertEqual(sem.acquire(), True)
1701 self.assertReturnsIfImplemented(0, get_value, sem)
1702 self.assertEqual(sem.acquire(False), False)
1703 self.assertReturnsIfImplemented(0, get_value, sem)
1704 self.assertEqual(sem.release(), None)
1705 self.assertReturnsIfImplemented(1, get_value, sem)
1706 self.assertEqual(sem.release(), None)
1707 self.assertReturnsIfImplemented(2, get_value, sem)
1708
1709 def test_semaphore(self):
1710 sem = self.Semaphore(2)
1711 self._test_semaphore(sem)
1712 self.assertEqual(sem.release(), None)
1713 self.assertReturnsIfImplemented(3, get_value, sem)
1714 self.assertEqual(sem.release(), None)
1715 self.assertReturnsIfImplemented(4, get_value, sem)
1716
1717 def test_bounded_semaphore(self):
1718 sem = self.BoundedSemaphore(2)
1719 self._test_semaphore(sem)
1720 # Currently fails on OS/X
1721 #if HAVE_GETVALUE:
1722 # self.assertRaises(ValueError, sem.release)
1723 # self.assertReturnsIfImplemented(2, get_value, sem)
1724
1725 def test_timeout(self):
1726 if self.TYPE != 'processes':
1727 self.skipTest('test not appropriate for {}'.format(self.TYPE))
1728
1729 sem = self.Semaphore(0)
1730 acquire = TimingWrapper(sem.acquire)
1731
1732 self.assertEqual(acquire(False), False)
1733 self.assertTimingAlmostEqual(acquire.elapsed, 0.0)
1734
1735 self.assertEqual(acquire(False, None), False)
1736 self.assertTimingAlmostEqual(acquire.elapsed, 0.0)
1737
1738 self.assertEqual(acquire(False, TIMEOUT1), False)
1739 self.assertTimingAlmostEqual(acquire.elapsed, 0)
1740
1741 self.assertEqual(acquire(True, TIMEOUT2), False)
1742 self.assertTimingAlmostEqual(acquire.elapsed, TIMEOUT2)
1743
1744 self.assertEqual(acquire(timeout=TIMEOUT3), False)
1745 self.assertTimingAlmostEqual(acquire.elapsed, TIMEOUT3)
1746
1747
1748 class _TestCondition(BaseTestCase):
1749
1750 @classmethod
1751 def f(cls, cond, sleeping, woken, timeout=None):
1752 cond.acquire()
1753 sleeping.release()
1754 cond.wait(timeout)
1755 woken.release()
1756 cond.release()
1757
1758 def assertReachesEventually(self, func, value):
1759 for _ in support.sleeping_retry(support.SHORT_TIMEOUT):
1760 try:
1761 if func() == value:
1762 break
1763 except NotImplementedError:
1764 break
1765
1766 self.assertReturnsIfImplemented(value, func)
1767
1768 def check_invariant(self, cond):
1769 # this is only supposed to succeed when there are no sleepers
1770 if self.TYPE == 'processes':
1771 try:
1772 sleepers = (cond._sleeping_count.get_value() -
1773 cond._woken_count.get_value())
1774 self.assertEqual(sleepers, 0)
1775 self.assertEqual(cond._wait_semaphore.get_value(), 0)
1776 except NotImplementedError:
1777 pass
1778
1779 def test_notify(self):
1780 cond = self.Condition()
1781 sleeping = self.Semaphore(0)
1782 woken = self.Semaphore(0)
1783
1784 p = self.Process(target=self.f, args=(cond, sleeping, woken))
1785 p.daemon = True
1786 p.start()
1787
1788 t = threading.Thread(target=self.f, args=(cond, sleeping, woken))
1789 t.daemon = True
1790 t.start()
1791
1792 # wait for both children to start sleeping
1793 sleeping.acquire()
1794 sleeping.acquire()
1795
1796 # check no process/thread has woken up
1797 self.assertReachesEventually(lambda: get_value(woken), 0)
1798
1799 # wake up one process/thread
1800 cond.acquire()
1801 cond.notify()
1802 cond.release()
1803
1804 # check one process/thread has woken up
1805 self.assertReachesEventually(lambda: get_value(woken), 1)
1806
1807 # wake up another
1808 cond.acquire()
1809 cond.notify()
1810 cond.release()
1811
1812 # check other has woken up
1813 self.assertReachesEventually(lambda: get_value(woken), 2)
1814
1815 # check state is not mucked up
1816 self.check_invariant(cond)
1817
1818 threading_helper.join_thread(t)
1819 join_process(p)
1820
1821 def test_notify_all(self):
1822 cond = self.Condition()
1823 sleeping = self.Semaphore(0)
1824 woken = self.Semaphore(0)
1825
1826 # start some threads/processes which will timeout
1827 workers = []
1828 for i in range(3):
1829 p = self.Process(target=self.f,
1830 args=(cond, sleeping, woken, TIMEOUT1))
1831 p.daemon = True
1832 p.start()
1833 workers.append(p)
1834
1835 t = threading.Thread(target=self.f,
1836 args=(cond, sleeping, woken, TIMEOUT1))
1837 t.daemon = True
1838 t.start()
1839 workers.append(t)
1840
1841 # wait for them all to sleep
1842 for i in range(6):
1843 sleeping.acquire()
1844
1845 # check they have all timed out
1846 for i in range(6):
1847 woken.acquire()
1848 self.assertReturnsIfImplemented(0, get_value, woken)
1849
1850 # check state is not mucked up
1851 self.check_invariant(cond)
1852
1853 # start some more threads/processes
1854 for i in range(3):
1855 p = self.Process(target=self.f, args=(cond, sleeping, woken))
1856 p.daemon = True
1857 p.start()
1858 workers.append(p)
1859
1860 t = threading.Thread(target=self.f, args=(cond, sleeping, woken))
1861 t.daemon = True
1862 t.start()
1863 workers.append(t)
1864
1865 # wait for them to all sleep
1866 for i in range(6):
1867 sleeping.acquire()
1868
1869 # check no process/thread has woken up
1870 time.sleep(DELTA)
1871 self.assertReturnsIfImplemented(0, get_value, woken)
1872
1873 # wake them all up
1874 cond.acquire()
1875 cond.notify_all()
1876 cond.release()
1877
1878 # check they have all woken
1879 for i in range(6):
1880 woken.acquire()
1881 self.assertReturnsIfImplemented(0, get_value, woken)
1882
1883 # check state is not mucked up
1884 self.check_invariant(cond)
1885
1886 for w in workers:
1887 # NOTE: join_process and join_thread are the same
1888 threading_helper.join_thread(w)
1889
1890 def test_notify_n(self):
1891 cond = self.Condition()
1892 sleeping = self.Semaphore(0)
1893 woken = self.Semaphore(0)
1894
1895 # start some threads/processes
1896 workers = []
1897 for i in range(3):
1898 p = self.Process(target=self.f, args=(cond, sleeping, woken))
1899 p.daemon = True
1900 p.start()
1901 workers.append(p)
1902
1903 t = threading.Thread(target=self.f, args=(cond, sleeping, woken))
1904 t.daemon = True
1905 t.start()
1906 workers.append(t)
1907
1908 # wait for them to all sleep
1909 for i in range(6):
1910 sleeping.acquire()
1911
1912 # check no process/thread has woken up
1913 time.sleep(DELTA)
1914 self.assertReturnsIfImplemented(0, get_value, woken)
1915
1916 # wake some of them up
1917 cond.acquire()
1918 cond.notify(n=2)
1919 cond.release()
1920
1921 # check 2 have woken
1922 self.assertReachesEventually(lambda: get_value(woken), 2)
1923
1924 # wake the rest of them
1925 cond.acquire()
1926 cond.notify(n=4)
1927 cond.release()
1928
1929 self.assertReachesEventually(lambda: get_value(woken), 6)
1930
1931 # doesn't do anything more
1932 cond.acquire()
1933 cond.notify(n=3)
1934 cond.release()
1935
1936 self.assertReturnsIfImplemented(6, get_value, woken)
1937
1938 # check state is not mucked up
1939 self.check_invariant(cond)
1940
1941 for w in workers:
1942 # NOTE: join_process and join_thread are the same
1943 threading_helper.join_thread(w)
1944
1945 def test_timeout(self):
1946 cond = self.Condition()
1947 wait = TimingWrapper(cond.wait)
1948 cond.acquire()
1949 res = wait(TIMEOUT1)
1950 cond.release()
1951 self.assertEqual(res, False)
1952 self.assertTimingAlmostEqual(wait.elapsed, TIMEOUT1)
1953
1954 @classmethod
1955 def _test_waitfor_f(cls, cond, state):
1956 with cond:
1957 state.value = 0
1958 cond.notify()
1959 result = cond.wait_for(lambda : state.value==4)
1960 if not result or state.value != 4:
1961 sys.exit(1)
1962
1963 @unittest.skipUnless(HAS_SHAREDCTYPES, 'needs sharedctypes')
1964 def test_waitfor(self):
1965 # based on test in test/lock_tests.py
1966 cond = self.Condition()
1967 state = self.Value('i', -1)
1968
1969 p = self.Process(target=self._test_waitfor_f, args=(cond, state))
1970 p.daemon = True
1971 p.start()
1972
1973 with cond:
1974 result = cond.wait_for(lambda : state.value==0)
1975 self.assertTrue(result)
1976 self.assertEqual(state.value, 0)
1977
1978 for i in range(4):
1979 time.sleep(0.01)
1980 with cond:
1981 state.value += 1
1982 cond.notify()
1983
1984 join_process(p)
1985 self.assertEqual(p.exitcode, 0)
1986
1987 @classmethod
1988 def _test_waitfor_timeout_f(cls, cond, state, success, sem):
1989 sem.release()
1990 with cond:
1991 expected = 0.100
1992 dt = time.monotonic()
1993 result = cond.wait_for(lambda : state.value==4, timeout=expected)
1994 dt = time.monotonic() - dt
1995 if not result and (expected - CLOCK_RES) <= dt:
1996 success.value = True
1997
1998 @unittest.skipUnless(HAS_SHAREDCTYPES, 'needs sharedctypes')
1999 def test_waitfor_timeout(self):
2000 # based on test in test/lock_tests.py
2001 cond = self.Condition()
2002 state = self.Value('i', 0)
2003 success = self.Value('i', False)
2004 sem = self.Semaphore(0)
2005
2006 p = self.Process(target=self._test_waitfor_timeout_f,
2007 args=(cond, state, success, sem))
2008 p.daemon = True
2009 p.start()
2010 self.assertTrue(sem.acquire(timeout=support.LONG_TIMEOUT))
2011
2012 # Only increment 3 times, so state == 4 is never reached.
2013 for i in range(3):
2014 time.sleep(0.010)
2015 with cond:
2016 state.value += 1
2017 cond.notify()
2018
2019 join_process(p)
2020 self.assertTrue(success.value)
2021
2022 @classmethod
2023 def _test_wait_result(cls, c, pid):
2024 with c:
2025 c.notify()
2026 time.sleep(1)
2027 if pid is not None:
2028 os.kill(pid, signal.SIGINT)
2029
2030 def test_wait_result(self):
2031 if isinstance(self, ProcessesMixin) and sys.platform != 'win32':
2032 pid = os.getpid()
2033 else:
2034 pid = None
2035
2036 c = self.Condition()
2037 with c:
2038 self.assertFalse(c.wait(0))
2039 self.assertFalse(c.wait(0.1))
2040
2041 p = self.Process(target=self._test_wait_result, args=(c, pid))
2042 p.start()
2043
2044 self.assertTrue(c.wait(60))
2045 if pid is not None:
2046 self.assertRaises(KeyboardInterrupt, c.wait, 60)
2047
2048 p.join()
2049
2050
2051 class _TestEvent(BaseTestCase):
2052
2053 @classmethod
2054 def _test_event(cls, event):
2055 time.sleep(TIMEOUT2)
2056 event.set()
2057
2058 def test_event(self):
2059 event = self.Event()
2060 wait = TimingWrapper(event.wait)
2061
2062 # Removed temporarily, due to API shear, this does not
2063 # work with threading._Event objects. is_set == isSet
2064 self.assertEqual(event.is_set(), False)
2065
2066 # Removed, threading.Event.wait() will return the value of the __flag
2067 # instead of None. API Shear with the semaphore backed mp.Event
2068 self.assertEqual(wait(0.0), False)
2069 self.assertTimingAlmostEqual(wait.elapsed, 0.0)
2070 self.assertEqual(wait(TIMEOUT1), False)
2071 self.assertTimingAlmostEqual(wait.elapsed, TIMEOUT1)
2072
2073 event.set()
2074
2075 # See note above on the API differences
2076 self.assertEqual(event.is_set(), True)
2077 self.assertEqual(wait(), True)
2078 self.assertTimingAlmostEqual(wait.elapsed, 0.0)
2079 self.assertEqual(wait(TIMEOUT1), True)
2080 self.assertTimingAlmostEqual(wait.elapsed, 0.0)
2081 # self.assertEqual(event.is_set(), True)
2082
2083 event.clear()
2084
2085 #self.assertEqual(event.is_set(), False)
2086
2087 p = self.Process(target=self._test_event, args=(event,))
2088 p.daemon = True
2089 p.start()
2090 self.assertEqual(wait(), True)
2091 p.join()
2092
2093 def test_repr(self) -> None:
2094 event = self.Event()
2095 if self.TYPE == 'processes':
2096 self.assertRegex(repr(event), r"<Event at .* unset>")
2097 event.set()
2098 self.assertRegex(repr(event), r"<Event at .* set>")
2099 event.clear()
2100 self.assertRegex(repr(event), r"<Event at .* unset>")
2101 elif self.TYPE == 'manager':
2102 self.assertRegex(repr(event), r"<EventProxy object, typeid 'Event' at .*")
2103 event.set()
2104 self.assertRegex(repr(event), r"<EventProxy object, typeid 'Event' at .*")
2105
2106
2107 # Tests for Barrier - adapted from tests in test/lock_tests.py
2108 #
2109
2110 # Many of the tests for threading.Barrier use a list as an atomic
2111 # counter: a value is appended to increment the counter, and the
2112 # length of the list gives the value. We use the class DummyList
2113 # for the same purpose.
2114
2115 class _DummyList(object):
2116
2117 def __init__(self):
2118 wrapper = multiprocessing.heap.BufferWrapper(struct.calcsize('i'))
2119 lock = multiprocessing.Lock()
2120 self.__setstate__((wrapper, lock))
2121 self._lengthbuf[0] = 0
2122
2123 def __setstate__(self, state):
2124 (self._wrapper, self._lock) = state
2125 self._lengthbuf = self._wrapper.create_memoryview().cast('i')
2126
2127 def __getstate__(self):
2128 return (self._wrapper, self._lock)
2129
2130 def append(self, _):
2131 with self._lock:
2132 self._lengthbuf[0] += 1
2133
2134 def __len__(self):
2135 with self._lock:
2136 return self._lengthbuf[0]
2137
2138 def _wait():
2139 # A crude wait/yield function not relying on synchronization primitives.
2140 time.sleep(0.01)
2141
2142
2143 class Bunch(object):
2144 """
2145 A bunch of threads.
2146 """
2147 def __init__(self, namespace, f, args, n, wait_before_exit=False):
2148 """
2149 Construct a bunch of `n` threads running the same function `f`.
2150 If `wait_before_exit` is True, the threads won't terminate until
2151 do_finish() is called.
2152 """
2153 self.f = f
2154 self.args = args
2155 self.n = n
2156 self.started = namespace.DummyList()
2157 self.finished = namespace.DummyList()
2158 self._can_exit = namespace.Event()
2159 if not wait_before_exit:
2160 self._can_exit.set()
2161
2162 threads = []
2163 for i in range(n):
2164 p = namespace.Process(target=self.task)
2165 p.daemon = True
2166 p.start()
2167 threads.append(p)
2168
2169 def finalize(threads):
2170 for p in threads:
2171 p.join()
2172
2173 self._finalizer = weakref.finalize(self, finalize, threads)
2174
2175 def task(self):
2176 pid = os.getpid()
2177 self.started.append(pid)
2178 try:
2179 self.f(*self.args)
2180 finally:
2181 self.finished.append(pid)
2182 self._can_exit.wait(30)
2183 assert self._can_exit.is_set()
2184
2185 def wait_for_started(self):
2186 while len(self.started) < self.n:
2187 _wait()
2188
2189 def wait_for_finished(self):
2190 while len(self.finished) < self.n:
2191 _wait()
2192
2193 def do_finish(self):
2194 self._can_exit.set()
2195
2196 def close(self):
2197 self._finalizer()
2198
2199
2200 class AppendTrue(object):
2201 def __init__(self, obj):
2202 self.obj = obj
2203 def __call__(self):
2204 self.obj.append(True)
2205
2206
2207 class _TestBarrier(BaseTestCase):
2208 """
2209 Tests for Barrier objects.
2210 """
2211 N = 5
2212 defaultTimeout = 30.0 # XXX Slow Windows buildbots need generous timeout
2213
2214 def setUp(self):
2215 self.barrier = self.Barrier(self.N, timeout=self.defaultTimeout)
2216
2217 def tearDown(self):
2218 self.barrier.abort()
2219 self.barrier = None
2220
2221 def DummyList(self):
2222 if self.TYPE == 'threads':
2223 return []
2224 elif self.TYPE == 'manager':
2225 return self.manager.list()
2226 else:
2227 return _DummyList()
2228
2229 def run_threads(self, f, args):
2230 b = Bunch(self, f, args, self.N-1)
2231 try:
2232 f(*args)
2233 b.wait_for_finished()
2234 finally:
2235 b.close()
2236
2237 @classmethod
2238 def multipass(cls, barrier, results, n):
2239 m = barrier.parties
2240 assert m == cls.N
2241 for i in range(n):
2242 results[0].append(True)
2243 assert len(results[1]) == i * m
2244 barrier.wait()
2245 results[1].append(True)
2246 assert len(results[0]) == (i + 1) * m
2247 barrier.wait()
2248 try:
2249 assert barrier.n_waiting == 0
2250 except NotImplementedError:
2251 pass
2252 assert not barrier.broken
2253
2254 def test_barrier(self, passes=1):
2255 """
2256 Test that a barrier is passed in lockstep
2257 """
2258 results = [self.DummyList(), self.DummyList()]
2259 self.run_threads(self.multipass, (self.barrier, results, passes))
2260
2261 def test_barrier_10(self):
2262 """
2263 Test that a barrier works for 10 consecutive runs
2264 """
2265 return self.test_barrier(10)
2266
2267 @classmethod
2268 def _test_wait_return_f(cls, barrier, queue):
2269 res = barrier.wait()
2270 queue.put(res)
2271
2272 def test_wait_return(self):
2273 """
2274 test the return value from barrier.wait
2275 """
2276 queue = self.Queue()
2277 self.run_threads(self._test_wait_return_f, (self.barrier, queue))
2278 results = [queue.get() for i in range(self.N)]
2279 self.assertEqual(results.count(0), 1)
2280 close_queue(queue)
2281
2282 @classmethod
2283 def _test_action_f(cls, barrier, results):
2284 barrier.wait()
2285 if len(results) != 1:
2286 raise RuntimeError
2287
2288 def test_action(self):
2289 """
2290 Test the 'action' callback
2291 """
2292 results = self.DummyList()
2293 barrier = self.Barrier(self.N, action=AppendTrue(results))
2294 self.run_threads(self._test_action_f, (barrier, results))
2295 self.assertEqual(len(results), 1)
2296
2297 @classmethod
2298 def _test_abort_f(cls, barrier, results1, results2):
2299 try:
2300 i = barrier.wait()
2301 if i == cls.N//2:
2302 raise RuntimeError
2303 barrier.wait()
2304 results1.append(True)
2305 except threading.BrokenBarrierError:
2306 results2.append(True)
2307 except RuntimeError:
2308 barrier.abort()
2309
2310 def test_abort(self):
2311 """
2312 Test that an abort will put the barrier in a broken state
2313 """
2314 results1 = self.DummyList()
2315 results2 = self.DummyList()
2316 self.run_threads(self._test_abort_f,
2317 (self.barrier, results1, results2))
2318 self.assertEqual(len(results1), 0)
2319 self.assertEqual(len(results2), self.N-1)
2320 self.assertTrue(self.barrier.broken)
2321
2322 @classmethod
2323 def _test_reset_f(cls, barrier, results1, results2, results3):
2324 i = barrier.wait()
2325 if i == cls.N//2:
2326 # Wait until the other threads are all in the barrier.
2327 while barrier.n_waiting < cls.N-1:
2328 time.sleep(0.001)
2329 barrier.reset()
2330 else:
2331 try:
2332 barrier.wait()
2333 results1.append(True)
2334 except threading.BrokenBarrierError:
2335 results2.append(True)
2336 # Now, pass the barrier again
2337 barrier.wait()
2338 results3.append(True)
2339
2340 def test_reset(self):
2341 """
2342 Test that a 'reset' on a barrier frees the waiting threads
2343 """
2344 results1 = self.DummyList()
2345 results2 = self.DummyList()
2346 results3 = self.DummyList()
2347 self.run_threads(self._test_reset_f,
2348 (self.barrier, results1, results2, results3))
2349 self.assertEqual(len(results1), 0)
2350 self.assertEqual(len(results2), self.N-1)
2351 self.assertEqual(len(results3), self.N)
2352
2353 @classmethod
2354 def _test_abort_and_reset_f(cls, barrier, barrier2,
2355 results1, results2, results3):
2356 try:
2357 i = barrier.wait()
2358 if i == cls.N//2:
2359 raise RuntimeError
2360 barrier.wait()
2361 results1.append(True)
2362 except threading.BrokenBarrierError:
2363 results2.append(True)
2364 except RuntimeError:
2365 barrier.abort()
2366 # Synchronize and reset the barrier. Must synchronize first so
2367 # that everyone has left it when we reset, and after so that no
2368 # one enters it before the reset.
2369 if barrier2.wait() == cls.N//2:
2370 barrier.reset()
2371 barrier2.wait()
2372 barrier.wait()
2373 results3.append(True)
2374
2375 def test_abort_and_reset(self):
2376 """
2377 Test that a barrier can be reset after being broken.
2378 """
2379 results1 = self.DummyList()
2380 results2 = self.DummyList()
2381 results3 = self.DummyList()
2382 barrier2 = self.Barrier(self.N)
2383
2384 self.run_threads(self._test_abort_and_reset_f,
2385 (self.barrier, barrier2, results1, results2, results3))
2386 self.assertEqual(len(results1), 0)
2387 self.assertEqual(len(results2), self.N-1)
2388 self.assertEqual(len(results3), self.N)
2389
2390 @classmethod
2391 def _test_timeout_f(cls, barrier, results):
2392 i = barrier.wait()
2393 if i == cls.N//2:
2394 # One thread is late!
2395 time.sleep(1.0)
2396 try:
2397 barrier.wait(0.5)
2398 except threading.BrokenBarrierError:
2399 results.append(True)
2400
2401 def test_timeout(self):
2402 """
2403 Test wait(timeout)
2404 """
2405 results = self.DummyList()
2406 self.run_threads(self._test_timeout_f, (self.barrier, results))
2407 self.assertEqual(len(results), self.barrier.parties)
2408
2409 @classmethod
2410 def _test_default_timeout_f(cls, barrier, results):
2411 i = barrier.wait(cls.defaultTimeout)
2412 if i == cls.N//2:
2413 # One thread is later than the default timeout
2414 time.sleep(1.0)
2415 try:
2416 barrier.wait()
2417 except threading.BrokenBarrierError:
2418 results.append(True)
2419
2420 def test_default_timeout(self):
2421 """
2422 Test the barrier's default timeout
2423 """
2424 barrier = self.Barrier(self.N, timeout=0.5)
2425 results = self.DummyList()
2426 self.run_threads(self._test_default_timeout_f, (barrier, results))
2427 self.assertEqual(len(results), barrier.parties)
2428
2429 def test_single_thread(self):
2430 b = self.Barrier(1)
2431 b.wait()
2432 b.wait()
2433
2434 @classmethod
2435 def _test_thousand_f(cls, barrier, passes, conn, lock):
2436 for i in range(passes):
2437 barrier.wait()
2438 with lock:
2439 conn.send(i)
2440
2441 def test_thousand(self):
2442 if self.TYPE == 'manager':
2443 self.skipTest('test not appropriate for {}'.format(self.TYPE))
2444 passes = 1000
2445 lock = self.Lock()
2446 conn, child_conn = self.Pipe(False)
2447 for j in range(self.N):
2448 p = self.Process(target=self._test_thousand_f,
2449 args=(self.barrier, passes, child_conn, lock))
2450 p.start()
2451 self.addCleanup(p.join)
2452
2453 for i in range(passes):
2454 for j in range(self.N):
2455 self.assertEqual(conn.recv(), i)
2456
2457 #
2458 #
2459 #
2460
2461 class _TestValue(BaseTestCase):
2462
2463 ALLOWED_TYPES = ('processes',)
2464
2465 codes_values = [
2466 ('i', 4343, 24234),
2467 ('d', 3.625, -4.25),
2468 ('h', -232, 234),
2469 ('q', 2 ** 33, 2 ** 34),
2470 ('c', latin('x'), latin('y'))
2471 ]
2472
2473 def setUp(self):
2474 if not HAS_SHAREDCTYPES:
2475 self.skipTest("requires multiprocessing.sharedctypes")
2476
2477 @classmethod
2478 def _test(cls, values):
2479 for sv, cv in zip(values, cls.codes_values):
2480 sv.value = cv[2]
2481
2482
2483 def test_value(self, raw=False):
2484 if raw:
2485 values = [self.RawValue(code, value)
2486 for code, value, _ in self.codes_values]
2487 else:
2488 values = [self.Value(code, value)
2489 for code, value, _ in self.codes_values]
2490
2491 for sv, cv in zip(values, self.codes_values):
2492 self.assertEqual(sv.value, cv[1])
2493
2494 proc = self.Process(target=self._test, args=(values,))
2495 proc.daemon = True
2496 proc.start()
2497 proc.join()
2498
2499 for sv, cv in zip(values, self.codes_values):
2500 self.assertEqual(sv.value, cv[2])
2501
2502 def test_rawvalue(self):
2503 self.test_value(raw=True)
2504
2505 def test_getobj_getlock(self):
2506 val1 = self.Value('i', 5)
2507 lock1 = val1.get_lock()
2508 obj1 = val1.get_obj()
2509
2510 val2 = self.Value('i', 5, lock=None)
2511 lock2 = val2.get_lock()
2512 obj2 = val2.get_obj()
2513
2514 lock = self.Lock()
2515 val3 = self.Value('i', 5, lock=lock)
2516 lock3 = val3.get_lock()
2517 obj3 = val3.get_obj()
2518 self.assertEqual(lock, lock3)
2519
2520 arr4 = self.Value('i', 5, lock=False)
2521 self.assertNotHasAttr(arr4, 'get_lock')
2522 self.assertNotHasAttr(arr4, 'get_obj')
2523
2524 self.assertRaises(AttributeError, self.Value, 'i', 5, lock='navalue')
2525
2526 arr5 = self.RawValue('i', 5)
2527 self.assertNotHasAttr(arr5, 'get_lock')
2528 self.assertNotHasAttr(arr5, 'get_obj')
2529
2530
2531 class _TestArray(BaseTestCase):
2532
2533 ALLOWED_TYPES = ('processes',)
2534
2535 @classmethod
2536 def f(cls, seq):
2537 for i in range(1, len(seq)):
2538 seq[i] += seq[i-1]
2539
2540 @unittest.skipIf(c_int is None, "requires _ctypes")
2541 def test_array(self, raw=False):
2542 seq = [680, 626, 934, 821, 150, 233, 548, 982, 714, 831]
2543 if raw:
2544 arr = self.RawArray('i', seq)
2545 else:
2546 arr = self.Array('i', seq)
2547
2548 self.assertEqual(len(arr), len(seq))
2549 self.assertEqual(arr[3], seq[3])
2550 self.assertEqual(list(arr[2:7]), list(seq[2:7]))
2551
2552 arr[4:8] = seq[4:8] = array.array('i', [1, 2, 3, 4])
2553
2554 self.assertEqual(list(arr[:]), seq)
2555
2556 self.f(seq)
2557
2558 p = self.Process(target=self.f, args=(arr,))
2559 p.daemon = True
2560 p.start()
2561 p.join()
2562
2563 self.assertEqual(list(arr[:]), seq)
2564
2565 @unittest.skipIf(c_int is None, "requires _ctypes")
2566 def test_array_from_size(self):
2567 size = 10
2568 # Test for zeroing (see issue #11675).
2569 # The repetition below strengthens the test by increasing the chances
2570 # of previously allocated non-zero memory being used for the new array
2571 # on the 2nd and 3rd loops.
2572 for _ in range(3):
2573 arr = self.Array('i', size)
2574 self.assertEqual(len(arr), size)
2575 self.assertEqual(list(arr), [0] * size)
2576 arr[:] = range(10)
2577 self.assertEqual(list(arr), list(range(10)))
2578 del arr
2579
2580 @unittest.skipIf(c_int is None, "requires _ctypes")
2581 def test_rawarray(self):
2582 self.test_array(raw=True)
2583
2584 @unittest.skipIf(c_int is None, "requires _ctypes")
2585 def test_getobj_getlock_obj(self):
2586 arr1 = self.Array('i', list(range(10)))
2587 lock1 = arr1.get_lock()
2588 obj1 = arr1.get_obj()
2589
2590 arr2 = self.Array('i', list(range(10)), lock=None)
2591 lock2 = arr2.get_lock()
2592 obj2 = arr2.get_obj()
2593
2594 lock = self.Lock()
2595 arr3 = self.Array('i', list(range(10)), lock=lock)
2596 lock3 = arr3.get_lock()
2597 obj3 = arr3.get_obj()
2598 self.assertEqual(lock, lock3)
2599
2600 arr4 = self.Array('i', range(10), lock=False)
2601 self.assertNotHasAttr(arr4, 'get_lock')
2602 self.assertNotHasAttr(arr4, 'get_obj')
2603 self.assertRaises(AttributeError,
2604 self.Array, 'i', range(10), lock='notalock')
2605
2606 arr5 = self.RawArray('i', range(10))
2607 self.assertNotHasAttr(arr5, 'get_lock')
2608 self.assertNotHasAttr(arr5, 'get_obj')
2609
2610 #
2611 #
2612 #
2613
2614 class _TestContainers(BaseTestCase):
2615
2616 ALLOWED_TYPES = ('manager',)
2617
2618 def test_list(self):
2619 a = self.list(list(range(10)))
2620 self.assertEqual(a[:], list(range(10)))
2621
2622 b = self.list()
2623 self.assertEqual(b[:], [])
2624
2625 b.extend(list(range(5)))
2626 self.assertEqual(b[:], list(range(5)))
2627
2628 self.assertEqual(b[2], 2)
2629 self.assertEqual(b[2:10], [2,3,4])
2630
2631 b *= 2
2632 self.assertEqual(b[:], [0, 1, 2, 3, 4, 0, 1, 2, 3, 4])
2633
2634 self.assertEqual(b + [5, 6], [0, 1, 2, 3, 4, 0, 1, 2, 3, 4, 5, 6])
2635
2636 self.assertEqual(a[:], list(range(10)))
2637
2638 d = [a, b]
2639 e = self.list(d)
2640 self.assertEqual(
2641 [element[:] for element in e],
2642 [[0, 1, 2, 3, 4, 5, 6, 7, 8, 9], [0, 1, 2, 3, 4, 0, 1, 2, 3, 4]]
2643 )
2644
2645 f = self.list([a])
2646 a.append('hello')
2647 self.assertEqual(f[0][:], [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 'hello'])
2648
2649 def test_list_isinstance(self):
2650 a = self.list()
2651 self.assertIsInstance(a, collections.abc.MutableSequence)
2652
2653 # MutableSequence also has __iter__, but we can iterate over
2654 # ListProxy using __getitem__ instead. Adding __iter__ to ListProxy
2655 # would change the behavior of a list modified during iteration.
2656 mutable_sequence_methods = (
2657 '__contains__', '__delitem__', '__getitem__', '__iadd__',
2658 '__len__', '__reversed__', '__setitem__', 'append',
2659 'clear', 'count', 'extend', 'index', 'insert', 'pop', 'remove',
2660 'reverse',
2661 )
2662 for name in mutable_sequence_methods:
2663 with self.subTest(name=name):
2664 self.assertTrue(callable(getattr(a, name)))
2665
2666 def test_list_iter(self):
2667 a = self.list(list(range(10)))
2668 it = iter(a)
2669 self.assertEqual(list(it), list(range(10)))
2670 self.assertEqual(list(it), []) # exhausted
2671 # list modified during iteration
2672 it = iter(a)
2673 a[0] = 100
2674 self.assertEqual(next(it), 100)
2675
2676 def test_list_proxy_in_list(self):
2677 a = self.list([self.list(range(3)) for _i in range(3)])
2678 self.assertEqual([inner[:] for inner in a], [[0, 1, 2]] * 3)
2679
2680 a[0][-1] = 55
2681 self.assertEqual(a[0][:], [0, 1, 55])
2682 for i in range(1, 3):
2683 self.assertEqual(a[i][:], [0, 1, 2])
2684
2685 self.assertEqual(a[1].pop(), 2)
2686 self.assertEqual(len(a[1]), 2)
2687 for i in range(0, 3, 2):
2688 self.assertEqual(len(a[i]), 3)
2689
2690 del a
2691
2692 b = self.list()
2693 b.append(b)
2694 del b
2695
2696 def test_dict(self):
2697 d = self.dict()
2698 indices = list(range(65, 70))
2699 for i in indices:
2700 d[i] = chr(i)
2701 self.assertEqual(d.copy(), dict((i, chr(i)) for i in indices))
2702 self.assertEqual(sorted(d.keys()), indices)
2703 self.assertEqual(sorted(d.values()), [chr(i) for i in indices])
2704 self.assertEqual(sorted(d.items()), [(i, chr(i)) for i in indices])
2705
2706 def test_dict_isinstance(self):
2707 a = self.dict()
2708 self.assertIsInstance(a, collections.abc.MutableMapping)
2709
2710 mutable_mapping_methods = (
2711 '__contains__', '__delitem__', '__eq__', '__getitem__', '__iter__',
2712 '__len__', '__ne__', '__setitem__', 'clear', 'get', 'items',
2713 'keys', 'pop', 'popitem', 'setdefault', 'update', 'values',
2714 )
2715 for name in mutable_mapping_methods:
2716 with self.subTest(name=name):
2717 self.assertTrue(callable(getattr(a, name)))
2718
2719 def test_dict_iter(self):
2720 d = self.dict()
2721 indices = list(range(65, 70))
2722 for i in indices:
2723 d[i] = chr(i)
2724 it = iter(d)
2725 self.assertEqual(list(it), indices)
2726 self.assertEqual(list(it), []) # exhausted
2727 # dictionary changed size during iteration
2728 it = iter(d)
2729 d.clear()
2730 self.assertRaises(RuntimeError, next, it)
2731
2732 def test_dict_proxy_nested(self):
2733 pets = self.dict(ferrets=2, hamsters=4)
2734 supplies = self.dict(water=10, feed=3)
2735 d = self.dict(pets=pets, supplies=supplies)
2736
2737 self.assertEqual(supplies['water'], 10)
2738 self.assertEqual(d['supplies']['water'], 10)
2739
2740 d['supplies']['blankets'] = 5
2741 self.assertEqual(supplies['blankets'], 5)
2742 self.assertEqual(d['supplies']['blankets'], 5)
2743
2744 d['supplies']['water'] = 7
2745 self.assertEqual(supplies['water'], 7)
2746 self.assertEqual(d['supplies']['water'], 7)
2747
2748 del pets
2749 del supplies
2750 self.assertEqual(d['pets']['ferrets'], 2)
2751 d['supplies']['blankets'] = 11
2752 self.assertEqual(d['supplies']['blankets'], 11)
2753
2754 pets = d['pets']
2755 supplies = d['supplies']
2756 supplies['water'] = 7
2757 self.assertEqual(supplies['water'], 7)
2758 self.assertEqual(d['supplies']['water'], 7)
2759
2760 d.clear()
2761 self.assertEqual(len(d), 0)
2762 self.assertEqual(supplies['water'], 7)
2763 self.assertEqual(pets['hamsters'], 4)
2764
2765 l = self.list([pets, supplies])
2766 l[0]['marmots'] = 1
2767 self.assertEqual(pets['marmots'], 1)
2768 self.assertEqual(l[0]['marmots'], 1)
2769
2770 del pets
2771 del supplies
2772 self.assertEqual(l[0]['marmots'], 1)
2773
2774 outer = self.list([[88, 99], l])
2775 self.assertIsInstance(outer[0], list) # Not a ListProxy
2776 self.assertEqual(outer[-1][-1]['feed'], 3)
2777
2778 def test_nested_queue(self):
2779 a = self.list() # Test queue inside list
2780 a.append(self.Queue())
2781 a[0].put(123)
2782 self.assertEqual(a[0].get(), 123)
2783 b = self.dict() # Test queue inside dict
2784 b[0] = self.Queue()
2785 b[0].put(456)
2786 self.assertEqual(b[0].get(), 456)
2787
2788 def test_namespace(self):
2789 n = self.Namespace()
2790 n.name = 'Bob'
2791 n.job = 'Builder'
2792 n._hidden = 'hidden'
2793 self.assertEqual((n.name, n.job), ('Bob', 'Builder'))
2794 del n.job
2795 self.assertEqual(str(n), "Namespace(name='Bob')")
2796 self.assertHasAttr(n, 'name')
2797 self.assertNotHasAttr(n, 'job')
2798
2799 #
2800 #
2801 #
2802
2803 def sqr(x, wait=0.0, event=None):
2804 if event is None:
2805 time.sleep(wait)
2806 else:
2807 event.wait(wait)
2808 return x*x
2809
2810 def mul(x, y):
2811 return x*y
2812
2813 def raise_large_valuerror(wait):
2814 time.sleep(wait)
2815 raise ValueError("x" * 1024**2)
2816
2817 def identity(x):
2818 return x
2819
2820 class CountedObject(object):
2821 n_instances = 0
2822
2823 def __new__(cls):
2824 cls.n_instances += 1
2825 return object.__new__(cls)
2826
2827 def __del__(self):
2828 type(self).n_instances -= 1
2829
2830 class SayWhenError(ValueError): pass
2831
2832 def exception_throwing_generator(total, when):
2833 if when == -1:
2834 raise SayWhenError("Somebody said when")
2835 for i in range(total):
2836 if i == when:
2837 raise SayWhenError("Somebody said when")
2838 yield i
2839
2840
2841 class _TestPool(BaseTestCase):
2842
2843 @classmethod
2844 def setUpClass(cls):
2845 super().setUpClass()
2846 cls.pool = cls.Pool(4)
2847
2848 @classmethod
2849 def tearDownClass(cls):
2850 cls.pool.terminate()
2851 cls.pool.join()
2852 cls.pool = None
2853 super().tearDownClass()
2854
2855 def test_apply(self):
2856 papply = self.pool.apply
2857 self.assertEqual(papply(sqr, (5,)), sqr(5))
2858 self.assertEqual(papply(sqr, (), {'x':3}), sqr(x=3))
2859
2860 def test_map(self):
2861 pmap = self.pool.map
2862 self.assertEqual(pmap(sqr, list(range(10))), list(map(sqr, list(range(10)))))
2863 self.assertEqual(pmap(sqr, list(range(100)), chunksize=20),
2864 list(map(sqr, list(range(100)))))
2865
2866 def test_starmap(self):
2867 psmap = self.pool.starmap
2868 tuples = list(zip(range(10), range(9,-1, -1)))
2869 self.assertEqual(psmap(mul, tuples),
2870 list(itertools.starmap(mul, tuples)))
2871 tuples = list(zip(range(100), range(99,-1, -1)))
2872 self.assertEqual(psmap(mul, tuples, chunksize=20),
2873 list(itertools.starmap(mul, tuples)))
2874
2875 def test_starmap_async(self):
2876 tuples = list(zip(range(100), range(99,-1, -1)))
2877 self.assertEqual(self.pool.starmap_async(mul, tuples).get(),
2878 list(itertools.starmap(mul, tuples)))
2879
2880 def test_map_async(self):
2881 self.assertEqual(self.pool.map_async(sqr, list(range(10))).get(),
2882 list(map(sqr, list(range(10)))))
2883
2884 def test_map_async_callbacks(self):
2885 call_args = self.manager.list() if self.TYPE == 'manager' else []
2886 self.pool.map_async(int, ['1'],
2887 callback=call_args.append,
2888 error_callback=call_args.append).wait()
2889 self.assertEqual(1, len(call_args))
2890 self.assertEqual([1], call_args[0])
2891 self.pool.map_async(int, ['a'],
2892 callback=call_args.append,
2893 error_callback=call_args.append).wait()
2894 self.assertEqual(2, len(call_args))
2895 self.assertIsInstance(call_args[1], ValueError)
2896
2897 def test_map_unplicklable(self):
2898 # Issue #19425 -- failure to pickle should not cause a hang
2899 if self.TYPE == 'threads':
2900 self.skipTest('test not appropriate for {}'.format(self.TYPE))
2901 class A(object):
2902 def __reduce__(self):
2903 raise RuntimeError('cannot pickle')
2904 with self.assertRaises(RuntimeError):
2905 self.pool.map(sqr, [A()]*10)
2906
2907 def test_map_chunksize(self):
2908 try:
2909 self.pool.map_async(sqr, [], chunksize=1).get(timeout=TIMEOUT1)
2910 except multiprocessing.TimeoutError:
2911 self.fail("pool.map_async with chunksize stalled on null list")
2912
2913 def test_map_handle_iterable_exception(self):
2914 if self.TYPE == 'manager':
2915 self.skipTest('test not appropriate for {}'.format(self.TYPE))
2916
2917 # SayWhenError seen at the very first of the iterable
2918 with self.assertRaises(SayWhenError):
2919 self.pool.map(sqr, exception_throwing_generator(1, -1), 1)
2920 # again, make sure it's reentrant
2921 with self.assertRaises(SayWhenError):
2922 self.pool.map(sqr, exception_throwing_generator(1, -1), 1)
2923
2924 with self.assertRaises(SayWhenError):
2925 self.pool.map(sqr, exception_throwing_generator(10, 3), 1)
2926
2927 class SpecialIterable:
2928 def __iter__(self):
2929 return self
2930 def __next__(self):
2931 raise SayWhenError
2932 def __len__(self):
2933 return 1
2934 with self.assertRaises(SayWhenError):
2935 self.pool.map(sqr, SpecialIterable(), 1)
2936 with self.assertRaises(SayWhenError):
2937 self.pool.map(sqr, SpecialIterable(), 1)
2938
2939 def test_async(self):
2940 res = self.pool.apply_async(sqr, (7, TIMEOUT1,))
2941 get = TimingWrapper(res.get)
2942 self.assertEqual(get(), 49)
2943 self.assertTimingAlmostEqual(get.elapsed, TIMEOUT1)
2944
2945 def test_async_timeout(self):
2946 p = self.Pool(3)
2947 try:
2948 event = threading.Event() if self.TYPE == 'threads' else None
2949 res = p.apply_async(sqr, (6, TIMEOUT2 + support.SHORT_TIMEOUT, event))
2950 get = TimingWrapper(res.get)
2951 self.assertRaises(multiprocessing.TimeoutError, get, timeout=TIMEOUT2)
2952 self.assertTimingAlmostEqual(get.elapsed, TIMEOUT2)
2953 finally:
2954 if event is not None:
2955 event.set()
2956 p.terminate()
2957 p.join()
2958
2959 def test_imap(self):
2960 it = self.pool.imap(sqr, list(range(10)))
2961 self.assertEqual(list(it), list(map(sqr, list(range(10)))))
2962
2963 it = self.pool.imap(sqr, list(range(10)))
2964 for i in range(10):
2965 self.assertEqual(next(it), i*i)
2966 self.assertRaises(StopIteration, it.__next__)
2967
2968 it = self.pool.imap(sqr, list(range(1000)), chunksize=100)
2969 for i in range(1000):
2970 self.assertEqual(next(it), i*i)
2971 self.assertRaises(StopIteration, it.__next__)
2972
2973 def test_imap_handle_iterable_exception(self):
2974 if self.TYPE == 'manager':
2975 self.skipTest('test not appropriate for {}'.format(self.TYPE))
2976
2977 # SayWhenError seen at the very first of the iterable
2978 it = self.pool.imap(sqr, exception_throwing_generator(1, -1), 1)
2979 self.assertRaises(SayWhenError, it.__next__)
2980 # again, make sure it's reentrant
2981 it = self.pool.imap(sqr, exception_throwing_generator(1, -1), 1)
2982 self.assertRaises(SayWhenError, it.__next__)
2983
2984 it = self.pool.imap(sqr, exception_throwing_generator(10, 3), 1)
2985 for i in range(3):
2986 self.assertEqual(next(it), i*i)
2987 self.assertRaises(SayWhenError, it.__next__)
2988
2989 # SayWhenError seen at start of problematic chunk's results
2990 it = self.pool.imap(sqr, exception_throwing_generator(20, 7), 2)
2991 for i in range(6):
2992 self.assertEqual(next(it), i*i)
2993 self.assertRaises(SayWhenError, it.__next__)
2994 it = self.pool.imap(sqr, exception_throwing_generator(20, 7), 4)
2995 for i in range(4):
2996 self.assertEqual(next(it), i*i)
2997 self.assertRaises(SayWhenError, it.__next__)
2998
2999 def test_imap_unordered(self):
3000 it = self.pool.imap_unordered(sqr, list(range(10)))
3001 self.assertEqual(sorted(it), list(map(sqr, list(range(10)))))
3002
3003 it = self.pool.imap_unordered(sqr, list(range(1000)), chunksize=100)
3004 self.assertEqual(sorted(it), list(map(sqr, list(range(1000)))))
3005
3006 def test_imap_unordered_handle_iterable_exception(self):
3007 if self.TYPE == 'manager':
3008 self.skipTest('test not appropriate for {}'.format(self.TYPE))
3009
3010 # SayWhenError seen at the very first of the iterable
3011 it = self.pool.imap_unordered(sqr,
3012 exception_throwing_generator(1, -1),
3013 1)
3014 self.assertRaises(SayWhenError, it.__next__)
3015 # again, make sure it's reentrant
3016 it = self.pool.imap_unordered(sqr,
3017 exception_throwing_generator(1, -1),
3018 1)
3019 self.assertRaises(SayWhenError, it.__next__)
3020
3021 it = self.pool.imap_unordered(sqr,
3022 exception_throwing_generator(10, 3),
3023 1)
3024 expected_values = list(map(sqr, list(range(10))))
3025 with self.assertRaises(SayWhenError):
3026 # imap_unordered makes it difficult to anticipate the SayWhenError
3027 for i in range(10):
3028 value = next(it)
3029 self.assertIn(value, expected_values)
3030 expected_values.remove(value)
3031
3032 it = self.pool.imap_unordered(sqr,
3033 exception_throwing_generator(20, 7),
3034 2)
3035 expected_values = list(map(sqr, list(range(20))))
3036 with self.assertRaises(SayWhenError):
3037 for i in range(20):
3038 value = next(it)
3039 self.assertIn(value, expected_values)
3040 expected_values.remove(value)
3041
3042 def test_make_pool(self):
3043 expected_error = (RemoteError if self.TYPE == 'manager'
3044 else ValueError)
3045
3046 self.assertRaises(expected_error, self.Pool, -1)
3047 self.assertRaises(expected_error, self.Pool, 0)
3048
3049 if self.TYPE != 'manager':
3050 p = self.Pool(3)
3051 try:
3052 self.assertEqual(3, len(p._pool))
3053 finally:
3054 p.close()
3055 p.join()
3056
3057 def test_terminate(self):
3058 # Simulate slow tasks which take "forever" to complete
3059 sleep_time = support.LONG_TIMEOUT
3060
3061 if self.TYPE == 'threads':
3062 # Thread pool workers can't be forced to quit, so if the first
3063 # task starts early enough, we will end up waiting for it.
3064 # Sleep for a shorter time, so the test doesn't block.
3065 sleep_time = 1
3066
3067 p = self.Pool(3)
3068 args = [sleep_time for i in range(10_000)]
3069 result = p.map_async(time.sleep, args, chunksize=1)
3070 time.sleep(0.2) # give some tasks a chance to start
3071 p.terminate()
3072 p.join()
3073
3074 def test_empty_iterable(self):
3075 # See Issue 12157
3076 p = self.Pool(1)
3077
3078 self.assertEqual(p.map(sqr, []), [])
3079 self.assertEqual(list(p.imap(sqr, [])), [])
3080 self.assertEqual(list(p.imap_unordered(sqr, [])), [])
3081 self.assertEqual(p.map_async(sqr, []).get(), [])
3082
3083 p.close()
3084 p.join()
3085
3086 def test_context(self):
3087 if self.TYPE == 'processes':
3088 L = list(range(10))
3089 expected = [sqr(i) for i in L]
3090 with self.Pool(2) as p:
3091 r = p.map_async(sqr, L)
3092 self.assertEqual(r.get(), expected)
3093 p.join()
3094 self.assertRaises(ValueError, p.map_async, sqr, L)
3095
3096 @classmethod
3097 def _test_traceback(cls):
3098 raise RuntimeError(123) # some comment
3099
3100 def test_traceback(self):
3101 # We want ensure that the traceback from the child process is
3102 # contained in the traceback raised in the main process.
3103 if self.TYPE == 'processes':
3104 with self.Pool(1) as p:
3105 try:
3106 p.apply(self._test_traceback)
3107 except Exception as e:
3108 exc = e
3109 else:
3110 self.fail('expected RuntimeError')
3111 p.join()
3112 self.assertIs(type(exc), RuntimeError)
3113 self.assertEqual(exc.args, (123,))
3114 cause = exc.__cause__
3115 self.assertIs(type(cause), multiprocessing.pool.RemoteTraceback)
3116 self.assertIn('raise RuntimeError(123) # some comment', cause.tb)
3117
3118 with test.support.captured_stderr() as f1:
3119 try:
3120 raise exc
3121 except RuntimeError:
3122 sys.excepthook(*sys.exc_info())
3123 self.assertIn('raise RuntimeError(123) # some comment',
3124 f1.getvalue())
3125 # _helper_reraises_exception should not make the error
3126 # a remote exception
3127 with self.Pool(1) as p:
3128 try:
3129 p.map(sqr, exception_throwing_generator(1, -1), 1)
3130 except Exception as e:
3131 exc = e
3132 else:
3133 self.fail('expected SayWhenError')
3134 self.assertIs(type(exc), SayWhenError)
3135 self.assertIs(exc.__cause__, None)
3136 p.join()
3137
3138 @classmethod
3139 def _test_wrapped_exception(cls):
3140 raise RuntimeError('foo')
3141
3142 def test_wrapped_exception(self):
3143 # Issue #20980: Should not wrap exception when using thread pool
3144 with self.Pool(1) as p:
3145 with self.assertRaises(RuntimeError):
3146 p.apply(self._test_wrapped_exception)
3147 p.join()
3148
3149 def test_map_no_failfast(self):
3150 # Issue #23992: the fail-fast behaviour when an exception is raised
3151 # during map() would make Pool.join() deadlock, because a worker
3152 # process would fill the result queue (after the result handler thread
3153 # terminated, hence not draining it anymore).
3154
3155 t_start = time.monotonic()
3156
3157 with self.assertRaises(ValueError):
3158 with self.Pool(2) as p:
3159 try:
3160 p.map(raise_large_valuerror, [0, 1])
3161 finally:
3162 time.sleep(0.5)
3163 p.close()
3164 p.join()
3165
3166 # check that we indeed waited for all jobs
3167 self.assertGreater(time.monotonic() - t_start, 0.9)
3168
3169 def test_release_task_refs(self):
3170 # Issue #29861: task arguments and results should not be kept
3171 # alive after we are done with them.
3172 objs = [CountedObject() for i in range(10)]
3173 refs = [weakref.ref(o) for o in objs]
3174 self.pool.map(identity, objs)
3175
3176 del objs
3177 time.sleep(DELTA) # let threaded cleanup code run
3178 support.gc_collect() # For PyPy or other GCs.
3179 self.assertEqual(set(wr() for wr in refs), {None})
3180 # With a process pool, copies of the objects are returned, check
3181 # they were released too.
3182 self.assertEqual(CountedObject.n_instances, 0)
3183
3184 def test_enter(self):
3185 if self.TYPE == 'manager':
3186 self.skipTest("test not applicable to manager")
3187
3188 pool = self.Pool(1)
3189 with pool:
3190 pass
3191 # call pool.terminate()
3192 # pool is no longer running
3193
3194 with self.assertRaises(ValueError):
3195 # bpo-35477: pool.__enter__() fails if the pool is not running
3196 with pool:
3197 pass
3198 pool.join()
3199
3200 def test_resource_warning(self):
3201 if self.TYPE == 'manager':
3202 self.skipTest("test not applicable to manager")
3203
3204 pool = self.Pool(1)
3205 pool.terminate()
3206 pool.join()
3207
3208 # force state to RUN to emit ResourceWarning in __del__()
3209 pool._state = multiprocessing.pool.RUN
3210
3211 with warnings_helper.check_warnings(
3212 ('unclosed running multiprocessing pool', ResourceWarning)):
3213 pool = None
3214 support.gc_collect()
3215
3216 def raising():
3217 raise KeyError("key")
3218
3219 def unpickleable_result():
3220 return lambda: 42
3221
3222 class _TestPoolWorkerErrors(BaseTestCase):
3223 ALLOWED_TYPES = ('processes', )
3224
3225 def test_async_error_callback(self):
3226 p = multiprocessing.Pool(2)
3227
3228 scratchpad = [None]
3229 def errback(exc):
3230 scratchpad[0] = exc
3231
3232 res = p.apply_async(raising, error_callback=errback)
3233 self.assertRaises(KeyError, res.get)
3234 self.assertTrue(scratchpad[0])
3235 self.assertIsInstance(scratchpad[0], KeyError)
3236
3237 p.close()
3238 p.join()
3239
3240 def test_unpickleable_result(self):
3241 from multiprocessing.pool import MaybeEncodingError
3242 p = multiprocessing.Pool(2)
3243
3244 # Make sure we don't lose pool processes because of encoding errors.
3245 for iteration in range(20):
3246
3247 scratchpad = [None]
3248 def errback(exc):
3249 scratchpad[0] = exc
3250
3251 res = p.apply_async(unpickleable_result, error_callback=errback)
3252 self.assertRaises(MaybeEncodingError, res.get)
3253 wrapped = scratchpad[0]
3254 self.assertTrue(wrapped)
3255 self.assertIsInstance(scratchpad[0], MaybeEncodingError)
3256 self.assertIsNotNone(wrapped.exc)
3257 self.assertIsNotNone(wrapped.value)
3258
3259 p.close()
3260 p.join()
3261
3262 class _TestPoolWorkerLifetime(BaseTestCase):
3263 ALLOWED_TYPES = ('processes', )
3264
3265 def test_pool_worker_lifetime(self):
3266 p = multiprocessing.Pool(3, maxtasksperchild=10)
3267 self.assertEqual(3, len(p._pool))
3268 origworkerpids = [w.pid for w in p._pool]
3269 # Run many tasks so each worker gets replaced (hopefully)
3270 results = []
3271 for i in range(100):
3272 results.append(p.apply_async(sqr, (i, )))
3273 # Fetch the results and verify we got the right answers,
3274 # also ensuring all the tasks have completed.
3275 for (j, res) in enumerate(results):
3276 self.assertEqual(res.get(), sqr(j))
3277 # Refill the pool
3278 p._repopulate_pool()
3279 # Wait until all workers are alive
3280 # (countdown * DELTA = 5 seconds max startup process time)
3281 countdown = 50
3282 while countdown and not all(w.is_alive() for w in p._pool):
3283 countdown -= 1
3284 time.sleep(DELTA)
3285 finalworkerpids = [w.pid for w in p._pool]
3286 # All pids should be assigned. See issue #7805.
3287 self.assertNotIn(None, origworkerpids)
3288 self.assertNotIn(None, finalworkerpids)
3289 # Finally, check that the worker pids have changed
3290 self.assertNotEqual(sorted(origworkerpids), sorted(finalworkerpids))
3291 p.close()
3292 p.join()
3293
3294 def test_pool_worker_lifetime_early_close(self):
3295 # Issue #10332: closing a pool whose workers have limited lifetimes
3296 # before all the tasks completed would make join() hang.
3297 p = multiprocessing.Pool(3, maxtasksperchild=1)
3298 results = []
3299 for i in range(6):
3300 results.append(p.apply_async(sqr, (i, 0.3)))
3301 p.close()
3302 p.join()
3303 # check the results
3304 for (j, res) in enumerate(results):
3305 self.assertEqual(res.get(), sqr(j))
3306
3307 def test_pool_maxtasksperchild_invalid(self):
3308 for value in [0, -1, 0.5, "12"]:
3309 with self.assertRaises(ValueError):
3310 multiprocessing.Pool(3, maxtasksperchild=value)
3311
3312 def test_worker_finalization_via_atexit_handler_of_multiprocessing(self):
3313 # tests cases against bpo-38744 and bpo-39360
3314 cmd = '''if 1:
3315 from multiprocessing import Pool
3316 problem = None
3317 class A:
3318 def __init__(self):
3319 self.pool = Pool(processes=1)
3320 def test():
3321 global problem
3322 problem = A()
3323 problem.pool.map(float, tuple(range(10)))
3324 if __name__ == "__main__":
3325 test()
3326 '''
3327 rc, out, err = test.support.script_helper.assert_python_ok('-c', cmd)
3328 self.assertEqual(rc, 0)
3329
3330 #
3331 # Test of creating a customized manager class
3332 #
3333
3334 from multiprocessing.managers import BaseManager, BaseProxy, RemoteError
3335
3336 class FooBar(object):
3337 def f(self):
3338 return 'f()'
3339 def g(self):
3340 raise ValueError
3341 def _h(self):
3342 return '_h()'
3343
3344 def baz():
3345 for i in range(10):
3346 yield i*i
3347
3348 class IteratorProxy(BaseProxy):
3349 _exposed_ = ('__next__',)
3350 def __iter__(self):
3351 return self
3352 def __next__(self):
3353 return self._callmethod('__next__')
3354
3355 class MyManager(BaseManager):
3356 pass
3357
3358 MyManager.register('Foo', callable=FooBar)
3359 MyManager.register('Bar', callable=FooBar, exposed=('f', '_h'))
3360 MyManager.register('baz', callable=baz, proxytype=IteratorProxy)
3361
3362
3363 class _TestMyManager(BaseTestCase):
3364
3365 ALLOWED_TYPES = ('manager',)
3366
3367 @support.skip_if_sanitizer('TSan: leaks threads', thread=True)
3368 def test_mymanager(self):
3369 manager = MyManager(shutdown_timeout=SHUTDOWN_TIMEOUT)
3370 manager.start()
3371 self.common(manager)
3372 manager.shutdown()
3373
3374 # bpo-30356: BaseManager._finalize_manager() sends SIGTERM
3375 # to the manager process if it takes longer than 1 second to stop,
3376 # which happens on slow buildbots.
3377 self.assertIn(manager._process.exitcode, (0, -signal.SIGTERM))
3378
3379 @support.skip_if_sanitizer('TSan: leaks threads', thread=True)
3380 def test_mymanager_context(self):
3381 manager = MyManager(shutdown_timeout=SHUTDOWN_TIMEOUT)
3382 with manager:
3383 self.common(manager)
3384 # bpo-30356: BaseManager._finalize_manager() sends SIGTERM
3385 # to the manager process if it takes longer than 1 second to stop,
3386 # which happens on slow buildbots.
3387 self.assertIn(manager._process.exitcode, (0, -signal.SIGTERM))
3388
3389 @support.skip_if_sanitizer('TSan: leaks threads', thread=True)
3390 def test_mymanager_context_prestarted(self):
3391 manager = MyManager(shutdown_timeout=SHUTDOWN_TIMEOUT)
3392 manager.start()
3393 with manager:
3394 self.common(manager)
3395 self.assertEqual(manager._process.exitcode, 0)
3396
3397 def common(self, manager):
3398 foo = manager.Foo()
3399 bar = manager.Bar()
3400 baz = manager.baz()
3401
3402 foo_methods = [name for name in ('f', 'g', '_h') if hasattr(foo, name)]
3403 bar_methods = [name for name in ('f', 'g', '_h') if hasattr(bar, name)]
3404
3405 self.assertEqual(foo_methods, ['f', 'g'])
3406 self.assertEqual(bar_methods, ['f', '_h'])
3407
3408 self.assertEqual(foo.f(), 'f()')
3409 self.assertRaises(ValueError, foo.g)
3410 self.assertEqual(foo._callmethod('f'), 'f()')
3411 self.assertRaises(RemoteError, foo._callmethod, '_h')
3412
3413 self.assertEqual(bar.f(), 'f()')
3414 self.assertEqual(bar._h(), '_h()')
3415 self.assertEqual(bar._callmethod('f'), 'f()')
3416 self.assertEqual(bar._callmethod('_h'), '_h()')
3417
3418 self.assertEqual(list(baz), [i*i for i in range(10)])
3419
3420
3421 #
3422 # Test of connecting to a remote server and using xmlrpclib for serialization
3423 #
3424
3425 _queue = pyqueue.Queue()
3426 def get_queue():
3427 return _queue
3428
3429 class QueueManager(BaseManager):
3430 '''manager class used by server process'''
3431 QueueManager.register('get_queue', callable=get_queue)
3432
3433 class QueueManager2(BaseManager):
3434 '''manager class which specifies the same interface as QueueManager'''
3435 QueueManager2.register('get_queue')
3436
3437
3438 SERIALIZER = 'xmlrpclib'
3439
3440 class _TestRemoteManager(BaseTestCase):
3441
3442 ALLOWED_TYPES = ('manager',)
3443 values = ['hello world', None, True, 2.25,
3444 'hall\xe5 v\xe4rlden',
3445 '\u043f\u0440\u0438\u0432\u0456\u0442 \u0441\u0432\u0456\u0442',
3446 b'hall\xe5 v\xe4rlden',
3447 ]
3448 result = values[:]
3449
3450 @classmethod
3451 def _putter(cls, address, authkey):
3452 manager = QueueManager2(
3453 address=address, authkey=authkey, serializer=SERIALIZER,
3454 shutdown_timeout=SHUTDOWN_TIMEOUT)
3455 manager.connect()
3456 queue = manager.get_queue()
3457 # Note that xmlrpclib will deserialize object as a list not a tuple
3458 queue.put(tuple(cls.values))
3459
3460 @support.skip_if_sanitizer('TSan: leaks threads', thread=True)
3461 def test_remote(self):
3462 authkey = os.urandom(32)
3463
3464 manager = QueueManager(
3465 address=(socket_helper.HOST, 0), authkey=authkey, serializer=SERIALIZER,
3466 shutdown_timeout=SHUTDOWN_TIMEOUT)
3467 manager.start()
3468 self.addCleanup(manager.shutdown)
3469
3470 p = self.Process(target=self._putter, args=(manager.address, authkey))
3471 p.daemon = True
3472 p.start()
3473
3474 manager2 = QueueManager2(
3475 address=manager.address, authkey=authkey, serializer=SERIALIZER,
3476 shutdown_timeout=SHUTDOWN_TIMEOUT)
3477 manager2.connect()
3478 queue = manager2.get_queue()
3479
3480 self.assertEqual(queue.get(), self.result)
3481
3482 # Because we are using xmlrpclib for serialization instead of
3483 # pickle this will cause a serialization error.
3484 self.assertRaises(Exception, queue.put, time.sleep)
3485
3486 # Make queue finalizer run before the server is stopped
3487 del queue
3488
3489
3490 @hashlib_helper.requires_hashdigest('sha256')
3491 class _TestManagerRestart(BaseTestCase):
3492
3493 @classmethod
3494 def _putter(cls, address, authkey):
3495 manager = QueueManager(
3496 address=address, authkey=authkey, serializer=SERIALIZER,
3497 shutdown_timeout=SHUTDOWN_TIMEOUT)
3498 manager.connect()
3499 queue = manager.get_queue()
3500 queue.put('hello world')
3501
3502 @support.skip_if_sanitizer("TSan: leaks threads", thread=True)
3503 def test_rapid_restart(self):
3504 authkey = os.urandom(32)
3505 manager = QueueManager(
3506 address=(socket_helper.HOST, 0), authkey=authkey,
3507 serializer=SERIALIZER, shutdown_timeout=SHUTDOWN_TIMEOUT)
3508 try:
3509 srvr = manager.get_server()
3510 addr = srvr.address
3511 # Close the connection.Listener socket which gets opened as a part
3512 # of manager.get_server(). It's not needed for the test.
3513 srvr.listener.close()
3514 manager.start()
3515
3516 p = self.Process(target=self._putter, args=(manager.address, authkey))
3517 p.start()
3518 p.join()
3519 queue = manager.get_queue()
3520 self.assertEqual(queue.get(), 'hello world')
3521 del queue
3522 finally:
3523 if hasattr(manager, "shutdown"):
3524 manager.shutdown()
3525
3526 manager = QueueManager(
3527 address=addr, authkey=authkey, serializer=SERIALIZER,
3528 shutdown_timeout=SHUTDOWN_TIMEOUT)
3529 try:
3530 manager.start()
3531 self.addCleanup(manager.shutdown)
3532 except OSError as e:
3533 if e.errno != errno.EADDRINUSE:
3534 raise
3535 # Retry after some time, in case the old socket was lingering
3536 # (sporadic failure on buildbots)
3537 time.sleep(1.0)
3538 manager = QueueManager(
3539 address=addr, authkey=authkey, serializer=SERIALIZER,
3540 shutdown_timeout=SHUTDOWN_TIMEOUT)
3541 if hasattr(manager, "shutdown"):
3542 self.addCleanup(manager.shutdown)
3543
3544
3545 class FakeConnection:
3546 def send(self, payload):
3547 pass
3548
3549 def recv(self):
3550 return '#ERROR', pyqueue.Empty()
3551
3552 class TestManagerExceptions(unittest.TestCase):
3553 # Issue 106558: Manager exceptions avoids creating cyclic references.
3554 def setUp(self):
3555 self.mgr = multiprocessing.Manager()
3556
3557 def tearDown(self):
3558 self.mgr.shutdown()
3559 self.mgr.join()
3560
3561 def test_queue_get(self):
3562 queue = self.mgr.Queue()
3563 if gc.isenabled():
3564 gc.disable()
3565 self.addCleanup(gc.enable)
3566 try:
3567 queue.get_nowait()
3568 except pyqueue.Empty as e:
3569 wr = weakref.ref(e)
3570 self.assertEqual(wr(), None)
3571
3572 def test_dispatch(self):
3573 if gc.isenabled():
3574 gc.disable()
3575 self.addCleanup(gc.enable)
3576 try:
3577 multiprocessing.managers.dispatch(FakeConnection(), None, None)
3578 except pyqueue.Empty as e:
3579 wr = weakref.ref(e)
3580 self.assertEqual(wr(), None)
3581
3582 #
3583 #
3584 #
3585
3586 SENTINEL = latin('')
3587
3588 class _TestConnection(BaseTestCase):
3589
3590 ALLOWED_TYPES = ('processes', 'threads')
3591
3592 @classmethod
3593 def _echo(cls, conn):
3594 for msg in iter(conn.recv_bytes, SENTINEL):
3595 conn.send_bytes(msg)
3596 conn.close()
3597
3598 def test_connection(self):
3599 conn, child_conn = self.Pipe()
3600
3601 p = self.Process(target=self._echo, args=(child_conn,))
3602 p.daemon = True
3603 p.start()
3604
3605 seq = [1, 2.25, None]
3606 msg = latin('hello world')
3607 longmsg = msg * 10
3608 arr = array.array('i', list(range(4)))
3609
3610 if self.TYPE == 'processes':
3611 self.assertEqual(type(conn.fileno()), int)
3612
3613 self.assertEqual(conn.send(seq), None)
3614 self.assertEqual(conn.recv(), seq)
3615
3616 self.assertEqual(conn.send_bytes(msg), None)
3617 self.assertEqual(conn.recv_bytes(), msg)
3618
3619 if self.TYPE == 'processes':
3620 buffer = array.array('i', [0]*10)
3621 expected = list(arr) + [0] * (10 - len(arr))
3622 self.assertEqual(conn.send_bytes(arr), None)
3623 self.assertEqual(conn.recv_bytes_into(buffer),
3624 len(arr) * buffer.itemsize)
3625 self.assertEqual(list(buffer), expected)
3626
3627 buffer = array.array('i', [0]*10)
3628 expected = [0] * 3 + list(arr) + [0] * (10 - 3 - len(arr))
3629 self.assertEqual(conn.send_bytes(arr), None)
3630 self.assertEqual(conn.recv_bytes_into(buffer, 3 * buffer.itemsize),
3631 len(arr) * buffer.itemsize)
3632 self.assertEqual(list(buffer), expected)
3633
3634 buffer = bytearray(latin(' ' * 40))
3635 self.assertEqual(conn.send_bytes(longmsg), None)
3636 try:
3637 res = conn.recv_bytes_into(buffer)
3638 except multiprocessing.BufferTooShort as e:
3639 self.assertEqual(e.args, (longmsg,))
3640 else:
3641 self.fail('expected BufferTooShort, got %s' % res)
3642
3643 poll = TimingWrapper(conn.poll)
3644
3645 self.assertEqual(poll(), False)
3646 self.assertTimingAlmostEqual(poll.elapsed, 0)
3647
3648 self.assertEqual(poll(-1), False)
3649 self.assertTimingAlmostEqual(poll.elapsed, 0)
3650
3651 self.assertEqual(poll(TIMEOUT1), False)
3652 self.assertTimingAlmostEqual(poll.elapsed, TIMEOUT1)
3653
3654 conn.send(None)
3655 time.sleep(.1)
3656
3657 self.assertEqual(poll(TIMEOUT1), True)
3658 self.assertTimingAlmostEqual(poll.elapsed, 0)
3659
3660 self.assertEqual(conn.recv(), None)
3661
3662 really_big_msg = latin('X') * (1024 * 1024 * 16) # 16Mb
3663 conn.send_bytes(really_big_msg)
3664 self.assertEqual(conn.recv_bytes(), really_big_msg)
3665
3666 conn.send_bytes(SENTINEL) # tell child to quit
3667 child_conn.close()
3668
3669 if self.TYPE == 'processes':
3670 self.assertEqual(conn.readable, True)
3671 self.assertEqual(conn.writable, True)
3672 self.assertRaises(EOFError, conn.recv)
3673 self.assertRaises(EOFError, conn.recv_bytes)
3674
3675 p.join()
3676
3677 def test_duplex_false(self):
3678 reader, writer = self.Pipe(duplex=False)
3679 self.assertEqual(writer.send(1), None)
3680 self.assertEqual(reader.recv(), 1)
3681 if self.TYPE == 'processes':
3682 self.assertEqual(reader.readable, True)
3683 self.assertEqual(reader.writable, False)
3684 self.assertEqual(writer.readable, False)
3685 self.assertEqual(writer.writable, True)
3686 self.assertRaises(OSError, reader.send, 2)
3687 self.assertRaises(OSError, writer.recv)
3688 self.assertRaises(OSError, writer.poll)
3689
3690 def test_spawn_close(self):
3691 # We test that a pipe connection can be closed by parent
3692 # process immediately after child is spawned. On Windows this
3693 # would have sometimes failed on old versions because
3694 # child_conn would be closed before the child got a chance to
3695 # duplicate it.
3696 conn, child_conn = self.Pipe()
3697
3698 p = self.Process(target=self._echo, args=(child_conn,))
3699 p.daemon = True
3700 p.start()
3701 child_conn.close() # this might complete before child initializes
3702
3703 msg = latin('hello')
3704 conn.send_bytes(msg)
3705 self.assertEqual(conn.recv_bytes(), msg)
3706
3707 conn.send_bytes(SENTINEL)
3708 conn.close()
3709 p.join()
3710
3711 def test_sendbytes(self):
3712 if self.TYPE != 'processes':
3713 self.skipTest('test not appropriate for {}'.format(self.TYPE))
3714
3715 msg = latin('abcdefghijklmnopqrstuvwxyz')
3716 a, b = self.Pipe()
3717
3718 a.send_bytes(msg)
3719 self.assertEqual(b.recv_bytes(), msg)
3720
3721 a.send_bytes(msg, 5)
3722 self.assertEqual(b.recv_bytes(), msg[5:])
3723
3724 a.send_bytes(msg, 7, 8)
3725 self.assertEqual(b.recv_bytes(), msg[7:7+8])
3726
3727 a.send_bytes(msg, 26)
3728 self.assertEqual(b.recv_bytes(), latin(''))
3729
3730 a.send_bytes(msg, 26, 0)
3731 self.assertEqual(b.recv_bytes(), latin(''))
3732
3733 self.assertRaises(ValueError, a.send_bytes, msg, 27)
3734
3735 self.assertRaises(ValueError, a.send_bytes, msg, 22, 5)
3736
3737 self.assertRaises(ValueError, a.send_bytes, msg, 26, 1)
3738
3739 self.assertRaises(ValueError, a.send_bytes, msg, -1)
3740
3741 self.assertRaises(ValueError, a.send_bytes, msg, 4, -1)
3742
3743 @classmethod
3744 def _is_fd_assigned(cls, fd):
3745 try:
3746 os.fstat(fd)
3747 except OSError as e:
3748 if e.errno == errno.EBADF:
3749 return False
3750 raise
3751 else:
3752 return True
3753
3754 @classmethod
3755 def _writefd(cls, conn, data, create_dummy_fds=False):
3756 if create_dummy_fds:
3757 for i in range(0, 256):
3758 if not cls._is_fd_assigned(i):
3759 os.dup2(conn.fileno(), i)
3760 fd = reduction.recv_handle(conn)
3761 if msvcrt:
3762 fd = msvcrt.open_osfhandle(fd, os.O_WRONLY)
3763 os.write(fd, data)
3764 os.close(fd)
3765
3766 @unittest.skipUnless(HAS_REDUCTION, "test needs multiprocessing.reduction")
3767 def test_fd_transfer(self):
3768 if self.TYPE != 'processes':
3769 self.skipTest("only makes sense with processes")
3770 conn, child_conn = self.Pipe(duplex=True)
3771
3772 p = self.Process(target=self._writefd, args=(child_conn, b"foo"))
3773 p.daemon = True
3774 p.start()
3775 self.addCleanup(os_helper.unlink, os_helper.TESTFN)
3776 with open(os_helper.TESTFN, "wb") as f:
3777 fd = f.fileno()
3778 if msvcrt:
3779 fd = msvcrt.get_osfhandle(fd)
3780 reduction.send_handle(conn, fd, p.pid)
3781 p.join()
3782 with open(os_helper.TESTFN, "rb") as f:
3783 self.assertEqual(f.read(), b"foo")
3784
3785 @unittest.skipUnless(HAS_REDUCTION, "test needs multiprocessing.reduction")
3786 @unittest.skipIf(sys.platform == "win32",
3787 "test semantics don't make sense on Windows")
3788 @unittest.skipIf(MAXFD <= 256,
3789 "largest assignable fd number is too small")
3790 @unittest.skipUnless(hasattr(os, "dup2"),
3791 "test needs os.dup2()")
3792 def test_large_fd_transfer(self):
3793 # With fd > 256 (issue #11657)
3794 if self.TYPE != 'processes':
3795 self.skipTest("only makes sense with processes")
3796 conn, child_conn = self.Pipe(duplex=True)
3797
3798 p = self.Process(target=self._writefd, args=(child_conn, b"bar", True))
3799 p.daemon = True
3800 p.start()
3801 self.addCleanup(os_helper.unlink, os_helper.TESTFN)
3802 with open(os_helper.TESTFN, "wb") as f:
3803 fd = f.fileno()
3804 for newfd in range(256, MAXFD):
3805 if not self._is_fd_assigned(newfd):
3806 break
3807 else:
3808 self.fail("could not find an unassigned large file descriptor")
3809 os.dup2(fd, newfd)
3810 try:
3811 reduction.send_handle(conn, newfd, p.pid)
3812 finally:
3813 os.close(newfd)
3814 p.join()
3815 with open(os_helper.TESTFN, "rb") as f:
3816 self.assertEqual(f.read(), b"bar")
3817
3818 @classmethod
3819 def _send_data_without_fd(self, conn):
3820 os.write(conn.fileno(), b"\0")
3821
3822 @unittest.skipUnless(HAS_REDUCTION, "test needs multiprocessing.reduction")
3823 @unittest.skipIf(sys.platform == "win32", "doesn't make sense on Windows")
3824 def test_missing_fd_transfer(self):
3825 # Check that exception is raised when received data is not
3826 # accompanied by a file descriptor in ancillary data.
3827 if self.TYPE != 'processes':
3828 self.skipTest("only makes sense with processes")
3829 conn, child_conn = self.Pipe(duplex=True)
3830
3831 p = self.Process(target=self._send_data_without_fd, args=(child_conn,))
3832 p.daemon = True
3833 p.start()
3834 self.assertRaises(RuntimeError, reduction.recv_handle, conn)
3835 p.join()
3836
3837 def test_context(self):
3838 a, b = self.Pipe()
3839
3840 with a, b:
3841 a.send(1729)
3842 self.assertEqual(b.recv(), 1729)
3843 if self.TYPE == 'processes':
3844 self.assertFalse(a.closed)
3845 self.assertFalse(b.closed)
3846
3847 if self.TYPE == 'processes':
3848 self.assertTrue(a.closed)
3849 self.assertTrue(b.closed)
3850 self.assertRaises(OSError, a.recv)
3851 self.assertRaises(OSError, b.recv)
3852
3853 class _TestListener(BaseTestCase):
3854
3855 ALLOWED_TYPES = ('processes',)
3856
3857 def test_multiple_bind(self):
3858 for family in self.connection.families:
3859 l = self.connection.Listener(family=family)
3860 self.addCleanup(l.close)
3861 self.assertRaises(OSError, self.connection.Listener,
3862 l.address, family)
3863
3864 def test_context(self):
3865 with self.connection.Listener() as l:
3866 with self.connection.Client(l.address) as c:
3867 with l.accept() as d:
3868 c.send(1729)
3869 self.assertEqual(d.recv(), 1729)
3870
3871 if self.TYPE == 'processes':
3872 self.assertRaises(OSError, l.accept)
3873
3874 def test_empty_authkey(self):
3875 # bpo-43952: allow empty bytes as authkey
3876 def handler(*args):
3877 raise RuntimeError('Connection took too long...')
3878
3879 def run(addr, authkey):
3880 client = self.connection.Client(addr, authkey=authkey)
3881 client.send(1729)
3882
3883 key = b''
3884
3885 with self.connection.Listener(authkey=key) as listener:
3886 thread = threading.Thread(target=run, args=(listener.address, key))
3887 thread.start()
3888 try:
3889 with listener.accept() as d:
3890 self.assertEqual(d.recv(), 1729)
3891 finally:
3892 thread.join()
3893
3894 if self.TYPE == 'processes':
3895 with self.assertRaises(OSError):
3896 listener.accept()
3897
3898 @unittest.skipUnless(util.abstract_sockets_supported,
3899 "test needs abstract socket support")
3900 def test_abstract_socket(self):
3901 with self.connection.Listener("\0something") as listener:
3902 with self.connection.Client(listener.address) as client:
3903 with listener.accept() as d:
3904 client.send(1729)
3905 self.assertEqual(d.recv(), 1729)
3906
3907 if self.TYPE == 'processes':
3908 self.assertRaises(OSError, listener.accept)
3909
3910
3911 class _TestListenerClient(BaseTestCase):
3912
3913 ALLOWED_TYPES = ('processes', 'threads')
3914
3915 @classmethod
3916 def _test(cls, address):
3917 conn = cls.connection.Client(address)
3918 conn.send('hello')
3919 conn.close()
3920
3921 def test_listener_client(self):
3922 for family in self.connection.families:
3923 l = self.connection.Listener(family=family)
3924 p = self.Process(target=self._test, args=(l.address,))
3925 p.daemon = True
3926 p.start()
3927 conn = l.accept()
3928 self.assertEqual(conn.recv(), 'hello')
3929 p.join()
3930 l.close()
3931
3932 def test_issue14725(self):
3933 l = self.connection.Listener()
3934 p = self.Process(target=self._test, args=(l.address,))
3935 p.daemon = True
3936 p.start()
3937 time.sleep(1)
3938 # On Windows the client process should by now have connected,
3939 # written data and closed the pipe handle by now. This causes
3940 # ConnectNamdedPipe() to fail with ERROR_NO_DATA. See Issue
3941 # 14725.
3942 conn = l.accept()
3943 self.assertEqual(conn.recv(), 'hello')
3944 conn.close()
3945 p.join()
3946 l.close()
3947
3948 def test_issue16955(self):
3949 for fam in self.connection.families:
3950 l = self.connection.Listener(family=fam)
3951 c = self.connection.Client(l.address)
3952 a = l.accept()
3953 a.send_bytes(b"hello")
3954 self.assertTrue(c.poll(1))
3955 a.close()
3956 c.close()
3957 l.close()
3958
3959 class _TestPoll(BaseTestCase):
3960
3961 ALLOWED_TYPES = ('processes', 'threads')
3962
3963 def test_empty_string(self):
3964 a, b = self.Pipe()
3965 self.assertEqual(a.poll(), False)
3966 b.send_bytes(b'')
3967 self.assertEqual(a.poll(), True)
3968 self.assertEqual(a.poll(), True)
3969
3970 @classmethod
3971 def _child_strings(cls, conn, strings):
3972 for s in strings:
3973 time.sleep(0.1)
3974 conn.send_bytes(s)
3975 conn.close()
3976
3977 def test_strings(self):
3978 strings = (b'hello', b'', b'a', b'b', b'', b'bye', b'', b'lop')
3979 a, b = self.Pipe()
3980 p = self.Process(target=self._child_strings, args=(b, strings))
3981 p.start()
3982
3983 for s in strings:
3984 for i in range(200):
3985 if a.poll(0.01):
3986 break
3987 x = a.recv_bytes()
3988 self.assertEqual(s, x)
3989
3990 p.join()
3991
3992 @classmethod
3993 def _child_boundaries(cls, r):
3994 # Polling may "pull" a message in to the child process, but we
3995 # don't want it to pull only part of a message, as that would
3996 # corrupt the pipe for any other processes which might later
3997 # read from it.
3998 r.poll(5)
3999
4000 def test_boundaries(self):
4001 r, w = self.Pipe(False)
4002 p = self.Process(target=self._child_boundaries, args=(r,))
4003 p.start()
4004 time.sleep(2)
4005 L = [b"first", b"second"]
4006 for obj in L:
4007 w.send_bytes(obj)
4008 w.close()
4009 p.join()
4010 self.assertIn(r.recv_bytes(), L)
4011
4012 @classmethod
4013 def _child_dont_merge(cls, b):
4014 b.send_bytes(b'a')
4015 b.send_bytes(b'b')
4016 b.send_bytes(b'cd')
4017
4018 def test_dont_merge(self):
4019 a, b = self.Pipe()
4020 self.assertEqual(a.poll(0.0), False)
4021 self.assertEqual(a.poll(0.1), False)
4022
4023 p = self.Process(target=self._child_dont_merge, args=(b,))
4024 p.start()
4025
4026 self.assertEqual(a.recv_bytes(), b'a')
4027 self.assertEqual(a.poll(1.0), True)
4028 self.assertEqual(a.poll(1.0), True)
4029 self.assertEqual(a.recv_bytes(), b'b')
4030 self.assertEqual(a.poll(1.0), True)
4031 self.assertEqual(a.poll(1.0), True)
4032 self.assertEqual(a.poll(0.0), True)
4033 self.assertEqual(a.recv_bytes(), b'cd')
4034
4035 p.join()
4036
4037 #
4038 # Test of sending connection and socket objects between processes
4039 #
4040
4041 @unittest.skipUnless(HAS_REDUCTION, "test needs multiprocessing.reduction")
4042 @hashlib_helper.requires_hashdigest('sha256')
4043 class _TestPicklingConnections(BaseTestCase):
4044
4045 ALLOWED_TYPES = ('processes',)
4046
4047 @classmethod
4048 def tearDownClass(cls):
4049 from multiprocessing import resource_sharer
4050 resource_sharer.stop(timeout=support.LONG_TIMEOUT)
4051
4052 @classmethod
4053 def _listener(cls, conn, families):
4054 for fam in families:
4055 l = cls.connection.Listener(family=fam)
4056 conn.send(l.address)
4057 new_conn = l.accept()
4058 conn.send(new_conn)
4059 new_conn.close()
4060 l.close()
4061
4062 l = socket.create_server((socket_helper.HOST, 0))
4063 conn.send(l.getsockname())
4064 new_conn, addr = l.accept()
4065 conn.send(new_conn)
4066 new_conn.close()
4067 l.close()
4068
4069 conn.recv()
4070
4071 @classmethod
4072 def _remote(cls, conn):
4073 for (address, msg) in iter(conn.recv, None):
4074 client = cls.connection.Client(address)
4075 client.send(msg.upper())
4076 client.close()
4077
4078 address, msg = conn.recv()
4079 client = socket.socket()
4080 client.connect(address)
4081 client.sendall(msg.upper())
4082 client.close()
4083
4084 conn.close()
4085
4086 def test_pickling(self):
4087 families = self.connection.families
4088
4089 lconn, lconn0 = self.Pipe()
4090 lp = self.Process(target=self._listener, args=(lconn0, families))
4091 lp.daemon = True
4092 lp.start()
4093 lconn0.close()
4094
4095 rconn, rconn0 = self.Pipe()
4096 rp = self.Process(target=self._remote, args=(rconn0,))
4097 rp.daemon = True
4098 rp.start()
4099 rconn0.close()
4100
4101 for fam in families:
4102 msg = ('This connection uses family %s' % fam).encode('ascii')
4103 address = lconn.recv()
4104 rconn.send((address, msg))
4105 new_conn = lconn.recv()
4106 self.assertEqual(new_conn.recv(), msg.upper())
4107
4108 rconn.send(None)
4109
4110 msg = latin('This connection uses a normal socket')
4111 address = lconn.recv()
4112 rconn.send((address, msg))
4113 new_conn = lconn.recv()
4114 buf = []
4115 while True:
4116 s = new_conn.recv(100)
4117 if not s:
4118 break
4119 buf.append(s)
4120 buf = b''.join(buf)
4121 self.assertEqual(buf, msg.upper())
4122 new_conn.close()
4123
4124 lconn.send(None)
4125
4126 rconn.close()
4127 lconn.close()
4128
4129 lp.join()
4130 rp.join()
4131
4132 @classmethod
4133 def child_access(cls, conn):
4134 w = conn.recv()
4135 w.send('all is well')
4136 w.close()
4137
4138 r = conn.recv()
4139 msg = r.recv()
4140 conn.send(msg*2)
4141
4142 conn.close()
4143
4144 def test_access(self):
4145 # On Windows, if we do not specify a destination pid when
4146 # using DupHandle then we need to be careful to use the
4147 # correct access flags for DuplicateHandle(), or else
4148 # DupHandle.detach() will raise PermissionError. For example,
4149 # for a read only pipe handle we should use
4150 # access=FILE_GENERIC_READ. (Unfortunately
4151 # DUPLICATE_SAME_ACCESS does not work.)
4152 conn, child_conn = self.Pipe()
4153 p = self.Process(target=self.child_access, args=(child_conn,))
4154 p.daemon = True
4155 p.start()
4156 child_conn.close()
4157
4158 r, w = self.Pipe(duplex=False)
4159 conn.send(w)
4160 w.close()
4161 self.assertEqual(r.recv(), 'all is well')
4162 r.close()
4163
4164 r, w = self.Pipe(duplex=False)
4165 conn.send(r)
4166 r.close()
4167 w.send('foobar')
4168 w.close()
4169 self.assertEqual(conn.recv(), 'foobar'*2)
4170
4171 p.join()
4172
4173 #
4174 #
4175 #
4176
4177 class _TestHeap(BaseTestCase):
4178
4179 ALLOWED_TYPES = ('processes',)
4180
4181 def setUp(self):
4182 super().setUp()
4183 # Make pristine heap for these tests
4184 self.old_heap = multiprocessing.heap.BufferWrapper._heap
4185 multiprocessing.heap.BufferWrapper._heap = multiprocessing.heap.Heap()
4186
4187 def tearDown(self):
4188 multiprocessing.heap.BufferWrapper._heap = self.old_heap
4189 super().tearDown()
4190
4191 def test_heap(self):
4192 iterations = 5000
4193 maxblocks = 50
4194 blocks = []
4195
4196 # get the heap object
4197 heap = multiprocessing.heap.BufferWrapper._heap
4198 heap._DISCARD_FREE_SPACE_LARGER_THAN = 0
4199
4200 # create and destroy lots of blocks of different sizes
4201 for i in range(iterations):
4202 size = int(random.lognormvariate(0, 1) * 1000)
4203 b = multiprocessing.heap.BufferWrapper(size)
4204 blocks.append(b)
4205 if len(blocks) > maxblocks:
4206 i = random.randrange(maxblocks)
4207 del blocks[i]
4208 del b
4209
4210 # verify the state of the heap
4211 with heap._lock:
4212 all = []
4213 free = 0
4214 occupied = 0
4215 for L in list(heap._len_to_seq.values()):
4216 # count all free blocks in arenas
4217 for arena, start, stop in L:
4218 all.append((heap._arenas.index(arena), start, stop,
4219 stop-start, 'free'))
4220 free += (stop-start)
4221 for arena, arena_blocks in heap._allocated_blocks.items():
4222 # count all allocated blocks in arenas
4223 for start, stop in arena_blocks:
4224 all.append((heap._arenas.index(arena), start, stop,
4225 stop-start, 'occupied'))
4226 occupied += (stop-start)
4227
4228 self.assertEqual(free + occupied,
4229 sum(arena.size for arena in heap._arenas))
4230
4231 all.sort()
4232
4233 for i in range(len(all)-1):
4234 (arena, start, stop) = all[i][:3]
4235 (narena, nstart, nstop) = all[i+1][:3]
4236 if arena != narena:
4237 # Two different arenas
4238 self.assertEqual(stop, heap._arenas[arena].size) # last block
4239 self.assertEqual(nstart, 0) # first block
4240 else:
4241 # Same arena: two adjacent blocks
4242 self.assertEqual(stop, nstart)
4243
4244 # test free'ing all blocks
4245 random.shuffle(blocks)
4246 while blocks:
4247 blocks.pop()
4248
4249 self.assertEqual(heap._n_frees, heap._n_mallocs)
4250 self.assertEqual(len(heap._pending_free_blocks), 0)
4251 self.assertEqual(len(heap._arenas), 0)
4252 self.assertEqual(len(heap._allocated_blocks), 0, heap._allocated_blocks)
4253 self.assertEqual(len(heap._len_to_seq), 0)
4254
4255 def test_free_from_gc(self):
4256 # Check that freeing of blocks by the garbage collector doesn't deadlock
4257 # (issue #12352).
4258 # Make sure the GC is enabled, and set lower collection thresholds to
4259 # make collections more frequent (and increase the probability of
4260 # deadlock).
4261 if not gc.isenabled():
4262 gc.enable()
4263 self.addCleanup(gc.disable)
4264 thresholds = gc.get_threshold()
4265 self.addCleanup(gc.set_threshold, *thresholds)
4266 gc.set_threshold(10)
4267
4268 # perform numerous block allocations, with cyclic references to make
4269 # sure objects are collected asynchronously by the gc
4270 for i in range(5000):
4271 a = multiprocessing.heap.BufferWrapper(1)
4272 b = multiprocessing.heap.BufferWrapper(1)
4273 # circular references
4274 a.buddy = b
4275 b.buddy = a
4276
4277 #
4278 #
4279 #
4280
4281 class _Foo(Structure):
4282 _fields_ = [
4283 ('x', c_int),
4284 ('y', c_double),
4285 ('z', c_longlong,)
4286 ]
4287
4288 class _TestSharedCTypes(BaseTestCase):
4289
4290 ALLOWED_TYPES = ('processes',)
4291
4292 def setUp(self):
4293 if not HAS_SHAREDCTYPES:
4294 self.skipTest("requires multiprocessing.sharedctypes")
4295
4296 @classmethod
4297 def _double(cls, x, y, z, foo, arr, string):
4298 x.value *= 2
4299 y.value *= 2
4300 z.value *= 2
4301 foo.x *= 2
4302 foo.y *= 2
4303 string.value *= 2
4304 for i in range(len(arr)):
4305 arr[i] *= 2
4306
4307 def test_sharedctypes(self, lock=False):
4308 x = Value('i', 7, lock=lock)
4309 y = Value(c_double, 1.0/3.0, lock=lock)
4310 z = Value(c_longlong, 2 ** 33, lock=lock)
4311 foo = Value(_Foo, 3, 2, lock=lock)
4312 arr = self.Array('d', list(range(10)), lock=lock)
4313 string = self.Array('c', 20, lock=lock)
4314 string.value = latin('hello')
4315
4316 p = self.Process(target=self._double, args=(x, y, z, foo, arr, string))
4317 p.daemon = True
4318 p.start()
4319 p.join()
4320
4321 self.assertEqual(x.value, 14)
4322 self.assertAlmostEqual(y.value, 2.0/3.0)
4323 self.assertEqual(z.value, 2 ** 34)
4324 self.assertEqual(foo.x, 6)
4325 self.assertAlmostEqual(foo.y, 4.0)
4326 for i in range(10):
4327 self.assertAlmostEqual(arr[i], i*2)
4328 self.assertEqual(string.value, latin('hellohello'))
4329
4330 def test_synchronize(self):
4331 self.test_sharedctypes(lock=True)
4332
4333 def test_copy(self):
4334 foo = _Foo(2, 5.0, 2 ** 33)
4335 bar = copy(foo)
4336 foo.x = 0
4337 foo.y = 0
4338 foo.z = 0
4339 self.assertEqual(bar.x, 2)
4340 self.assertAlmostEqual(bar.y, 5.0)
4341 self.assertEqual(bar.z, 2 ** 33)
4342
4343
4344 def resource_tracker_format_subtests(func):
4345 """Run given test using both resource tracker communication formats"""
4346 def _inner(self, *args, **kwargs):
4347 tracker = resource_tracker._resource_tracker
4348 for use_simple_format in False, True:
4349 with (
4350 self.subTest(use_simple_format=use_simple_format),
4351 unittest.mock.patch.object(
4352 tracker, '_use_simple_format', use_simple_format)
4353 ):
4354 func(self, *args, **kwargs)
4355 return _inner
4356
4357 @unittest.skipUnless(HAS_SHMEM, "requires multiprocessing.shared_memory")
4358 @hashlib_helper.requires_hashdigest('sha256')
4359 class _TestSharedMemory(BaseTestCase):
4360
4361 ALLOWED_TYPES = ('processes',)
4362
4363 @staticmethod
4364 def _attach_existing_shmem_then_write(shmem_name_or_obj, binary_data):
4365 if isinstance(shmem_name_or_obj, str):
4366 local_sms = shared_memory.SharedMemory(shmem_name_or_obj)
4367 else:
4368 local_sms = shmem_name_or_obj
4369 local_sms.buf[:len(binary_data)] = binary_data
4370 local_sms.close()
4371
4372 def _new_shm_name(self, prefix):
4373 # Add a PID to the name of a POSIX shared memory object to allow
4374 # running multiprocessing tests (test_multiprocessing_fork,
4375 # test_multiprocessing_spawn, etc) in parallel.
4376 return prefix + str(os.getpid())
4377
4378 def test_shared_memory_name_with_embedded_null(self):
4379 name_tsmb = self._new_shm_name('test01_null')
4380 sms = shared_memory.SharedMemory(name_tsmb, create=True, size=512)
4381 self.addCleanup(sms.unlink)
4382 with self.assertRaises(ValueError):
4383 shared_memory.SharedMemory(name_tsmb + '\0a', create=False, size=512)
4384 if shared_memory._USE_POSIX:
4385 orig_name = sms._name
4386 try:
4387 sms._name = orig_name + '\0a'
4388 with self.assertRaises(ValueError):
4389 sms.unlink()
4390 finally:
4391 sms._name = orig_name
4392
4393 def test_shared_memory_basics(self):
4394 name_tsmb = self._new_shm_name('test01_tsmb')
4395 sms = shared_memory.SharedMemory(name_tsmb, create=True, size=512)
4396 self.addCleanup(sms.unlink)
4397
4398 # Verify attributes are readable.
4399 self.assertEqual(sms.name, name_tsmb)
4400 self.assertGreaterEqual(sms.size, 512)
4401 self.assertGreaterEqual(len(sms.buf), sms.size)
4402
4403 # Verify __repr__
4404 self.assertIn(sms.name, str(sms))
4405 self.assertIn(str(sms.size), str(sms))
4406
4407 # Modify contents of shared memory segment through memoryview.
4408 sms.buf[0] = 42
4409 self.assertEqual(sms.buf[0], 42)
4410
4411 # Attach to existing shared memory segment.
4412 also_sms = shared_memory.SharedMemory(name_tsmb)
4413 self.assertEqual(also_sms.buf[0], 42)
4414 also_sms.close()
4415
4416 # Attach to existing shared memory segment but specify a new size.
4417 same_sms = shared_memory.SharedMemory(name_tsmb, size=20*sms.size)
4418 self.assertLess(same_sms.size, 20*sms.size) # Size was ignored.
4419 same_sms.close()
4420
4421 # Creating Shared Memory Segment with -ve size
4422 with self.assertRaises(ValueError):
4423 shared_memory.SharedMemory(create=True, size=-2)
4424
4425 # Attaching Shared Memory Segment without a name
4426 with self.assertRaises(ValueError):
4427 shared_memory.SharedMemory(create=False)
4428
4429 # Test if shared memory segment is created properly,
4430 # when _make_filename returns an existing shared memory segment name
4431 with unittest.mock.patch(
4432 'multiprocessing.shared_memory._make_filename') as mock_make_filename:
4433
4434 NAME_PREFIX = shared_memory._SHM_NAME_PREFIX
4435 names = [self._new_shm_name('test01_fn'), self._new_shm_name('test02_fn')]
4436 # Prepend NAME_PREFIX which can be '/psm_' or 'wnsm_', necessary
4437 # because some POSIX compliant systems require name to start with /
4438 names = [NAME_PREFIX + name for name in names]
4439
4440 mock_make_filename.side_effect = names
4441 shm1 = shared_memory.SharedMemory(create=True, size=1)
4442 self.addCleanup(shm1.unlink)
4443 self.assertEqual(shm1._name, names[0])
4444
4445 mock_make_filename.side_effect = names
4446 shm2 = shared_memory.SharedMemory(create=True, size=1)
4447 self.addCleanup(shm2.unlink)
4448 self.assertEqual(shm2._name, names[1])
4449
4450 if shared_memory._USE_POSIX:
4451 # Posix Shared Memory can only be unlinked once. Here we
4452 # test an implementation detail that is not observed across
4453 # all supported platforms (since WindowsNamedSharedMemory
4454 # manages unlinking on its own and unlink() does nothing).
4455 # True release of shared memory segment does not necessarily
4456 # happen until process exits, depending on the OS platform.
4457 name_dblunlink = self._new_shm_name('test01_dblunlink')
4458 sms_uno = shared_memory.SharedMemory(
4459 name_dblunlink,
4460 create=True,
4461 size=5000
4462 )
4463 with self.assertRaises(FileNotFoundError):
4464 try:
4465 self.assertGreaterEqual(sms_uno.size, 5000)
4466
4467 sms_duo = shared_memory.SharedMemory(name_dblunlink)
4468 sms_duo.unlink() # First shm_unlink() call.
4469 sms_duo.close()
4470 sms_uno.close()
4471
4472 finally:
4473 sms_uno.unlink() # A second shm_unlink() call is bad.
4474
4475 with self.assertRaises(FileExistsError):
4476 # Attempting to create a new shared memory segment with a
4477 # name that is already in use triggers an exception.
4478 there_can_only_be_one_sms = shared_memory.SharedMemory(
4479 name_tsmb,
4480 create=True,
4481 size=512
4482 )
4483
4484 if shared_memory._USE_POSIX:
4485 # Requesting creation of a shared memory segment with the option
4486 # to attach to an existing segment, if that name is currently in
4487 # use, should not trigger an exception.
4488 # Note: Using a smaller size could possibly cause truncation of
4489 # the existing segment but is OS platform dependent. In the
4490 # case of MacOS/darwin, requesting a smaller size is disallowed.
4491 class OptionalAttachSharedMemory(shared_memory.SharedMemory):
4492 _flags = os.O_CREAT | os.O_RDWR
4493 ok_if_exists_sms = OptionalAttachSharedMemory(name_tsmb)
4494 self.assertEqual(ok_if_exists_sms.size, sms.size)
4495 ok_if_exists_sms.close()
4496
4497 # Attempting to attach to an existing shared memory segment when
4498 # no segment exists with the supplied name triggers an exception.
4499 with self.assertRaises(FileNotFoundError):
4500 nonexisting_sms = shared_memory.SharedMemory('test01_notthere')
4501 nonexisting_sms.unlink() # Error should occur on prior line.
4502
4503 sms.close()
4504
4505 def test_shared_memory_recreate(self):
4506 # Test if shared memory segment is created properly,
4507 # when _make_filename returns an existing shared memory segment name
4508 with unittest.mock.patch(
4509 'multiprocessing.shared_memory._make_filename') as mock_make_filename:
4510
4511 NAME_PREFIX = shared_memory._SHM_NAME_PREFIX
4512 names = [self._new_shm_name('test03_fn'), self._new_shm_name('test04_fn')]
4513 # Prepend NAME_PREFIX which can be '/psm_' or 'wnsm_', necessary
4514 # because some POSIX compliant systems require name to start with /
4515 names = [NAME_PREFIX + name for name in names]
4516
4517 mock_make_filename.side_effect = names
4518 shm1 = shared_memory.SharedMemory(create=True, size=1)
4519 self.addCleanup(shm1.unlink)
4520 self.assertEqual(shm1._name, names[0])
4521
4522 mock_make_filename.side_effect = names
4523 shm2 = shared_memory.SharedMemory(create=True, size=1)
4524 self.addCleanup(shm2.unlink)
4525 self.assertEqual(shm2._name, names[1])
4526
4527 def test_invalid_shared_memory_creation(self):
4528 # Test creating a shared memory segment with negative size
4529 with self.assertRaises(ValueError):
4530 sms_invalid = shared_memory.SharedMemory(create=True, size=-1)
4531
4532 # Test creating a shared memory segment with size 0
4533 with self.assertRaises(ValueError):
4534 sms_invalid = shared_memory.SharedMemory(create=True, size=0)
4535
4536 # Test creating a shared memory segment without size argument
4537 with self.assertRaises(ValueError):
4538 sms_invalid = shared_memory.SharedMemory(create=True)
4539
4540 def test_shared_memory_pickle_unpickle(self):
4541 for proto in range(pickle.HIGHEST_PROTOCOL + 1):
4542 with self.subTest(proto=proto):
4543 sms = shared_memory.SharedMemory(create=True, size=512)
4544 self.addCleanup(sms.unlink)
4545 sms.buf[0:6] = b'pickle'
4546
4547 # Test pickling
4548 pickled_sms = pickle.dumps(sms, protocol=proto)
4549
4550 # Test unpickling
4551 sms2 = pickle.loads(pickled_sms)
4552 self.assertIsInstance(sms2, shared_memory.SharedMemory)
4553 self.assertEqual(sms.name, sms2.name)
4554 self.assertEqual(bytes(sms.buf[0:6]), b'pickle')
4555 self.assertEqual(bytes(sms2.buf[0:6]), b'pickle')
4556
4557 # Test that unpickled version is still the same SharedMemory
4558 sms.buf[0:6] = b'newval'
4559 self.assertEqual(bytes(sms.buf[0:6]), b'newval')
4560 self.assertEqual(bytes(sms2.buf[0:6]), b'newval')
4561
4562 sms2.buf[0:6] = b'oldval'
4563 self.assertEqual(bytes(sms.buf[0:6]), b'oldval')
4564 self.assertEqual(bytes(sms2.buf[0:6]), b'oldval')
4565
4566 def test_shared_memory_pickle_unpickle_dead_object(self):
4567 for proto in range(pickle.HIGHEST_PROTOCOL + 1):
4568 with self.subTest(proto=proto):
4569 sms = shared_memory.SharedMemory(create=True, size=512)
4570 sms.buf[0:6] = b'pickle'
4571 pickled_sms = pickle.dumps(sms, protocol=proto)
4572
4573 # Now, we are going to kill the original object.
4574 # So, unpickled one won't be able to attach to it.
4575 sms.close()
4576 sms.unlink()
4577
4578 with self.assertRaises(FileNotFoundError):
4579 pickle.loads(pickled_sms)
4580
4581 def test_shared_memory_across_processes(self):
4582 # bpo-40135: don't define shared memory block's name in case of
4583 # the failure when we run multiprocessing tests in parallel.
4584 sms = shared_memory.SharedMemory(create=True, size=512)
4585 self.addCleanup(sms.unlink)
4586
4587 # Verify remote attachment to existing block by name is working.
4588 p = self.Process(
4589 target=self._attach_existing_shmem_then_write,
4590 args=(sms.name, b'howdy')
4591 )
4592 p.daemon = True
4593 p.start()
4594 p.join()
4595 self.assertEqual(bytes(sms.buf[:5]), b'howdy')
4596
4597 # Verify pickling of SharedMemory instance also works.
4598 p = self.Process(
4599 target=self._attach_existing_shmem_then_write,
4600 args=(sms, b'HELLO')
4601 )
4602 p.daemon = True
4603 p.start()
4604 p.join()
4605 self.assertEqual(bytes(sms.buf[:5]), b'HELLO')
4606
4607 sms.close()
4608
4609 @unittest.skipIf(os.name != "posix", "not feasible in non-posix platforms")
4610 def test_shared_memory_SharedMemoryServer_ignores_sigint(self):
4611 # bpo-36368: protect SharedMemoryManager server process from
4612 # KeyboardInterrupt signals.
4613 smm = multiprocessing.managers.SharedMemoryManager()
4614 smm.start()
4615
4616 # make sure the manager works properly at the beginning
4617 sl = smm.ShareableList(range(10))
4618
4619 # the manager's server should ignore KeyboardInterrupt signals, and
4620 # maintain its connection with the current process, and success when
4621 # asked to deliver memory segments.
4622 os.kill(smm._process.pid, signal.SIGINT)
4623
4624 sl2 = smm.ShareableList(range(10))
4625
4626 # test that the custom signal handler registered in the Manager does
4627 # not affect signal handling in the parent process.
4628 with self.assertRaises(KeyboardInterrupt):
4629 os.kill(os.getpid(), signal.SIGINT)
4630
4631 smm.shutdown()
4632
4633 @unittest.skipIf(os.name != "posix", "resource_tracker is posix only")
4634 @resource_tracker_format_subtests
4635 def test_shared_memory_SharedMemoryManager_reuses_resource_tracker(self):
4636 # bpo-36867: test that a SharedMemoryManager uses the
4637 # same resource_tracker process as its parent.
4638 cmd = '''if 1:
4639 from multiprocessing.managers import SharedMemoryManager
4640
4641
4642 smm = SharedMemoryManager()
4643 smm.start()
4644 sl = smm.ShareableList(range(10))
4645 smm.shutdown()
4646 '''
4647 rc, out, err = test.support.script_helper.assert_python_ok('-c', cmd)
4648
4649 # Before bpo-36867 was fixed, a SharedMemoryManager not using the same
4650 # resource_tracker process as its parent would make the parent's
4651 # tracker complain about sl being leaked even though smm.shutdown()
4652 # properly released sl.
4653 self.assertFalse(err)
4654
4655 def test_shared_memory_SharedMemoryManager_basics(self):
4656 smm1 = multiprocessing.managers.SharedMemoryManager()
4657 with self.assertRaises(ValueError):
4658 smm1.SharedMemory(size=9) # Fails if SharedMemoryServer not started
4659 smm1.start()
4660 lol = [ smm1.ShareableList(range(i)) for i in range(5, 10) ]
4661 lom = [ smm1.SharedMemory(size=j) for j in range(32, 128, 16) ]
4662 doppleganger_list0 = shared_memory.ShareableList(name=lol[0].shm.name)
4663 self.assertEqual(len(doppleganger_list0), 5)
4664 doppleganger_shm0 = shared_memory.SharedMemory(name=lom[0].name)
4665 self.assertGreaterEqual(len(doppleganger_shm0.buf), 32)
4666 held_name = lom[0].name
4667 smm1.shutdown()
4668 if sys.platform != "win32":
4669 # Calls to unlink() have no effect on Windows platform; shared
4670 # memory will only be released once final process exits.
4671 with self.assertRaises(FileNotFoundError):
4672 # No longer there to be attached to again.
4673 absent_shm = shared_memory.SharedMemory(name=held_name)
4674
4675 with multiprocessing.managers.SharedMemoryManager() as smm2:
4676 sl = smm2.ShareableList("howdy")
4677 shm = smm2.SharedMemory(size=128)
4678 held_name = sl.shm.name
4679 if sys.platform != "win32":
4680 with self.assertRaises(FileNotFoundError):
4681 # No longer there to be attached to again.
4682 absent_sl = shared_memory.ShareableList(name=held_name)
4683
4684
4685 def test_shared_memory_ShareableList_basics(self):
4686 sl = shared_memory.ShareableList(
4687 ['howdy', b'HoWdY', -273.154, 100, None, True, 42]
4688 )
4689 self.addCleanup(sl.shm.unlink)
4690
4691 # Verify __repr__
4692 self.assertIn(sl.shm.name, str(sl))
4693 self.assertIn(str(list(sl)), str(sl))
4694
4695 # Index Out of Range (get)
4696 with self.assertRaises(IndexError):
4697 sl[7]
4698
4699 # Index Out of Range (set)
4700 with self.assertRaises(IndexError):
4701 sl[7] = 2
4702
4703 # Assign value without format change (str -> str)
4704 current_format = sl._get_packing_format(0)
4705 sl[0] = 'howdy'
4706 self.assertEqual(current_format, sl._get_packing_format(0))
4707
4708 # Verify attributes are readable.
4709 self.assertEqual(sl.format, '8s8sdqxxxxxx?xxxxxxxx?q')
4710
4711 # Exercise len().
4712 self.assertEqual(len(sl), 7)
4713
4714 # Exercise index().
4715 with warnings.catch_warnings():
4716 # Suppress BytesWarning when comparing against b'HoWdY'.
4717 warnings.simplefilter('ignore')
4718 with self.assertRaises(ValueError):
4719 sl.index('100')
4720 self.assertEqual(sl.index(100), 3)
4721
4722 # Exercise retrieving individual values.
4723 self.assertEqual(sl[0], 'howdy')
4724 self.assertEqual(sl[-2], True)
4725
4726 # Exercise iterability.
4727 self.assertEqual(
4728 tuple(sl),
4729 ('howdy', b'HoWdY', -273.154, 100, None, True, 42)
4730 )
4731
4732 # Exercise modifying individual values.
4733 sl[3] = 42
4734 self.assertEqual(sl[3], 42)
4735 sl[4] = 'some' # Change type at a given position.
4736 self.assertEqual(sl[4], 'some')
4737 self.assertEqual(sl.format, '8s8sdq8sxxxxxxx?q')
4738 with self.assertRaisesRegex(ValueError,
4739 "exceeds available storage"):
4740 sl[4] = 'far too many'
4741 self.assertEqual(sl[4], 'some')
4742 sl[0] = 'encodés' # Exactly 8 bytes of UTF-8 data
4743 self.assertEqual(sl[0], 'encodés')
4744 self.assertEqual(sl[1], b'HoWdY') # no spillage
4745 with self.assertRaisesRegex(ValueError,
4746 "exceeds available storage"):
4747 sl[0] = 'encodées' # Exactly 9 bytes of UTF-8 data
4748 self.assertEqual(sl[1], b'HoWdY')
4749 with self.assertRaisesRegex(ValueError,
4750 "exceeds available storage"):
4751 sl[1] = b'123456789'
4752 self.assertEqual(sl[1], b'HoWdY')
4753
4754 # Exercise count().
4755 with warnings.catch_warnings():
4756 # Suppress BytesWarning when comparing against b'HoWdY'.
4757 warnings.simplefilter('ignore')
4758 self.assertEqual(sl.count(42), 2)
4759 self.assertEqual(sl.count(b'HoWdY'), 1)
4760 self.assertEqual(sl.count(b'adios'), 0)
4761
4762 # Exercise creating a duplicate.
4763 name_duplicate = self._new_shm_name('test03_duplicate')
4764 sl_copy = shared_memory.ShareableList(sl, name=name_duplicate)
4765 try:
4766 self.assertNotEqual(sl.shm.name, sl_copy.shm.name)
4767 self.assertEqual(name_duplicate, sl_copy.shm.name)
4768 self.assertEqual(list(sl), list(sl_copy))
4769 self.assertEqual(sl.format, sl_copy.format)
4770 sl_copy[-1] = 77
4771 self.assertEqual(sl_copy[-1], 77)
4772 self.assertNotEqual(sl[-1], 77)
4773 sl_copy.shm.close()
4774 finally:
4775 sl_copy.shm.unlink()
4776
4777 # Obtain a second handle on the same ShareableList.
4778 sl_tethered = shared_memory.ShareableList(name=sl.shm.name)
4779 self.assertEqual(sl.shm.name, sl_tethered.shm.name)
4780 sl_tethered[-1] = 880
4781 self.assertEqual(sl[-1], 880)
4782 sl_tethered.shm.close()
4783
4784 sl.shm.close()
4785
4786 # Exercise creating an empty ShareableList.
4787 empty_sl = shared_memory.ShareableList()
4788 try:
4789 self.assertEqual(len(empty_sl), 0)
4790 self.assertEqual(empty_sl.format, '')
4791 self.assertEqual(empty_sl.count('any'), 0)
4792 with self.assertRaises(ValueError):
4793 empty_sl.index(None)
4794 empty_sl.shm.close()
4795 finally:
4796 empty_sl.shm.unlink()
4797
4798 def test_shared_memory_ShareableList_pickling(self):
4799 for proto in range(pickle.HIGHEST_PROTOCOL + 1):
4800 with self.subTest(proto=proto):
4801 sl = shared_memory.ShareableList(range(10))
4802 self.addCleanup(sl.shm.unlink)
4803
4804 serialized_sl = pickle.dumps(sl, protocol=proto)
4805 deserialized_sl = pickle.loads(serialized_sl)
4806 self.assertIsInstance(
4807 deserialized_sl, shared_memory.ShareableList)
4808 self.assertEqual(deserialized_sl[-1], 9)
4809 self.assertIsNot(sl, deserialized_sl)
4810
4811 deserialized_sl[4] = "changed"
4812 self.assertEqual(sl[4], "changed")
4813 sl[3] = "newvalue"
4814 self.assertEqual(deserialized_sl[3], "newvalue")
4815
4816 larger_sl = shared_memory.ShareableList(range(400))
4817 self.addCleanup(larger_sl.shm.unlink)
4818 serialized_larger_sl = pickle.dumps(larger_sl, protocol=proto)
4819 self.assertEqual(len(serialized_sl), len(serialized_larger_sl))
4820 larger_sl.shm.close()
4821
4822 deserialized_sl.shm.close()
4823 sl.shm.close()
4824
4825 def test_shared_memory_ShareableList_pickling_dead_object(self):
4826 for proto in range(pickle.HIGHEST_PROTOCOL + 1):
4827 with self.subTest(proto=proto):
4828 sl = shared_memory.ShareableList(range(10))
4829 serialized_sl = pickle.dumps(sl, protocol=proto)
4830
4831 # Now, we are going to kill the original object.
4832 # So, unpickled one won't be able to attach to it.
4833 sl.shm.close()
4834 sl.shm.unlink()
4835
4836 with self.assertRaises(FileNotFoundError):
4837 pickle.loads(serialized_sl)
4838
4839 def test_shared_memory_cleaned_after_process_termination(self):
4840 cmd = '''if 1:
4841 import os, time, sys
4842 from multiprocessing import shared_memory
4843
4844 # Create a shared_memory segment, and send the segment name
4845 sm = shared_memory.SharedMemory(create=True, size=10)
4846 sys.stdout.write(sm.name + '\\n')
4847 sys.stdout.flush()
4848 time.sleep(100)
4849 '''
4850 with subprocess.Popen([sys.executable, '-E', '-c', cmd],
4851 stdout=subprocess.PIPE,
4852 stderr=subprocess.PIPE) as p:
4853 name = p.stdout.readline().strip().decode()
4854
4855 # killing abruptly processes holding reference to a shared memory
4856 # segment should not leak the given memory segment.
4857 p.terminate()
4858 p.wait()
4859
4860 err_msg = ("A SharedMemory segment was leaked after "
4861 "a process was abruptly terminated")
4862 for _ in support.sleeping_retry(support.LONG_TIMEOUT, err_msg):
4863 try:
4864 smm = shared_memory.SharedMemory(name, create=False)
4865 except FileNotFoundError:
4866 break
4867
4868 if os.name == 'posix':
4869 # Without this line it was raising warnings like:
4870 # UserWarning: resource_tracker:
4871 # There appear to be 1 leaked shared_memory
4872 # objects to clean up at shutdown
4873 # See: https://bugs.python.org/issue45209
4874 resource_tracker.unregister(f"/{name}", "shared_memory")
4875
4876 # A warning was emitted by the subprocess' own
4877 # resource_tracker (on Windows, shared memory segments
4878 # are released automatically by the OS).
4879 err = p.stderr.read().decode()
4880 self.assertIn(
4881 "resource_tracker: There appear to be 1 leaked "
4882 "shared_memory objects to clean up at shutdown", err)
4883
4884 @unittest.skipIf(os.name != "posix", "resource_tracker is posix only")
4885 @resource_tracker_format_subtests
4886 def test_shared_memory_untracking(self):
4887 # gh-82300: When a separate Python process accesses shared memory
4888 # with track=False, it must not cause the memory to be deleted
4889 # when terminating.
4890 cmd = '''if 1:
4891 import sys
4892 from multiprocessing.shared_memory import SharedMemory
4893 mem = SharedMemory(create=False, name=sys.argv[1], track=False)
4894 mem.close()
4895 '''
4896 mem = shared_memory.SharedMemory(create=True, size=10)
4897 # The resource tracker shares pipes with the subprocess, and so
4898 # err existing means that the tracker process has terminated now.
4899 try:
4900 rc, out, err = script_helper.assert_python_ok("-c", cmd, mem.name)
4901 self.assertNotIn(b"resource_tracker", err)
4902 self.assertEqual(rc, 0)
4903 mem2 = shared_memory.SharedMemory(create=False, name=mem.name)
4904 mem2.close()
4905 finally:
4906 try:
4907 mem.unlink()
4908 except OSError:
4909 pass
4910 mem.close()
4911
4912 @unittest.skipIf(os.name != "posix", "resource_tracker is posix only")
4913 @resource_tracker_format_subtests
4914 def test_shared_memory_tracking(self):
4915 # gh-82300: When a separate Python process accesses shared memory
4916 # with track=True, it must cause the memory to be deleted when
4917 # terminating.
4918 cmd = '''if 1:
4919 import sys
4920 from multiprocessing.shared_memory import SharedMemory
4921 mem = SharedMemory(create=False, name=sys.argv[1], track=True)
4922 mem.close()
4923 '''
4924 mem = shared_memory.SharedMemory(create=True, size=10)
4925 try:
4926 rc, out, err = script_helper.assert_python_ok("-c", cmd, mem.name)
4927 self.assertEqual(rc, 0)
4928 self.assertIn(
4929 b"resource_tracker: There appear to be 1 leaked "
4930 b"shared_memory objects to clean up at shutdown", err)
4931 finally:
4932 try:
4933 mem.unlink()
4934 except OSError:
4935 pass
4936 resource_tracker.unregister(mem._name, "shared_memory")
4937 mem.close()
4938
4939 #
4940 # Test to verify that `Finalize` works.
4941 #
4942
4943 class _TestFinalize(BaseTestCase):
4944
4945 ALLOWED_TYPES = ('processes',)
4946
4947 def setUp(self):
4948 self.registry_backup = util._finalizer_registry.copy()
4949 util._finalizer_registry.clear()
4950
4951 def tearDown(self):
4952 gc.collect() # For PyPy or other GCs.
4953 self.assertFalse(util._finalizer_registry)
4954 util._finalizer_registry.update(self.registry_backup)
4955
4956 @classmethod
4957 def _test_finalize(cls, conn):
4958 class Foo(object):
4959 pass
4960
4961 a = Foo()
4962 util.Finalize(a, conn.send, args=('a',))
4963 del a # triggers callback for a
4964 gc.collect() # For PyPy or other GCs.
4965
4966 b = Foo()
4967 close_b = util.Finalize(b, conn.send, args=('b',))
4968 close_b() # triggers callback for b
4969 close_b() # does nothing because callback has already been called
4970 del b # does nothing because callback has already been called
4971 gc.collect() # For PyPy or other GCs.
4972
4973 c = Foo()
4974 util.Finalize(c, conn.send, args=('c',))
4975
4976 d10 = Foo()
4977 util.Finalize(d10, conn.send, args=('d10',), exitpriority=1)
4978
4979 d01 = Foo()
4980 util.Finalize(d01, conn.send, args=('d01',), exitpriority=0)
4981 d02 = Foo()
4982 util.Finalize(d02, conn.send, args=('d02',), exitpriority=0)
4983 d03 = Foo()
4984 util.Finalize(d03, conn.send, args=('d03',), exitpriority=0)
4985
4986 util.Finalize(None, conn.send, args=('e',), exitpriority=-10)
4987
4988 util.Finalize(None, conn.send, args=('STOP',), exitpriority=-100)
4989
4990 # call multiprocessing's cleanup function then exit process without
4991 # garbage collecting locals
4992 util._exit_function()
4993 conn.close()
4994 os._exit(0)
4995
4996 def test_finalize(self):
4997 conn, child_conn = self.Pipe()
4998
4999 p = self.Process(target=self._test_finalize, args=(child_conn,))
5000 p.daemon = True
5001 p.start()
5002 p.join()
5003
5004 result = [obj for obj in iter(conn.recv, 'STOP')]
5005 self.assertEqual(result, ['a', 'b', 'd10', 'd03', 'd02', 'd01', 'e'])
5006
5007 @support.requires_resource('cpu')
5008 def test_thread_safety(self):
5009 # bpo-24484: _run_finalizers() should be thread-safe
5010 def cb():
5011 pass
5012
5013 class Foo(object):
5014 def __init__(self):
5015 self.ref = self # create reference cycle
5016 # insert finalizer at random key
5017 util.Finalize(self, cb, exitpriority=random.randint(1, 100))
5018
5019 finish = False
5020 exc = None
5021
5022 def run_finalizers():
5023 nonlocal exc
5024 while not finish:
5025 time.sleep(random.random() * 1e-1)
5026 try:
5027 # A GC run will eventually happen during this,
5028 # collecting stale Foo's and mutating the registry
5029 util._run_finalizers()
5030 except Exception as e:
5031 exc = e
5032
5033 def make_finalizers():
5034 nonlocal exc
5035 d = {}
5036 while not finish:
5037 try:
5038 # Old Foo's get gradually replaced and later
5039 # collected by the GC (because of the cyclic ref)
5040 d[random.getrandbits(5)] = {Foo() for i in range(10)}
5041 except Exception as e:
5042 exc = e
5043 d.clear()
5044
5045 old_interval = sys.getswitchinterval()
5046 old_threshold = gc.get_threshold()
5047 try:
5048 support.setswitchinterval(1e-6)
5049 gc.set_threshold(5, 5, 5)
5050 threads = [threading.Thread(target=run_finalizers),
5051 threading.Thread(target=make_finalizers)]
5052 with threading_helper.start_threads(threads):
5053 time.sleep(4.0) # Wait a bit to trigger race condition
5054 finish = True
5055 if exc is not None:
5056 raise exc
5057 finally:
5058 sys.setswitchinterval(old_interval)
5059 gc.set_threshold(*old_threshold)
5060 gc.collect() # Collect remaining Foo's
5061
5062
5063 #
5064 # Test that from ... import * works for each module
5065 #
5066
5067 class _TestImportStar(unittest.TestCase):
5068
5069 def get_module_names(self):
5070 import glob
5071 folder = os.path.dirname(multiprocessing.__file__)
5072 pattern = os.path.join(glob.escape(folder), '*.py')
5073 files = glob.glob(pattern)
5074 modules = [os.path.splitext(os.path.split(f)[1])[0] for f in files]
5075 modules = ['multiprocessing.' + m for m in modules]
5076 modules.remove('multiprocessing.__init__')
5077 modules.append('multiprocessing')
5078 return modules
5079
5080 def test_import(self):
5081 modules = self.get_module_names()
5082 if sys.platform == 'win32':
5083 modules.remove('multiprocessing.popen_fork')
5084 modules.remove('multiprocessing.popen_forkserver')
5085 modules.remove('multiprocessing.popen_spawn_posix')
5086 else:
5087 modules.remove('multiprocessing.popen_spawn_win32')
5088 if not HAS_REDUCTION:
5089 modules.remove('multiprocessing.popen_forkserver')
5090
5091 if c_int is None:
5092 # This module requires _ctypes
5093 modules.remove('multiprocessing.sharedctypes')
5094
5095 for name in modules:
5096 __import__(name)
5097 mod = sys.modules[name]
5098 self.assertHasAttr(mod, '__all__', name)
5099 for attr in mod.__all__:
5100 self.assertHasAttr(mod, attr)
5101
5102 #
5103 # Quick test that logging works -- does not test logging output
5104 #
5105
5106 class _TestLogging(BaseTestCase):
5107
5108 ALLOWED_TYPES = ('processes',)
5109
5110 def test_enable_logging(self):
5111 logger = multiprocessing.get_logger()
5112 logger.setLevel(util.SUBWARNING)
5113 self.assertIsNotNone(logger)
5114 logger.debug('this will not be printed')
5115 logger.info('nor will this')
5116 logger.setLevel(LOG_LEVEL)
5117
5118 @classmethod
5119 def _test_level(cls, conn):
5120 logger = multiprocessing.get_logger()
5121 conn.send(logger.getEffectiveLevel())
5122
5123 def test_level(self):
5124 LEVEL1 = 32
5125 LEVEL2 = 37
5126
5127 logger = multiprocessing.get_logger()
5128 root_logger = logging.getLogger()
5129 root_level = root_logger.level
5130
5131 reader, writer = multiprocessing.Pipe(duplex=False)
5132
5133 logger.setLevel(LEVEL1)
5134 p = self.Process(target=self._test_level, args=(writer,))
5135 p.start()
5136 self.assertEqual(LEVEL1, reader.recv())
5137 p.join()
5138 p.close()
5139
5140 logger.setLevel(logging.NOTSET)
5141 root_logger.setLevel(LEVEL2)
5142 p = self.Process(target=self._test_level, args=(writer,))
5143 p.start()
5144 self.assertEqual(LEVEL2, reader.recv())
5145 p.join()
5146 p.close()
5147
5148 root_logger.setLevel(root_level)
5149 logger.setLevel(level=LOG_LEVEL)
5150
5151 def test_filename(self):
5152 logger = multiprocessing.get_logger()
5153 original_level = logger.level
5154 try:
5155 logger.setLevel(util.DEBUG)
5156 stream = io.StringIO()
5157 handler = logging.StreamHandler(stream)
5158 logging_format = '[%(levelname)s] [%(filename)s] %(message)s'
5159 handler.setFormatter(logging.Formatter(logging_format))
5160 logger.addHandler(handler)
5161 logger.info('1')
5162 util.info('2')
5163 logger.debug('3')
5164 filename = os.path.basename(__file__)
5165 log_record = stream.getvalue()
5166 self.assertIn(f'[INFO] [{filename}] 1', log_record)
5167 self.assertIn(f'[INFO] [{filename}] 2', log_record)
5168 self.assertIn(f'[DEBUG] [{filename}] 3', log_record)
5169 finally:
5170 logger.setLevel(original_level)
5171 logger.removeHandler(handler)
5172 handler.close()
5173
5174
5175 # class _TestLoggingProcessName(BaseTestCase):
5176 #
5177 # def handle(self, record):
5178 # assert record.processName == multiprocessing.current_process().name
5179 # self.__handled = True
5180 #
5181 # def test_logging(self):
5182 # handler = logging.Handler()
5183 # handler.handle = self.handle
5184 # self.__handled = False
5185 # # Bypass getLogger() and side-effects
5186 # logger = logging.getLoggerClass()(
5187 # 'multiprocessing.test.TestLoggingProcessName')
5188 # logger.addHandler(handler)
5189 # logger.propagate = False
5190 #
5191 # logger.warn('foo')
5192 # assert self.__handled
5193
5194 #
5195 # Check that Process.join() retries if os.waitpid() fails with EINTR
5196 #
5197
5198 class _TestPollEintr(BaseTestCase):
5199
5200 ALLOWED_TYPES = ('processes',)
5201
5202 @classmethod
5203 def _killer(cls, pid):
5204 time.sleep(0.1)
5205 os.kill(pid, signal.SIGUSR1)
5206
5207 @unittest.skipUnless(hasattr(signal, 'SIGUSR1'), 'requires SIGUSR1')
5208 def test_poll_eintr(self):
5209 got_signal = [False]
5210 def record(*args):
5211 got_signal[0] = True
5212 pid = os.getpid()
5213 oldhandler = signal.signal(signal.SIGUSR1, record)
5214 try:
5215 killer = self.Process(target=self._killer, args=(pid,))
5216 killer.start()
5217 try:
5218 p = self.Process(target=time.sleep, args=(2,))
5219 p.start()
5220 p.join()
5221 finally:
5222 killer.join()
5223 self.assertTrue(got_signal[0])
5224 self.assertEqual(p.exitcode, 0)
5225 finally:
5226 signal.signal(signal.SIGUSR1, oldhandler)
5227
5228 #
5229 # Test to verify handle verification, see issue 3321
5230 #
5231
5232 class TestInvalidHandle(unittest.TestCase):
5233
5234 @unittest.skipIf(WIN32, "skipped on Windows")
5235 def test_invalid_handles(self):
5236 conn = multiprocessing.connection.Connection(44977608)
5237 # check that poll() doesn't crash
5238 try:
5239 conn.poll()
5240 except (ValueError, OSError):
5241 pass
5242 finally:
5243 # Hack private attribute _handle to avoid printing an error
5244 # in conn.__del__
5245 conn._handle = None
5246 self.assertRaises((ValueError, OSError),
5247 multiprocessing.connection.Connection, -1)
5248
5249
5250 #
5251 # Regression tests for BaseProcess kwargs handling
5252 #
5253
5254 class TestBaseProcessKwargs(unittest.TestCase):
5255 def test_default_kwargs_not_shared_between_instances(self):
5256 # Creating multiple Process instances without passing kwargs
5257 # must create independent empty dicts (no shared state).
5258 p1 = multiprocessing.Process(target=lambda: None)
5259 p2 = multiprocessing.Process(target=lambda: None)
5260 self.assertIsInstance(p1._kwargs, dict)
5261 self.assertIsInstance(p2._kwargs, dict)
5262 self.assertIsNot(p1._kwargs, p2._kwargs)
5263 # Mutating one should not affect the other
5264 p1._kwargs['x'] = 1
5265 self.assertNotIn('x', p2._kwargs)
5266
5267
5268 @hashlib_helper.requires_hashdigest('sha256')
5269 class OtherTest(unittest.TestCase):
5270 # TODO: add more tests for deliver/answer challenge.
5271 def test_deliver_challenge_auth_failure(self):
5272 class _FakeConnection(object):
5273 def recv_bytes(self, size):
5274 return b'something bogus'
5275 def send_bytes(self, data):
5276 pass
5277 self.assertRaises(multiprocessing.AuthenticationError,
5278 multiprocessing.connection.deliver_challenge,
5279 _FakeConnection(), b'abc')
5280
5281 def test_answer_challenge_auth_failure(self):
5282 class _FakeConnection(object):
5283 def __init__(self):
5284 self.count = 0
5285 def recv_bytes(self, size):
5286 self.count += 1
5287 if self.count == 1:
5288 return multiprocessing.connection._CHALLENGE
5289 elif self.count == 2:
5290 return b'something bogus'
5291 return b''
5292 def send_bytes(self, data):
5293 pass
5294 self.assertRaises(multiprocessing.AuthenticationError,
5295 multiprocessing.connection.answer_challenge,
5296 _FakeConnection(), b'abc')
5297
5298
5299 @hashlib_helper.requires_hashdigest('md5')
5300 @hashlib_helper.requires_hashdigest('sha256')
5301 class ChallengeResponseTest(unittest.TestCase):
5302 authkey = b'supadupasecretkey'
5303
5304 def create_response(self, message):
5305 return multiprocessing.connection._create_response(
5306 self.authkey, message
5307 )
5308
5309 def verify_challenge(self, message, response):
5310 return multiprocessing.connection._verify_challenge(
5311 self.authkey, message, response
5312 )
5313
5314 def test_challengeresponse(self):
5315 for algo in [None, "md5", "sha256"]:
5316 with self.subTest(f"{algo=}"):
5317 msg = b'is-twenty-bytes-long' # The length of a legacy message.
5318 if algo:
5319 prefix = b'{%s}' % algo.encode("ascii")
5320 else:
5321 prefix = b''
5322 msg = prefix + msg
5323 response = self.create_response(msg)
5324 if not response.startswith(prefix):
5325 self.fail(response)
5326 self.verify_challenge(msg, response)
5327
5328 # TODO(gpshead): We need integration tests for handshakes between modern
5329 # deliver_challenge() and verify_response() code and connections running a
5330 # test-local copy of the legacy Python <=3.11 implementations.
5331
5332 # TODO(gpshead): properly annotate tests for requires_hashdigest rather than
5333 # only running these on a platform supporting everything. otherwise logic
5334 # issues preventing it from working on FIPS mode setups will be hidden.
5335
5336 #
5337 # Test Manager.start()/Pool.__init__() initializer feature - see issue 5585
5338 #
5339
5340 def initializer(ns):
5341 ns.test += 1
5342
5343 @hashlib_helper.requires_hashdigest('sha256')
5344 class TestInitializers(unittest.TestCase):
5345 def setUp(self):
5346 self.mgr = multiprocessing.Manager()
5347 self.ns = self.mgr.Namespace()
5348 self.ns.test = 0
5349
5350 def tearDown(self):
5351 self.mgr.shutdown()
5352 self.mgr.join()
5353
5354 def test_manager_initializer(self):
5355 m = multiprocessing.managers.SyncManager()
5356 self.assertRaises(TypeError, m.start, 1)
5357 m.start(initializer, (self.ns,))
5358 self.assertEqual(self.ns.test, 1)
5359 m.shutdown()
5360 m.join()
5361
5362 def test_pool_initializer(self):
5363 self.assertRaises(TypeError, multiprocessing.Pool, initializer=1)
5364 p = multiprocessing.Pool(1, initializer, (self.ns,))
5365 p.close()
5366 p.join()
5367 self.assertEqual(self.ns.test, 1)
5368
5369 #
5370 # Issue 5155, 5313, 5331: Test process in processes
5371 # Verifies os.close(sys.stdin.fileno) vs. sys.stdin.close() behavior
5372 #
5373
5374 def _this_sub_process(q):
5375 try:
5376 item = q.get(block=False)
5377 except pyqueue.Empty:
5378 pass
5379
5380 def _test_process():
5381 queue = multiprocessing.Queue()
5382 subProc = multiprocessing.Process(target=_this_sub_process, args=(queue,))
5383 subProc.daemon = True
5384 subProc.start()
5385 subProc.join()
5386
5387 def _afunc(x):
5388 return x*x
5389
5390 def pool_in_process():
5391 pool = multiprocessing.Pool(processes=4)
5392 x = pool.map(_afunc, [1, 2, 3, 4, 5, 6, 7])
5393 pool.close()
5394 pool.join()
5395
5396 class _file_like(object):
5397 def __init__(self, delegate):
5398 self._delegate = delegate
5399 self._pid = None
5400
5401 @property
5402 def cache(self):
5403 pid = os.getpid()
5404 # There are no race conditions since fork keeps only the running thread
5405 if pid != self._pid:
5406 self._pid = pid
5407 self._cache = []
5408 return self._cache
5409
5410 def write(self, data):
5411 self.cache.append(data)
5412
5413 def flush(self):
5414 self._delegate.write(''.join(self.cache))
5415 self._cache = []
5416
5417 class TestStdinBadfiledescriptor(unittest.TestCase):
5418
5419 def test_queue_in_process(self):
5420 proc = multiprocessing.Process(target=_test_process)
5421 proc.start()
5422 proc.join()
5423
5424 def test_pool_in_process(self):
5425 p = multiprocessing.Process(target=pool_in_process)
5426 p.start()
5427 p.join()
5428
5429 def test_flushing(self):
5430 sio = io.StringIO()
5431 flike = _file_like(sio)
5432 flike.write('foo')
5433 proc = multiprocessing.Process(target=lambda: flike.flush())
5434 flike.flush()
5435 assert sio.getvalue() == 'foo'
5436
5437
5438 class TestWait(unittest.TestCase):
5439
5440 @classmethod
5441 def _child_test_wait(cls, w, slow):
5442 for i in range(10):
5443 if slow:
5444 time.sleep(random.random() * 0.100)
5445 w.send((i, os.getpid()))
5446 w.close()
5447
5448 def test_wait(self, slow=False):
5449 from multiprocessing.connection import wait
5450 readers = []
5451 procs = []
5452 messages = []
5453
5454 for i in range(4):
5455 r, w = multiprocessing.Pipe(duplex=False)
5456 p = multiprocessing.Process(target=self._child_test_wait, args=(w, slow))
5457 p.daemon = True
5458 p.start()
5459 w.close()
5460 readers.append(r)
5461 procs.append(p)
5462 self.addCleanup(p.join)
5463
5464 while readers:
5465 for r in wait(readers):
5466 try:
5467 msg = r.recv()
5468 except EOFError:
5469 readers.remove(r)
5470 r.close()
5471 else:
5472 messages.append(msg)
5473
5474 messages.sort()
5475 expected = sorted((i, p.pid) for i in range(10) for p in procs)
5476 self.assertEqual(messages, expected)
5477
5478 @classmethod
5479 def _child_test_wait_socket(cls, address, slow):
5480 s = socket.socket()
5481 s.connect(address)
5482 for i in range(10):
5483 if slow:
5484 time.sleep(random.random() * 0.100)
5485 s.sendall(('%s\n' % i).encode('ascii'))
5486 s.close()
5487
5488 def test_wait_socket(self, slow=False):
5489 from multiprocessing.connection import wait
5490 l = socket.create_server((socket_helper.HOST, 0))
5491 addr = l.getsockname()
5492 readers = []
5493 procs = []
5494 dic = {}
5495
5496 for i in range(4):
5497 p = multiprocessing.Process(target=self._child_test_wait_socket,
5498 args=(addr, slow))
5499 p.daemon = True
5500 p.start()
5501 procs.append(p)
5502 self.addCleanup(p.join)
5503
5504 for i in range(4):
5505 r, _ = l.accept()
5506 readers.append(r)
5507 dic[r] = []
5508 l.close()
5509
5510 while readers:
5511 for r in wait(readers):
5512 msg = r.recv(32)
5513 if not msg:
5514 readers.remove(r)
5515 r.close()
5516 else:
5517 dic[r].append(msg)
5518
5519 expected = ''.join('%s\n' % i for i in range(10)).encode('ascii')
5520 for v in dic.values():
5521 self.assertEqual(b''.join(v), expected)
5522
5523 def test_wait_slow(self):
5524 self.test_wait(True)
5525
5526 def test_wait_socket_slow(self):
5527 self.test_wait_socket(True)
5528
5529 @support.requires_resource('walltime')
5530 def test_wait_timeout(self):
5531 from multiprocessing.connection import wait
5532
5533 timeout = 5.0 # seconds
5534 a, b = multiprocessing.Pipe()
5535
5536 start = time.monotonic()
5537 res = wait([a, b], timeout)
5538 delta = time.monotonic() - start
5539
5540 self.assertEqual(res, [])
5541 self.assertGreater(delta, timeout - CLOCK_RES)
5542
5543 b.send(None)
5544 res = wait([a, b], 20)
5545 self.assertEqual(res, [a])
5546
5547 @classmethod
5548 def signal_and_sleep(cls, sem, period):
5549 sem.release()
5550 time.sleep(period)
5551
5552 @support.requires_resource('walltime')
5553 def test_wait_integer(self):
5554 from multiprocessing.connection import wait
5555
5556 expected = 3
5557 sorted_ = lambda l: sorted(l, key=lambda x: id(x))
5558 sem = multiprocessing.Semaphore(0)
5559 a, b = multiprocessing.Pipe()
5560 p = multiprocessing.Process(target=self.signal_and_sleep,
5561 args=(sem, expected))
5562
5563 p.start()
5564 self.assertIsInstance(p.sentinel, int)
5565 self.assertTrue(sem.acquire(timeout=20))
5566
5567 start = time.monotonic()
5568 res = wait([a, p.sentinel, b], expected + 20)
5569 delta = time.monotonic() - start
5570
5571 self.assertEqual(res, [p.sentinel])
5572 self.assertLess(delta, expected + 2)
5573 self.assertGreater(delta, expected - 2)
5574
5575 a.send(None)
5576
5577 start = time.monotonic()
5578 res = wait([a, p.sentinel, b], 20)
5579 delta = time.monotonic() - start
5580
5581 self.assertEqual(sorted_(res), sorted_([p.sentinel, b]))
5582 self.assertLess(delta, 0.4)
5583
5584 b.send(None)
5585
5586 start = time.monotonic()
5587 res = wait([a, p.sentinel, b], 20)
5588 delta = time.monotonic() - start
5589
5590 self.assertEqual(sorted_(res), sorted_([a, p.sentinel, b]))
5591 self.assertLess(delta, 0.4)
5592
5593 p.terminate()
5594 p.join()
5595
5596 def test_neg_timeout(self):
5597 from multiprocessing.connection import wait
5598 a, b = multiprocessing.Pipe()
5599 t = time.monotonic()
5600 res = wait([a], timeout=-1)
5601 t = time.monotonic() - t
5602 self.assertEqual(res, [])
5603 self.assertLess(t, 1)
5604 a.close()
5605 b.close()
5606
5607 #
5608 # Issue 14151: Test invalid family on invalid environment
5609 #
5610
5611 class TestInvalidFamily(unittest.TestCase):
5612
5613 @unittest.skipIf(WIN32, "skipped on Windows")
5614 def test_invalid_family(self):
5615 with self.assertRaises(ValueError):
5616 multiprocessing.connection.Listener(r'\\.\test')
5617
5618 @unittest.skipUnless(WIN32, "skipped on non-Windows platforms")
5619 def test_invalid_family_win32(self):
5620 with self.assertRaises(ValueError):
5621 multiprocessing.connection.Listener('/var/test.pipe')
5622
5623 #
5624 # Issue 12098: check sys.flags of child matches that for parent
5625 #
5626
5627 class TestFlags(unittest.TestCase):
5628 @classmethod
5629 def run_in_grandchild(cls, conn):
5630 conn.send(tuple(sys.flags))
5631
5632 @classmethod
5633 def run_in_child(cls, start_method):
5634 import json
5635 mp = multiprocessing.get_context(start_method)
5636 r, w = mp.Pipe(duplex=False)
5637 p = mp.Process(target=cls.run_in_grandchild, args=(w,))
5638 with warnings.catch_warnings(category=DeprecationWarning):
5639 p.start()
5640 grandchild_flags = r.recv()
5641 p.join()
5642 r.close()
5643 w.close()
5644 flags = (tuple(sys.flags), grandchild_flags)
5645 print(json.dumps(flags))
5646
5647 def test_flags(self):
5648 import json
5649 # start child process using unusual flags
5650 prog = (
5651 'from test._test_multiprocessing import TestFlags; '
5652 f'TestFlags.run_in_child({multiprocessing.get_start_method()!r})'
5653 )
5654 data = subprocess.check_output(
5655 [sys.executable, '-E', '-S', '-O', '-c', prog])
5656 child_flags, grandchild_flags = json.loads(data.decode('ascii'))
5657 self.assertEqual(child_flags, grandchild_flags)
5658
5659 #
5660 # Test interaction with socket timeouts - see Issue #6056
5661 #
5662
5663 class TestTimeouts(unittest.TestCase):
5664 @classmethod
5665 def _test_timeout(cls, child, address):
5666 time.sleep(1)
5667 child.send(123)
5668 child.close()
5669 conn = multiprocessing.connection.Client(address)
5670 conn.send(456)
5671 conn.close()
5672
5673 def test_timeout(self):
5674 old_timeout = socket.getdefaulttimeout()
5675 try:
5676 socket.setdefaulttimeout(0.1)
5677 parent, child = multiprocessing.Pipe(duplex=True)
5678 l = multiprocessing.connection.Listener(family='AF_INET')
5679 p = multiprocessing.Process(target=self._test_timeout,
5680 args=(child, l.address))
5681 p.start()
5682 child.close()
5683 self.assertEqual(parent.recv(), 123)
5684 parent.close()
5685 conn = l.accept()
5686 self.assertEqual(conn.recv(), 456)
5687 conn.close()
5688 l.close()
5689 join_process(p)
5690 finally:
5691 socket.setdefaulttimeout(old_timeout)
5692
5693 #
5694 # Test what happens with no "if __name__ == '__main__'"
5695 #
5696
5697 class TestNoForkBomb(unittest.TestCase):
5698 def test_noforkbomb(self):
5699 sm = multiprocessing.get_start_method()
5700 name = os.path.join(os.path.dirname(__file__), 'mp_fork_bomb.py')
5701 if sm != 'fork':
5702 rc, out, err = test.support.script_helper.assert_python_failure(name, sm)
5703 self.assertEqual(out, b'')
5704 self.assertIn(b'RuntimeError', err)
5705 else:
5706 rc, out, err = test.support.script_helper.assert_python_ok(name, sm)
5707 self.assertEqual(out.rstrip(), b'123')
5708 self.assertEqual(err, b'')
5709
5710 #
5711 # Issue #17555: ForkAwareThreadLock
5712 #
5713
5714 class TestForkAwareThreadLock(unittest.TestCase):
5715 # We recursively start processes. Issue #17555 meant that the
5716 # after fork registry would get duplicate entries for the same
5717 # lock. The size of the registry at generation n was ~2**n.
5718
5719 @classmethod
5720 def child(cls, n, conn):
5721 if n > 1:
5722 p = multiprocessing.Process(target=cls.child, args=(n-1, conn))
5723 p.start()
5724 conn.close()
5725 join_process(p)
5726 else:
5727 conn.send(len(util._afterfork_registry))
5728 conn.close()
5729
5730 def test_lock(self):
5731 r, w = multiprocessing.Pipe(False)
5732 l = util.ForkAwareThreadLock()
5733 old_size = len(util._afterfork_registry)
5734 p = multiprocessing.Process(target=self.child, args=(5, w))
5735 p.start()
5736 w.close()
5737 new_size = r.recv()
5738 join_process(p)
5739 self.assertLessEqual(new_size, old_size)
5740
5741 #
5742 # Check that non-forked child processes do not inherit unneeded fds/handles
5743 #
5744
5745 class TestCloseFds(unittest.TestCase):
5746
5747 def get_high_socket_fd(self):
5748 if WIN32:
5749 # The child process will not have any socket handles, so
5750 # calling socket.fromfd() should produce WSAENOTSOCK even
5751 # if there is a handle of the same number.
5752 return socket.socket().detach()
5753 else:
5754 # We want to produce a socket with an fd high enough that a
5755 # freshly created child process will not have any fds as high.
5756 fd = socket.socket().detach()
5757 to_close = []
5758 while fd < 50:
5759 to_close.append(fd)
5760 fd = os.dup(fd)
5761 for x in to_close:
5762 os.close(x)
5763 return fd
5764
5765 def close(self, fd):
5766 if WIN32:
5767 socket.socket(socket.AF_INET, socket.SOCK_STREAM, fileno=fd).close()
5768 else:
5769 os.close(fd)
5770
5771 @classmethod
5772 def _test_closefds(cls, conn, fd):
5773 try:
5774 s = socket.fromfd(fd, socket.AF_INET, socket.SOCK_STREAM)
5775 except Exception as e:
5776 conn.send(e)
5777 else:
5778 s.close()
5779 conn.send(None)
5780
5781 def test_closefd(self):
5782 if not HAS_REDUCTION:
5783 raise unittest.SkipTest('requires fd pickling')
5784
5785 reader, writer = multiprocessing.Pipe()
5786 fd = self.get_high_socket_fd()
5787 try:
5788 p = multiprocessing.Process(target=self._test_closefds,
5789 args=(writer, fd))
5790 p.start()
5791 writer.close()
5792 e = reader.recv()
5793 join_process(p)
5794 finally:
5795 self.close(fd)
5796 writer.close()
5797 reader.close()
5798
5799 if multiprocessing.get_start_method() == 'fork':
5800 self.assertIs(e, None)
5801 else:
5802 WSAENOTSOCK = 10038
5803 self.assertIsInstance(e, OSError)
5804 self.assertTrue(e.errno == errno.EBADF or
5805 e.winerror == WSAENOTSOCK, e)
5806
5807 #
5808 # Issue #17097: EINTR should be ignored by recv(), send(), accept() etc
5809 #
5810
5811 class TestIgnoreEINTR(unittest.TestCase):
5812
5813 # Sending CONN_MAX_SIZE bytes into a multiprocessing pipe must block
5814 CONN_MAX_SIZE = max(support.PIPE_MAX_SIZE, support.SOCK_MAX_SIZE)
5815
5816 @classmethod
5817 def _test_ignore(cls, conn):
5818 def handler(signum, frame):
5819 pass
5820 signal.signal(signal.SIGUSR1, handler)
5821 conn.send('ready')
5822 x = conn.recv()
5823 conn.send(x)
5824 conn.send_bytes(b'x' * cls.CONN_MAX_SIZE)
5825
5826 @unittest.skipUnless(hasattr(signal, 'SIGUSR1'), 'requires SIGUSR1')
5827 def test_ignore(self):
5828 conn, child_conn = multiprocessing.Pipe()
5829 try:
5830 p = multiprocessing.Process(target=self._test_ignore,
5831 args=(child_conn,))
5832 p.daemon = True
5833 p.start()
5834 child_conn.close()
5835 self.assertEqual(conn.recv(), 'ready')
5836 time.sleep(0.1)
5837 os.kill(p.pid, signal.SIGUSR1)
5838 time.sleep(0.1)
5839 conn.send(1234)
5840 self.assertEqual(conn.recv(), 1234)
5841 time.sleep(0.1)
5842 os.kill(p.pid, signal.SIGUSR1)
5843 self.assertEqual(conn.recv_bytes(), b'x' * self.CONN_MAX_SIZE)
5844 time.sleep(0.1)
5845 p.join()
5846 finally:
5847 conn.close()
5848
5849 @classmethod
5850 def _test_ignore_listener(cls, conn):
5851 def handler(signum, frame):
5852 pass
5853 signal.signal(signal.SIGUSR1, handler)
5854 with multiprocessing.connection.Listener() as l:
5855 conn.send(l.address)
5856 a = l.accept()
5857 a.send('welcome')
5858
5859 @unittest.skipUnless(hasattr(signal, 'SIGUSR1'), 'requires SIGUSR1')
5860 def test_ignore_listener(self):
5861 conn, child_conn = multiprocessing.Pipe()
5862 try:
5863 p = multiprocessing.Process(target=self._test_ignore_listener,
5864 args=(child_conn,))
5865 p.daemon = True
5866 p.start()
5867 child_conn.close()
5868 address = conn.recv()
5869 time.sleep(0.1)
5870 os.kill(p.pid, signal.SIGUSR1)
5871 time.sleep(0.1)
5872 client = multiprocessing.connection.Client(address)
5873 self.assertEqual(client.recv(), 'welcome')
5874 p.join()
5875 finally:
5876 conn.close()
5877
5878 class TestStartMethod(unittest.TestCase):
5879 @classmethod
5880 def _check_context(cls, conn):
5881 conn.send(multiprocessing.get_start_method())
5882
5883 def check_context(self, ctx):
5884 r, w = ctx.Pipe(duplex=False)
5885 p = ctx.Process(target=self._check_context, args=(w,))
5886 p.start()
5887 w.close()
5888 child_method = r.recv()
5889 r.close()
5890 p.join()
5891 self.assertEqual(child_method, ctx.get_start_method())
5892
5893 def test_context(self):
5894 for method in ('fork', 'spawn', 'forkserver'):
5895 try:
5896 ctx = multiprocessing.get_context(method)
5897 except ValueError:
5898 continue
5899 self.assertEqual(ctx.get_start_method(), method)
5900 self.assertIs(ctx.get_context(), ctx)
5901 self.assertRaises(ValueError, ctx.set_start_method, 'spawn')
5902 self.assertRaises(ValueError, ctx.set_start_method, None)
5903 self.check_context(ctx)
5904
5905 @staticmethod
5906 def _dummy_func():
5907 pass
5908
5909 def test_spawn_dont_set_context(self):
5910 # Run a process with spawn or forkserver context may change
5911 # the global start method, see gh-109263.
5912 for method in ('fork', 'spawn', 'forkserver'):
5913 multiprocessing.set_start_method(None, force=True)
5914
5915 try:
5916 ctx = multiprocessing.get_context(method)
5917 except ValueError:
5918 continue
5919 process = ctx.Process(target=self._dummy_func)
5920 process.start()
5921 process.join()
5922 self.assertIsNone(multiprocessing.get_start_method(allow_none=True))
5923
5924 @only_run_in_spawn_testsuite("freeze_support is not start method specific")
5925 def test_freeze_support_dont_set_context(self):
5926 # gh-140814: freeze_support() should not set the start method
5927 # as a side effect, so a later set_start_method() still works.
5928 multiprocessing.set_start_method(None, force=True)
5929 try:
5930 multiprocessing.freeze_support()
5931 self.assertIsNone(
5932 multiprocessing.get_start_method(allow_none=True))
5933 # Should not raise "context has already been set"
5934 multiprocessing.set_start_method('spawn')
5935 finally:
5936 multiprocessing.set_start_method(None, force=True)
5937
5938 def test_context_check_module_types(self):
5939 try:
5940 ctx = multiprocessing.get_context('forkserver')
5941 except ValueError:
5942 raise unittest.SkipTest('forkserver should be available')
5943 with self.assertRaisesRegex(TypeError, 'module_names must be a list of strings'):
5944 ctx.set_forkserver_preload([1, 2, 3])
5945
5946 def test_set_get(self):
5947 multiprocessing.set_forkserver_preload(PRELOAD)
5948 count = 0
5949 old_method = multiprocessing.get_start_method()
5950 try:
5951 for method in ('fork', 'spawn', 'forkserver'):
5952 try:
5953 multiprocessing.set_start_method(method, force=True)
5954 except ValueError:
5955 continue
5956 self.assertEqual(multiprocessing.get_start_method(), method)
5957 ctx = multiprocessing.get_context()
5958 self.assertEqual(ctx.get_start_method(), method)
5959 self.assertStartsWith(type(ctx).__name__.lower(), method)
5960 self.assertStartsWith(ctx.Process.__name__.lower(), method)
5961 self.check_context(multiprocessing)
5962 count += 1
5963 finally:
5964 multiprocessing.set_start_method(old_method, force=True)
5965 self.assertGreaterEqual(count, 1)
5966
5967 def test_get_all_start_methods(self):
5968 methods = multiprocessing.get_all_start_methods()
5969 self.assertIn('spawn', methods)
5970 if sys.platform == 'win32':
5971 self.assertEqual(methods, ['spawn'])
5972 elif sys.platform == 'darwin':
5973 self.assertEqual(methods[0], 'spawn') # The default is first.
5974 # Whether these work or not, they remain available on macOS.
5975 self.assertIn('fork', methods)
5976 self.assertIn('forkserver', methods)
5977 else:
5978 # POSIX
5979 self.assertIn('fork', methods)
5980 if other_methods := set(methods) - {'fork', 'spawn'}:
5981 # If there are more than those two, forkserver must be one.
5982 self.assertEqual({'forkserver'}, other_methods)
5983 # The default is the first method in the list.
5984 self.assertIn(methods[0], {'forkserver', 'spawn'},
5985 msg='3.14+ default must not be fork')
5986 if methods[0] == 'spawn':
5987 # Confirm that the current default selection logic prefers
5988 # forkserver vs spawn when available.
5989 self.assertNotIn('forkserver', methods)
5990
5991 def test_preload_resources(self):
5992 if multiprocessing.get_start_method() != 'forkserver':
5993 self.skipTest("test only relevant for 'forkserver' method")
5994 name = os.path.join(os.path.dirname(__file__), 'mp_preload.py')
5995 rc, out, err = test.support.script_helper.assert_python_ok(name)
5996 out = out.decode()
5997 err = err.decode()
5998 if out.rstrip() != 'ok' or err != '':
5999 print(out)
6000 print(err)
6001 self.fail("failed spawning forkserver or grandchild")
6002
6003 @unittest.skipIf(sys.platform == "win32",
6004 "Only Spawn on windows so no risk of mixing")
6005 @only_run_in_spawn_testsuite("avoids redundant testing.")
6006 def test_mixed_startmethod(self):
6007 # Fork-based locks cannot be used with spawned process
6008 for process_method in ["spawn", "forkserver"]:
6009 queue = multiprocessing.get_context("fork").Queue()
6010 process_ctx = multiprocessing.get_context(process_method)
6011 p = process_ctx.Process(target=close_queue, args=(queue,))
6012 err_msg = "A SemLock created in a fork"
6013 with self.assertRaisesRegex(RuntimeError, err_msg):
6014 p.start()
6015
6016 # non-fork-based locks can be used with all other start methods
6017 for queue_method in ["spawn", "forkserver"]:
6018 for process_method in multiprocessing.get_all_start_methods():
6019 queue = multiprocessing.get_context(queue_method).Queue()
6020 process_ctx = multiprocessing.get_context(process_method)
6021 p = process_ctx.Process(target=close_queue, args=(queue,))
6022 p.start()
6023 p.join()
6024
6025 @classmethod
6026 def _put_one_in_queue(cls, queue):
6027 queue.put(1)
6028
6029 @classmethod
6030 def _put_two_and_nest_once(cls, queue):
6031 queue.put(2)
6032 process = multiprocessing.Process(target=cls._put_one_in_queue, args=(queue,))
6033 process.start()
6034 process.join()
6035
6036 def test_nested_startmethod(self):
6037 # gh-108520: Regression test to ensure that child process can send its
6038 # arguments to another process
6039 queue = multiprocessing.Queue()
6040
6041 process = multiprocessing.Process(target=self._put_two_and_nest_once, args=(queue,))
6042 process.start()
6043 process.join()
6044
6045 results = []
6046 while not queue.empty():
6047 results.append(queue.get())
6048
6049 # gh-109706: queue.put(1) can write into the queue before queue.put(2),
6050 # there is no synchronization in the test.
6051 self.assertSetEqual(set(results), set([2, 1]))
6052
6053
6054 @unittest.skipIf(sys.platform == "win32",
6055 "test semantics don't make sense on Windows")
6056 class TestResourceTracker(unittest.TestCase):
6057
6058 def test_resource_tracker(self):
6059 #
6060 # Check that killing process does not leak named semaphores
6061 #
6062 cmd = '''if 1:
6063 import time, os
6064 import multiprocessing as mp
6065 from multiprocessing import resource_tracker
6066 from multiprocessing.shared_memory import SharedMemory
6067
6068 mp.set_start_method("spawn")
6069
6070
6071 def create_and_register_resource(rtype):
6072 if rtype == "semaphore":
6073 lock = mp.Lock()
6074 return lock, lock._semlock.name
6075 elif rtype == "shared_memory":
6076 sm = SharedMemory(create=True, size=10)
6077 return sm, sm._name
6078 else:
6079 raise ValueError(
6080 "Resource type {{}} not understood".format(rtype))
6081
6082
6083 resource1, rname1 = create_and_register_resource("{rtype}")
6084 resource2, rname2 = create_and_register_resource("{rtype}")
6085
6086 os.write({w}, rname1.encode("ascii") + b"\\n")
6087 os.write({w}, rname2.encode("ascii") + b"\\n")
6088
6089 time.sleep(10)
6090 '''
6091 for rtype in resource_tracker._CLEANUP_FUNCS:
6092 with self.subTest(rtype=rtype):
6093 if rtype in ("noop", "dummy"):
6094 # Artefact resource type used by the resource_tracker
6095 # or tests
6096 continue
6097 r, w = os.pipe()
6098 p = subprocess.Popen([sys.executable,
6099 '-E', '-c', cmd.format(w=w, rtype=rtype)],
6100 pass_fds=[w],
6101 stderr=subprocess.PIPE)
6102 os.close(w)
6103 with open(r, 'rb', closefd=True) as f:
6104 name1 = f.readline().rstrip().decode('ascii')
6105 name2 = f.readline().rstrip().decode('ascii')
6106 _resource_unlink(name1, rtype)
6107 p.terminate()
6108 p.wait()
6109
6110 err_msg = (f"A {rtype} resource was leaked after a process was "
6111 f"abruptly terminated")
6112 for _ in support.sleeping_retry(support.SHORT_TIMEOUT,
6113 err_msg):
6114 try:
6115 _resource_unlink(name2, rtype)
6116 except OSError as e:
6117 # docs say it should be ENOENT, but OSX seems to give
6118 # EINVAL
6119 self.assertIn(e.errno, (errno.ENOENT, errno.EINVAL))
6120 break
6121
6122 err = p.stderr.read().decode('utf-8')
6123 p.stderr.close()
6124 expected = ('resource_tracker: There appear to be 2 leaked {} '
6125 'objects'.format(
6126 rtype))
6127 self.assertRegex(err, expected)
6128 self.assertRegex(err, r'resource_tracker: %r: \[Errno' % name1)
6129
6130 def check_resource_tracker_death(self, signum, should_die):
6131 # bpo-31310: if the semaphore tracker process has died, it should
6132 # be restarted implicitly.
6133 from multiprocessing.resource_tracker import _resource_tracker
6134 pid = _resource_tracker._pid
6135 if pid is not None:
6136 os.kill(pid, signal.SIGKILL)
6137 support.wait_process(pid, exitcode=-signal.SIGKILL)
6138 with warnings.catch_warnings():
6139 warnings.simplefilter("ignore")
6140 _resource_tracker.ensure_running()
6141 pid = _resource_tracker._pid
6142
6143 os.kill(pid, signum)
6144 time.sleep(1.0) # give it time to die
6145
6146 ctx = multiprocessing.get_context("spawn")
6147 with warnings.catch_warnings(record=True) as all_warn:
6148 warnings.simplefilter("always")
6149 sem = ctx.Semaphore()
6150 sem.acquire()
6151 sem.release()
6152 wr = weakref.ref(sem)
6153 # ensure `sem` gets collected, which triggers communication with
6154 # the semaphore tracker
6155 del sem
6156 gc.collect()
6157 self.assertIsNone(wr())
6158 if should_die:
6159 self.assertEqual(len(all_warn), 1)
6160 the_warn = all_warn[0]
6161 self.assertIsSubclass(the_warn.category, UserWarning)
6162 self.assertIn("resource_tracker: process died",
6163 str(the_warn.message))
6164 else:
6165 self.assertEqual(len(all_warn), 0)
6166
6167 def test_resource_tracker_sigint(self):
6168 # Catchable signal (ignored by semaphore tracker)
6169 self.check_resource_tracker_death(signal.SIGINT, False)
6170
6171 def test_resource_tracker_sigterm(self):
6172 # Catchable signal (ignored by semaphore tracker)
6173 self.check_resource_tracker_death(signal.SIGTERM, False)
6174
6175 @unittest.skipIf(sys.platform.startswith("netbsd"),
6176 "gh-125620: Skip on NetBSD due to long wait for SIGKILL process termination.")
6177 def test_resource_tracker_sigkill(self):
6178 # Uncatchable signal.
6179 self.check_resource_tracker_death(signal.SIGKILL, True)
6180
6181 @staticmethod
6182 def _is_resource_tracker_reused(conn, pid):
6183 from multiprocessing.resource_tracker import _resource_tracker
6184 _resource_tracker.ensure_running()
6185 # The pid should be None in the child (the at-fork handler clears
6186 # it for fork; spawn/forkserver children never had it set). It
6187 # should not be a new value.
6188 reused = _resource_tracker._pid in (None, pid)
6189 reused &= _resource_tracker._check_alive()
6190 conn.send(reused)
6191
6192 def test_resource_tracker_reused(self):
6193 from multiprocessing.resource_tracker import _resource_tracker
6194 _resource_tracker.ensure_running()
6195 pid = _resource_tracker._pid
6196
6197 r, w = multiprocessing.Pipe(duplex=False)
6198 p = multiprocessing.Process(target=self._is_resource_tracker_reused,
6199 args=(w, pid))
6200 p.start()
6201 is_resource_tracker_reused = r.recv()
6202
6203 # Clean up
6204 p.join()
6205 w.close()
6206 r.close()
6207
6208 self.assertTrue(is_resource_tracker_reused)
6209
6210 def test_too_long_name_resource(self):
6211 # gh-96819: Resource names that will make the length of a write to a pipe
6212 # greater than PIPE_BUF are not allowed
6213 rtype = "shared_memory"
6214 too_long_name_resource = "a" * (512 - len(rtype))
6215 with self.assertRaises(ValueError):
6216 resource_tracker.register(too_long_name_resource, rtype)
6217
6218 def _test_resource_tracker_leak_resources(self, cleanup):
6219 # We use a separate instance for testing, since the main global
6220 # _resource_tracker may be used to watch test infrastructure.
6221 from multiprocessing.resource_tracker import ResourceTracker
6222 tracker = ResourceTracker()
6223 tracker.ensure_running()
6224 self.assertTrue(tracker._check_alive())
6225
6226 self.assertIsNone(tracker._exitcode)
6227 tracker.register('somename', 'dummy')
6228 if cleanup:
6229 tracker.unregister('somename', 'dummy')
6230 expected_exit_code = 0
6231 else:
6232 expected_exit_code = 1
6233
6234 self.assertTrue(tracker._check_alive())
6235 self.assertIsNone(tracker._exitcode)
6236 tracker._stop()
6237 self.assertEqual(tracker._exitcode, expected_exit_code)
6238
6239 def test_resource_tracker_exit_code(self):
6240 """
6241 Test the exit code of the resource tracker.
6242
6243 If no leaked resources were found, exit code should be 0, otherwise 1
6244 """
6245 for cleanup in [True, False]:
6246 with self.subTest(cleanup=cleanup):
6247 self._test_resource_tracker_leak_resources(
6248 cleanup=cleanup,
6249 )
6250
6251 @unittest.skipUnless(hasattr(signal, "pthread_sigmask"), "pthread_sigmask is not available")
6252 def test_resource_tracker_blocked_signals(self):
6253 #
6254 # gh-127586: Check that resource_tracker does not override blocked signals of caller.
6255 #
6256 from multiprocessing.resource_tracker import ResourceTracker
6257 orig_sigmask = signal.pthread_sigmask(signal.SIG_BLOCK, set())
6258 signals = {signal.SIGTERM, signal.SIGINT, signal.SIGUSR1}
6259
6260 try:
6261 for sig in signals:
6262 signal.pthread_sigmask(signal.SIG_SETMASK, {sig})
6263 self.assertEqual(signal.pthread_sigmask(signal.SIG_BLOCK, set()), {sig})
6264 tracker = ResourceTracker()
6265 tracker.ensure_running()
6266 self.assertEqual(signal.pthread_sigmask(signal.SIG_BLOCK, set()), {sig})
6267 tracker._stop()
6268 finally:
6269 # restore sigmask to what it was before executing test
6270 signal.pthread_sigmask(signal.SIG_SETMASK, orig_sigmask)
6271
6272 @only_run_in_forkserver_testsuite("avoids redundant testing.")
6273 def test_resource_tracker_fork_deadlock(self):
6274 # gh-146313: ResourceTracker.__del__ used to deadlock if a forked
6275 # child still held the pipe's write end open when the parent
6276 # exited, because the parent would block in waitpid() waiting for
6277 # the tracker to exit, but the tracker would never see EOF.
6278 cmd = '''if 1:
6279 import os, signal
6280 from multiprocessing.resource_tracker import ensure_running
6281 ensure_running()
6282 if os.fork() == 0:
6283 signal.pause()
6284 os._exit(0)
6285 # parent falls through and exits, triggering __del__
6286 '''
6287 proc = subprocess.Popen([sys.executable, '-c', cmd],
6288 start_new_session=True)
6289 try:
6290 try:
6291 proc.wait(timeout=support.SHORT_TIMEOUT)
6292 except subprocess.TimeoutExpired:
6293 self.fail(
6294 "Parent process deadlocked in ResourceTracker.__del__"
6295 )
6296 self.assertEqual(proc.returncode, 0)
6297 finally:
6298 try:
6299 os.killpg(proc.pid, signal.SIGKILL)
6300 except ProcessLookupError:
6301 pass
6302 proc.wait()
6303
6304 @only_run_in_forkserver_testsuite("avoids redundant testing.")
6305 def test_resource_tracker_mp_fork_reuse_and_prompt_reap(self):
6306 # gh-146313 / gh-80849: A child started via multiprocessing.Process
6307 # with the 'fork' start method should reuse the parent's resource
6308 # tracker (the at-fork handler preserves the inherited pipe fd),
6309 # *and* the parent should be able to reap the tracker promptly
6310 # after joining the child, without hitting the waitpid timeout.
6311 cmd = textwrap.dedent('''
6312 import multiprocessing as mp
6313 from multiprocessing.resource_tracker import _resource_tracker
6314
6315 def child(conn):
6316 # Prove we can talk to the parent's tracker by registering
6317 # and unregistering a dummy resource over the inherited fd.
6318 # If the fd were closed, ensure_running would launch a new
6319 # tracker and _pid would be non-None.
6320 _resource_tracker.register("x", "dummy")
6321 _resource_tracker.unregister("x", "dummy")
6322 conn.send((_resource_tracker._fd is not None,
6323 _resource_tracker._pid is None,
6324 _resource_tracker._check_alive()))
6325
6326 if __name__ == "__main__":
6327 mp.set_start_method("fork")
6328 _resource_tracker.ensure_running()
6329 r, w = mp.Pipe(duplex=False)
6330 p = mp.Process(target=child, args=(w,))
6331 p.start()
6332 child_has_fd, child_pid_none, child_alive = r.recv()
6333 p.join()
6334 w.close(); r.close()
6335
6336 # Now simulate __del__: the child has exited and released
6337 # its fd copy, so the tracker should see EOF and exit
6338 # promptly -- no timeout.
6339 _resource_tracker._stop(wait_timeout=5.0)
6340 print(child_has_fd, child_pid_none, child_alive,
6341 _resource_tracker._waitpid_timed_out,
6342 _resource_tracker._exitcode)
6343 ''')
6344 rc, out, err = script_helper.assert_python_ok('-c', cmd)
6345 parts = out.decode().split()
6346 self.assertEqual(parts, ['True', 'True', 'True', 'False', '0'],
6347 f"unexpected: {parts!r} stderr={err!r}")
6348
6349 @only_run_in_forkserver_testsuite("avoids redundant testing.")
6350 def test_resource_tracker_raw_fork_prompt_reap(self):
6351 # gh-146313: After a raw os.fork() the at-fork handler closes the
6352 # child's inherited fd, so the parent can reap the tracker
6353 # immediately -- even while the child is still alive -- rather
6354 # than waiting out the 1s timeout.
6355 cmd = textwrap.dedent('''
6356 import os, signal
6357 from multiprocessing.resource_tracker import _resource_tracker
6358
6359 _resource_tracker.ensure_running()
6360 r, w = os.pipe()
6361 pid = os.fork()
6362 if pid == 0:
6363 os.close(r)
6364 # Report whether our fd was closed by the at-fork handler.
6365 os.write(w, b"1" if _resource_tracker._fd is None else b"0")
6366 os.close(w)
6367 signal.pause() # stay alive so parent's reap is meaningful
6368 os._exit(0)
6369 os.close(w)
6370 child_fd_closed = os.read(r, 1) == b"1"
6371 os.close(r)
6372
6373 # Child is still alive and paused. Because it closed its fd
6374 # copy, our close below is the last one and the tracker exits.
6375 _resource_tracker._stop(wait_timeout=5.0)
6376
6377 os.kill(pid, signal.SIGKILL)
6378 os.waitpid(pid, 0)
6379 print(child_fd_closed,
6380 _resource_tracker._waitpid_timed_out,
6381 _resource_tracker._exitcode)
6382 ''')
6383 rc, out, err = script_helper.assert_python_ok('-c', cmd)
6384 parts = out.decode().split()
6385 self.assertEqual(parts, ['True', 'False', '0'],
6386 f"unexpected: {parts!r} stderr={err!r}")
6387
6388 @only_run_in_forkserver_testsuite("avoids redundant testing.")
6389 def test_resource_tracker_lock_reinit_after_fork(self):
6390 # gh-146313: If a parent thread held the tracker's lock at fork
6391 # time, the child would inherit the held lock and deadlock on
6392 # its next ensure_running(). The at-fork handler reinits it.
6393 cmd = textwrap.dedent('''
6394 import os, threading
6395 from multiprocessing.resource_tracker import _resource_tracker
6396
6397 held = threading.Event()
6398 release = threading.Event()
6399 def hold():
6400 with _resource_tracker._lock:
6401 held.set()
6402 release.wait()
6403 t = threading.Thread(target=hold)
6404 t.start()
6405 held.wait()
6406
6407 pid = os.fork()
6408 if pid == 0:
6409 ok = _resource_tracker._lock.acquire(timeout=5.0)
6410 os._exit(0 if ok else 1)
6411
6412 release.set()
6413 t.join()
6414 _, status = os.waitpid(pid, 0)
6415 print(os.waitstatus_to_exitcode(status))
6416 ''')
6417 rc, out, err = script_helper.assert_python_ok(
6418 '-W', 'ignore::DeprecationWarning', '-c', cmd)
6419 self.assertEqual(out.strip(), b'0',
6420 f"child failed to acquire lock: stderr={err!r}")
6421
6422 @only_run_in_forkserver_testsuite("avoids redundant testing.")
6423 def test_resource_tracker_safety_net_timeout(self):
6424 # gh-146313: When an mp.Process(fork) child holds the preserved
6425 # fd and the parent calls _stop() without joining (simulating
6426 # abnormal shutdown), the safety-net timeout should fire rather
6427 # than deadlocking.
6428 cmd = textwrap.dedent('''
6429 import multiprocessing as mp
6430 import signal
6431 from multiprocessing.resource_tracker import _resource_tracker
6432
6433 if __name__ == "__main__":
6434 mp.set_start_method("fork")
6435 _resource_tracker.ensure_running()
6436 p = mp.Process(target=signal.pause)
6437 p.start()
6438 # Stop WITHOUT joining -- child still holds preserved fd
6439 _resource_tracker._stop(wait_timeout=0.5)
6440 print(_resource_tracker._waitpid_timed_out)
6441 p.terminate()
6442 p.join()
6443 ''')
6444 rc, out, err = script_helper.assert_python_ok('-c', cmd)
6445 self.assertEqual(out.strip(), b'True',
6446 f"safety-net timeout did not fire: stderr={err!r}")
6447
6448
6449 class TestSimpleQueue(unittest.TestCase):
6450
6451 @classmethod
6452 def _test_empty(cls, queue, child_can_start, parent_can_continue):
6453 child_can_start.wait()
6454 # issue 30301, could fail under spawn and forkserver
6455 try:
6456 queue.put(queue.empty())
6457 queue.put(queue.empty())
6458 finally:
6459 parent_can_continue.set()
6460
6461 def test_empty_exceptions(self):
6462 # Assert that checking emptiness of a closed queue raises
6463 # an OSError, independently of whether the queue was used
6464 # or not. This differs from Queue and JoinableQueue.
6465 q = multiprocessing.SimpleQueue()
6466 q.close() # close the pipe
6467 with self.assertRaisesRegex(OSError, 'is closed'):
6468 q.empty()
6469
6470 def test_empty(self):
6471 queue = multiprocessing.SimpleQueue()
6472 child_can_start = multiprocessing.Event()
6473 parent_can_continue = multiprocessing.Event()
6474
6475 proc = multiprocessing.Process(
6476 target=self._test_empty,
6477 args=(queue, child_can_start, parent_can_continue)
6478 )
6479 proc.daemon = True
6480 proc.start()
6481
6482 self.assertTrue(queue.empty())
6483
6484 child_can_start.set()
6485 parent_can_continue.wait()
6486
6487 self.assertFalse(queue.empty())
6488 self.assertEqual(queue.get(), True)
6489 self.assertEqual(queue.get(), False)
6490 self.assertTrue(queue.empty())
6491
6492 proc.join()
6493
6494 def test_close(self):
6495 queue = multiprocessing.SimpleQueue()
6496 queue.close()
6497 # closing a queue twice should not fail
6498 queue.close()
6499
6500 # Test specific to CPython since it tests private attributes
6501 @test.support.cpython_only
6502 def test_closed(self):
6503 queue = multiprocessing.SimpleQueue()
6504 queue.close()
6505 self.assertTrue(queue._reader.closed)
6506 self.assertTrue(queue._writer.closed)
6507
6508
6509 class TestPoolNotLeakOnFailure(unittest.TestCase):
6510
6511 def test_release_unused_processes(self):
6512 # Issue #19675: During pool creation, if we can't create a process,
6513 # don't leak already created ones.
6514 will_fail_in = 3
6515 forked_processes = []
6516
6517 class FailingForkProcess:
6518 def __init__(self, **kwargs):
6519 self.name = 'Fake Process'
6520 self.exitcode = None
6521 self.state = None
6522 forked_processes.append(self)
6523
6524 def start(self):
6525 nonlocal will_fail_in
6526 if will_fail_in <= 0:
6527 raise OSError("Manually induced OSError")
6528 will_fail_in -= 1
6529 self.state = 'started'
6530
6531 def terminate(self):
6532 self.state = 'stopping'
6533
6534 def join(self):
6535 if self.state == 'stopping':
6536 self.state = 'stopped'
6537
6538 def is_alive(self):
6539 return self.state == 'started' or self.state == 'stopping'
6540
6541 with self.assertRaisesRegex(OSError, 'Manually induced OSError'):
6542 p = multiprocessing.pool.Pool(5, context=unittest.mock.MagicMock(
6543 Process=FailingForkProcess))
6544 p.close()
6545 p.join()
6546 for process in forked_processes:
6547 self.assertFalse(process.is_alive(), process)
6548
6549
6550 @hashlib_helper.requires_hashdigest('sha256')
6551 class TestSyncManagerTypes(unittest.TestCase):
6552 """Test all the types which can be shared between a parent and a
6553 child process by using a manager which acts as an intermediary
6554 between them.
6555
6556 In the following unit-tests the base type is created in the parent
6557 process, the @classmethod represents the worker process and the
6558 shared object is readable and editable between the two.
6559
6560 # The child.
6561 @classmethod
6562 def _test_list(cls, obj):
6563 assert obj[0] == 5
6564 assert obj.append(6)
6565
6566 # The parent.
6567 def test_list(self):
6568 o = self.manager.list()
6569 o.append(5)
6570 self.run_worker(self._test_list, o)
6571 assert o[1] == 6
6572 """
6573 manager_class = multiprocessing.managers.SyncManager
6574
6575 def setUp(self):
6576 self.manager = self.manager_class()
6577 self.manager.start()
6578 self.proc = None
6579
6580 def tearDown(self):
6581 if self.proc is not None and self.proc.is_alive():
6582 self.proc.terminate()
6583 self.proc.join()
6584 self.manager.shutdown()
6585 self.manager = None
6586 self.proc = None
6587
6588 @classmethod
6589 def setUpClass(cls):
6590 support.reap_children()
6591
6592 tearDownClass = setUpClass
6593
6594 def wait_proc_exit(self):
6595 # Only the manager process should be returned by active_children()
6596 # but this can take a bit on slow machines, so wait a few seconds
6597 # if there are other children too (see #17395).
6598 join_process(self.proc)
6599
6600 timeout = WAIT_ACTIVE_CHILDREN_TIMEOUT
6601 start_time = time.monotonic()
6602 for _ in support.sleeping_retry(timeout, error=False):
6603 if len(multiprocessing.active_children()) <= 1:
6604 break
6605 else:
6606 dt = time.monotonic() - start_time
6607 support.environment_altered = True
6608 support.print_warning(f"multiprocessing.Manager still has "
6609 f"{multiprocessing.active_children()} "
6610 f"active children after {dt:.1f} seconds")
6611
6612 def run_worker(self, worker, obj):
6613 self.proc = multiprocessing.Process(target=worker, args=(obj, ))
6614 self.proc.daemon = True
6615 self.proc.start()
6616 self.wait_proc_exit()
6617 self.assertEqual(self.proc.exitcode, 0)
6618
6619 @classmethod
6620 def _test_event(cls, obj):
6621 assert obj.is_set()
6622 obj.wait()
6623 obj.clear()
6624 obj.wait(0.001)
6625
6626 def test_event(self):
6627 o = self.manager.Event()
6628 o.set()
6629 self.run_worker(self._test_event, o)
6630 assert not o.is_set()
6631 o.wait(0.001)
6632
6633 @classmethod
6634 def _test_lock(cls, obj):
6635 obj.acquire()
6636 obj.locked()
6637
6638 def test_lock(self, lname="Lock"):
6639 o = getattr(self.manager, lname)()
6640 self.run_worker(self._test_lock, o)
6641 o.release()
6642 self.assertRaises(RuntimeError, o.release) # already released
6643
6644 @classmethod
6645 def _test_rlock(cls, obj):
6646 obj.acquire()
6647 obj.release()
6648 obj.locked()
6649
6650 def test_rlock(self, lname="RLock"):
6651 o = getattr(self.manager, lname)()
6652 self.run_worker(self._test_rlock, o)
6653
6654 @classmethod
6655 def _test_semaphore(cls, obj):
6656 obj.acquire()
6657
6658 def test_semaphore(self, sname="Semaphore"):
6659 o = getattr(self.manager, sname)()
6660 self.run_worker(self._test_semaphore, o)
6661 o.release()
6662
6663 def test_bounded_semaphore(self):
6664 self.test_semaphore(sname="BoundedSemaphore")
6665
6666 @classmethod
6667 def _test_condition(cls, obj):
6668 obj.acquire()
6669 obj.release()
6670
6671 def test_condition(self):
6672 o = self.manager.Condition()
6673 self.run_worker(self._test_condition, o)
6674
6675 @classmethod
6676 def _test_barrier(cls, obj):
6677 assert obj.parties == 5
6678 obj.reset()
6679
6680 def test_barrier(self):
6681 o = self.manager.Barrier(5)
6682 self.run_worker(self._test_barrier, o)
6683
6684 @classmethod
6685 def _test_pool(cls, obj):
6686 # TODO: fix https://bugs.python.org/issue35919
6687 with obj:
6688 pass
6689
6690 def test_pool(self):
6691 o = self.manager.Pool(processes=4)
6692 self.run_worker(self._test_pool, o)
6693
6694 @classmethod
6695 def _test_queue(cls, obj):
6696 assert obj.qsize() == 2
6697 assert obj.full()
6698 assert not obj.empty()
6699 assert obj.get() == 5
6700 assert not obj.empty()
6701 assert obj.get() == 6
6702 assert obj.empty()
6703
6704 def test_queue(self, qname="Queue"):
6705 o = getattr(self.manager, qname)(2)
6706 o.put(5)
6707 o.put(6)
6708 self.run_worker(self._test_queue, o)
6709 assert o.empty()
6710 assert not o.full()
6711
6712 def test_joinable_queue(self):
6713 self.test_queue("JoinableQueue")
6714
6715 @classmethod
6716 def _test_list(cls, obj):
6717 case = unittest.TestCase()
6718 case.assertEqual(obj[0], 5)
6719 case.assertEqual(obj.count(5), 1)
6720 case.assertEqual(obj.index(5), 0)
6721 obj += [7]
6722 case.assertIsInstance(obj, multiprocessing.managers.ListProxy)
6723 case.assertListEqual(list(obj), [5, 7])
6724 obj *= 2
6725 case.assertIsInstance(obj, multiprocessing.managers.ListProxy)
6726 case.assertListEqual(list(obj), [5, 7, 5, 7])
6727 double_obj = obj * 2
6728 case.assertIsInstance(double_obj, list)
6729 case.assertListEqual(list(double_obj), [5, 7, 5, 7, 5, 7, 5, 7])
6730 double_obj = 2 * obj
6731 case.assertIsInstance(double_obj, list)
6732 case.assertListEqual(list(double_obj), [5, 7, 5, 7, 5, 7, 5, 7])
6733 copied_obj = obj.copy()
6734 case.assertIsInstance(copied_obj, list)
6735 case.assertListEqual(list(copied_obj), [5, 7, 5, 7])
6736 obj.extend(double_obj + copied_obj)
6737 obj.sort()
6738 obj.reverse()
6739 for x in obj:
6740 pass
6741 case.assertEqual(len(obj), 16)
6742 case.assertEqual(obj.pop(0), 7)
6743 obj.clear()
6744 case.assertEqual(len(obj), 0)
6745
6746 def test_list(self):
6747 o = self.manager.list()
6748 o.append(5)
6749 self.run_worker(self._test_list, o)
6750 self.assertIsNotNone(o)
6751 self.assertEqual(len(o), 0)
6752
6753 @classmethod
6754 def _test_dict(cls, obj):
6755 case = unittest.TestCase()
6756 case.assertEqual(len(obj), 1)
6757 case.assertEqual(obj['foo'], 5)
6758 case.assertEqual(obj.get('foo'), 5)
6759 case.assertListEqual(list(obj.items()), [('foo', 5)])
6760 case.assertListEqual(list(obj.keys()), ['foo'])
6761 case.assertListEqual(list(obj.values()), [5])
6762 case.assertDictEqual(obj.copy(), {'foo': 5})
6763 obj |= {'bar': 6}
6764 case.assertIsInstance(obj, multiprocessing.managers.DictProxy)
6765 case.assertDictEqual(dict(obj), {'foo': 5, 'bar': 6})
6766 x = reversed(obj)
6767 case.assertIsInstance(x, type(iter([])))
6768 case.assertListEqual(list(x), ['bar', 'foo'])
6769 x = {'bar': 7, 'baz': 7} | obj
6770 case.assertIsInstance(x, dict)
6771 case.assertDictEqual(dict(x), {'foo': 5, 'bar': 6, 'baz': 7})
6772 x = obj | {'bar': 7, 'baz': 7}
6773 case.assertIsInstance(x, dict)
6774 case.assertDictEqual(dict(x), {'foo': 5, 'bar': 7, 'baz': 7})
6775 x = obj.fromkeys(['bar'], 6)
6776 case.assertIsInstance(x, dict)
6777 case.assertDictEqual(x, {'bar': 6})
6778 x = obj.popitem()
6779 case.assertIsInstance(x, tuple)
6780 case.assertTupleEqual(x, ('bar', 6))
6781 obj.setdefault('bar', 0)
6782 obj.update({'bar': 7})
6783 case.assertEqual(obj.pop('bar'), 7)
6784 obj.clear()
6785 case.assertEqual(len(obj), 0)
6786
6787 def test_dict(self):
6788 o = self.manager.dict()
6789 o['foo'] = 5
6790 self.run_worker(self._test_dict, o)
6791 self.assertIsNotNone(o)
6792 self.assertEqual(len(o), 0)
6793
6794 @classmethod
6795 def _test_value(cls, obj):
6796 case = unittest.TestCase()
6797 case.assertEqual(obj.value, 1)
6798 case.assertEqual(obj.get(), 1)
6799 obj.set(2)
6800
6801 def test_value(self):
6802 o = self.manager.Value('i', 1)
6803 self.run_worker(self._test_value, o)
6804 self.assertEqual(o.value, 2)
6805 self.assertEqual(o.get(), 2)
6806
6807 @classmethod
6808 def _test_array(cls, obj):
6809 case = unittest.TestCase()
6810 case.assertEqual(obj[0], 0)
6811 case.assertEqual(obj[1], 1)
6812 case.assertEqual(len(obj), 2)
6813 case.assertListEqual(list(obj), [0, 1])
6814
6815 def test_array(self):
6816 o = self.manager.Array('i', [0, 1])
6817 self.run_worker(self._test_array, o)
6818
6819 @classmethod
6820 def _test_namespace(cls, obj):
6821 case = unittest.TestCase()
6822 case.assertEqual(obj.x, 0)
6823 case.assertEqual(obj.y, 1)
6824
6825 def test_namespace(self):
6826 o = self.manager.Namespace()
6827 o.x = 0
6828 o.y = 1
6829 self.run_worker(self._test_namespace, o)
6830
6831 @classmethod
6832 def _test_set_operator_symbols(cls, obj):
6833 case = unittest.TestCase()
6834 obj.update(['a', 'b', 'c'])
6835 case.assertEqual(len(obj), 3)
6836 case.assertIn('a', obj)
6837 case.assertNotIn('d', obj)
6838 result = obj | {'d', 'e'}
6839 case.assertSetEqual(result, {'a', 'b', 'c', 'd', 'e'})
6840 result = {'d', 'e'} | obj
6841 case.assertSetEqual(result, {'a', 'b', 'c', 'd', 'e'})
6842 obj |= {'d', 'e'}
6843 case.assertSetEqual(obj, {'a', 'b', 'c', 'd', 'e'})
6844 case.assertIsInstance(obj, multiprocessing.managers.SetProxy)
6845
6846 obj.clear()
6847 obj.update(['a', 'b', 'c'])
6848 result = {'a', 'b', 'd'} - obj
6849 case.assertSetEqual(result, {'d'})
6850 result = obj - {'a', 'b'}
6851 case.assertSetEqual(result, {'c'})
6852 obj -= {'a', 'b'}
6853 case.assertSetEqual(obj, {'c'})
6854 case.assertIsInstance(obj, multiprocessing.managers.SetProxy)
6855
6856 obj.clear()
6857 obj.update(['a', 'b', 'c'])
6858 result = {'b', 'c', 'd'} ^ obj
6859 case.assertSetEqual(result, {'a', 'd'})
6860 result = obj ^ {'b', 'c', 'd'}
6861 case.assertSetEqual(result, {'a', 'd'})
6862 obj ^= {'b', 'c', 'd'}
6863 case.assertSetEqual(obj, {'a', 'd'})
6864 case.assertIsInstance(obj, multiprocessing.managers.SetProxy)
6865
6866 obj.clear()
6867 obj.update(['a', 'b', 'c'])
6868 result = obj & {'b', 'c', 'd'}
6869 case.assertSetEqual(result, {'b', 'c'})
6870 result = {'b', 'c', 'd'} & obj
6871 case.assertSetEqual(result, {'b', 'c'})
6872 obj &= {'b', 'c', 'd'}
6873 case.assertSetEqual(obj, {'b', 'c'})
6874 case.assertIsInstance(obj, multiprocessing.managers.SetProxy)
6875
6876 obj.clear()
6877 obj.update(['a', 'b', 'c'])
6878 case.assertSetEqual(set(obj), {'a', 'b', 'c'})
6879
6880 @classmethod
6881 def _test_set_operator_methods(cls, obj):
6882 case = unittest.TestCase()
6883 obj.add('d')
6884 case.assertIn('d', obj)
6885
6886 obj.clear()
6887 obj.update(['a', 'b', 'c'])
6888 copy_obj = obj.copy()
6889 case.assertSetEqual(copy_obj, obj)
6890 obj.remove('a')
6891 case.assertNotIn('a', obj)
6892 case.assertRaises(KeyError, obj.remove, 'a')
6893
6894 obj.clear()
6895 obj.update(['a'])
6896 obj.discard('a')
6897 case.assertNotIn('a', obj)
6898 obj.discard('a')
6899 case.assertNotIn('a', obj)
6900 obj.update(['a'])
6901 popped = obj.pop()
6902 case.assertNotIn(popped, obj)
6903
6904 obj.clear()
6905 obj.update(['a', 'b', 'c'])
6906 result = obj.intersection({'b', 'c', 'd'})
6907 case.assertSetEqual(result, {'b', 'c'})
6908 obj.intersection_update({'b', 'c', 'd'})
6909 case.assertSetEqual(obj, {'b', 'c'})
6910
6911 obj.clear()
6912 obj.update(['a', 'b', 'c'])
6913 result = obj.difference({'a', 'b'})
6914 case.assertSetEqual(result, {'c'})
6915 obj.difference_update({'a', 'b'})
6916 case.assertSetEqual(obj, {'c'})
6917
6918 obj.clear()
6919 obj.update(['a', 'b', 'c'])
6920 result = obj.symmetric_difference({'b', 'c', 'd'})
6921 case.assertSetEqual(result, {'a', 'd'})
6922 obj.symmetric_difference_update({'b', 'c', 'd'})
6923 case.assertSetEqual(obj, {'a', 'd'})
6924
6925 @classmethod
6926 def _test_set_comparisons(cls, obj):
6927 case = unittest.TestCase()
6928 obj.update(['a', 'b', 'c'])
6929 result = obj.union({'d', 'e'})
6930 case.assertSetEqual(result, {'a', 'b', 'c', 'd', 'e'})
6931 case.assertTrue(obj.isdisjoint({'d', 'e'}))
6932 case.assertFalse(obj.isdisjoint({'a', 'd'}))
6933
6934 case.assertTrue(obj.issubset({'a', 'b', 'c', 'd'}))
6935 case.assertFalse(obj.issubset({'a', 'b'}))
6936 case.assertLess(obj, {'a', 'b', 'c', 'd'})
6937 case.assertLessEqual(obj, {'a', 'b', 'c'})
6938
6939 case.assertTrue(obj.issuperset({'a', 'b'}))
6940 case.assertFalse(obj.issuperset({'a', 'b', 'd'}))
6941 case.assertGreater(obj, {'a'})
6942 case.assertGreaterEqual(obj, {'a', 'b'})
6943
6944 def test_set(self):
6945 o = self.manager.set()
6946 self.run_worker(self._test_set_operator_symbols, o)
6947 o = self.manager.set()
6948 self.run_worker(self._test_set_operator_methods, o)
6949 o = self.manager.set()
6950 self.run_worker(self._test_set_comparisons, o)
6951
6952 def test_set_init(self):
6953 o = self.manager.set({'a', 'b', 'c'})
6954 self.assertSetEqual(o, {'a', 'b', 'c'})
6955 o = self.manager.set(["a", "b", "c"])
6956 self.assertSetEqual(o, {"a", "b", "c"})
6957 o = self.manager.set({"a": 1, "b": 2, "c": 3})
6958 self.assertSetEqual(o, {"a", "b", "c"})
6959 self.assertRaises(RemoteError, self.manager.set, 1234)
6960
6961 def test_set_contain_all_method(self):
6962 o = self.manager.set()
6963 set_methods = {
6964 '__and__', '__class_getitem__', '__contains__', '__iand__', '__ior__',
6965 '__isub__', '__iter__', '__ixor__', '__len__', '__or__', '__rand__',
6966 '__ror__', '__rsub__', '__rxor__', '__sub__', '__xor__',
6967 '__ge__', '__gt__', '__le__', '__lt__',
6968 'add', 'clear', 'copy', 'difference', 'difference_update', 'discard',
6969 'intersection', 'intersection_update', 'isdisjoint', 'issubset',
6970 'issuperset', 'pop', 'remove', 'symmetric_difference',
6971 'symmetric_difference_update', 'union', 'update',
6972 }
6973 self.assertLessEqual(set_methods, set(dir(o)))
6974
6975
6976 class TestNamedResource(unittest.TestCase):
6977 @only_run_in_spawn_testsuite("spawn specific test.")
6978 def test_global_named_resource_spawn(self):
6979 #
6980 # gh-90549: Check that global named resources in main module
6981 # will not leak by a subprocess, in spawn context.
6982 #
6983 testfn = os_helper.TESTFN
6984 self.addCleanup(os_helper.unlink, testfn)
6985 with open(testfn, 'w', encoding='utf-8') as f:
6986 f.write(textwrap.dedent('''\
6987 import multiprocessing as mp
6988 ctx = mp.get_context('spawn')
6989 global_resource = ctx.Semaphore()
6990 def submain(): pass
6991 if __name__ == '__main__':
6992 p = ctx.Process(target=submain)
6993 p.start()
6994 p.join()
6995 '''))
6996 rc, out, err = script_helper.assert_python_ok(testfn)
6997 # on error, err = 'UserWarning: resource_tracker: There appear to
6998 # be 1 leaked semaphore objects to clean up at shutdown'
6999 self.assertFalse(err, msg=err.decode('utf-8'))
7000
7001
7002 class _TestAtExit(BaseTestCase):
7003
7004 ALLOWED_TYPES = ('processes',)
7005
7006 @classmethod
7007 def _write_file_at_exit(self, output_path):
7008 import atexit
7009 def exit_handler():
7010 with open(output_path, 'w') as f:
7011 f.write("deadbeef")
7012 atexit.register(exit_handler)
7013
7014 def test_atexit(self):
7015 # gh-83856
7016 with os_helper.temp_dir() as temp_dir:
7017 output_path = os.path.join(temp_dir, 'output.txt')
7018 p = self.Process(target=self._write_file_at_exit, args=(output_path,))
7019 p.start()
7020 p.join()
7021 with open(output_path) as f:
7022 self.assertEqual(f.read(), 'deadbeef')
7023
7024
7025 class _TestSpawnedSysPath(BaseTestCase):
7026 """Test that sys.path is setup in forkserver and spawn processes."""
7027
7028 ALLOWED_TYPES = {'processes'}
7029 # Not applicable to fork which inherits everything from the process as is.
7030 START_METHODS = {"forkserver", "spawn"}
7031
7032 def setUp(self):
7033 self._orig_sys_path = list(sys.path)
7034 self._temp_dir = tempfile.mkdtemp(prefix="test_sys_path-")
7035 self._mod_name = "unique_test_mod"
7036 module_path = os.path.join(self._temp_dir, f"{self._mod_name}.py")
7037 with open(module_path, "w", encoding="utf-8") as mod:
7038 mod.write("# A simple test module\n")
7039 sys.path[:] = [p for p in sys.path if p] # remove any existing ""s
7040 sys.path.insert(0, self._temp_dir)
7041 sys.path.insert(0, "") # Replaced with an abspath in child.
7042 self.assertIn(self.start_method, self.START_METHODS)
7043 self._ctx = multiprocessing.get_context(self.start_method)
7044
7045 def tearDown(self):
7046 sys.path[:] = self._orig_sys_path
7047 shutil.rmtree(self._temp_dir, ignore_errors=True)
7048
7049 @staticmethod
7050 def enq_imported_module_names(queue):
7051 queue.put(tuple(sys.modules))
7052
7053 def test_forkserver_preload_imports_sys_path(self):
7054 if self._ctx.get_start_method() != "forkserver":
7055 self.skipTest("forkserver specific test.")
7056 self.assertNotIn(self._mod_name, sys.modules)
7057 multiprocessing.forkserver._forkserver._stop() # Must be fresh.
7058 self._ctx.set_forkserver_preload(
7059 ["test.test_multiprocessing_forkserver", self._mod_name])
7060 q = self._ctx.Queue()
7061 proc = self._ctx.Process(
7062 target=self.enq_imported_module_names, args=(q,))
7063 proc.start()
7064 proc.join()
7065 child_imported_modules = q.get()
7066 q.close()
7067 self.assertIn(self._mod_name, child_imported_modules)
7068
7069 @staticmethod
7070 def enq_sys_path_and_import(queue, mod_name):
7071 queue.put(sys.path)
7072 try:
7073 importlib.import_module(mod_name)
7074 except ImportError as exc:
7075 queue.put(exc)
7076 else:
7077 queue.put(None)
7078
7079 def test_child_sys_path(self):
7080 q = self._ctx.Queue()
7081 proc = self._ctx.Process(
7082 target=self.enq_sys_path_and_import, args=(q, self._mod_name))
7083 proc.start()
7084 proc.join()
7085 child_sys_path = q.get()
7086 import_error = q.get()
7087 q.close()
7088 self.assertNotIn("", child_sys_path) # replaced by an abspath
7089 self.assertIn(self._temp_dir, child_sys_path) # our addition
7090 # ignore the first element, it is the absolute "" replacement
7091 self.assertEqual(child_sys_path[1:], sys.path[1:])
7092 self.assertIsNone(import_error, msg=f"child could not import {self._mod_name}")
7093
7094 def test_std_streams_flushed_after_preload(self):
7095 # gh-135335: Check fork server flushes standard streams after
7096 # preloading modules
7097 if multiprocessing.get_start_method() != "forkserver":
7098 self.skipTest("forkserver specific test")
7099
7100 name = os.path.join(os.path.dirname(__file__), 'mp_preload_flush.py')
7101 _, out, err = test.support.script_helper.assert_python_ok(name)
7102
7103 # Check stderr first, as it is more likely to be useful to see in the
7104 # event of a failure.
7105 self.assertEqual(err.decode().rstrip(), '__main____mp_main__')
7106 self.assertEqual(out.decode().rstrip(), '__main____mp_main__')
7107
7108
7109 class MiscTestCase(unittest.TestCase):
7110 def test__all__(self):
7111 # Just make sure names in not_exported are excluded
7112 support.check__all__(self, multiprocessing, extra=multiprocessing.__all__,
7113 not_exported=['SUBDEBUG', 'SUBWARNING'])
7114
7115 @only_run_in_spawn_testsuite("avoids redundant testing.")
7116 def test_spawn_sys_executable_none_allows_import(self):
7117 # Regression test for a bug introduced in
7118 # https://github.com/python/cpython/issues/90876 that caused an
7119 # ImportError in multiprocessing when sys.executable was None.
7120 # This can be true in embedded environments.
7121 rc, out, err = script_helper.assert_python_ok(
7122 "-c",
7123 """if 1:
7124 import sys
7125 sys.executable = None
7126 assert "multiprocessing" not in sys.modules, "already imported!"
7127 import multiprocessing
7128 import multiprocessing.spawn # This should not fail\n""",
7129 )
7130 self.assertEqual(rc, 0)
7131 self.assertFalse(err, msg=err.decode('utf-8'))
7132
7133 def test_large_pool(self):
7134 #
7135 # gh-89240: Check that large pools are always okay
7136 #
7137 testfn = os_helper.TESTFN
7138 self.addCleanup(os_helper.unlink, testfn)
7139 with open(testfn, 'w', encoding='utf-8') as f:
7140 f.write(textwrap.dedent('''\
7141 import multiprocessing
7142 def f(x): return x*x
7143 if __name__ == '__main__':
7144 with multiprocessing.Pool(200) as p:
7145 print(sum(p.map(f, range(1000))))
7146 '''))
7147 rc, out, err = script_helper.assert_python_ok(testfn)
7148 self.assertEqual("332833500", out.decode('utf-8').strip())
7149 self.assertFalse(err, msg=err.decode('utf-8'))
7150
7151 def test_forked_thread_not_started(self):
7152 # gh-134381: Ensure that a thread that has not been started yet in
7153 # the parent process can be started within a forked child process.
7154
7155 if multiprocessing.get_start_method() != "fork":
7156 self.skipTest("fork specific test")
7157
7158 q = multiprocessing.Queue()
7159 t = threading.Thread(target=lambda: q.put("done"), daemon=True)
7160
7161 def child():
7162 t.start()
7163 t.join()
7164
7165 p = multiprocessing.Process(target=child)
7166 p.start()
7167 p.join(support.SHORT_TIMEOUT)
7168
7169 self.assertEqual(p.exitcode, 0)
7170 self.assertEqual(q.get_nowait(), "done")
7171 close_queue(q)
7172
7173 def test_preload_main(self):
7174 # gh-126631: Check that __main__ can be pre-loaded
7175 if multiprocessing.get_start_method() != "forkserver":
7176 self.skipTest("forkserver specific test")
7177
7178 name = os.path.join(os.path.dirname(__file__), 'mp_preload_main.py')
7179 _, out, err = test.support.script_helper.assert_python_ok(name)
7180 self.assertEqual(err, b'')
7181
7182 # The trailing empty string comes from split() on output ending with \n
7183 out = out.decode().split("\n")
7184 self.assertEqual(out, ['__main__', '__mp_main__', 'f', 'f', ''])
7185
7186 def test_preload_main_sys_argv(self):
7187 # gh-143706: Check that sys.argv is set before __main__ is pre-loaded
7188 if multiprocessing.get_start_method() != "forkserver":
7189 self.skipTest("forkserver specific test")
7190
7191 name = os.path.join(os.path.dirname(__file__), 'mp_preload_sysargv.py')
7192 _, out, err = test.support.script_helper.assert_python_ok(
7193 name, 'foo', 'bar')
7194 self.assertEqual(err, b'')
7195
7196 out = out.decode().split("\n")
7197 expected_argv = "['foo', 'bar']"
7198 self.assertEqual(out, [
7199 f"module:{expected_argv}",
7200 f"fun:{expected_argv}",
7201 f"module:{expected_argv}",
7202 f"fun:{expected_argv}",
7203 '',
7204 ])
7205
7206 @only_run_in_forkserver_testsuite("forkserver specific test.")
7207 def test_preload_main_large_sys_argv(self):
7208 # gh-144503: a very large parent sys.argv must not prevent the
7209 # forkserver from starting (it previously overflowed the OS
7210 # per-argument length limit when repr'd into the -c command string).
7211 name = os.path.join(os.path.dirname(__file__),
7212 'mp_preload_large_sysargv.py')
7213 _, out, err = test.support.script_helper.assert_python_ok(name)
7214 self.assertEqual(err, b'')
7215
7216 out = out.decode().split("\n")
7217 self.assertEqual(out, [
7218 'preload:5002:sentinel',
7219 'worker:5002:sentinel',
7220 '',
7221 ])
7222
7223 #
7224 # Mixins
7225 #
7226
7227 class BaseMixin(object):
7228 @classmethod
7229 def setUpClass(cls):
7230 cls.dangling = (multiprocessing.process._dangling.copy(),
7231 threading._dangling.copy())
7232
7233 @classmethod
7234 def tearDownClass(cls):
7235 # bpo-26762: Some multiprocessing objects like Pool create reference
7236 # cycles. Trigger a garbage collection to break these cycles.
7237 test.support.gc_collect()
7238
7239 processes = set(multiprocessing.process._dangling) - set(cls.dangling[0])
7240 if processes:
7241 test.support.environment_altered = True
7242 support.print_warning(f'Dangling processes: {processes}')
7243 processes = None
7244
7245 threads = set(threading._dangling) - set(cls.dangling[1])
7246 if threads:
7247 test.support.environment_altered = True
7248 support.print_warning(f'Dangling threads: {threads}')
7249 threads = None
7250
7251
7252 class ProcessesMixin(BaseMixin):
7253 TYPE = 'processes'
7254 Process = multiprocessing.Process
7255 connection = multiprocessing.connection
7256 current_process = staticmethod(multiprocessing.current_process)
7257 parent_process = staticmethod(multiprocessing.parent_process)
7258 active_children = staticmethod(multiprocessing.active_children)
7259 set_executable = staticmethod(multiprocessing.set_executable)
7260 Pool = staticmethod(multiprocessing.Pool)
7261 Pipe = staticmethod(multiprocessing.Pipe)
7262 Queue = staticmethod(multiprocessing.Queue)
7263 JoinableQueue = staticmethod(multiprocessing.JoinableQueue)
7264 Lock = staticmethod(multiprocessing.Lock)
7265 RLock = staticmethod(multiprocessing.RLock)
7266 Semaphore = staticmethod(multiprocessing.Semaphore)
7267 BoundedSemaphore = staticmethod(multiprocessing.BoundedSemaphore)
7268 Condition = staticmethod(multiprocessing.Condition)
7269 Event = staticmethod(multiprocessing.Event)
7270 Barrier = staticmethod(multiprocessing.Barrier)
7271 Value = staticmethod(multiprocessing.Value)
7272 Array = staticmethod(multiprocessing.Array)
7273 RawValue = staticmethod(multiprocessing.RawValue)
7274 RawArray = staticmethod(multiprocessing.RawArray)
7275
7276
7277 class ManagerMixin(BaseMixin):
7278 TYPE = 'manager'
7279 Process = multiprocessing.Process
7280 Queue = property(operator.attrgetter('manager.Queue'))
7281 JoinableQueue = property(operator.attrgetter('manager.JoinableQueue'))
7282 Lock = property(operator.attrgetter('manager.Lock'))
7283 RLock = property(operator.attrgetter('manager.RLock'))
7284 Semaphore = property(operator.attrgetter('manager.Semaphore'))
7285 BoundedSemaphore = property(operator.attrgetter('manager.BoundedSemaphore'))
7286 Condition = property(operator.attrgetter('manager.Condition'))
7287 Event = property(operator.attrgetter('manager.Event'))
7288 Barrier = property(operator.attrgetter('manager.Barrier'))
7289 Value = property(operator.attrgetter('manager.Value'))
7290 Array = property(operator.attrgetter('manager.Array'))
7291 list = property(operator.attrgetter('manager.list'))
7292 dict = property(operator.attrgetter('manager.dict'))
7293 Namespace = property(operator.attrgetter('manager.Namespace'))
7294
7295 @classmethod
7296 def Pool(cls, *args, **kwds):
7297 return cls.manager.Pool(*args, **kwds)
7298
7299 @classmethod
7300 def setUpClass(cls):
7301 super().setUpClass()
7302 cls.manager = multiprocessing.Manager()
7303
7304 @classmethod
7305 def tearDownClass(cls):
7306 # only the manager process should be returned by active_children()
7307 # but this can take a bit on slow machines, so wait a few seconds
7308 # if there are other children too (see #17395)
7309 timeout = WAIT_ACTIVE_CHILDREN_TIMEOUT
7310 start_time = time.monotonic()
7311 for _ in support.sleeping_retry(timeout, error=False):
7312 if len(multiprocessing.active_children()) <= 1:
7313 break
7314 else:
7315 dt = time.monotonic() - start_time
7316 support.environment_altered = True
7317 support.print_warning(f"multiprocessing.Manager still has "
7318 f"{multiprocessing.active_children()} "
7319 f"active children after {dt:.1f} seconds")
7320
7321 gc.collect() # do garbage collection
7322 if cls.manager._number_of_objects() != 0:
7323 # This is not really an error since some tests do not
7324 # ensure that all processes which hold a reference to a
7325 # managed object have been joined.
7326 test.support.environment_altered = True
7327 support.print_warning('Shared objects which still exist '
7328 'at manager shutdown:')
7329 support.print_warning(cls.manager._debug_info())
7330 cls.manager.shutdown()
7331 cls.manager.join()
7332 cls.manager = None
7333
7334 super().tearDownClass()
7335
7336
7337 class ThreadsMixin(BaseMixin):
7338 TYPE = 'threads'
7339 Process = multiprocessing.dummy.Process
7340 connection = multiprocessing.dummy.connection
7341 current_process = staticmethod(multiprocessing.dummy.current_process)
7342 active_children = staticmethod(multiprocessing.dummy.active_children)
7343 Pool = staticmethod(multiprocessing.dummy.Pool)
7344 Pipe = staticmethod(multiprocessing.dummy.Pipe)
7345 Queue = staticmethod(multiprocessing.dummy.Queue)
7346 JoinableQueue = staticmethod(multiprocessing.dummy.JoinableQueue)
7347 Lock = staticmethod(multiprocessing.dummy.Lock)
7348 RLock = staticmethod(multiprocessing.dummy.RLock)
7349 Semaphore = staticmethod(multiprocessing.dummy.Semaphore)
7350 BoundedSemaphore = staticmethod(multiprocessing.dummy.BoundedSemaphore)
7351 Condition = staticmethod(multiprocessing.dummy.Condition)
7352 Event = staticmethod(multiprocessing.dummy.Event)
7353 Barrier = staticmethod(multiprocessing.dummy.Barrier)
7354 Value = staticmethod(multiprocessing.dummy.Value)
7355 Array = staticmethod(multiprocessing.dummy.Array)
7356
7357 #
7358 # Functions used to create test cases from the base ones in this module
7359 #
7360
7361 def install_tests_in_module_dict(remote_globs, start_method,
7362 only_type=None, exclude_types=False):
7363 __module__ = remote_globs['__name__']
7364 local_globs = globals()
7365 ALL_TYPES = {'processes', 'threads', 'manager'}
7366
7367 for name, base in local_globs.items():
7368 if not isinstance(base, type):
7369 continue
7370 if issubclass(base, BaseTestCase):
7371 if base is BaseTestCase:
7372 continue
7373 assert set(base.ALLOWED_TYPES) <= ALL_TYPES, base.ALLOWED_TYPES
7374 if base.START_METHODS and start_method not in base.START_METHODS:
7375 continue # class not intended for this start method.
7376 for type_ in base.ALLOWED_TYPES:
7377 if only_type and type_ != only_type:
7378 continue
7379 if exclude_types:
7380 continue
7381 newname = 'With' + type_.capitalize() + name[1:]
7382 Mixin = local_globs[type_.capitalize() + 'Mixin']
7383 class Temp(base, Mixin, unittest.TestCase):
7384 pass
7385 if type_ == 'manager':
7386 Temp = hashlib_helper.requires_hashdigest('sha256')(Temp)
7387 Temp.__name__ = Temp.__qualname__ = newname
7388 Temp.__module__ = __module__
7389 Temp.start_method = start_method
7390 remote_globs[newname] = Temp
7391 elif issubclass(base, unittest.TestCase):
7392 if only_type:
7393 continue
7394
7395 class Temp(base, object):
7396 pass
7397 Temp.__name__ = Temp.__qualname__ = name
7398 Temp.__module__ = __module__
7399 remote_globs[name] = Temp
7400
7401 dangling = [None, None]
7402 old_start_method = [None]
7403
7404 def setUpModule():
7405 multiprocessing.set_forkserver_preload(PRELOAD)
7406 multiprocessing.process._cleanup()
7407 dangling[0] = multiprocessing.process._dangling.copy()
7408 dangling[1] = threading._dangling.copy()
7409 old_start_method[0] = multiprocessing.get_start_method(allow_none=True)
7410 try:
7411 multiprocessing.set_start_method(start_method, force=True)
7412 except ValueError:
7413 raise unittest.SkipTest(start_method +
7414 ' start method not supported')
7415
7416 if sys.platform.startswith("linux"):
7417 try:
7418 lock = multiprocessing.RLock()
7419 except OSError:
7420 raise unittest.SkipTest("OSError raises on RLock creation, "
7421 "see issue 3111!")
7422 check_enough_semaphores()
7423 util.get_temp_dir() # creates temp directory
7424 multiprocessing.get_logger().setLevel(LOG_LEVEL)
7425
7426 def tearDownModule():
7427 need_sleep = False
7428
7429 # bpo-26762: Some multiprocessing objects like Pool create reference
7430 # cycles. Trigger a garbage collection to break these cycles.
7431 test.support.gc_collect()
7432
7433 multiprocessing.set_start_method(old_start_method[0], force=True)
7434 # pause a bit so we don't get warning about dangling threads/processes
7435 processes = set(multiprocessing.process._dangling) - set(dangling[0])
7436 if processes:
7437 need_sleep = True
7438 test.support.environment_altered = True
7439 support.print_warning(f'Dangling processes: {processes}')
7440 processes = None
7441
7442 threads = set(threading._dangling) - set(dangling[1])
7443 if threads:
7444 need_sleep = True
7445 test.support.environment_altered = True
7446 support.print_warning(f'Dangling threads: {threads}')
7447 threads = None
7448
7449 # Sleep 500 ms to give time to child processes to complete.
7450 if need_sleep:
7451 time.sleep(0.5)
7452
7453 multiprocessing.util._cleanup_tests()
7454
7455 remote_globs['setUpModule'] = setUpModule
7456 remote_globs['tearDownModule'] = tearDownModule
7457
7458
7459 @unittest.skipIf(not hasattr(_multiprocessing, 'SemLock'), 'SemLock not available')
7460 @unittest.skipIf(sys.platform != "linux", "Linux only")
7461 class SemLockTests(unittest.TestCase):
7462
7463 def test_semlock_subclass(self):
7464 class SemLock(_multiprocessing.SemLock):
7465 pass
7466 name = f'test_semlock_subclass-{os.getpid()}'
7467 s = SemLock(1, 0, 10, name, False)
7468 _multiprocessing.sem_unlink(name)
7469
7470
7471 @unittest.skipUnless(HAS_SHMEM, "requires multiprocessing.shared_memory")
7472 class TestSharedMemoryNames(unittest.TestCase):
7473 @subTests('use_simple_format', (True, False))
7474 def test_that_shared_memory_name_with_colons_has_no_resource_tracker_errors(
7475 self, use_simple_format):
7476 # Test script that creates and cleans up shared memory with colon in name
7477 test_script = textwrap.dedent("""
7478 import sys
7479 from multiprocessing import shared_memory
7480 from multiprocessing import resource_tracker
7481 import time
7482
7483 resource_tracker._resource_tracker._use_simple_format = %s
7484
7485 # Test various patterns of colons in names
7486 test_names = [
7487 "a:b",
7488 "a:b:c",
7489 "test:name:with:many:colons",
7490 ":starts:with:colon",
7491 "ends:with:colon:",
7492 "::double::colons::",
7493 "name\\nwithnewline",
7494 "name-with-trailing-newline\\n",
7495 "\\nname-starts-with-newline",
7496 "colons:and\\nnewlines:mix",
7497 "multi\\nline\\nname",
7498 ]
7499
7500 for name in test_names:
7501 try:
7502 shm = shared_memory.SharedMemory(create=True, size=100, name=name)
7503 shm.buf[:5] = b'hello' # Write something to the shared memory
7504 shm.close()
7505 shm.unlink()
7506
7507 except Exception as e:
7508 print(f"Error with name '{name}': {e}", file=sys.stderr)
7509 sys.exit(1)
7510
7511 print("SUCCESS")
7512 """ % use_simple_format)
7513
7514 rc, out, err = script_helper.assert_python_ok("-c", test_script)
7515 self.assertIn(b"SUCCESS", out)
7516 self.assertNotIn(b"traceback", err.lower(), err)
7517 self.assertNotIn(b"resource_tracker.py", err, err)