Debian Codegraph @

Lib/test/test_queue.py

raw
1 # Some simple queue module tests, plus some failure conditions
2 # to ensure the Queue locks remain stable.
3 import itertools
4 import random
5 import threading
6 import time
7 import unittest
8 import weakref
9 from test.support import gc_collect, bigmemtest
10 from test.support import import_helper
11 from test.support import threading_helper
12
13 # queue module depends on threading primitives
14 threading_helper.requires_working_threading(module=True)
15
16 py_queue = import_helper.import_fresh_module('queue', blocked=['_queue'])
17 c_queue = import_helper.import_fresh_module('queue', fresh=['_queue'])
18 need_c_queue = unittest.skipUnless(c_queue, "No _queue module found")
19
20 QUEUE_SIZE = 5
21
22 def qfull(q):
23 return q.maxsize > 0 and q.qsize() == q.maxsize
24
25 # A thread to run a function that unclogs a blocked Queue.
26 class _TriggerThread(threading.Thread):
27 def __init__(self, fn, args):
28 self.fn = fn
29 self.args = args
30 self.startedEvent = threading.Event()
31 threading.Thread.__init__(self)
32
33 def run(self):
34 # The sleep isn't necessary, but is intended to give the blocking
35 # function in the main thread a chance at actually blocking before
36 # we unclog it. But if the sleep is longer than the timeout-based
37 # tests wait in their blocking functions, those tests will fail.
38 # So we give them much longer timeout values compared to the
39 # sleep here (I aimed at 10 seconds for blocking functions --
40 # they should never actually wait that long - they should make
41 # progress as soon as we call self.fn()).
42 time.sleep(0.1)
43 self.startedEvent.set()
44 self.fn(*self.args)
45
46
47 # Execute a function that blocks, and in a separate thread, a function that
48 # triggers the release. Returns the result of the blocking function. Caution:
49 # block_func must guarantee to block until trigger_func is called, and
50 # trigger_func must guarantee to change queue state so that block_func can make
51 # enough progress to return. In particular, a block_func that just raises an
52 # exception regardless of whether trigger_func is called will lead to
53 # timing-dependent sporadic failures, and one of those went rarely seen but
54 # undiagnosed for years. Now block_func must be unexceptional. If block_func
55 # is supposed to raise an exception, call do_exceptional_blocking_test()
56 # instead.
57
58 class BlockingTestMixin:
59
60 def do_blocking_test(self, block_func, block_args, trigger_func, trigger_args):
61 thread = _TriggerThread(trigger_func, trigger_args)
62 thread.start()
63 try:
64 self.result = block_func(*block_args)
65 # If block_func returned before our thread made the call, we failed!
66 if not thread.startedEvent.is_set():
67 self.fail("blocking function %r appeared not to block" %
68 block_func)
69 return self.result
70 finally:
71 threading_helper.join_thread(thread) # make sure the thread terminates
72
73 # Call this instead if block_func is supposed to raise an exception.
74 def do_exceptional_blocking_test(self,block_func, block_args, trigger_func,
75 trigger_args, expected_exception_class):
76 thread = _TriggerThread(trigger_func, trigger_args)
77 thread.start()
78 try:
79 try:
80 block_func(*block_args)
81 except expected_exception_class:
82 raise
83 else:
84 self.fail("expected exception of kind %r" %
85 expected_exception_class)
86 finally:
87 threading_helper.join_thread(thread) # make sure the thread terminates
88 if not thread.startedEvent.is_set():
89 self.fail("trigger thread ended but event never set")
90
91
92 class BaseQueueTestMixin(BlockingTestMixin):
93 def setUp(self):
94 self.cum = 0
95 self.cumlock = threading.Lock()
96
97 def basic_queue_test(self, q):
98 if q.qsize():
99 raise RuntimeError("Call this function with an empty queue")
100 self.assertTrue(q.empty())
101 self.assertFalse(q.full())
102 # I guess we better check things actually queue correctly a little :)
103 q.put(111)
104 q.put(333)
105 q.put(222)
106 target_order = dict(Queue = [111, 333, 222],
107 LifoQueue = [222, 333, 111],
108 PriorityQueue = [111, 222, 333])
109 actual_order = [q.get(), q.get(), q.get()]
110 self.assertEqual(actual_order, target_order[q.__class__.__name__],
111 "Didn't seem to queue the correct data!")
112 for i in range(QUEUE_SIZE-1):
113 q.put(i)
114 self.assertTrue(q.qsize(), "Queue should not be empty")
115 self.assertTrue(not qfull(q), "Queue should not be full")
116 last = 2 * QUEUE_SIZE
117 full = 3 * 2 * QUEUE_SIZE
118 q.put(last)
119 self.assertTrue(qfull(q), "Queue should be full")
120 self.assertFalse(q.empty())
121 self.assertTrue(q.full())
122 try:
123 q.put(full, block=0)
124 self.fail("Didn't appear to block with a full queue")
125 except self.queue.Full:
126 pass
127 try:
128 q.put(full, timeout=0.01)
129 self.fail("Didn't appear to time-out with a full queue")
130 except self.queue.Full:
131 pass
132 # Test a blocking put
133 self.do_blocking_test(q.put, (full,), q.get, ())
134 self.do_blocking_test(q.put, (full, True, 10), q.get, ())
135 # Empty it
136 for i in range(QUEUE_SIZE):
137 q.get()
138 self.assertTrue(not q.qsize(), "Queue should be empty")
139 try:
140 q.get(block=0)
141 self.fail("Didn't appear to block with an empty queue")
142 except self.queue.Empty:
143 pass
144 try:
145 q.get(timeout=0.01)
146 self.fail("Didn't appear to time-out with an empty queue")
147 except self.queue.Empty:
148 pass
149 # Test a blocking get
150 self.do_blocking_test(q.get, (), q.put, ('empty',))
151 self.do_blocking_test(q.get, (True, 10), q.put, ('empty',))
152
153
154 def worker(self, q):
155 while True:
156 x = q.get()
157 if x < 0:
158 q.task_done()
159 return
160 with self.cumlock:
161 self.cum += x
162 q.task_done()
163
164 def queue_join_test(self, q):
165 self.cum = 0
166 threads = []
167 for i in (0,1):
168 thread = threading.Thread(target=self.worker, args=(q,))
169 thread.start()
170 threads.append(thread)
171 for i in range(100):
172 q.put(i)
173 q.join()
174 self.assertEqual(self.cum, sum(range(100)),
175 "q.join() did not block until all tasks were done")
176 for i in (0,1):
177 q.put(-1) # instruct the threads to close
178 q.join() # verify that you can join twice
179 for thread in threads:
180 thread.join()
181
182 def test_queue_task_done(self):
183 # Test to make sure a queue task completed successfully.
184 q = self.type2test()
185 try:
186 q.task_done()
187 except ValueError:
188 pass
189 else:
190 self.fail("Did not detect task count going negative")
191
192 def test_queue_join(self):
193 # Test that a queue join()s successfully, and before anything else
194 # (done twice for insurance).
195 q = self.type2test()
196 self.queue_join_test(q)
197 self.queue_join_test(q)
198 try:
199 q.task_done()
200 except ValueError:
201 pass
202 else:
203 self.fail("Did not detect task count going negative")
204
205 def test_basic(self):
206 # Do it a couple of times on the same queue.
207 # Done twice to make sure works with same instance reused.
208 q = self.type2test(QUEUE_SIZE)
209 self.basic_queue_test(q)
210 self.basic_queue_test(q)
211
212 def test_negative_timeout_raises_exception(self):
213 q = self.type2test(QUEUE_SIZE)
214 with self.assertRaises(ValueError):
215 q.put(1, timeout=-1)
216 with self.assertRaises(ValueError):
217 q.get(1, timeout=-1)
218
219 def test_nowait(self):
220 q = self.type2test(QUEUE_SIZE)
221 for i in range(QUEUE_SIZE):
222 q.put_nowait(1)
223 with self.assertRaises(self.queue.Full):
224 q.put_nowait(1)
225
226 for i in range(QUEUE_SIZE):
227 q.get_nowait()
228 with self.assertRaises(self.queue.Empty):
229 q.get_nowait()
230
231 def test_shrinking_queue(self):
232 # issue 10110
233 q = self.type2test(3)
234 q.put(1)
235 q.put(2)
236 q.put(3)
237 with self.assertRaises(self.queue.Full):
238 q.put_nowait(4)
239 self.assertEqual(q.qsize(), 3)
240 q.maxsize = 2 # shrink the queue
241 with self.assertRaises(self.queue.Full):
242 q.put_nowait(4)
243
244 def test_shutdown_empty(self):
245 q = self.type2test()
246 q.shutdown()
247 with self.assertRaises(self.queue.ShutDown):
248 q.put("data")
249 with self.assertRaises(self.queue.ShutDown):
250 q.get()
251
252 def test_shutdown_nonempty(self):
253 q = self.type2test()
254 q.put("data")
255 q.shutdown()
256 q.get()
257 with self.assertRaises(self.queue.ShutDown):
258 q.get()
259
260 def test_shutdown_immediate(self):
261 q = self.type2test()
262 q.put("data")
263 q.shutdown(immediate=True)
264 with self.assertRaises(self.queue.ShutDown):
265 q.get()
266
267 def test_shutdown_allowed_transitions(self):
268 # allowed transitions would be from alive via shutdown to immediate
269 q = self.type2test()
270 self.assertFalse(q.is_shutdown)
271
272 q.shutdown()
273 self.assertTrue(q.is_shutdown)
274
275 q.shutdown(immediate=True)
276 self.assertTrue(q.is_shutdown)
277
278 q.shutdown(immediate=False)
279
280 def _shutdown_all_methods_in_one_thread(self, immediate):
281 q = self.type2test(2)
282 q.put("L")
283 q.put_nowait("O")
284 q.shutdown(immediate)
285
286 with self.assertRaises(self.queue.ShutDown):
287 q.put("E")
288 with self.assertRaises(self.queue.ShutDown):
289 q.put_nowait("W")
290 if immediate:
291 with self.assertRaises(self.queue.ShutDown):
292 q.get()
293 with self.assertRaises(self.queue.ShutDown):
294 q.get_nowait()
295 with self.assertRaises(ValueError):
296 q.task_done()
297 q.join()
298 else:
299 self.assertIn(q.get(), "LO")
300 q.task_done()
301 self.assertIn(q.get(), "LO")
302 q.task_done()
303 q.join()
304 # on shutdown(immediate=False)
305 # when queue is empty, should raise ShutDown Exception
306 with self.assertRaises(self.queue.ShutDown):
307 q.get() # p.get(True)
308 with self.assertRaises(self.queue.ShutDown):
309 q.get_nowait() # p.get(False)
310 with self.assertRaises(self.queue.ShutDown):
311 q.get(True, 1.0)
312
313 def test_shutdown_all_methods_in_one_thread(self):
314 return self._shutdown_all_methods_in_one_thread(False)
315
316 def test_shutdown_immediate_all_methods_in_one_thread(self):
317 return self._shutdown_all_methods_in_one_thread(True)
318
319 def _write_msg_thread(self, q, n, results,
320 i_when_exec_shutdown, event_shutdown,
321 barrier_start):
322 # All `write_msg_threads`
323 # put several items into the queue.
324 for i in range(0, i_when_exec_shutdown//2):
325 q.put((i, 'LOYD'))
326 # Wait for the barrier to be complete.
327 barrier_start.wait()
328
329 for i in range(i_when_exec_shutdown//2, n):
330 try:
331 q.put((i, "YDLO"))
332 except self.queue.ShutDown:
333 results.append(False)
334 break
335
336 # Trigger queue shutdown.
337 if i == i_when_exec_shutdown:
338 # Only one thread should call shutdown().
339 if not event_shutdown.is_set():
340 event_shutdown.set()
341 results.append(True)
342
343 def _read_msg_thread(self, q, results, barrier_start):
344 # Get at least one item.
345 q.get(True)
346 q.task_done()
347 # Wait for the barrier to be complete.
348 barrier_start.wait()
349 while True:
350 try:
351 q.get(False)
352 q.task_done()
353 except self.queue.ShutDown:
354 results.append(True)
355 break
356 except self.queue.Empty:
357 pass
358
359 def _shutdown_thread(self, q, results, event_end, immediate):
360 event_end.wait()
361 q.shutdown(immediate)
362 results.append(q.qsize() == 0)
363
364 def _join_thread(self, q, barrier_start):
365 # Wait for the barrier to be complete.
366 barrier_start.wait()
367 q.join()
368
369 def _shutdown_all_methods_in_many_threads(self, immediate):
370 # Run a 'multi-producers/consumers queue' use case,
371 # with enough items into the queue.
372 # When shutdown, all running threads will be joined.
373 q = self.type2test()
374 ps = []
375 res_puts = []
376 res_gets = []
377 res_shutdown = []
378 write_threads = 4
379 read_threads = 6
380 join_threads = 2
381 nb_msgs = 1024*64
382 nb_msgs_w = nb_msgs // write_threads
383 when_exec_shutdown = nb_msgs_w // 2
384 # Use of a Barrier to ensure that
385 # - all write threads put all their items into the queue,
386 # - all read thread get at least one item from the queue,
387 # and keep on running until shutdown.
388 # The join thread is started only when shutdown is immediate.
389 nparties = write_threads + read_threads
390 if immediate:
391 nparties += join_threads
392 barrier_start = threading.Barrier(nparties)
393 ev_exec_shutdown = threading.Event()
394 lprocs = [
395 (self._write_msg_thread, write_threads, (q, nb_msgs_w, res_puts,
396 when_exec_shutdown, ev_exec_shutdown,
397 barrier_start)),
398 (self._read_msg_thread, read_threads, (q, res_gets, barrier_start)),
399 (self._shutdown_thread, 1, (q, res_shutdown, ev_exec_shutdown, immediate)),
400 ]
401 if immediate:
402 lprocs.append((self._join_thread, join_threads, (q, barrier_start)))
403 # start all threads.
404 for func, n, args in lprocs:
405 for i in range(n):
406 ps.append(threading.Thread(target=func, args=args))
407 ps[-1].start()
408 for thread in ps:
409 thread.join()
410
411 self.assertTrue(True in res_puts)
412 self.assertEqual(res_gets.count(True), read_threads)
413 if immediate:
414 self.assertListEqual(res_shutdown, [True])
415 self.assertTrue(q.empty())
416
417 def test_shutdown_all_methods_in_many_threads(self):
418 return self._shutdown_all_methods_in_many_threads(False)
419
420 def test_shutdown_immediate_all_methods_in_many_threads(self):
421 return self._shutdown_all_methods_in_many_threads(True)
422
423 def _get(self, q, go, results, shutdown=False):
424 go.wait()
425 try:
426 msg = q.get()
427 results.append(not shutdown)
428 return not shutdown
429 except self.queue.ShutDown:
430 results.append(shutdown)
431 return shutdown
432
433 def _get_shutdown(self, q, go, results):
434 return self._get(q, go, results, True)
435
436 def _get_task_done(self, q, go, results):
437 go.wait()
438 try:
439 msg = q.get()
440 q.task_done()
441 results.append(True)
442 return msg
443 except self.queue.ShutDown:
444 results.append(False)
445 return False
446
447 def _put(self, q, msg, go, results, shutdown=False):
448 go.wait()
449 try:
450 q.put(msg)
451 results.append(not shutdown)
452 return not shutdown
453 except self.queue.ShutDown:
454 results.append(shutdown)
455 return shutdown
456
457 def _put_shutdown(self, q, msg, go, results):
458 return self._put(q, msg, go, results, True)
459
460 def _join(self, q, results, shutdown=False):
461 try:
462 q.join()
463 results.append(not shutdown)
464 return not shutdown
465 except self.queue.ShutDown:
466 results.append(shutdown)
467 return shutdown
468
469 def _join_shutdown(self, q, results):
470 return self._join(q, results, True)
471
472 def _shutdown_get(self, immediate):
473 q = self.type2test(2)
474 results = []
475 go = threading.Event()
476 q.put("Y")
477 q.put("D")
478 # queue full
479
480 if immediate:
481 thrds = (
482 (self._get_shutdown, (q, go, results)),
483 (self._get_shutdown, (q, go, results)),
484 )
485 else:
486 thrds = (
487 # on shutdown(immediate=False)
488 # one of these threads should raise Shutdown
489 (self._get, (q, go, results)),
490 (self._get, (q, go, results)),
491 (self._get, (q, go, results)),
492 )
493 threads = []
494 for func, params in thrds:
495 threads.append(threading.Thread(target=func, args=params))
496 threads[-1].start()
497 q.shutdown(immediate)
498 go.set()
499 for t in threads:
500 t.join()
501 if immediate:
502 self.assertListEqual(results, [True, True])
503 else:
504 self.assertListEqual(sorted(results), [False] + [True]*(len(thrds)-1))
505
506 def test_shutdown_get(self):
507 return self._shutdown_get(False)
508
509 def test_shutdown_immediate_get(self):
510 return self._shutdown_get(True)
511
512 def _shutdown_put(self, immediate):
513 q = self.type2test(2)
514 results = []
515 go = threading.Event()
516 q.put("Y")
517 q.put("D")
518 # queue fulled
519
520 thrds = (
521 (self._put_shutdown, (q, "E", go, results)),
522 (self._put_shutdown, (q, "W", go, results)),
523 )
524 threads = []
525 for func, params in thrds:
526 threads.append(threading.Thread(target=func, args=params))
527 threads[-1].start()
528 q.shutdown()
529 go.set()
530 for t in threads:
531 t.join()
532
533 self.assertEqual(results, [True]*len(thrds))
534
535 def test_shutdown_put(self):
536 return self._shutdown_put(False)
537
538 def test_shutdown_immediate_put(self):
539 return self._shutdown_put(True)
540
541 def _shutdown_join(self, immediate):
542 q = self.type2test()
543 results = []
544 q.put("Y")
545 go = threading.Event()
546 nb = q.qsize()
547
548 thrds = (
549 (self._join, (q, results)),
550 (self._join, (q, results)),
551 )
552 threads = []
553 for func, params in thrds:
554 threads.append(threading.Thread(target=func, args=params))
555 threads[-1].start()
556 if not immediate:
557 res = []
558 for i in range(nb):
559 threads.append(threading.Thread(target=self._get_task_done, args=(q, go, res)))
560 threads[-1].start()
561 q.shutdown(immediate)
562 go.set()
563 for t in threads:
564 t.join()
565
566 self.assertEqual(results, [True]*len(thrds))
567
568 def test_shutdown_immediate_join(self):
569 return self._shutdown_join(True)
570
571 def test_shutdown_join(self):
572 return self._shutdown_join(False)
573
574 def _shutdown_put_join(self, immediate):
575 q = self.type2test(2)
576 results = []
577 go = threading.Event()
578 q.put("Y")
579 # queue not fulled
580
581 thrds = (
582 (self._put_shutdown, (q, "E", go, results)),
583 (self._join, (q, results)),
584 )
585 threads = []
586 for func, params in thrds:
587 threads.append(threading.Thread(target=func, args=params))
588 threads[-1].start()
589 self.assertEqual(q.unfinished_tasks, 1)
590
591 q.shutdown(immediate)
592 go.set()
593
594 if immediate:
595 with self.assertRaises(self.queue.ShutDown):
596 q.get_nowait()
597 else:
598 result = q.get()
599 self.assertEqual(result, "Y")
600 q.task_done()
601
602 for t in threads:
603 t.join()
604
605 self.assertEqual(results, [True]*len(thrds))
606
607 def test_shutdown_immediate_put_join(self):
608 return self._shutdown_put_join(True)
609
610 def test_shutdown_put_join(self):
611 return self._shutdown_put_join(False)
612
613 def test_shutdown_get_task_done_join(self):
614 q = self.type2test(2)
615 results = []
616 go = threading.Event()
617 q.put("Y")
618 q.put("D")
619 self.assertEqual(q.unfinished_tasks, q.qsize())
620
621 thrds = (
622 (self._get_task_done, (q, go, results)),
623 (self._get_task_done, (q, go, results)),
624 (self._join, (q, results)),
625 (self._join, (q, results)),
626 )
627 threads = []
628 for func, params in thrds:
629 threads.append(threading.Thread(target=func, args=params))
630 threads[-1].start()
631 go.set()
632 q.shutdown(False)
633 for t in threads:
634 t.join()
635
636 self.assertEqual(results, [True]*len(thrds))
637
638 def test_shutdown_pending_get(self):
639 def get():
640 try:
641 results.append(q.get())
642 except Exception as e:
643 results.append(e)
644
645 q = self.type2test()
646 results = []
647 get_thread = threading.Thread(target=get)
648 get_thread.start()
649 q.shutdown(immediate=False)
650 get_thread.join(timeout=10.0)
651 self.assertFalse(get_thread.is_alive())
652 self.assertEqual(len(results), 1)
653 self.assertIsInstance(results[0], self.queue.ShutDown)
654
655
656 class QueueTest(BaseQueueTestMixin):
657
658 def setUp(self):
659 self.type2test = self.queue.Queue
660 super().setUp()
661
662 class PyQueueTest(QueueTest, unittest.TestCase):
663 queue = py_queue
664
665
666 @need_c_queue
667 class CQueueTest(QueueTest, unittest.TestCase):
668 queue = c_queue
669
670
671 class LifoQueueTest(BaseQueueTestMixin):
672
673 def setUp(self):
674 self.type2test = self.queue.LifoQueue
675 super().setUp()
676
677
678 class PyLifoQueueTest(LifoQueueTest, unittest.TestCase):
679 queue = py_queue
680
681
682 @need_c_queue
683 class CLifoQueueTest(LifoQueueTest, unittest.TestCase):
684 queue = c_queue
685
686
687 class PriorityQueueTest(BaseQueueTestMixin):
688
689 def setUp(self):
690 self.type2test = self.queue.PriorityQueue
691 super().setUp()
692
693
694 class PyPriorityQueueTest(PriorityQueueTest, unittest.TestCase):
695 queue = py_queue
696
697
698 @need_c_queue
699 class CPriorityQueueTest(PriorityQueueTest, unittest.TestCase):
700 queue = c_queue
701
702
703 # A Queue subclass that can provoke failure at a moment's notice :)
704 class FailingQueueException(Exception): pass
705
706
707 class FailingQueueTest(BlockingTestMixin):
708
709 def setUp(self):
710
711 Queue = self.queue.Queue
712
713 class FailingQueue(Queue):
714 def __init__(self, *args):
715 self.fail_next_put = False
716 self.fail_next_get = False
717 Queue.__init__(self, *args)
718 def _put(self, item):
719 if self.fail_next_put:
720 self.fail_next_put = False
721 raise FailingQueueException("You Lose")
722 return Queue._put(self, item)
723 def _get(self):
724 if self.fail_next_get:
725 self.fail_next_get = False
726 raise FailingQueueException("You Lose")
727 return Queue._get(self)
728
729 self.FailingQueue = FailingQueue
730
731 super().setUp()
732
733 def failing_queue_test(self, q):
734 if q.qsize():
735 raise RuntimeError("Call this function with an empty queue")
736 for i in range(QUEUE_SIZE-1):
737 q.put(i)
738 # Test a failing non-blocking put.
739 q.fail_next_put = True
740 try:
741 q.put("oops", block=0)
742 self.fail("The queue didn't fail when it should have")
743 except FailingQueueException:
744 pass
745 q.fail_next_put = True
746 try:
747 q.put("oops", timeout=0.1)
748 self.fail("The queue didn't fail when it should have")
749 except FailingQueueException:
750 pass
751 q.put("last")
752 self.assertTrue(qfull(q), "Queue should be full")
753 # Test a failing blocking put
754 q.fail_next_put = True
755 try:
756 self.do_blocking_test(q.put, ("full",), q.get, ())
757 self.fail("The queue didn't fail when it should have")
758 except FailingQueueException:
759 pass
760 # Check the Queue isn't damaged.
761 # put failed, but get succeeded - re-add
762 q.put("last")
763 # Test a failing timeout put
764 q.fail_next_put = True
765 try:
766 self.do_exceptional_blocking_test(q.put, ("full", True, 10), q.get, (),
767 FailingQueueException)
768 self.fail("The queue didn't fail when it should have")
769 except FailingQueueException:
770 pass
771 # Check the Queue isn't damaged.
772 # put failed, but get succeeded - re-add
773 q.put("last")
774 self.assertTrue(qfull(q), "Queue should be full")
775 q.get()
776 self.assertTrue(not qfull(q), "Queue should not be full")
777 q.put("last")
778 self.assertTrue(qfull(q), "Queue should be full")
779 # Test a blocking put
780 self.do_blocking_test(q.put, ("full",), q.get, ())
781 # Empty it
782 for i in range(QUEUE_SIZE):
783 q.get()
784 self.assertTrue(not q.qsize(), "Queue should be empty")
785 q.put("first")
786 q.fail_next_get = True
787 try:
788 q.get()
789 self.fail("The queue didn't fail when it should have")
790 except FailingQueueException:
791 pass
792 self.assertTrue(q.qsize(), "Queue should not be empty")
793 q.fail_next_get = True
794 try:
795 q.get(timeout=0.1)
796 self.fail("The queue didn't fail when it should have")
797 except FailingQueueException:
798 pass
799 self.assertTrue(q.qsize(), "Queue should not be empty")
800 q.get()
801 self.assertTrue(not q.qsize(), "Queue should be empty")
802 q.fail_next_get = True
803 try:
804 self.do_exceptional_blocking_test(q.get, (), q.put, ('empty',),
805 FailingQueueException)
806 self.fail("The queue didn't fail when it should have")
807 except FailingQueueException:
808 pass
809 # put succeeded, but get failed.
810 self.assertTrue(q.qsize(), "Queue should not be empty")
811 q.get()
812 self.assertTrue(not q.qsize(), "Queue should be empty")
813
814 def test_failing_queue(self):
815
816 # Test to make sure a queue is functioning correctly.
817 # Done twice to the same instance.
818 q = self.FailingQueue(QUEUE_SIZE)
819 self.failing_queue_test(q)
820 self.failing_queue_test(q)
821
822
823
824 class PyFailingQueueTest(FailingQueueTest, unittest.TestCase):
825 queue = py_queue
826
827
828 @need_c_queue
829 class CFailingQueueTest(FailingQueueTest, unittest.TestCase):
830 queue = c_queue
831
832
833 class BaseSimpleQueueTest:
834
835 def setUp(self):
836 self.q = self.type2test()
837
838 def feed(self, q, seq, rnd, sentinel):
839 while True:
840 try:
841 val = seq.pop()
842 except IndexError:
843 q.put(sentinel)
844 return
845 q.put(val)
846 if rnd.random() > 0.5:
847 time.sleep(rnd.random() * 1e-3)
848
849 def consume(self, q, results, sentinel):
850 while True:
851 val = q.get()
852 if val == sentinel:
853 return
854 results.append(val)
855
856 def consume_nonblock(self, q, results, sentinel):
857 while True:
858 while True:
859 try:
860 val = q.get(block=False)
861 except self.queue.Empty:
862 time.sleep(1e-5)
863 else:
864 break
865 if val == sentinel:
866 return
867 results.append(val)
868
869 def consume_timeout(self, q, results, sentinel):
870 while True:
871 while True:
872 try:
873 val = q.get(timeout=1e-5)
874 except self.queue.Empty:
875 pass
876 else:
877 break
878 if val == sentinel:
879 return
880 results.append(val)
881
882 def run_threads(self, n_threads, q, inputs, feed_func, consume_func):
883 results = []
884 sentinel = None
885 seq = inputs.copy()
886 seq.reverse()
887 rnd = random.Random(42)
888
889 exceptions = []
890 def log_exceptions(f):
891 def wrapper(*args, **kwargs):
892 try:
893 f(*args, **kwargs)
894 except BaseException as e:
895 exceptions.append(e)
896 return wrapper
897
898 feeders = [threading.Thread(target=log_exceptions(feed_func),
899 args=(q, seq, rnd, sentinel))
900 for i in range(n_threads)]
901 consumers = [threading.Thread(target=log_exceptions(consume_func),
902 args=(q, results, sentinel))
903 for i in range(n_threads)]
904
905 with threading_helper.start_threads(feeders + consumers):
906 pass
907
908 self.assertFalse(exceptions)
909 self.assertTrue(q.empty())
910 self.assertEqual(q.qsize(), 0)
911
912 return results
913
914 def test_basic(self):
915 # Basic tests for get(), put() etc.
916 q = self.q
917 self.assertTrue(q.empty())
918 self.assertEqual(q.qsize(), 0)
919 q.put(1)
920 self.assertFalse(q.empty())
921 self.assertEqual(q.qsize(), 1)
922 q.put(2)
923 q.put_nowait(3)
924 q.put(4)
925 self.assertFalse(q.empty())
926 self.assertEqual(q.qsize(), 4)
927
928 self.assertEqual(q.get(), 1)
929 self.assertEqual(q.qsize(), 3)
930
931 self.assertEqual(q.get_nowait(), 2)
932 self.assertEqual(q.qsize(), 2)
933
934 self.assertEqual(q.get(block=False), 3)
935 self.assertFalse(q.empty())
936 self.assertEqual(q.qsize(), 1)
937
938 self.assertEqual(q.get(timeout=0.1), 4)
939 self.assertTrue(q.empty())
940 self.assertEqual(q.qsize(), 0)
941
942 with self.assertRaises(self.queue.Empty):
943 q.get(block=False)
944 with self.assertRaises(self.queue.Empty):
945 q.get(timeout=1e-3)
946 with self.assertRaises(self.queue.Empty):
947 q.get_nowait()
948 self.assertTrue(q.empty())
949 self.assertEqual(q.qsize(), 0)
950
951 def test_negative_timeout_raises_exception(self):
952 q = self.q
953 q.put(1)
954 with self.assertRaises(ValueError):
955 q.get(timeout=-1)
956
957 def test_order(self):
958 # Test a pair of concurrent put() and get()
959 q = self.q
960 inputs = list(range(100))
961 results = self.run_threads(1, q, inputs, self.feed, self.consume)
962
963 # One producer, one consumer => results appended in well-defined order
964 self.assertEqual(results, inputs)
965
966 @bigmemtest(size=50, memuse=100*2**20, dry_run=False)
967 def test_many_threads(self, size):
968 # Test multiple concurrent put() and get()
969 q = self.q
970 inputs = list(range(10000))
971 results = self.run_threads(size, q, inputs, self.feed, self.consume)
972
973 # Multiple consumers without synchronization append the
974 # results in random order
975 self.assertEqual(sorted(results), inputs)
976
977 @bigmemtest(size=50, memuse=100*2**20, dry_run=False)
978 def test_many_threads_nonblock(self, size):
979 # Test multiple concurrent put() and get(block=False)
980 q = self.q
981 inputs = list(range(10000))
982 results = self.run_threads(size, q, inputs,
983 self.feed, self.consume_nonblock)
984
985 self.assertEqual(sorted(results), inputs)
986
987 @bigmemtest(size=50, memuse=100*2**20, dry_run=False)
988 def test_many_threads_timeout(self, size):
989 # Test multiple concurrent put() and get(timeout=...)
990 q = self.q
991 inputs = list(range(1000))
992 results = self.run_threads(size, q, inputs,
993 self.feed, self.consume_timeout)
994
995 self.assertEqual(sorted(results), inputs)
996
997 def test_references(self):
998 # The queue should lose references to each item as soon as
999 # it leaves the queue.
1000 class C:
1001 pass
1002
1003 N = 20
1004 q = self.q
1005 for i in range(N):
1006 q.put(C())
1007 for i in range(N):
1008 wr = weakref.ref(q.get())
1009 gc_collect() # For PyPy or other GCs.
1010 self.assertIsNone(wr())
1011
1012
1013 class PySimpleQueueTest(BaseSimpleQueueTest, unittest.TestCase):
1014
1015 queue = py_queue
1016 def setUp(self):
1017 self.type2test = self.queue._PySimpleQueue
1018 super().setUp()
1019
1020
1021 @need_c_queue
1022 class CSimpleQueueTest(BaseSimpleQueueTest, unittest.TestCase):
1023
1024 queue = c_queue
1025
1026 def setUp(self):
1027 self.type2test = self.queue.SimpleQueue
1028 super().setUp()
1029
1030 def test_is_default(self):
1031 self.assertIs(self.type2test, self.queue.SimpleQueue)
1032 self.assertIs(self.type2test, self.queue.SimpleQueue)
1033
1034 def test_reentrancy(self):
1035 # bpo-14976: put() may be called reentrantly in an asynchronous
1036 # callback.
1037 q = self.q
1038 gen = itertools.count()
1039 N = 10000
1040 results = []
1041
1042 # This test exploits the fact that __del__ in a reference cycle
1043 # can be called any time the GC may run.
1044
1045 class Circular(object):
1046 def __init__(self):
1047 self.circular = self
1048
1049 def __del__(self):
1050 q.put(next(gen))
1051
1052 while True:
1053 o = Circular()
1054 q.put(next(gen))
1055 del o
1056 results.append(q.get())
1057 if results[-1] >= N:
1058 break
1059
1060 self.assertEqual(results, list(range(N + 1)))
1061
1062
1063 if __name__ == "__main__":
1064 unittest.main()