| 1 |
# Some simple queue module tests, plus some failure conditions |
| 2 |
# to ensure the Queue locks remain stable. |
| 3 |
import itertools |
| 4 |
import random |
| 5 |
import threading |
| 6 |
import time |
| 7 |
import unittest |
| 8 |
import weakref |
| 9 |
from test.support import gc_collect, bigmemtest |
| 10 |
from test.support import import_helper |
| 11 |
from test.support import threading_helper |
| 12 |
|
| 13 |
# queue module depends on threading primitives |
| 14 |
threading_helper.requires_working_threading(module=True) |
| 15 |
|
| 16 |
py_queue = import_helper.import_fresh_module('queue', blocked=['_queue']) |
| 17 |
c_queue = import_helper.import_fresh_module('queue', fresh=['_queue']) |
| 18 |
need_c_queue = unittest.skipUnless(c_queue, "No _queue module found") |
| 19 |
|
| 20 |
QUEUE_SIZE = 5 |
| 21 |
|
| 22 |
def qfull(q): |
| 23 |
return q.maxsize > 0 and q.qsize() == q.maxsize |
| 24 |
|
| 25 |
# A thread to run a function that unclogs a blocked Queue. |
| 26 |
class _TriggerThread(threading.Thread): |
| 27 |
def __init__(self, fn, args): |
| 28 |
self.fn = fn |
| 29 |
self.args = args |
| 30 |
self.startedEvent = threading.Event() |
| 31 |
threading.Thread.__init__(self) |
| 32 |
|
| 33 |
def run(self): |
| 34 |
# The sleep isn't necessary, but is intended to give the blocking |
| 35 |
# function in the main thread a chance at actually blocking before |
| 36 |
# we unclog it. But if the sleep is longer than the timeout-based |
| 37 |
# tests wait in their blocking functions, those tests will fail. |
| 38 |
# So we give them much longer timeout values compared to the |
| 39 |
# sleep here (I aimed at 10 seconds for blocking functions -- |
| 40 |
# they should never actually wait that long - they should make |
| 41 |
# progress as soon as we call self.fn()). |
| 42 |
time.sleep(0.1) |
| 43 |
self.startedEvent.set() |
| 44 |
self.fn(*self.args) |
| 45 |
|
| 46 |
|
| 47 |
# Execute a function that blocks, and in a separate thread, a function that |
| 48 |
# triggers the release. Returns the result of the blocking function. Caution: |
| 49 |
# block_func must guarantee to block until trigger_func is called, and |
| 50 |
# trigger_func must guarantee to change queue state so that block_func can make |
| 51 |
# enough progress to return. In particular, a block_func that just raises an |
| 52 |
# exception regardless of whether trigger_func is called will lead to |
| 53 |
# timing-dependent sporadic failures, and one of those went rarely seen but |
| 54 |
# undiagnosed for years. Now block_func must be unexceptional. If block_func |
| 55 |
# is supposed to raise an exception, call do_exceptional_blocking_test() |
| 56 |
# instead. |
| 57 |
|
| 58 |
class BlockingTestMixin: |
| 59 |
|
| 60 |
def do_blocking_test(self, block_func, block_args, trigger_func, trigger_args): |
| 61 |
thread = _TriggerThread(trigger_func, trigger_args) |
| 62 |
thread.start() |
| 63 |
try: |
| 64 |
self.result = block_func(*block_args) |
| 65 |
# If block_func returned before our thread made the call, we failed! |
| 66 |
if not thread.startedEvent.is_set(): |
| 67 |
self.fail("blocking function %r appeared not to block" % |
| 68 |
block_func) |
| 69 |
return self.result |
| 70 |
finally: |
| 71 |
threading_helper.join_thread(thread) # make sure the thread terminates |
| 72 |
|
| 73 |
# Call this instead if block_func is supposed to raise an exception. |
| 74 |
def do_exceptional_blocking_test(self,block_func, block_args, trigger_func, |
| 75 |
trigger_args, expected_exception_class): |
| 76 |
thread = _TriggerThread(trigger_func, trigger_args) |
| 77 |
thread.start() |
| 78 |
try: |
| 79 |
try: |
| 80 |
block_func(*block_args) |
| 81 |
except expected_exception_class: |
| 82 |
raise |
| 83 |
else: |
| 84 |
self.fail("expected exception of kind %r" % |
| 85 |
expected_exception_class) |
| 86 |
finally: |
| 87 |
threading_helper.join_thread(thread) # make sure the thread terminates |
| 88 |
if not thread.startedEvent.is_set(): |
| 89 |
self.fail("trigger thread ended but event never set") |
| 90 |
|
| 91 |
|
| 92 |
class BaseQueueTestMixin(BlockingTestMixin): |
| 93 |
def setUp(self): |
| 94 |
self.cum = 0 |
| 95 |
self.cumlock = threading.Lock() |
| 96 |
|
| 97 |
def basic_queue_test(self, q): |
| 98 |
if q.qsize(): |
| 99 |
raise RuntimeError("Call this function with an empty queue") |
| 100 |
self.assertTrue(q.empty()) |
| 101 |
self.assertFalse(q.full()) |
| 102 |
# I guess we better check things actually queue correctly a little :) |
| 103 |
q.put(111) |
| 104 |
q.put(333) |
| 105 |
q.put(222) |
| 106 |
target_order = dict(Queue = [111, 333, 222], |
| 107 |
LifoQueue = [222, 333, 111], |
| 108 |
PriorityQueue = [111, 222, 333]) |
| 109 |
actual_order = [q.get(), q.get(), q.get()] |
| 110 |
self.assertEqual(actual_order, target_order[q.__class__.__name__], |
| 111 |
"Didn't seem to queue the correct data!") |
| 112 |
for i in range(QUEUE_SIZE-1): |
| 113 |
q.put(i) |
| 114 |
self.assertTrue(q.qsize(), "Queue should not be empty") |
| 115 |
self.assertTrue(not qfull(q), "Queue should not be full") |
| 116 |
last = 2 * QUEUE_SIZE |
| 117 |
full = 3 * 2 * QUEUE_SIZE |
| 118 |
q.put(last) |
| 119 |
self.assertTrue(qfull(q), "Queue should be full") |
| 120 |
self.assertFalse(q.empty()) |
| 121 |
self.assertTrue(q.full()) |
| 122 |
try: |
| 123 |
q.put(full, block=0) |
| 124 |
self.fail("Didn't appear to block with a full queue") |
| 125 |
except self.queue.Full: |
| 126 |
pass |
| 127 |
try: |
| 128 |
q.put(full, timeout=0.01) |
| 129 |
self.fail("Didn't appear to time-out with a full queue") |
| 130 |
except self.queue.Full: |
| 131 |
pass |
| 132 |
# Test a blocking put |
| 133 |
self.do_blocking_test(q.put, (full,), q.get, ()) |
| 134 |
self.do_blocking_test(q.put, (full, True, 10), q.get, ()) |
| 135 |
# Empty it |
| 136 |
for i in range(QUEUE_SIZE): |
| 137 |
q.get() |
| 138 |
self.assertTrue(not q.qsize(), "Queue should be empty") |
| 139 |
try: |
| 140 |
q.get(block=0) |
| 141 |
self.fail("Didn't appear to block with an empty queue") |
| 142 |
except self.queue.Empty: |
| 143 |
pass |
| 144 |
try: |
| 145 |
q.get(timeout=0.01) |
| 146 |
self.fail("Didn't appear to time-out with an empty queue") |
| 147 |
except self.queue.Empty: |
| 148 |
pass |
| 149 |
# Test a blocking get |
| 150 |
self.do_blocking_test(q.get, (), q.put, ('empty',)) |
| 151 |
self.do_blocking_test(q.get, (True, 10), q.put, ('empty',)) |
| 152 |
|
| 153 |
|
| 154 |
def worker(self, q): |
| 155 |
while True: |
| 156 |
x = q.get() |
| 157 |
if x < 0: |
| 158 |
q.task_done() |
| 159 |
return |
| 160 |
with self.cumlock: |
| 161 |
self.cum += x |
| 162 |
q.task_done() |
| 163 |
|
| 164 |
def queue_join_test(self, q): |
| 165 |
self.cum = 0 |
| 166 |
threads = [] |
| 167 |
for i in (0,1): |
| 168 |
thread = threading.Thread(target=self.worker, args=(q,)) |
| 169 |
thread.start() |
| 170 |
threads.append(thread) |
| 171 |
for i in range(100): |
| 172 |
q.put(i) |
| 173 |
q.join() |
| 174 |
self.assertEqual(self.cum, sum(range(100)), |
| 175 |
"q.join() did not block until all tasks were done") |
| 176 |
for i in (0,1): |
| 177 |
q.put(-1) # instruct the threads to close |
| 178 |
q.join() # verify that you can join twice |
| 179 |
for thread in threads: |
| 180 |
thread.join() |
| 181 |
|
| 182 |
def test_queue_task_done(self): |
| 183 |
# Test to make sure a queue task completed successfully. |
| 184 |
q = self.type2test() |
| 185 |
try: |
| 186 |
q.task_done() |
| 187 |
except ValueError: |
| 188 |
pass |
| 189 |
else: |
| 190 |
self.fail("Did not detect task count going negative") |
| 191 |
|
| 192 |
def test_queue_join(self): |
| 193 |
# Test that a queue join()s successfully, and before anything else |
| 194 |
# (done twice for insurance). |
| 195 |
q = self.type2test() |
| 196 |
self.queue_join_test(q) |
| 197 |
self.queue_join_test(q) |
| 198 |
try: |
| 199 |
q.task_done() |
| 200 |
except ValueError: |
| 201 |
pass |
| 202 |
else: |
| 203 |
self.fail("Did not detect task count going negative") |
| 204 |
|
| 205 |
def test_basic(self): |
| 206 |
# Do it a couple of times on the same queue. |
| 207 |
# Done twice to make sure works with same instance reused. |
| 208 |
q = self.type2test(QUEUE_SIZE) |
| 209 |
self.basic_queue_test(q) |
| 210 |
self.basic_queue_test(q) |
| 211 |
|
| 212 |
def test_negative_timeout_raises_exception(self): |
| 213 |
q = self.type2test(QUEUE_SIZE) |
| 214 |
with self.assertRaises(ValueError): |
| 215 |
q.put(1, timeout=-1) |
| 216 |
with self.assertRaises(ValueError): |
| 217 |
q.get(1, timeout=-1) |
| 218 |
|
| 219 |
def test_nowait(self): |
| 220 |
q = self.type2test(QUEUE_SIZE) |
| 221 |
for i in range(QUEUE_SIZE): |
| 222 |
q.put_nowait(1) |
| 223 |
with self.assertRaises(self.queue.Full): |
| 224 |
q.put_nowait(1) |
| 225 |
|
| 226 |
for i in range(QUEUE_SIZE): |
| 227 |
q.get_nowait() |
| 228 |
with self.assertRaises(self.queue.Empty): |
| 229 |
q.get_nowait() |
| 230 |
|
| 231 |
def test_shrinking_queue(self): |
| 232 |
# issue 10110 |
| 233 |
q = self.type2test(3) |
| 234 |
q.put(1) |
| 235 |
q.put(2) |
| 236 |
q.put(3) |
| 237 |
with self.assertRaises(self.queue.Full): |
| 238 |
q.put_nowait(4) |
| 239 |
self.assertEqual(q.qsize(), 3) |
| 240 |
q.maxsize = 2 # shrink the queue |
| 241 |
with self.assertRaises(self.queue.Full): |
| 242 |
q.put_nowait(4) |
| 243 |
|
| 244 |
def test_shutdown_empty(self): |
| 245 |
q = self.type2test() |
| 246 |
q.shutdown() |
| 247 |
with self.assertRaises(self.queue.ShutDown): |
| 248 |
q.put("data") |
| 249 |
with self.assertRaises(self.queue.ShutDown): |
| 250 |
q.get() |
| 251 |
|
| 252 |
def test_shutdown_nonempty(self): |
| 253 |
q = self.type2test() |
| 254 |
q.put("data") |
| 255 |
q.shutdown() |
| 256 |
q.get() |
| 257 |
with self.assertRaises(self.queue.ShutDown): |
| 258 |
q.get() |
| 259 |
|
| 260 |
def test_shutdown_immediate(self): |
| 261 |
q = self.type2test() |
| 262 |
q.put("data") |
| 263 |
q.shutdown(immediate=True) |
| 264 |
with self.assertRaises(self.queue.ShutDown): |
| 265 |
q.get() |
| 266 |
|
| 267 |
def test_shutdown_allowed_transitions(self): |
| 268 |
# allowed transitions would be from alive via shutdown to immediate |
| 269 |
q = self.type2test() |
| 270 |
self.assertFalse(q.is_shutdown) |
| 271 |
|
| 272 |
q.shutdown() |
| 273 |
self.assertTrue(q.is_shutdown) |
| 274 |
|
| 275 |
q.shutdown(immediate=True) |
| 276 |
self.assertTrue(q.is_shutdown) |
| 277 |
|
| 278 |
q.shutdown(immediate=False) |
| 279 |
|
| 280 |
def _shutdown_all_methods_in_one_thread(self, immediate): |
| 281 |
q = self.type2test(2) |
| 282 |
q.put("L") |
| 283 |
q.put_nowait("O") |
| 284 |
q.shutdown(immediate) |
| 285 |
|
| 286 |
with self.assertRaises(self.queue.ShutDown): |
| 287 |
q.put("E") |
| 288 |
with self.assertRaises(self.queue.ShutDown): |
| 289 |
q.put_nowait("W") |
| 290 |
if immediate: |
| 291 |
with self.assertRaises(self.queue.ShutDown): |
| 292 |
q.get() |
| 293 |
with self.assertRaises(self.queue.ShutDown): |
| 294 |
q.get_nowait() |
| 295 |
with self.assertRaises(ValueError): |
| 296 |
q.task_done() |
| 297 |
q.join() |
| 298 |
else: |
| 299 |
self.assertIn(q.get(), "LO") |
| 300 |
q.task_done() |
| 301 |
self.assertIn(q.get(), "LO") |
| 302 |
q.task_done() |
| 303 |
q.join() |
| 304 |
# on shutdown(immediate=False) |
| 305 |
# when queue is empty, should raise ShutDown Exception |
| 306 |
with self.assertRaises(self.queue.ShutDown): |
| 307 |
q.get() # p.get(True) |
| 308 |
with self.assertRaises(self.queue.ShutDown): |
| 309 |
q.get_nowait() # p.get(False) |
| 310 |
with self.assertRaises(self.queue.ShutDown): |
| 311 |
q.get(True, 1.0) |
| 312 |
|
| 313 |
def test_shutdown_all_methods_in_one_thread(self): |
| 314 |
return self._shutdown_all_methods_in_one_thread(False) |
| 315 |
|
| 316 |
def test_shutdown_immediate_all_methods_in_one_thread(self): |
| 317 |
return self._shutdown_all_methods_in_one_thread(True) |
| 318 |
|
| 319 |
def _write_msg_thread(self, q, n, results, |
| 320 |
i_when_exec_shutdown, event_shutdown, |
| 321 |
barrier_start): |
| 322 |
# All `write_msg_threads` |
| 323 |
# put several items into the queue. |
| 324 |
for i in range(0, i_when_exec_shutdown//2): |
| 325 |
q.put((i, 'LOYD')) |
| 326 |
# Wait for the barrier to be complete. |
| 327 |
barrier_start.wait() |
| 328 |
|
| 329 |
for i in range(i_when_exec_shutdown//2, n): |
| 330 |
try: |
| 331 |
q.put((i, "YDLO")) |
| 332 |
except self.queue.ShutDown: |
| 333 |
results.append(False) |
| 334 |
break |
| 335 |
|
| 336 |
# Trigger queue shutdown. |
| 337 |
if i == i_when_exec_shutdown: |
| 338 |
# Only one thread should call shutdown(). |
| 339 |
if not event_shutdown.is_set(): |
| 340 |
event_shutdown.set() |
| 341 |
results.append(True) |
| 342 |
|
| 343 |
def _read_msg_thread(self, q, results, barrier_start): |
| 344 |
# Get at least one item. |
| 345 |
q.get(True) |
| 346 |
q.task_done() |
| 347 |
# Wait for the barrier to be complete. |
| 348 |
barrier_start.wait() |
| 349 |
while True: |
| 350 |
try: |
| 351 |
q.get(False) |
| 352 |
q.task_done() |
| 353 |
except self.queue.ShutDown: |
| 354 |
results.append(True) |
| 355 |
break |
| 356 |
except self.queue.Empty: |
| 357 |
pass |
| 358 |
|
| 359 |
def _shutdown_thread(self, q, results, event_end, immediate): |
| 360 |
event_end.wait() |
| 361 |
q.shutdown(immediate) |
| 362 |
results.append(q.qsize() == 0) |
| 363 |
|
| 364 |
def _join_thread(self, q, barrier_start): |
| 365 |
# Wait for the barrier to be complete. |
| 366 |
barrier_start.wait() |
| 367 |
q.join() |
| 368 |
|
| 369 |
def _shutdown_all_methods_in_many_threads(self, immediate): |
| 370 |
# Run a 'multi-producers/consumers queue' use case, |
| 371 |
# with enough items into the queue. |
| 372 |
# When shutdown, all running threads will be joined. |
| 373 |
q = self.type2test() |
| 374 |
ps = [] |
| 375 |
res_puts = [] |
| 376 |
res_gets = [] |
| 377 |
res_shutdown = [] |
| 378 |
write_threads = 4 |
| 379 |
read_threads = 6 |
| 380 |
join_threads = 2 |
| 381 |
nb_msgs = 1024*64 |
| 382 |
nb_msgs_w = nb_msgs // write_threads |
| 383 |
when_exec_shutdown = nb_msgs_w // 2 |
| 384 |
# Use of a Barrier to ensure that |
| 385 |
# - all write threads put all their items into the queue, |
| 386 |
# - all read thread get at least one item from the queue, |
| 387 |
# and keep on running until shutdown. |
| 388 |
# The join thread is started only when shutdown is immediate. |
| 389 |
nparties = write_threads + read_threads |
| 390 |
if immediate: |
| 391 |
nparties += join_threads |
| 392 |
barrier_start = threading.Barrier(nparties) |
| 393 |
ev_exec_shutdown = threading.Event() |
| 394 |
lprocs = [ |
| 395 |
(self._write_msg_thread, write_threads, (q, nb_msgs_w, res_puts, |
| 396 |
when_exec_shutdown, ev_exec_shutdown, |
| 397 |
barrier_start)), |
| 398 |
(self._read_msg_thread, read_threads, (q, res_gets, barrier_start)), |
| 399 |
(self._shutdown_thread, 1, (q, res_shutdown, ev_exec_shutdown, immediate)), |
| 400 |
] |
| 401 |
if immediate: |
| 402 |
lprocs.append((self._join_thread, join_threads, (q, barrier_start))) |
| 403 |
# start all threads. |
| 404 |
for func, n, args in lprocs: |
| 405 |
for i in range(n): |
| 406 |
ps.append(threading.Thread(target=func, args=args)) |
| 407 |
ps[-1].start() |
| 408 |
for thread in ps: |
| 409 |
thread.join() |
| 410 |
|
| 411 |
self.assertTrue(True in res_puts) |
| 412 |
self.assertEqual(res_gets.count(True), read_threads) |
| 413 |
if immediate: |
| 414 |
self.assertListEqual(res_shutdown, [True]) |
| 415 |
self.assertTrue(q.empty()) |
| 416 |
|
| 417 |
def test_shutdown_all_methods_in_many_threads(self): |
| 418 |
return self._shutdown_all_methods_in_many_threads(False) |
| 419 |
|
| 420 |
def test_shutdown_immediate_all_methods_in_many_threads(self): |
| 421 |
return self._shutdown_all_methods_in_many_threads(True) |
| 422 |
|
| 423 |
def _get(self, q, go, results, shutdown=False): |
| 424 |
go.wait() |
| 425 |
try: |
| 426 |
msg = q.get() |
| 427 |
results.append(not shutdown) |
| 428 |
return not shutdown |
| 429 |
except self.queue.ShutDown: |
| 430 |
results.append(shutdown) |
| 431 |
return shutdown |
| 432 |
|
| 433 |
def _get_shutdown(self, q, go, results): |
| 434 |
return self._get(q, go, results, True) |
| 435 |
|
| 436 |
def _get_task_done(self, q, go, results): |
| 437 |
go.wait() |
| 438 |
try: |
| 439 |
msg = q.get() |
| 440 |
q.task_done() |
| 441 |
results.append(True) |
| 442 |
return msg |
| 443 |
except self.queue.ShutDown: |
| 444 |
results.append(False) |
| 445 |
return False |
| 446 |
|
| 447 |
def _put(self, q, msg, go, results, shutdown=False): |
| 448 |
go.wait() |
| 449 |
try: |
| 450 |
q.put(msg) |
| 451 |
results.append(not shutdown) |
| 452 |
return not shutdown |
| 453 |
except self.queue.ShutDown: |
| 454 |
results.append(shutdown) |
| 455 |
return shutdown |
| 456 |
|
| 457 |
def _put_shutdown(self, q, msg, go, results): |
| 458 |
return self._put(q, msg, go, results, True) |
| 459 |
|
| 460 |
def _join(self, q, results, shutdown=False): |
| 461 |
try: |
| 462 |
q.join() |
| 463 |
results.append(not shutdown) |
| 464 |
return not shutdown |
| 465 |
except self.queue.ShutDown: |
| 466 |
results.append(shutdown) |
| 467 |
return shutdown |
| 468 |
|
| 469 |
def _join_shutdown(self, q, results): |
| 470 |
return self._join(q, results, True) |
| 471 |
|
| 472 |
def _shutdown_get(self, immediate): |
| 473 |
q = self.type2test(2) |
| 474 |
results = [] |
| 475 |
go = threading.Event() |
| 476 |
q.put("Y") |
| 477 |
q.put("D") |
| 478 |
# queue full |
| 479 |
|
| 480 |
if immediate: |
| 481 |
thrds = ( |
| 482 |
(self._get_shutdown, (q, go, results)), |
| 483 |
(self._get_shutdown, (q, go, results)), |
| 484 |
) |
| 485 |
else: |
| 486 |
thrds = ( |
| 487 |
# on shutdown(immediate=False) |
| 488 |
# one of these threads should raise Shutdown |
| 489 |
(self._get, (q, go, results)), |
| 490 |
(self._get, (q, go, results)), |
| 491 |
(self._get, (q, go, results)), |
| 492 |
) |
| 493 |
threads = [] |
| 494 |
for func, params in thrds: |
| 495 |
threads.append(threading.Thread(target=func, args=params)) |
| 496 |
threads[-1].start() |
| 497 |
q.shutdown(immediate) |
| 498 |
go.set() |
| 499 |
for t in threads: |
| 500 |
t.join() |
| 501 |
if immediate: |
| 502 |
self.assertListEqual(results, [True, True]) |
| 503 |
else: |
| 504 |
self.assertListEqual(sorted(results), [False] + [True]*(len(thrds)-1)) |
| 505 |
|
| 506 |
def test_shutdown_get(self): |
| 507 |
return self._shutdown_get(False) |
| 508 |
|
| 509 |
def test_shutdown_immediate_get(self): |
| 510 |
return self._shutdown_get(True) |
| 511 |
|
| 512 |
def _shutdown_put(self, immediate): |
| 513 |
q = self.type2test(2) |
| 514 |
results = [] |
| 515 |
go = threading.Event() |
| 516 |
q.put("Y") |
| 517 |
q.put("D") |
| 518 |
# queue fulled |
| 519 |
|
| 520 |
thrds = ( |
| 521 |
(self._put_shutdown, (q, "E", go, results)), |
| 522 |
(self._put_shutdown, (q, "W", go, results)), |
| 523 |
) |
| 524 |
threads = [] |
| 525 |
for func, params in thrds: |
| 526 |
threads.append(threading.Thread(target=func, args=params)) |
| 527 |
threads[-1].start() |
| 528 |
q.shutdown() |
| 529 |
go.set() |
| 530 |
for t in threads: |
| 531 |
t.join() |
| 532 |
|
| 533 |
self.assertEqual(results, [True]*len(thrds)) |
| 534 |
|
| 535 |
def test_shutdown_put(self): |
| 536 |
return self._shutdown_put(False) |
| 537 |
|
| 538 |
def test_shutdown_immediate_put(self): |
| 539 |
return self._shutdown_put(True) |
| 540 |
|
| 541 |
def _shutdown_join(self, immediate): |
| 542 |
q = self.type2test() |
| 543 |
results = [] |
| 544 |
q.put("Y") |
| 545 |
go = threading.Event() |
| 546 |
nb = q.qsize() |
| 547 |
|
| 548 |
thrds = ( |
| 549 |
(self._join, (q, results)), |
| 550 |
(self._join, (q, results)), |
| 551 |
) |
| 552 |
threads = [] |
| 553 |
for func, params in thrds: |
| 554 |
threads.append(threading.Thread(target=func, args=params)) |
| 555 |
threads[-1].start() |
| 556 |
if not immediate: |
| 557 |
res = [] |
| 558 |
for i in range(nb): |
| 559 |
threads.append(threading.Thread(target=self._get_task_done, args=(q, go, res))) |
| 560 |
threads[-1].start() |
| 561 |
q.shutdown(immediate) |
| 562 |
go.set() |
| 563 |
for t in threads: |
| 564 |
t.join() |
| 565 |
|
| 566 |
self.assertEqual(results, [True]*len(thrds)) |
| 567 |
|
| 568 |
def test_shutdown_immediate_join(self): |
| 569 |
return self._shutdown_join(True) |
| 570 |
|
| 571 |
def test_shutdown_join(self): |
| 572 |
return self._shutdown_join(False) |
| 573 |
|
| 574 |
def _shutdown_put_join(self, immediate): |
| 575 |
q = self.type2test(2) |
| 576 |
results = [] |
| 577 |
go = threading.Event() |
| 578 |
q.put("Y") |
| 579 |
# queue not fulled |
| 580 |
|
| 581 |
thrds = ( |
| 582 |
(self._put_shutdown, (q, "E", go, results)), |
| 583 |
(self._join, (q, results)), |
| 584 |
) |
| 585 |
threads = [] |
| 586 |
for func, params in thrds: |
| 587 |
threads.append(threading.Thread(target=func, args=params)) |
| 588 |
threads[-1].start() |
| 589 |
self.assertEqual(q.unfinished_tasks, 1) |
| 590 |
|
| 591 |
q.shutdown(immediate) |
| 592 |
go.set() |
| 593 |
|
| 594 |
if immediate: |
| 595 |
with self.assertRaises(self.queue.ShutDown): |
| 596 |
q.get_nowait() |
| 597 |
else: |
| 598 |
result = q.get() |
| 599 |
self.assertEqual(result, "Y") |
| 600 |
q.task_done() |
| 601 |
|
| 602 |
for t in threads: |
| 603 |
t.join() |
| 604 |
|
| 605 |
self.assertEqual(results, [True]*len(thrds)) |
| 606 |
|
| 607 |
def test_shutdown_immediate_put_join(self): |
| 608 |
return self._shutdown_put_join(True) |
| 609 |
|
| 610 |
def test_shutdown_put_join(self): |
| 611 |
return self._shutdown_put_join(False) |
| 612 |
|
| 613 |
def test_shutdown_get_task_done_join(self): |
| 614 |
q = self.type2test(2) |
| 615 |
results = [] |
| 616 |
go = threading.Event() |
| 617 |
q.put("Y") |
| 618 |
q.put("D") |
| 619 |
self.assertEqual(q.unfinished_tasks, q.qsize()) |
| 620 |
|
| 621 |
thrds = ( |
| 622 |
(self._get_task_done, (q, go, results)), |
| 623 |
(self._get_task_done, (q, go, results)), |
| 624 |
(self._join, (q, results)), |
| 625 |
(self._join, (q, results)), |
| 626 |
) |
| 627 |
threads = [] |
| 628 |
for func, params in thrds: |
| 629 |
threads.append(threading.Thread(target=func, args=params)) |
| 630 |
threads[-1].start() |
| 631 |
go.set() |
| 632 |
q.shutdown(False) |
| 633 |
for t in threads: |
| 634 |
t.join() |
| 635 |
|
| 636 |
self.assertEqual(results, [True]*len(thrds)) |
| 637 |
|
| 638 |
def test_shutdown_pending_get(self): |
| 639 |
def get(): |
| 640 |
try: |
| 641 |
results.append(q.get()) |
| 642 |
except Exception as e: |
| 643 |
results.append(e) |
| 644 |
|
| 645 |
q = self.type2test() |
| 646 |
results = [] |
| 647 |
get_thread = threading.Thread(target=get) |
| 648 |
get_thread.start() |
| 649 |
q.shutdown(immediate=False) |
| 650 |
get_thread.join(timeout=10.0) |
| 651 |
self.assertFalse(get_thread.is_alive()) |
| 652 |
self.assertEqual(len(results), 1) |
| 653 |
self.assertIsInstance(results[0], self.queue.ShutDown) |
| 654 |
|
| 655 |
|
| 656 |
class QueueTest(BaseQueueTestMixin): |
| 657 |
|
| 658 |
def setUp(self): |
| 659 |
self.type2test = self.queue.Queue |
| 660 |
super().setUp() |
| 661 |
|
| 662 |
class PyQueueTest(QueueTest, unittest.TestCase): |
| 663 |
queue = py_queue |
| 664 |
|
| 665 |
|
| 666 |
@need_c_queue |
| 667 |
class CQueueTest(QueueTest, unittest.TestCase): |
| 668 |
queue = c_queue |
| 669 |
|
| 670 |
|
| 671 |
class LifoQueueTest(BaseQueueTestMixin): |
| 672 |
|
| 673 |
def setUp(self): |
| 674 |
self.type2test = self.queue.LifoQueue |
| 675 |
super().setUp() |
| 676 |
|
| 677 |
|
| 678 |
class PyLifoQueueTest(LifoQueueTest, unittest.TestCase): |
| 679 |
queue = py_queue |
| 680 |
|
| 681 |
|
| 682 |
@need_c_queue |
| 683 |
class CLifoQueueTest(LifoQueueTest, unittest.TestCase): |
| 684 |
queue = c_queue |
| 685 |
|
| 686 |
|
| 687 |
class PriorityQueueTest(BaseQueueTestMixin): |
| 688 |
|
| 689 |
def setUp(self): |
| 690 |
self.type2test = self.queue.PriorityQueue |
| 691 |
super().setUp() |
| 692 |
|
| 693 |
|
| 694 |
class PyPriorityQueueTest(PriorityQueueTest, unittest.TestCase): |
| 695 |
queue = py_queue |
| 696 |
|
| 697 |
|
| 698 |
@need_c_queue |
| 699 |
class CPriorityQueueTest(PriorityQueueTest, unittest.TestCase): |
| 700 |
queue = c_queue |
| 701 |
|
| 702 |
|
| 703 |
# A Queue subclass that can provoke failure at a moment's notice :) |
| 704 |
class FailingQueueException(Exception): pass |
| 705 |
|
| 706 |
|
| 707 |
class FailingQueueTest(BlockingTestMixin): |
| 708 |
|
| 709 |
def setUp(self): |
| 710 |
|
| 711 |
Queue = self.queue.Queue |
| 712 |
|
| 713 |
class FailingQueue(Queue): |
| 714 |
def __init__(self, *args): |
| 715 |
self.fail_next_put = False |
| 716 |
self.fail_next_get = False |
| 717 |
Queue.__init__(self, *args) |
| 718 |
def _put(self, item): |
| 719 |
if self.fail_next_put: |
| 720 |
self.fail_next_put = False |
| 721 |
raise FailingQueueException("You Lose") |
| 722 |
return Queue._put(self, item) |
| 723 |
def _get(self): |
| 724 |
if self.fail_next_get: |
| 725 |
self.fail_next_get = False |
| 726 |
raise FailingQueueException("You Lose") |
| 727 |
return Queue._get(self) |
| 728 |
|
| 729 |
self.FailingQueue = FailingQueue |
| 730 |
|
| 731 |
super().setUp() |
| 732 |
|
| 733 |
def failing_queue_test(self, q): |
| 734 |
if q.qsize(): |
| 735 |
raise RuntimeError("Call this function with an empty queue") |
| 736 |
for i in range(QUEUE_SIZE-1): |
| 737 |
q.put(i) |
| 738 |
# Test a failing non-blocking put. |
| 739 |
q.fail_next_put = True |
| 740 |
try: |
| 741 |
q.put("oops", block=0) |
| 742 |
self.fail("The queue didn't fail when it should have") |
| 743 |
except FailingQueueException: |
| 744 |
pass |
| 745 |
q.fail_next_put = True |
| 746 |
try: |
| 747 |
q.put("oops", timeout=0.1) |
| 748 |
self.fail("The queue didn't fail when it should have") |
| 749 |
except FailingQueueException: |
| 750 |
pass |
| 751 |
q.put("last") |
| 752 |
self.assertTrue(qfull(q), "Queue should be full") |
| 753 |
# Test a failing blocking put |
| 754 |
q.fail_next_put = True |
| 755 |
try: |
| 756 |
self.do_blocking_test(q.put, ("full",), q.get, ()) |
| 757 |
self.fail("The queue didn't fail when it should have") |
| 758 |
except FailingQueueException: |
| 759 |
pass |
| 760 |
# Check the Queue isn't damaged. |
| 761 |
# put failed, but get succeeded - re-add |
| 762 |
q.put("last") |
| 763 |
# Test a failing timeout put |
| 764 |
q.fail_next_put = True |
| 765 |
try: |
| 766 |
self.do_exceptional_blocking_test(q.put, ("full", True, 10), q.get, (), |
| 767 |
FailingQueueException) |
| 768 |
self.fail("The queue didn't fail when it should have") |
| 769 |
except FailingQueueException: |
| 770 |
pass |
| 771 |
# Check the Queue isn't damaged. |
| 772 |
# put failed, but get succeeded - re-add |
| 773 |
q.put("last") |
| 774 |
self.assertTrue(qfull(q), "Queue should be full") |
| 775 |
q.get() |
| 776 |
self.assertTrue(not qfull(q), "Queue should not be full") |
| 777 |
q.put("last") |
| 778 |
self.assertTrue(qfull(q), "Queue should be full") |
| 779 |
# Test a blocking put |
| 780 |
self.do_blocking_test(q.put, ("full",), q.get, ()) |
| 781 |
# Empty it |
| 782 |
for i in range(QUEUE_SIZE): |
| 783 |
q.get() |
| 784 |
self.assertTrue(not q.qsize(), "Queue should be empty") |
| 785 |
q.put("first") |
| 786 |
q.fail_next_get = True |
| 787 |
try: |
| 788 |
q.get() |
| 789 |
self.fail("The queue didn't fail when it should have") |
| 790 |
except FailingQueueException: |
| 791 |
pass |
| 792 |
self.assertTrue(q.qsize(), "Queue should not be empty") |
| 793 |
q.fail_next_get = True |
| 794 |
try: |
| 795 |
q.get(timeout=0.1) |
| 796 |
self.fail("The queue didn't fail when it should have") |
| 797 |
except FailingQueueException: |
| 798 |
pass |
| 799 |
self.assertTrue(q.qsize(), "Queue should not be empty") |
| 800 |
q.get() |
| 801 |
self.assertTrue(not q.qsize(), "Queue should be empty") |
| 802 |
q.fail_next_get = True |
| 803 |
try: |
| 804 |
self.do_exceptional_blocking_test(q.get, (), q.put, ('empty',), |
| 805 |
FailingQueueException) |
| 806 |
self.fail("The queue didn't fail when it should have") |
| 807 |
except FailingQueueException: |
| 808 |
pass |
| 809 |
# put succeeded, but get failed. |
| 810 |
self.assertTrue(q.qsize(), "Queue should not be empty") |
| 811 |
q.get() |
| 812 |
self.assertTrue(not q.qsize(), "Queue should be empty") |
| 813 |
|
| 814 |
def test_failing_queue(self): |
| 815 |
|
| 816 |
# Test to make sure a queue is functioning correctly. |
| 817 |
# Done twice to the same instance. |
| 818 |
q = self.FailingQueue(QUEUE_SIZE) |
| 819 |
self.failing_queue_test(q) |
| 820 |
self.failing_queue_test(q) |
| 821 |
|
| 822 |
|
| 823 |
|
| 824 |
class PyFailingQueueTest(FailingQueueTest, unittest.TestCase): |
| 825 |
queue = py_queue |
| 826 |
|
| 827 |
|
| 828 |
@need_c_queue |
| 829 |
class CFailingQueueTest(FailingQueueTest, unittest.TestCase): |
| 830 |
queue = c_queue |
| 831 |
|
| 832 |
|
| 833 |
class BaseSimpleQueueTest: |
| 834 |
|
| 835 |
def setUp(self): |
| 836 |
self.q = self.type2test() |
| 837 |
|
| 838 |
def feed(self, q, seq, rnd, sentinel): |
| 839 |
while True: |
| 840 |
try: |
| 841 |
val = seq.pop() |
| 842 |
except IndexError: |
| 843 |
q.put(sentinel) |
| 844 |
return |
| 845 |
q.put(val) |
| 846 |
if rnd.random() > 0.5: |
| 847 |
time.sleep(rnd.random() * 1e-3) |
| 848 |
|
| 849 |
def consume(self, q, results, sentinel): |
| 850 |
while True: |
| 851 |
val = q.get() |
| 852 |
if val == sentinel: |
| 853 |
return |
| 854 |
results.append(val) |
| 855 |
|
| 856 |
def consume_nonblock(self, q, results, sentinel): |
| 857 |
while True: |
| 858 |
while True: |
| 859 |
try: |
| 860 |
val = q.get(block=False) |
| 861 |
except self.queue.Empty: |
| 862 |
time.sleep(1e-5) |
| 863 |
else: |
| 864 |
break |
| 865 |
if val == sentinel: |
| 866 |
return |
| 867 |
results.append(val) |
| 868 |
|
| 869 |
def consume_timeout(self, q, results, sentinel): |
| 870 |
while True: |
| 871 |
while True: |
| 872 |
try: |
| 873 |
val = q.get(timeout=1e-5) |
| 874 |
except self.queue.Empty: |
| 875 |
pass |
| 876 |
else: |
| 877 |
break |
| 878 |
if val == sentinel: |
| 879 |
return |
| 880 |
results.append(val) |
| 881 |
|
| 882 |
def run_threads(self, n_threads, q, inputs, feed_func, consume_func): |
| 883 |
results = [] |
| 884 |
sentinel = None |
| 885 |
seq = inputs.copy() |
| 886 |
seq.reverse() |
| 887 |
rnd = random.Random(42) |
| 888 |
|
| 889 |
exceptions = [] |
| 890 |
def log_exceptions(f): |
| 891 |
def wrapper(*args, **kwargs): |
| 892 |
try: |
| 893 |
f(*args, **kwargs) |
| 894 |
except BaseException as e: |
| 895 |
exceptions.append(e) |
| 896 |
return wrapper |
| 897 |
|
| 898 |
feeders = [threading.Thread(target=log_exceptions(feed_func), |
| 899 |
args=(q, seq, rnd, sentinel)) |
| 900 |
for i in range(n_threads)] |
| 901 |
consumers = [threading.Thread(target=log_exceptions(consume_func), |
| 902 |
args=(q, results, sentinel)) |
| 903 |
for i in range(n_threads)] |
| 904 |
|
| 905 |
with threading_helper.start_threads(feeders + consumers): |
| 906 |
pass |
| 907 |
|
| 908 |
self.assertFalse(exceptions) |
| 909 |
self.assertTrue(q.empty()) |
| 910 |
self.assertEqual(q.qsize(), 0) |
| 911 |
|
| 912 |
return results |
| 913 |
|
| 914 |
def test_basic(self): |
| 915 |
# Basic tests for get(), put() etc. |
| 916 |
q = self.q |
| 917 |
self.assertTrue(q.empty()) |
| 918 |
self.assertEqual(q.qsize(), 0) |
| 919 |
q.put(1) |
| 920 |
self.assertFalse(q.empty()) |
| 921 |
self.assertEqual(q.qsize(), 1) |
| 922 |
q.put(2) |
| 923 |
q.put_nowait(3) |
| 924 |
q.put(4) |
| 925 |
self.assertFalse(q.empty()) |
| 926 |
self.assertEqual(q.qsize(), 4) |
| 927 |
|
| 928 |
self.assertEqual(q.get(), 1) |
| 929 |
self.assertEqual(q.qsize(), 3) |
| 930 |
|
| 931 |
self.assertEqual(q.get_nowait(), 2) |
| 932 |
self.assertEqual(q.qsize(), 2) |
| 933 |
|
| 934 |
self.assertEqual(q.get(block=False), 3) |
| 935 |
self.assertFalse(q.empty()) |
| 936 |
self.assertEqual(q.qsize(), 1) |
| 937 |
|
| 938 |
self.assertEqual(q.get(timeout=0.1), 4) |
| 939 |
self.assertTrue(q.empty()) |
| 940 |
self.assertEqual(q.qsize(), 0) |
| 941 |
|
| 942 |
with self.assertRaises(self.queue.Empty): |
| 943 |
q.get(block=False) |
| 944 |
with self.assertRaises(self.queue.Empty): |
| 945 |
q.get(timeout=1e-3) |
| 946 |
with self.assertRaises(self.queue.Empty): |
| 947 |
q.get_nowait() |
| 948 |
self.assertTrue(q.empty()) |
| 949 |
self.assertEqual(q.qsize(), 0) |
| 950 |
|
| 951 |
def test_negative_timeout_raises_exception(self): |
| 952 |
q = self.q |
| 953 |
q.put(1) |
| 954 |
with self.assertRaises(ValueError): |
| 955 |
q.get(timeout=-1) |
| 956 |
|
| 957 |
def test_order(self): |
| 958 |
# Test a pair of concurrent put() and get() |
| 959 |
q = self.q |
| 960 |
inputs = list(range(100)) |
| 961 |
results = self.run_threads(1, q, inputs, self.feed, self.consume) |
| 962 |
|
| 963 |
# One producer, one consumer => results appended in well-defined order |
| 964 |
self.assertEqual(results, inputs) |
| 965 |
|
| 966 |
@bigmemtest(size=50, memuse=100*2**20, dry_run=False) |
| 967 |
def test_many_threads(self, size): |
| 968 |
# Test multiple concurrent put() and get() |
| 969 |
q = self.q |
| 970 |
inputs = list(range(10000)) |
| 971 |
results = self.run_threads(size, q, inputs, self.feed, self.consume) |
| 972 |
|
| 973 |
# Multiple consumers without synchronization append the |
| 974 |
# results in random order |
| 975 |
self.assertEqual(sorted(results), inputs) |
| 976 |
|
| 977 |
@bigmemtest(size=50, memuse=100*2**20, dry_run=False) |
| 978 |
def test_many_threads_nonblock(self, size): |
| 979 |
# Test multiple concurrent put() and get(block=False) |
| 980 |
q = self.q |
| 981 |
inputs = list(range(10000)) |
| 982 |
results = self.run_threads(size, q, inputs, |
| 983 |
self.feed, self.consume_nonblock) |
| 984 |
|
| 985 |
self.assertEqual(sorted(results), inputs) |
| 986 |
|
| 987 |
@bigmemtest(size=50, memuse=100*2**20, dry_run=False) |
| 988 |
def test_many_threads_timeout(self, size): |
| 989 |
# Test multiple concurrent put() and get(timeout=...) |
| 990 |
q = self.q |
| 991 |
inputs = list(range(1000)) |
| 992 |
results = self.run_threads(size, q, inputs, |
| 993 |
self.feed, self.consume_timeout) |
| 994 |
|
| 995 |
self.assertEqual(sorted(results), inputs) |
| 996 |
|
| 997 |
def test_references(self): |
| 998 |
# The queue should lose references to each item as soon as |
| 999 |
# it leaves the queue. |
| 1000 |
class C: |
| 1001 |
pass |
| 1002 |
|
| 1003 |
N = 20 |
| 1004 |
q = self.q |
| 1005 |
for i in range(N): |
| 1006 |
q.put(C()) |
| 1007 |
for i in range(N): |
| 1008 |
wr = weakref.ref(q.get()) |
| 1009 |
gc_collect() # For PyPy or other GCs. |
| 1010 |
self.assertIsNone(wr()) |
| 1011 |
|
| 1012 |
|
| 1013 |
class PySimpleQueueTest(BaseSimpleQueueTest, unittest.TestCase): |
| 1014 |
|
| 1015 |
queue = py_queue |
| 1016 |
def setUp(self): |
| 1017 |
self.type2test = self.queue._PySimpleQueue |
| 1018 |
super().setUp() |
| 1019 |
|
| 1020 |
|
| 1021 |
@need_c_queue |
| 1022 |
class CSimpleQueueTest(BaseSimpleQueueTest, unittest.TestCase): |
| 1023 |
|
| 1024 |
queue = c_queue |
| 1025 |
|
| 1026 |
def setUp(self): |
| 1027 |
self.type2test = self.queue.SimpleQueue |
| 1028 |
super().setUp() |
| 1029 |
|
| 1030 |
def test_is_default(self): |
| 1031 |
self.assertIs(self.type2test, self.queue.SimpleQueue) |
| 1032 |
self.assertIs(self.type2test, self.queue.SimpleQueue) |
| 1033 |
|
| 1034 |
def test_reentrancy(self): |
| 1035 |
# bpo-14976: put() may be called reentrantly in an asynchronous |
| 1036 |
# callback. |
| 1037 |
q = self.q |
| 1038 |
gen = itertools.count() |
| 1039 |
N = 10000 |
| 1040 |
results = [] |
| 1041 |
|
| 1042 |
# This test exploits the fact that __del__ in a reference cycle |
| 1043 |
# can be called any time the GC may run. |
| 1044 |
|
| 1045 |
class Circular(object): |
| 1046 |
def __init__(self): |
| 1047 |
self.circular = self |
| 1048 |
|
| 1049 |
def __del__(self): |
| 1050 |
q.put(next(gen)) |
| 1051 |
|
| 1052 |
while True: |
| 1053 |
o = Circular() |
| 1054 |
q.put(next(gen)) |
| 1055 |
del o |
| 1056 |
results.append(q.get()) |
| 1057 |
if results[-1] >= N: |
| 1058 |
break |
| 1059 |
|
| 1060 |
self.assertEqual(results, list(range(N + 1))) |
| 1061 |
|
| 1062 |
|
| 1063 |
if __name__ == "__main__": |
| 1064 |
unittest.main() |