Debian Codegraph @

Lib/test/test_interpreters/test_queues.py

raw
1 import importlib
2 import pickle
3 import threading
4 from textwrap import dedent
5 import unittest
6
7 from test.support import import_helper, Py_DEBUG
8 # Raise SkipTest if subinterpreters not supported.
9 _queues = import_helper.import_module('_interpqueues')
10 from concurrent import interpreters
11 from concurrent.interpreters import _queues as queues, _crossinterp
12 import test._crossinterp_definitions as defs
13 from .utils import _run_output, TestBase as _TestBase
14
15 HUGE_TIMEOUT = 3600
16 REPLACE = _crossinterp._UNBOUND_CONSTANT_TO_FLAG[_crossinterp.UNBOUND]
17
18
19 def get_num_queues():
20 return len(_queues.list_all())
21
22
23 class TestBase(_TestBase):
24 def tearDown(self):
25 for qid, _, _ in _queues.list_all():
26 try:
27 _queues.destroy(qid)
28 except Exception:
29 pass
30
31
32 class LowLevelTests(TestBase):
33
34 # The behaviors in the low-level module are important in as much
35 # as they are exercised by the high-level module. Therefore the
36 # most important testing happens in the high-level tests.
37 # These low-level tests cover corner cases that are not
38 # encountered by the high-level module, thus they
39 # mostly shouldn't matter as much.
40
41 def test_highlevel_reloaded(self):
42 # See gh-115490 (https://github.com/python/cpython/issues/115490).
43 importlib.reload(queues)
44
45 def test_create_destroy(self):
46 qid = _queues.create(2, REPLACE, -1)
47 _queues.destroy(qid)
48 self.assertEqual(get_num_queues(), 0)
49 with self.assertRaises(queues.QueueNotFoundError):
50 _queues.get(qid)
51 with self.assertRaises(queues.QueueNotFoundError):
52 _queues.destroy(qid)
53
54 def test_not_destroyed(self):
55 # It should have cleaned up any remaining queues.
56 stdout, stderr = self.assert_python_ok(
57 '-c',
58 dedent(f"""
59 import {_queues.__name__} as _queues
60 _queues.create(2, {REPLACE}, -1)
61 """),
62 )
63 self.assertEqual(stdout, '')
64 if Py_DEBUG:
65 self.assertNotEqual(stderr, '')
66 else:
67 self.assertEqual(stderr, '')
68
69 def test_bind_release(self):
70 with self.subTest('typical'):
71 qid = _queues.create(2, REPLACE, -1)
72 _queues.bind(qid)
73 _queues.release(qid)
74 self.assertEqual(get_num_queues(), 0)
75
76 with self.subTest('bind too much'):
77 qid = _queues.create(2, REPLACE, -1)
78 _queues.bind(qid)
79 _queues.bind(qid)
80 _queues.release(qid)
81 _queues.destroy(qid)
82 self.assertEqual(get_num_queues(), 0)
83
84 with self.subTest('nested'):
85 qid = _queues.create(2, REPLACE, -1)
86 _queues.bind(qid)
87 _queues.bind(qid)
88 _queues.release(qid)
89 _queues.release(qid)
90 self.assertEqual(get_num_queues(), 0)
91
92 with self.subTest('release without binding'):
93 qid = _queues.create(2, REPLACE, -1)
94 with self.assertRaises(queues.QueueError):
95 _queues.release(qid)
96
97
98 class QueueTests(TestBase):
99
100 def test_create(self):
101 with self.subTest('vanilla'):
102 queue = queues.create()
103 self.assertEqual(queue.maxsize, 0)
104
105 with self.subTest('small maxsize'):
106 queue = queues.create(3)
107 self.assertEqual(queue.maxsize, 3)
108
109 with self.subTest('big maxsize'):
110 queue = queues.create(100)
111 self.assertEqual(queue.maxsize, 100)
112
113 with self.subTest('no maxsize'):
114 queue = queues.create(0)
115 self.assertEqual(queue.maxsize, 0)
116
117 with self.subTest('negative maxsize'):
118 queue = queues.create(-10)
119 self.assertEqual(queue.maxsize, -10)
120
121 with self.subTest('bad maxsize'):
122 with self.assertRaises(TypeError):
123 queues.create('1')
124
125 def test_shareable(self):
126 queue1 = queues.create()
127
128 interp = interpreters.create()
129 interp.exec(dedent(f"""
130 from concurrent.interpreters import _queues as queues
131 queue1 = queues.Queue({queue1.id})
132 """));
133
134 with self.subTest('same interpreter'):
135 queue2 = queues.create()
136 queue1.put(queue2)
137 queue3 = queue1.get()
138 self.assertIs(queue3, queue2)
139
140 with self.subTest('from current interpreter'):
141 queue4 = queues.create()
142 queue1.put(queue4)
143 out = _run_output(interp, dedent("""
144 queue4 = queue1.get()
145 print(queue4.id)
146 """))
147 qid = int(out)
148 self.assertEqual(qid, queue4.id)
149
150 with self.subTest('from subinterpreter'):
151 out = _run_output(interp, dedent("""
152 queue5 = queues.create()
153 queue1.put(queue5)
154 print(queue5.id)
155 """))
156 qid = int(out)
157 queue5 = queue1.get()
158 self.assertEqual(queue5.id, qid)
159
160 def test_id_type(self):
161 queue = queues.create()
162 self.assertIsInstance(queue.id, int)
163
164 def test_custom_id(self):
165 with self.assertRaises(queues.QueueNotFoundError):
166 queues.Queue(1_000_000)
167
168 def test_id_readonly(self):
169 queue = queues.create()
170 with self.assertRaises(AttributeError):
171 queue.id = 1_000_000
172
173 def test_maxsize_readonly(self):
174 queue = queues.create(10)
175 with self.assertRaises(AttributeError):
176 queue.maxsize = 1_000_000
177
178 def test_hashable(self):
179 queue = queues.create()
180 expected = hash(queue.id)
181 actual = hash(queue)
182 self.assertEqual(actual, expected)
183
184 def test_equality(self):
185 queue1 = queues.create()
186 queue2 = queues.create()
187 self.assertEqual(queue1, queue1)
188 self.assertNotEqual(queue1, queue2)
189
190 def test_pickle(self):
191 queue = queues.create()
192 for protocol in range(pickle.HIGHEST_PROTOCOL + 1):
193 with self.subTest(protocol=protocol):
194 data = pickle.dumps(queue, protocol)
195 unpickled = pickle.loads(data)
196 self.assertEqual(unpickled, queue)
197
198
199 class TestQueueOps(TestBase):
200
201 def test_empty(self):
202 queue = queues.create()
203 before = queue.empty()
204 queue.put(None)
205 during = queue.empty()
206 queue.get()
207 after = queue.empty()
208
209 self.assertIs(before, True)
210 self.assertIs(during, False)
211 self.assertIs(after, True)
212
213 def test_full(self):
214 for maxsize in [1, 3, 11]:
215 with self.subTest(f'maxsize={maxsize}'):
216 num_to_add = maxsize
217 expected = [False] * (num_to_add * 2 + 3)
218 expected[maxsize] = True
219 expected[maxsize + 1] = True
220
221 queue = queues.create(maxsize)
222 actual = []
223 empty = [queue.empty()]
224
225 for _ in range(num_to_add):
226 actual.append(queue.full())
227 queue.put_nowait(None)
228 actual.append(queue.full())
229 with self.assertRaises(queues.QueueFull):
230 queue.put_nowait(None)
231 empty.append(queue.empty())
232
233 for _ in range(num_to_add):
234 actual.append(queue.full())
235 queue.get_nowait()
236 actual.append(queue.full())
237 with self.assertRaises(queues.QueueEmpty):
238 queue.get_nowait()
239 actual.append(queue.full())
240 empty.append(queue.empty())
241
242 self.assertEqual(actual, expected)
243 self.assertEqual(empty, [True, False, True])
244
245 # no max size
246 for args in [(), (0,), (-1,), (-10,)]:
247 with self.subTest(f'maxsize={args[0]}' if args else '<default>'):
248 num_to_add = 13
249 expected = [False] * (num_to_add * 2 + 3)
250
251 queue = queues.create(*args)
252 actual = []
253 empty = [queue.empty()]
254
255 for _ in range(num_to_add):
256 actual.append(queue.full())
257 queue.put_nowait(None)
258 actual.append(queue.full())
259 empty.append(queue.empty())
260
261 for _ in range(num_to_add):
262 actual.append(queue.full())
263 queue.get_nowait()
264 actual.append(queue.full())
265 with self.assertRaises(queues.QueueEmpty):
266 queue.get_nowait()
267 actual.append(queue.full())
268 empty.append(queue.empty())
269
270 self.assertEqual(actual, expected)
271 self.assertEqual(empty, [True, False, True])
272
273 def test_qsize(self):
274 expected = [0, 1, 2, 3, 2, 3, 2, 1, 0, 1, 0]
275 actual = []
276 queue = queues.create()
277 for _ in range(3):
278 actual.append(queue.qsize())
279 queue.put(None)
280 actual.append(queue.qsize())
281 queue.get()
282 actual.append(queue.qsize())
283 queue.put(None)
284 actual.append(queue.qsize())
285 for _ in range(3):
286 queue.get()
287 actual.append(queue.qsize())
288 queue.put(None)
289 actual.append(queue.qsize())
290 queue.get()
291 actual.append(queue.qsize())
292
293 self.assertEqual(actual, expected)
294
295 def test_put_get_main(self):
296 expected = list(range(20))
297 queue = queues.create()
298 for i in range(20):
299 queue.put(i)
300 actual = [queue.get() for _ in range(20)]
301
302 self.assertEqual(actual, expected)
303
304 def test_put_timeout(self):
305 queue = queues.create(2)
306 queue.put(None)
307 queue.put(None)
308 with self.assertRaises(queues.QueueFull):
309 queue.put(None, timeout=0.1)
310 with self.assertRaises(queues.QueueFull):
311 queue.put(None, HUGE_TIMEOUT, 0.1)
312 queue.get()
313 queue.put(None)
314
315 def test_put_nowait(self):
316 queue = queues.create(2)
317 queue.put_nowait(None)
318 queue.put_nowait(None)
319 with self.assertRaises(queues.QueueFull):
320 queue.put_nowait(None)
321 with self.assertRaises(queues.QueueFull):
322 queue.put(None, False)
323 with self.assertRaises(queues.QueueFull):
324 queue.put(None, False, timeout=HUGE_TIMEOUT)
325 queue.get()
326 queue.put_nowait(None)
327
328 def test_put_full_fallback(self):
329 for obj in [
330 None,
331 True,
332 10,
333 'spam',
334 b'spam',
335 (0, 'a'),
336 # not shareable
337 [1, 2, 3],
338 {'a': 13, 'b': 17},
339 ]:
340 with self.subTest(repr(obj)):
341 queue = queues.create()
342
343 queue.put(obj)
344 obj2 = queue.get()
345 self.assertEqual(obj2, obj)
346
347 queue.put(obj)
348 obj2 = queue.get_nowait()
349 self.assertEqual(obj2, obj)
350
351 def test_get_timeout(self):
352 queue = queues.create()
353 with self.assertRaises(queues.QueueEmpty):
354 queue.get(timeout=0.1)
355 with self.assertRaises(queues.QueueEmpty):
356 queue.get(HUGE_TIMEOUT, 0.1)
357
358 def test_get_nowait(self):
359 queue = queues.create()
360 with self.assertRaises(queues.QueueEmpty):
361 queue.get_nowait()
362 with self.assertRaises(queues.QueueEmpty):
363 queue.get(False)
364 with self.assertRaises(queues.QueueEmpty):
365 queue.get(False, timeout=HUGE_TIMEOUT)
366
367 def test_put_get_full_fallback(self):
368 expected = list(range(20))
369 queue = queues.create()
370 for methname in ('get', 'get_nowait'):
371 with self.subTest(f'{methname}()'):
372 get = getattr(queue, methname)
373
374 for i in range(20):
375 queue.put(i)
376 actual = [get() for _ in range(20)]
377 self.assertEqual(actual, expected)
378
379 obj = [1, 2, 3] # lists are not shareable
380 queue.put(obj)
381 obj2 = get()
382 self.assertEqual(obj, obj2)
383 self.assertIsNot(obj, obj2)
384
385 def test_put_get_same_interpreter(self):
386 interp = interpreters.create()
387 interp.exec(dedent("""
388 from concurrent.interpreters import _queues as queues
389 queue = queues.create()
390 """))
391 for methname in ('get', 'get_nowait'):
392 with self.subTest(f'{methname}()'):
393 interp.exec(dedent(f"""
394 orig = b'spam'
395 queue.put(orig)
396 obj = queue.{methname}()
397 assert obj == orig, 'expected: obj == orig'
398 assert obj is not orig, 'expected: obj is not orig'
399 """))
400
401 def test_put_get_different_interpreters(self):
402 interp = interpreters.create()
403 queue1 = queues.create()
404 queue2 = queues.create()
405 self.assertEqual(len(queues.list_all()), 2)
406
407 for methname in ('get', 'get_nowait'):
408 with self.subTest(f'{methname}()'):
409 obj1 = b'spam'
410 queue1.put(obj1)
411
412 out = _run_output(
413 interp,
414 dedent(f"""
415 from concurrent.interpreters import _queues as queues
416 queue1 = queues.Queue({queue1.id})
417 queue2 = queues.Queue({queue2.id})
418 assert queue1.qsize() == 1, 'expected: queue1.qsize() == 1'
419 obj = queue1.{methname}()
420 assert queue1.qsize() == 0, 'expected: queue1.qsize() == 0'
421 assert obj == b'spam', 'expected: obj == obj1'
422 # When going to another interpreter we get a copy.
423 assert id(obj) != {id(obj1)}, 'expected: obj is not obj1'
424 obj2 = b'eggs'
425 print(id(obj2))
426 assert queue2.qsize() == 0, 'expected: queue2.qsize() == 0'
427 queue2.put(obj2)
428 assert queue2.qsize() == 1, 'expected: queue2.qsize() == 1'
429 """))
430 self.assertEqual(len(queues.list_all()), 2)
431 self.assertEqual(queue1.qsize(), 0)
432 self.assertEqual(queue2.qsize(), 1)
433
434 get = getattr(queue2, methname)
435 obj2 = get()
436 self.assertEqual(obj2, b'eggs')
437 self.assertNotEqual(id(obj2), int(out))
438
439 def test_put_cleared_with_subinterpreter(self):
440 def common(queue, unbound=None, presize=0):
441 if not unbound:
442 extraargs = ''
443 elif unbound is queues.UNBOUND:
444 extraargs = ', unbounditems=queues.UNBOUND'
445 elif unbound is queues.UNBOUND_ERROR:
446 extraargs = ', unbounditems=queues.UNBOUND_ERROR'
447 elif unbound is queues.UNBOUND_REMOVE:
448 extraargs = ', unbounditems=queues.UNBOUND_REMOVE'
449 else:
450 raise NotImplementedError(repr(unbound))
451 interp = interpreters.create()
452
453 _run_output(interp, dedent(f"""
454 from concurrent.interpreters import _queues as queues
455 queue = queues.Queue({queue.id})
456 obj1 = b'spam'
457 obj2 = b'eggs'
458 queue.put(obj1{extraargs})
459 queue.put(obj2{extraargs})
460 """))
461 self.assertEqual(queue.qsize(), presize + 2)
462
463 if presize == 0:
464 obj1 = queue.get()
465 self.assertEqual(obj1, b'spam')
466 self.assertEqual(queue.qsize(), presize + 1)
467
468 return interp
469
470 with self.subTest('default'): # UNBOUND
471 queue = queues.create()
472 interp = common(queue)
473 del interp
474 obj1 = queue.get()
475 self.assertIs(obj1, queues.UNBOUND)
476 self.assertEqual(queue.qsize(), 0)
477 with self.assertRaises(queues.QueueEmpty):
478 queue.get_nowait()
479
480 with self.subTest('UNBOUND'):
481 queue = queues.create()
482 interp = common(queue, queues.UNBOUND)
483 del interp
484 obj1 = queue.get()
485 self.assertIs(obj1, queues.UNBOUND)
486 self.assertEqual(queue.qsize(), 0)
487 with self.assertRaises(queues.QueueEmpty):
488 queue.get_nowait()
489
490 with self.subTest('UNBOUND_ERROR'):
491 queue = queues.create()
492 interp = common(queue, queues.UNBOUND_ERROR)
493
494 del interp
495 self.assertEqual(queue.qsize(), 1)
496 with self.assertRaises(queues.ItemInterpreterDestroyed):
497 queue.get()
498
499 self.assertEqual(queue.qsize(), 0)
500 with self.assertRaises(queues.QueueEmpty):
501 queue.get_nowait()
502
503 with self.subTest('UNBOUND_REMOVE'):
504 queue = queues.create()
505
506 interp = common(queue, queues.UNBOUND_REMOVE)
507 del interp
508 self.assertEqual(queue.qsize(), 0)
509 with self.assertRaises(queues.QueueEmpty):
510 queue.get_nowait()
511
512 queue.put(b'ham', unbounditems=queues.UNBOUND_REMOVE)
513 self.assertEqual(queue.qsize(), 1)
514 interp = common(queue, queues.UNBOUND_REMOVE, 1)
515 self.assertEqual(queue.qsize(), 3)
516 queue.put(42, unbounditems=queues.UNBOUND_REMOVE)
517 self.assertEqual(queue.qsize(), 4)
518 del interp
519 self.assertEqual(queue.qsize(), 2)
520 obj1 = queue.get()
521 obj2 = queue.get()
522 self.assertEqual(obj1, b'ham')
523 self.assertEqual(obj2, 42)
524 self.assertEqual(queue.qsize(), 0)
525 with self.assertRaises(queues.QueueEmpty):
526 queue.get_nowait()
527
528 def test_put_cleared_with_subinterpreter_mixed(self):
529 queue = queues.create()
530 interp = interpreters.create()
531 _run_output(interp, dedent(f"""
532 from concurrent.interpreters import _queues as queues
533 queue = queues.Queue({queue.id})
534 queue.put(1, unbounditems=queues.UNBOUND)
535 queue.put(2, unbounditems=queues.UNBOUND_ERROR)
536 queue.put(3)
537 queue.put(4, unbounditems=queues.UNBOUND_REMOVE)
538 queue.put(5, unbounditems=queues.UNBOUND)
539 """))
540 self.assertEqual(queue.qsize(), 5)
541
542 del interp
543 self.assertEqual(queue.qsize(), 4)
544
545 obj1 = queue.get()
546 self.assertIs(obj1, queues.UNBOUND)
547 self.assertEqual(queue.qsize(), 3)
548
549 with self.assertRaises(queues.ItemInterpreterDestroyed):
550 queue.get()
551 self.assertEqual(queue.qsize(), 2)
552
553 obj2 = queue.get()
554 self.assertIs(obj2, queues.UNBOUND)
555 self.assertEqual(queue.qsize(), 1)
556
557 obj3 = queue.get()
558 self.assertIs(obj3, queues.UNBOUND)
559 self.assertEqual(queue.qsize(), 0)
560
561 def test_put_cleared_with_subinterpreter_multiple(self):
562 queue = queues.create()
563 interp1 = interpreters.create()
564 interp2 = interpreters.create()
565
566 queue.put(1)
567 _run_output(interp1, dedent(f"""
568 from concurrent.interpreters import _queues as queues
569 queue = queues.Queue({queue.id})
570 obj1 = queue.get()
571 queue.put(2, unbounditems=queues.UNBOUND)
572 queue.put(obj1, unbounditems=queues.UNBOUND_REMOVE)
573 """))
574 _run_output(interp2, dedent(f"""
575 from concurrent.interpreters import _queues as queues
576 queue = queues.Queue({queue.id})
577 obj2 = queue.get()
578 obj1 = queue.get()
579 """))
580 self.assertEqual(queue.qsize(), 0)
581 queue.put(3)
582 _run_output(interp1, dedent("""
583 queue.put(4, unbounditems=queues.UNBOUND)
584 # interp closed here
585 queue.put(5, unbounditems=queues.UNBOUND_REMOVE)
586 queue.put(6, unbounditems=queues.UNBOUND)
587 """))
588 _run_output(interp2, dedent("""
589 queue.put(7, unbounditems=queues.UNBOUND_ERROR)
590 # interp closed here
591 queue.put(obj1, unbounditems=queues.UNBOUND_ERROR)
592 queue.put(obj2, unbounditems=queues.UNBOUND_REMOVE)
593 queue.put(8, unbounditems=queues.UNBOUND)
594 """))
595 _run_output(interp1, dedent("""
596 queue.put(9, unbounditems=queues.UNBOUND_REMOVE)
597 queue.put(10, unbounditems=queues.UNBOUND)
598 """))
599 self.assertEqual(queue.qsize(), 10)
600
601 obj3 = queue.get()
602 self.assertEqual(obj3, 3)
603 self.assertEqual(queue.qsize(), 9)
604
605 obj4 = queue.get()
606 self.assertEqual(obj4, 4)
607 self.assertEqual(queue.qsize(), 8)
608
609 del interp1
610 self.assertEqual(queue.qsize(), 6)
611
612 # obj5 was removed
613
614 obj6 = queue.get()
615 self.assertIs(obj6, queues.UNBOUND)
616 self.assertEqual(queue.qsize(), 5)
617
618 obj7 = queue.get()
619 self.assertEqual(obj7, 7)
620 self.assertEqual(queue.qsize(), 4)
621
622 del interp2
623 self.assertEqual(queue.qsize(), 3)
624
625 # obj1
626 with self.assertRaises(queues.ItemInterpreterDestroyed):
627 queue.get()
628 self.assertEqual(queue.qsize(), 2)
629
630 # obj2 was removed
631
632 obj8 = queue.get()
633 self.assertIs(obj8, queues.UNBOUND)
634 self.assertEqual(queue.qsize(), 1)
635
636 # obj9 was removed
637
638 obj10 = queue.get()
639 self.assertIs(obj10, queues.UNBOUND)
640 self.assertEqual(queue.qsize(), 0)
641
642 def test_put_get_different_threads(self):
643 queue1 = queues.create()
644 queue2 = queues.create()
645
646 def f():
647 while True:
648 try:
649 obj = queue1.get(timeout=0.1)
650 break
651 except queues.QueueEmpty:
652 continue
653 queue2.put(obj)
654 t = threading.Thread(target=f)
655 t.start()
656
657 orig = b'spam'
658 queue1.put(orig)
659 obj = queue2.get()
660 t.join()
661
662 self.assertEqual(obj, orig)
663 self.assertIsNot(obj, orig)
664
665
666 if __name__ == '__main__':
667 # Test needs to be a package, so we can do relative imports.
668 unittest.main()