Debian Codegraph @

Lib/concurrent/interpreters/_queues.py

raw
1 """Cross-interpreter Queues High Level Module."""
2
3 import pickle
4 import queue
5 import time
6 import weakref
7 import _interpqueues as _queues
8 from . import _crossinterp
9
10 # aliases:
11 from _interpqueues import (
12 QueueError, QueueNotFoundError,
13 )
14 from ._crossinterp import (
15 UNBOUND_ERROR, UNBOUND_REMOVE,
16 )
17
18 __all__ = [
19 'UNBOUND', 'UNBOUND_ERROR', 'UNBOUND_REMOVE',
20 'create', 'list_all',
21 'Queue',
22 'QueueError', 'QueueNotFoundError', 'QueueEmpty', 'QueueFull',
23 'ItemInterpreterDestroyed',
24 ]
25
26
27 class QueueEmpty(QueueError, queue.Empty):
28 """Raised from get_nowait() when the queue is empty.
29
30 It is also raised from get() if it times out.
31 """
32
33
34 class QueueFull(QueueError, queue.Full):
35 """Raised from put_nowait() when the queue is full.
36
37 It is also raised from put() if it times out.
38 """
39
40
41 class ItemInterpreterDestroyed(QueueError,
42 _crossinterp.ItemInterpreterDestroyed):
43 """Raised from get() and get_nowait()."""
44
45
46 _SHARED_ONLY = 0
47 _PICKLED = 1
48
49
50 UNBOUND = _crossinterp.UnboundItem.singleton('queue', __name__)
51
52
53 def _serialize_unbound(unbound):
54 if unbound is UNBOUND:
55 unbound = _crossinterp.UNBOUND
56 return _crossinterp.serialize_unbound(unbound)
57
58
59 def _resolve_unbound(flag):
60 resolved = _crossinterp.resolve_unbound(flag, ItemInterpreterDestroyed)
61 if resolved is _crossinterp.UNBOUND:
62 resolved = UNBOUND
63 return resolved
64
65
66 def create(maxsize=0, *, unbounditems=UNBOUND):
67 """Return a new cross-interpreter queue.
68
69 The queue may be used to pass data safely between interpreters.
70
71 "unbounditems" sets the default for Queue.put(); see that method for
72 supported values. The default value is UNBOUND, which replaces
73 the unbound item.
74 """
75 unbound = _serialize_unbound(unbounditems)
76 unboundop, = unbound
77 qid = _queues.create(maxsize, unboundop, -1)
78 self = Queue(qid)
79 self._set_unbound(unboundop, unbounditems)
80 return self
81
82
83 def list_all():
84 """Return a list of all open queues."""
85 queues = []
86 for qid, unboundop, _ in _queues.list_all():
87 self = Queue(qid)
88 if not hasattr(self, '_unbound'):
89 self._set_unbound(unboundop)
90 else:
91 assert self._unbound[0] == unboundop
92 queues.append(self)
93 return queues
94
95
96 _known_queues = weakref.WeakValueDictionary()
97
98 class Queue:
99 """A cross-interpreter queue."""
100
101 def __new__(cls, id, /):
102 # There is only one instance for any given ID.
103 if isinstance(id, int):
104 id = int(id)
105 else:
106 raise TypeError(f'id must be an int, got {id!r}')
107 try:
108 self = _known_queues[id]
109 except KeyError:
110 self = super().__new__(cls)
111 self._id = id
112 _known_queues[id] = self
113 _queues.bind(id)
114 return self
115
116 def __del__(self):
117 try:
118 _queues.release(self._id)
119 except QueueNotFoundError:
120 pass
121 try:
122 del _known_queues[self._id]
123 except KeyError:
124 pass
125
126 def __repr__(self):
127 return f'{type(self).__name__}({self.id})'
128
129 def __hash__(self):
130 return hash(self._id)
131
132 # for pickling:
133 def __reduce__(self):
134 return (type(self), (self._id,))
135
136 def _set_unbound(self, op, items=None):
137 assert not hasattr(self, '_unbound')
138 if items is None:
139 items = _resolve_unbound(op)
140 unbound = (op, items)
141 self._unbound = unbound
142 return unbound
143
144 @property
145 def id(self):
146 return self._id
147
148 @property
149 def unbounditems(self):
150 try:
151 _, items = self._unbound
152 except AttributeError:
153 op, _ = _queues.get_queue_defaults(self._id)
154 _, items = self._set_unbound(op)
155 return items
156
157 @property
158 def maxsize(self):
159 try:
160 return self._maxsize
161 except AttributeError:
162 self._maxsize = _queues.get_maxsize(self._id)
163 return self._maxsize
164
165 def empty(self):
166 return self.qsize() == 0
167
168 def full(self):
169 return _queues.is_full(self._id)
170
171 def qsize(self):
172 return _queues.get_count(self._id)
173
174 def put(self, obj, block=True, timeout=None, *,
175 unbounditems=None,
176 _delay=10 / 1000, # 10 milliseconds
177 ):
178 """Add the object to the queue.
179
180 If "block" is true, this blocks while the queue is full.
181
182 For most objects, the object received through Queue.get() will
183 be a new one, equivalent to the original and not sharing any
184 actual underlying data. The notable exceptions include
185 cross-interpreter types (like Queue) and memoryview, where the
186 underlying data is actually shared. Furthermore, some types
187 can be sent through a queue more efficiently than others. This
188 group includes various immutable types like int, str, bytes, and
189 tuple (if the items are likewise efficiently shareable). See interpreters.is_shareable().
190
191 "unbounditems" controls the behavior of Queue.get() for the given
192 object if the current interpreter (calling put()) is later
193 destroyed.
194
195 If "unbounditems" is None (the default) then it uses the
196 queue's default, set with create_queue(),
197 which is usually UNBOUND.
198
199 If "unbounditems" is UNBOUND_ERROR then get() will raise an
200 ItemInterpreterDestroyed exception if the original interpreter
201 has been destroyed. This does not otherwise affect the queue;
202 the next call to put() will work like normal, returning the next
203 item in the queue.
204
205 If "unbounditems" is UNBOUND_REMOVE then the item will be removed
206 from the queue as soon as the original interpreter is destroyed.
207 Be aware that this will introduce an imbalance between put()
208 and get() calls.
209
210 If "unbounditems" is UNBOUND then it is returned by get() in place
211 of the unbound item.
212 """
213 if not block:
214 return self.put_nowait(obj, unbounditems=unbounditems)
215 if unbounditems is None:
216 unboundop = -1
217 else:
218 unboundop, = _serialize_unbound(unbounditems)
219 if timeout is not None:
220 timeout = int(timeout)
221 if timeout < 0:
222 raise ValueError(f'timeout value must be non-negative')
223 end = time.time() + timeout
224 while True:
225 try:
226 _queues.put(self._id, obj, unboundop)
227 except QueueFull as exc:
228 if timeout is not None and time.time() >= end:
229 raise # re-raise
230 time.sleep(_delay)
231 else:
232 break
233
234 def put_nowait(self, obj, *, unbounditems=None):
235 if unbounditems is None:
236 unboundop = -1
237 else:
238 unboundop, = _serialize_unbound(unbounditems)
239 _queues.put(self._id, obj, unboundop)
240
241 def get(self, block=True, timeout=None, *,
242 _delay=10 / 1000, # 10 milliseconds
243 ):
244 """Return the next object from the queue.
245
246 If "block" is true, this blocks while the queue is empty.
247
248 If the next item's original interpreter has been destroyed
249 then the "next object" is determined by the value of the
250 "unbounditems" argument to put().
251 """
252 if not block:
253 return self.get_nowait()
254 if timeout is not None:
255 timeout = int(timeout)
256 if timeout < 0:
257 raise ValueError(f'timeout value must be non-negative')
258 end = time.time() + timeout
259 while True:
260 try:
261 obj, unboundop = _queues.get(self._id)
262 except QueueEmpty as exc:
263 if timeout is not None and time.time() >= end:
264 raise # re-raise
265 time.sleep(_delay)
266 else:
267 break
268 if unboundop is not None:
269 assert obj is None, repr(obj)
270 return _resolve_unbound(unboundop)
271 return obj
272
273 def get_nowait(self):
274 """Return the next object from the channel.
275
276 If the queue is empty then raise QueueEmpty. Otherwise this
277 is the same as get().
278 """
279 try:
280 obj, unboundop = _queues.get(self._id)
281 except QueueEmpty as exc:
282 raise # re-raise
283 if unboundop is not None:
284 assert obj is None, repr(obj)
285 return _resolve_unbound(unboundop)
286 return obj
287
288
289 _queues._register_heap_types(Queue, QueueEmpty, QueueFull)