Debian Codegraph @

Lib/concurrent/interpreters/_queues.py

raw
1 """Cross-interpreter Queues High Level Module."""
2
3 import pickle
4 import queue
5 import time
6 import weakref
7 import _interpqueues as _queues
8 from . import _crossinterp
9
10 # aliases:
11 from _interpqueues import (
12 QueueError, QueueNotFoundError,
13 )
14 from ._crossinterp import (
15 UNBOUND_ERROR, UNBOUND_REMOVE,
16 )
17
18 __all__ = [
19 'UNBOUND', 'UNBOUND_ERROR', 'UNBOUND_REMOVE',
20 'create', 'list_all',
21 'Queue',
22 'QueueError', 'QueueNotFoundError', 'QueueEmpty', 'QueueFull',
23 'ItemInterpreterDestroyed',
24 ]
25
26
27 class QueueEmpty(QueueError, queue.Empty):
28 """Raised from get_nowait() when the queue is empty.
29
30 It is also raised from get() if it times out.
31 """
32
33
34 class QueueFull(QueueError, queue.Full):
35 """Raised from put_nowait() when the queue is full.
36
37 It is also raised from put() if it times out.
38 """
39
40
41 class ItemInterpreterDestroyed(QueueError,
42 _crossinterp.ItemInterpreterDestroyed):
43 """Raised from get() and get_nowait()."""
44
45
46 _SHARED_ONLY = 0
47 _PICKLED = 1
48
49
50 UNBOUND = _crossinterp.UnboundItem.singleton('queue', __name__)
51
52
53 def _serialize_unbound(unbound):
54 if unbound is UNBOUND:
55 unbound = _crossinterp.UNBOUND
56 return _crossinterp.serialize_unbound(unbound)
57
58
59 def _resolve_unbound(flag):
60 resolved = _crossinterp.resolve_unbound(flag, ItemInterpreterDestroyed)
61 if resolved is _crossinterp.UNBOUND:
62 resolved = UNBOUND
63 return resolved
64
65
66 def create(maxsize=0, *, unbounditems=UNBOUND):
67 """Return a new cross-interpreter queue.
68
69 The queue may be used to pass data safely between interpreters.
70
71 "unbounditems" sets the default for Queue.put(); see that method for
72 supported values. The default value is UNBOUND, which replaces
73 the unbound item.
74 """
75 unbound = _serialize_unbound(unbounditems)
76 unboundop, = unbound
77 qid = _queues.create(maxsize, unboundop, -1)
78 self = Queue(qid)
79 self._set_unbound(unboundop, unbounditems)
80 return self
81
82
83 def list_all():
84 """Return a list of all open queues."""
85 queues = []
86 for qid, unboundop, _ in _queues.list_all():
87 self = Queue(qid)
88 if not hasattr(self, '_unbound'):
89 self._set_unbound(unboundop)
90 else:
91 assert self._unbound[0] == unboundop
92 queues.append(self)
93 return queues
94
95
96 _known_queues = weakref.WeakValueDictionary()
97
98 class Queue:
99 """A cross-interpreter queue."""
100
101 def __new__(cls, id, /):
102 # There is only one instance for any given ID.
103 if isinstance(id, int):
104 id = int(id)
105 else:
106 raise TypeError(f'id must be an int, got {id!r}')
107 try:
108 self = _known_queues[id]
109 except KeyError:
110 self = super().__new__(cls)
111 self._id = id
112 _known_queues[id] = self
113 _queues.bind(id)
114 return self
115
116 def __del__(self):
117 try:
118 _queues.release(self._id)
119 except QueueNotFoundError:
120 pass
121 try:
122 del _known_queues[self._id]
123 except KeyError:
124 pass
125
126 def __repr__(self):
127 return f'{type(self).__name__}({self.id})'
128
129 def __hash__(self):
130 return hash(self._id)
131
132 # for pickling:
133 def __reduce__(self):
134 return (type(self), (self._id,))
135
136 def _set_unbound(self, op, items=None):
137 assert not hasattr(self, '_unbound')
138 if items is None:
139 items = _resolve_unbound(op)
140 unbound = (op, items)
141 self._unbound = unbound
142 return unbound
143
144 @property
145 def id(self):
146 return self._id
147
148 @property
149 def unbounditems(self):
150 try:
151 _, items = self._unbound
152 except AttributeError:
153 op, _ = _queues.get_queue_defaults(self._id)
154 _, items = self._set_unbound(op)
155 return items
156
157 @property
158 def maxsize(self):
159 try:
160 return self._maxsize
161 except AttributeError:
162 self._maxsize = _queues.get_maxsize(self._id)
163 return self._maxsize
164
165 def empty(self):
166 return self.qsize() == 0
167
168 def full(self):
169 return _queues.is_full(self._id)
170
171 def qsize(self):
172 return _queues.get_count(self._id)
173
174 def put(self, obj, block=True, timeout=None, *,
175 unbounditems=None,
176 _delay=10 / 1000, # 10 milliseconds
177 ):
178 """Add the object to the queue.
179
180 If "block" is true, this blocks while the queue is full.
181
182 For most objects, the object received through Queue.get() will
183 be a new one, equivalent to the original and not sharing any
184 actual underlying data. The notable exceptions include
185 cross-interpreter types (like Queue) and memoryview, where the
186 underlying data is actually shared. Furthermore, some types
187 can be sent through a queue more efficiently than others. This
188 group includes various immutable types like int, str, bytes, and
189 tuple (if the items are likewise efficiently shareable).
190 See interpreters.is_shareable().
191
192 "unbounditems" controls the behavior of Queue.get() for the given
193 object if the current interpreter (calling put()) is later
194 destroyed.
195
196 If "unbounditems" is None (the default) then it uses the
197 queue's default, set with create_queue(),
198 which is usually UNBOUND.
199
200 If "unbounditems" is UNBOUND_ERROR then get() will raise an
201 ItemInterpreterDestroyed exception if the original interpreter
202 has been destroyed. This does not otherwise affect the queue;
203 the next call to put() will work like normal, returning the next
204 item in the queue.
205
206 If "unbounditems" is UNBOUND_REMOVE then the item will be removed
207 from the queue as soon as the original interpreter is destroyed.
208 Be aware that this will introduce an imbalance between put()
209 and get() calls.
210
211 If "unbounditems" is UNBOUND then it is returned by get() in place
212 of the unbound item.
213 """
214 if not block:
215 return self.put_nowait(obj, unbounditems=unbounditems)
216 if unbounditems is None:
217 unboundop = -1
218 else:
219 unboundop, = _serialize_unbound(unbounditems)
220 if timeout is not None:
221 timeout = int(timeout)
222 if timeout < 0:
223 raise ValueError(f'timeout value must be non-negative')
224 end = time.time() + timeout
225 while True:
226 try:
227 _queues.put(self._id, obj, unboundop)
228 except QueueFull as exc:
229 if timeout is not None and time.time() >= end:
230 raise # re-raise
231 time.sleep(_delay)
232 else:
233 break
234
235 def put_nowait(self, obj, *, unbounditems=None):
236 if unbounditems is None:
237 unboundop = -1
238 else:
239 unboundop, = _serialize_unbound(unbounditems)
240 _queues.put(self._id, obj, unboundop)
241
242 def get(self, block=True, timeout=None, *,
243 _delay=10 / 1000, # 10 milliseconds
244 ):
245 """Return the next object from the queue.
246
247 If "block" is true, this blocks while the queue is empty.
248
249 If the next item's original interpreter has been destroyed
250 then the "next object" is determined by the value of the
251 "unbounditems" argument to put().
252 """
253 if not block:
254 return self.get_nowait()
255 if timeout is not None:
256 timeout = int(timeout)
257 if timeout < 0:
258 raise ValueError(f'timeout value must be non-negative')
259 end = time.time() + timeout
260 while True:
261 try:
262 obj, unboundop = _queues.get(self._id)
263 except QueueEmpty as exc:
264 if timeout is not None and time.time() >= end:
265 raise # re-raise
266 time.sleep(_delay)
267 else:
268 break
269 if unboundop is not None:
270 assert obj is None, repr(obj)
271 return _resolve_unbound(unboundop)
272 return obj
273
274 def get_nowait(self):
275 """Return the next object from the channel.
276
277 If the queue is empty then raise QueueEmpty. Otherwise this
278 is the same as get().
279 """
280 try:
281 obj, unboundop = _queues.get(self._id)
282 except QueueEmpty as exc:
283 raise # re-raise
284 if unboundop is not None:
285 assert obj is None, repr(obj)
286 return _resolve_unbound(unboundop)
287 return obj
288
289
290 _queues._register_heap_types(Queue, QueueEmpty, QueueFull)