Debian Codegraph @

loky/backend/synchronize.py

raw
1 ###############################################################################
2 # Synchronization primitives based on our SemLock implementation
3 #
4 # author: Thomas Moreau and Olivier Grisel
5 #
6 # adapted from multiprocessing/synchronize.py (17/02/2017)
7 # * Remove ctx argument for compatibility reason
8 # * Registers a cleanup function with the loky resource_tracker to remove the
9 # semaphore when the process dies instead.
10 #
11 # TODO: investigate which Python version is required to be able to use
12 # multiprocessing.resource_tracker and therefore multiprocessing.synchronize
13 # instead of a loky-specific fork.
14
15 import os
16 import sys
17 import tempfile
18 import threading
19 import _multiprocessing
20 from time import time as _time
21 from multiprocessing import process, util
22 from multiprocessing.context import assert_spawning
23
24 from . import resource_tracker
25
26 __all__ = [
27 "Lock",
28 "RLock",
29 "Semaphore",
30 "BoundedSemaphore",
31 "Condition",
32 "Event",
33 ]
34 # Try to import the mp.synchronize module cleanly, if it fails
35 # raise ImportError for platforms lacking a working sem_open implementation.
36 # See issue 3770
37 try:
38 from _multiprocessing import SemLock as _SemLock
39 from _multiprocessing import sem_unlink
40 except ImportError:
41 raise ImportError(
42 "This platform lacks a functioning sem_open"
43 " implementation, therefore, the required"
44 " synchronization primitives needed will not"
45 " function, see issue 3770."
46 )
47
48 #
49 # Constants
50 #
51
52 RECURSIVE_MUTEX, SEMAPHORE = range(2)
53 SEM_VALUE_MAX = _multiprocessing.SemLock.SEM_VALUE_MAX
54
55
56 #
57 # Base class for semaphores and mutexes; wraps `_multiprocessing.SemLock`
58 #
59
60
61 class SemLock:
62
63 _rand = tempfile._RandomNameSequence()
64
65 def __init__(self, kind, value, maxvalue, name=None):
66 # unlink_now is only used on win32 or when we are using fork.
67 unlink_now = False
68 if name is None:
69 # Try to find an unused name for the SemLock instance.
70 for _ in range(100):
71 try:
72 self._semlock = _SemLock(
73 kind, value, maxvalue, SemLock._make_name(), unlink_now
74 )
75 except FileExistsError: # pragma: no cover
76 pass
77 else:
78 break
79 else: # pragma: no cover
80 raise FileExistsError("cannot find name for semaphore")
81 else:
82 self._semlock = _SemLock(kind, value, maxvalue, name, unlink_now)
83 self.name = name
84 util.debug(
85 f"created semlock with handle {self._semlock.handle} and name "
86 f'"{self.name}"'
87 )
88
89 self._make_methods()
90
91 def _after_fork(obj):
92 obj._semlock._after_fork()
93
94 util.register_after_fork(self, _after_fork)
95
96 # When the object is garbage collected or the
97 # process shuts down we unlink the semaphore name
98 resource_tracker.register(self._semlock.name, "semlock")
99 util.Finalize(
100 self, SemLock._cleanup, (self._semlock.name,), exitpriority=0
101 )
102
103 @staticmethod
104 def _cleanup(name):
105 try:
106 sem_unlink(name)
107 except FileNotFoundError:
108 # Already unlinked, possibly by user code: ignore and make sure to
109 # unregister the semaphore from the resource tracker.
110 pass
111 finally:
112 resource_tracker.unregister(name, "semlock")
113
114 def _make_methods(self):
115 self.acquire = self._semlock.acquire
116 self.release = self._semlock.release
117
118 def __enter__(self):
119 return self._semlock.acquire()
120
121 def __exit__(self, *args):
122 return self._semlock.release()
123
124 def __getstate__(self):
125 assert_spawning(self)
126 sl = self._semlock
127 h = sl.handle
128 return (h, sl.kind, sl.maxvalue, sl.name)
129
130 def __setstate__(self, state):
131 self._semlock = _SemLock._rebuild(*state)
132 util.debug(
133 f'recreated blocker with handle {state[0]!r} and name "{state[3]}"'
134 )
135 self._make_methods()
136
137 @staticmethod
138 def _make_name():
139 # OSX does not support long names for semaphores
140 return f"/loky-{os.getpid()}-{next(SemLock._rand)}"
141
142
143 #
144 # Semaphore
145 #
146
147
148 class Semaphore(SemLock):
149 def __init__(self, value=1):
150 SemLock.__init__(self, SEMAPHORE, value, SEM_VALUE_MAX)
151
152 def get_value(self):
153 if sys.platform == "darwin":
154 raise NotImplementedError("OSX does not implement sem_getvalue")
155 return self._semlock._get_value()
156
157 def __repr__(self):
158 try:
159 value = self._semlock._get_value()
160 except Exception:
161 value = "unknown"
162 return f"<{self.__class__.__name__}(value={value})>"
163
164
165 #
166 # Bounded semaphore
167 #
168
169
170 class BoundedSemaphore(Semaphore):
171 def __init__(self, value=1):
172 SemLock.__init__(self, SEMAPHORE, value, value)
173
174 def __repr__(self):
175 try:
176 value = self._semlock._get_value()
177 except Exception:
178 value = "unknown"
179 return (
180 f"<{self.__class__.__name__}(value={value}, "
181 f"maxvalue={self._semlock.maxvalue})>"
182 )
183
184
185 #
186 # Non-recursive lock
187 #
188
189
190 class Lock(SemLock):
191 def __init__(self):
192 super().__init__(SEMAPHORE, 1, 1)
193
194 def __repr__(self):
195 try:
196 if self._semlock._is_mine():
197 name = process.current_process().name
198 if threading.current_thread().name != "MainThread":
199 name = f"{name}|{threading.current_thread().name}"
200 elif self._semlock._get_value() == 1:
201 name = "None"
202 elif self._semlock._count() > 0:
203 name = "SomeOtherThread"
204 else:
205 name = "SomeOtherProcess"
206 except Exception:
207 name = "unknown"
208 return f"<{self.__class__.__name__}(owner={name})>"
209
210
211 #
212 # Recursive lock
213 #
214
215
216 class RLock(SemLock):
217 def __init__(self):
218 super().__init__(RECURSIVE_MUTEX, 1, 1)
219
220 def __repr__(self):
221 try:
222 if self._semlock._is_mine():
223 name = process.current_process().name
224 if threading.current_thread().name != "MainThread":
225 name = f"{name}|{threading.current_thread().name}"
226 count = self._semlock._count()
227 elif self._semlock._get_value() == 1:
228 name, count = "None", 0
229 elif self._semlock._count() > 0:
230 name, count = "SomeOtherThread", "nonzero"
231 else:
232 name, count = "SomeOtherProcess", "nonzero"
233 except Exception:
234 name, count = "unknown", "unknown"
235 return f"<{self.__class__.__name__}({name}, {count})>"
236
237
238 #
239 # Condition variable
240 #
241
242
243 class Condition:
244 def __init__(self, lock=None):
245 self._lock = lock or RLock()
246 self._sleeping_count = Semaphore(0)
247 self._woken_count = Semaphore(0)
248 self._wait_semaphore = Semaphore(0)
249 self._make_methods()
250
251 def __getstate__(self):
252 assert_spawning(self)
253 return (
254 self._lock,
255 self._sleeping_count,
256 self._woken_count,
257 self._wait_semaphore,
258 )
259
260 def __setstate__(self, state):
261 (
262 self._lock,
263 self._sleeping_count,
264 self._woken_count,
265 self._wait_semaphore,
266 ) = state
267 self._make_methods()
268
269 def __enter__(self):
270 return self._lock.__enter__()
271
272 def __exit__(self, *args):
273 return self._lock.__exit__(*args)
274
275 def _make_methods(self):
276 self.acquire = self._lock.acquire
277 self.release = self._lock.release
278
279 def __repr__(self):
280 try:
281 num_waiters = (
282 self._sleeping_count._semlock._get_value()
283 - self._woken_count._semlock._get_value()
284 )
285 except Exception:
286 num_waiters = "unknown"
287 return f"<{self.__class__.__name__}({self._lock}, {num_waiters})>"
288
289 def wait(self, timeout=None):
290 assert (
291 self._lock._semlock._is_mine()
292 ), "must acquire() condition before using wait()"
293
294 # indicate that this thread is going to sleep
295 self._sleeping_count.release()
296
297 # release lock
298 count = self._lock._semlock._count()
299 for _ in range(count):
300 self._lock.release()
301
302 try:
303 # wait for notification or timeout
304 return self._wait_semaphore.acquire(True, timeout)
305 finally:
306 # indicate that this thread has woken
307 self._woken_count.release()
308
309 # reacquire lock
310 for _ in range(count):
311 self._lock.acquire()
312
313 def notify(self):
314 assert self._lock._semlock._is_mine(), "lock is not owned"
315 assert not self._wait_semaphore.acquire(False)
316
317 # to take account of timeouts since last notify() we subtract
318 # woken_count from sleeping_count and rezero woken_count
319 while self._woken_count.acquire(False):
320 res = self._sleeping_count.acquire(False)
321 assert res
322
323 if self._sleeping_count.acquire(False): # try grabbing a sleeper
324 self._wait_semaphore.release() # wake up one sleeper
325 self._woken_count.acquire() # wait for the sleeper to wake
326
327 # rezero _wait_semaphore in case a timeout just happened
328 self._wait_semaphore.acquire(False)
329
330 def notify_all(self):
331 assert self._lock._semlock._is_mine(), "lock is not owned"
332 assert not self._wait_semaphore.acquire(False)
333
334 # to take account of timeouts since last notify*() we subtract
335 # woken_count from sleeping_count and rezero woken_count
336 while self._woken_count.acquire(False):
337 res = self._sleeping_count.acquire(False)
338 assert res
339
340 sleepers = 0
341 while self._sleeping_count.acquire(False):
342 self._wait_semaphore.release() # wake up one sleeper
343 sleepers += 1
344
345 if sleepers:
346 for _ in range(sleepers):
347 self._woken_count.acquire() # wait for a sleeper to wake
348
349 # rezero wait_semaphore in case some timeouts just happened
350 while self._wait_semaphore.acquire(False):
351 pass
352
353 def wait_for(self, predicate, timeout=None):
354 result = predicate()
355 if result:
356 return result
357 if timeout is not None:
358 endtime = _time() + timeout
359 else:
360 endtime = None
361 waittime = None
362 while not result:
363 if endtime is not None:
364 waittime = endtime - _time()
365 if waittime <= 0:
366 break
367 self.wait(waittime)
368 result = predicate()
369 return result
370
371
372 #
373 # Event
374 #
375
376
377 class Event:
378 def __init__(self):
379 self._cond = Condition(Lock())
380 self._flag = Semaphore(0)
381
382 def is_set(self):
383 with self._cond:
384 if self._flag.acquire(False):
385 self._flag.release()
386 return True
387 return False
388
389 def set(self):
390 with self._cond:
391 self._flag.acquire(False)
392 self._flag.release()
393 self._cond.notify_all()
394
395 def clear(self):
396 with self._cond:
397 self._flag.acquire(False)
398
399 def wait(self, timeout=None):
400 with self._cond:
401 if self._flag.acquire(False):
402 self._flag.release()
403 else:
404 self._cond.wait(timeout)
405
406 if self._flag.acquire(False):
407 self._flag.release()
408 return True
409 return False