Debian Codegraph @

Lib/multiprocessing/managers.py

raw
1 #
2 # Module providing manager classes for dealing
3 # with shared objects
4 #
5 # multiprocessing/managers.py
6 #
7 # Copyright (c) 2006-2008, R Oudkerk
8 # Licensed to PSF under a Contributor Agreement.
9 #
10
11 __all__ = [ 'BaseManager', 'SyncManager', 'BaseProxy', 'Token' ]
12
13 #
14 # Imports
15 #
16
17 import sys
18 import threading
19 import signal
20 import array
21 import collections.abc
22 import queue
23 import time
24 import types
25 import os
26 from os import getpid
27
28 from traceback import format_exc
29
30 from . import connection
31 from .context import reduction, get_spawning_popen, ProcessError
32 from . import pool
33 from . import process
34 from . import util
35 from . import get_context
36 try:
37 from . import shared_memory
38 except ImportError:
39 HAS_SHMEM = False
40 else:
41 HAS_SHMEM = True
42 __all__.append('SharedMemoryManager')
43
44 #
45 # Register some things for pickling
46 #
47
48 def reduce_array(a):
49 return array.array, (a.typecode, a.tobytes())
50 reduction.register(array.array, reduce_array)
51
52 view_types = [type(getattr({}, name)()) for name in ('items','keys','values')]
53 def rebuild_as_list(obj):
54 return list, (list(obj),)
55 for view_type in view_types:
56 reduction.register(view_type, rebuild_as_list)
57 del view_type, view_types
58
59 #
60 # Type for identifying shared objects
61 #
62
63 class Token(object):
64 '''
65 Type to uniquely identify a shared object
66 '''
67 __slots__ = ('typeid', 'address', 'id')
68
69 def __init__(self, typeid, address, id):
70 (self.typeid, self.address, self.id) = (typeid, address, id)
71
72 def __getstate__(self):
73 return (self.typeid, self.address, self.id)
74
75 def __setstate__(self, state):
76 (self.typeid, self.address, self.id) = state
77
78 def __repr__(self):
79 return '%s(typeid=%r, address=%r, id=%r)' % \
80 (self.__class__.__name__, self.typeid, self.address, self.id)
81
82 #
83 # Function for communication with a manager's server process
84 #
85
86 def dispatch(c, id, methodname, args=(), kwds={}):
87 '''
88 Send a message to manager using connection `c` and return response
89 '''
90 c.send((id, methodname, args, kwds))
91 kind, result = c.recv()
92 if kind == '#RETURN':
93 return result
94 try:
95 raise convert_to_error(kind, result)
96 finally:
97 del result # break reference cycle
98
99 def convert_to_error(kind, result):
100 if kind == '#ERROR':
101 return result
102 elif kind in ('#TRACEBACK', '#UNSERIALIZABLE'):
103 if not isinstance(result, str):
104 raise TypeError(
105 "Result {0!r} (kind '{1}') type is {2}, not str".format(
106 result, kind, type(result)))
107 if kind == '#UNSERIALIZABLE':
108 return RemoteError('Unserializable message: %s\n' % result)
109 else:
110 return RemoteError(result)
111 else:
112 return ValueError('Unrecognized message type {!r}'.format(kind))
113
114 class RemoteError(Exception):
115 def __str__(self):
116 return ('\n' + '-'*75 + '\n' + str(self.args[0]) + '-'*75)
117
118 #
119 # Functions for finding the method names of an object
120 #
121
122 def all_methods(obj):
123 '''
124 Return a list of names of methods of `obj`
125 '''
126 temp = []
127 for name in dir(obj):
128 func = getattr(obj, name)
129 if callable(func):
130 temp.append(name)
131 return temp
132
133 def public_methods(obj):
134 '''
135 Return a list of names of methods of `obj` which do not start with '_'
136 '''
137 return [name for name in all_methods(obj) if name[0] != '_']
138
139 #
140 # Server which is run in a process controlled by a manager
141 #
142
143 class Server(object):
144 '''
145 Server class which runs in a process controlled by a manager object
146 '''
147 public = ['shutdown', 'create', 'accept_connection', 'get_methods',
148 'debug_info', 'number_of_objects', 'dummy', 'incref', 'decref']
149
150 def __init__(self, registry, address, authkey, serializer):
151 if not isinstance(authkey, bytes):
152 raise TypeError(
153 "Authkey {0!r} is type {1!s}, not bytes".format(
154 authkey, type(authkey)))
155 self.registry = registry
156 self.authkey = process.AuthenticationString(authkey)
157 Listener, Client = listener_client[serializer]
158
159 # do authentication later
160 self.listener = Listener(address=address, backlog=128)
161 self.address = self.listener.address
162
163 self.id_to_obj = {'0': (None, ())}
164 self.id_to_refcount = {}
165 self.id_to_local_proxy_obj = {}
166 self.mutex = threading.Lock()
167
168 def serve_forever(self):
169 '''
170 Run the server forever
171 '''
172 self.stop_event = threading.Event()
173 process.current_process()._manager_server = self
174 try:
175 accepter = threading.Thread(target=self.accepter)
176 accepter.daemon = True
177 accepter.start()
178 try:
179 while not self.stop_event.is_set():
180 self.stop_event.wait(1)
181 except (KeyboardInterrupt, SystemExit):
182 pass
183 finally:
184 if sys.stdout != sys.__stdout__: # what about stderr?
185 util.debug('resetting stdout, stderr')
186 sys.stdout = sys.__stdout__
187 sys.stderr = sys.__stderr__
188 sys.exit(0)
189
190 def accepter(self):
191 while True:
192 try:
193 c = self.listener.accept()
194 except OSError:
195 continue
196 t = threading.Thread(target=self.handle_request, args=(c,))
197 t.daemon = True
198 t.start()
199
200 def _handle_request(self, c):
201 request = None
202 try:
203 connection.deliver_challenge(c, self.authkey)
204 connection.answer_challenge(c, self.authkey)
205 request = c.recv()
206 ignore, funcname, args, kwds = request
207 assert funcname in self.public, '%r unrecognized' % funcname
208 func = getattr(self, funcname)
209 except Exception:
210 msg = ('#TRACEBACK', format_exc())
211 else:
212 try:
213 result = func(c, *args, **kwds)
214 except Exception:
215 msg = ('#TRACEBACK', format_exc())
216 else:
217 msg = ('#RETURN', result)
218
219 try:
220 c.send(msg)
221 except Exception as e:
222 try:
223 c.send(('#TRACEBACK', format_exc()))
224 except Exception:
225 pass
226 util.info('Failure to send message: %r', msg)
227 util.info(' ... request was %r', request)
228 util.info(' ... exception was %r', e)
229
230 def handle_request(self, conn):
231 '''
232 Handle a new connection
233 '''
234 try:
235 self._handle_request(conn)
236 except SystemExit:
237 # Server.serve_client() calls sys.exit(0) on EOF
238 pass
239 finally:
240 conn.close()
241
242 def serve_client(self, conn):
243 '''
244 Handle requests from the proxies in a particular process/thread
245 '''
246 util.debug('starting server thread to service %r',
247 threading.current_thread().name)
248
249 recv = conn.recv
250 send = conn.send
251 id_to_obj = self.id_to_obj
252
253 while not self.stop_event.is_set():
254
255 try:
256 methodname = obj = None
257 request = recv()
258 ident, methodname, args, kwds = request
259 try:
260 obj, exposed, gettypeid = id_to_obj[ident]
261 except KeyError as ke:
262 try:
263 obj, exposed, gettypeid = \
264 self.id_to_local_proxy_obj[ident]
265 except KeyError:
266 raise ke
267
268 if methodname not in exposed:
269 raise AttributeError(
270 'method %r of %r object is not in exposed=%r' %
271 (methodname, type(obj), exposed)
272 )
273
274 function = getattr(obj, methodname)
275
276 try:
277 res = function(*args, **kwds)
278 except Exception as e:
279 msg = ('#ERROR', e)
280 else:
281 typeid = gettypeid and gettypeid.get(methodname, None)
282 if typeid:
283 rident, rexposed = self.create(conn, typeid, res)
284 token = Token(typeid, self.address, rident)
285 msg = ('#PROXY', (rexposed, token))
286 else:
287 msg = ('#RETURN', res)
288
289 except AttributeError:
290 if methodname is None:
291 msg = ('#TRACEBACK', format_exc())
292 else:
293 try:
294 fallback_func = self.fallback_mapping[methodname]
295 result = fallback_func(
296 self, conn, ident, obj, *args, **kwds
297 )
298 msg = ('#RETURN', result)
299 except Exception:
300 msg = ('#TRACEBACK', format_exc())
301
302 except EOFError:
303 util.debug('got EOF -- exiting thread serving %r',
304 threading.current_thread().name)
305 sys.exit(0)
306
307 except Exception:
308 msg = ('#TRACEBACK', format_exc())
309
310 try:
311 try:
312 send(msg)
313 except Exception:
314 send(('#UNSERIALIZABLE', format_exc()))
315 except Exception as e:
316 util.info('exception in thread serving %r',
317 threading.current_thread().name)
318 util.info(' ... message was %r', msg)
319 util.info(' ... exception was %r', e)
320 conn.close()
321 sys.exit(1)
322
323 def fallback_getvalue(self, conn, ident, obj):
324 return obj
325
326 def fallback_str(self, conn, ident, obj):
327 return str(obj)
328
329 def fallback_repr(self, conn, ident, obj):
330 return repr(obj)
331
332 fallback_mapping = {
333 '__str__':fallback_str,
334 '__repr__':fallback_repr,
335 '#GETVALUE':fallback_getvalue
336 }
337
338 def dummy(self, c):
339 pass
340
341 def debug_info(self, c):
342 '''
343 Return some info --- useful to spot problems with refcounting
344 '''
345 # Perhaps include debug info about 'c'?
346 with self.mutex:
347 result = []
348 keys = list(self.id_to_refcount.keys())
349 keys.sort()
350 for ident in keys:
351 if ident != '0':
352 result.append(' %s: refcount=%s\n %s' %
353 (ident, self.id_to_refcount[ident],
354 str(self.id_to_obj[ident][0])[:75]))
355 return '\n'.join(result)
356
357 def number_of_objects(self, c):
358 '''
359 Number of shared objects
360 '''
361 # Doesn't use (len(self.id_to_obj) - 1) as we shouldn't count ident='0'
362 return len(self.id_to_refcount)
363
364 def shutdown(self, c):
365 '''
366 Shutdown this process
367 '''
368 try:
369 util.debug('manager received shutdown message')
370 c.send(('#RETURN', None))
371 except:
372 import traceback
373 traceback.print_exc()
374 finally:
375 self.stop_event.set()
376
377 def create(self, c, typeid, /, *args, **kwds):
378 '''
379 Create a new shared object and return its id
380 '''
381 with self.mutex:
382 callable, exposed, method_to_typeid, proxytype = \
383 self.registry[typeid]
384
385 if callable is None:
386 if kwds or (len(args) != 1):
387 raise ValueError(
388 "Without callable, must have one non-keyword argument")
389 obj = args[0]
390 else:
391 obj = callable(*args, **kwds)
392
393 if exposed is None:
394 exposed = public_methods(obj)
395 if method_to_typeid is not None:
396 if not isinstance(method_to_typeid, dict):
397 raise TypeError(
398 "Method_to_typeid {0!r}: type {1!s}, not dict".format(
399 method_to_typeid, type(method_to_typeid)))
400 exposed = list(exposed) + list(method_to_typeid)
401
402 ident = '%x' % id(obj) # convert to string because xmlrpclib
403 # only has 32 bit signed integers
404 util.debug('%r callable returned object with id %r', typeid, ident)
405
406 self.id_to_obj[ident] = (obj, set(exposed), method_to_typeid)
407 if ident not in self.id_to_refcount:
408 self.id_to_refcount[ident] = 0
409
410 self.incref(c, ident)
411 return ident, tuple(exposed)
412
413 def get_methods(self, c, token):
414 '''
415 Return the methods of the shared object indicated by token
416 '''
417 return tuple(self.id_to_obj[token.id][1])
418
419 def accept_connection(self, c, name):
420 '''
421 Spawn a new thread to serve this connection
422 '''
423 threading.current_thread().name = name
424 c.send(('#RETURN', None))
425 self.serve_client(c)
426
427 def incref(self, c, ident):
428 with self.mutex:
429 try:
430 self.id_to_refcount[ident] += 1
431 except KeyError as ke:
432 # If no external references exist but an internal (to the
433 # manager) still does and a new external reference is created
434 # from it, restore the manager's tracking of it from the
435 # previously stashed internal ref.
436 if ident in self.id_to_local_proxy_obj:
437 self.id_to_refcount[ident] = 1
438 self.id_to_obj[ident] = \
439 self.id_to_local_proxy_obj[ident]
440 util.debug('Server re-enabled tracking & INCREF %r', ident)
441 else:
442 raise ke
443
444 def decref(self, c, ident):
445 if ident not in self.id_to_refcount and \
446 ident in self.id_to_local_proxy_obj:
447 util.debug('Server DECREF skipping %r', ident)
448 return
449
450 with self.mutex:
451 if self.id_to_refcount[ident] <= 0:
452 raise AssertionError(
453 "Id {0!s} ({1!r}) has refcount {2:n}, not 1+".format(
454 ident, self.id_to_obj[ident],
455 self.id_to_refcount[ident]))
456 self.id_to_refcount[ident] -= 1
457 if self.id_to_refcount[ident] == 0:
458 del self.id_to_refcount[ident]
459
460 if ident not in self.id_to_refcount:
461 # Two-step process in case the object turns out to contain other
462 # proxy objects (e.g. a managed list of managed lists).
463 # Otherwise, deleting self.id_to_obj[ident] would trigger the
464 # deleting of the stored value (another managed object) which would
465 # in turn attempt to acquire the mutex that is already held here.
466 self.id_to_obj[ident] = (None, (), None) # thread-safe
467 util.debug('disposing of obj with id %r', ident)
468 with self.mutex:
469 del self.id_to_obj[ident]
470
471
472 #
473 # Class to represent state of a manager
474 #
475
476 class State(object):
477 __slots__ = ['value']
478 INITIAL = 0
479 STARTED = 1
480 SHUTDOWN = 2
481
482 #
483 # Mapping from serializer name to Listener and Client types
484 #
485
486 listener_client = {
487 'pickle' : (connection.Listener, connection.Client),
488 'xmlrpclib' : (connection.XmlListener, connection.XmlClient)
489 }
490
491 #
492 # Definition of BaseManager
493 #
494
495 class BaseManager(object):
496 '''
497 Base class for managers
498 '''
499 _registry = {}
500 _Server = Server
501
502 def __init__(self, address=None, authkey=None, serializer='pickle',
503 ctx=None, *, shutdown_timeout=1.0):
504 if authkey is None:
505 authkey = process.current_process().authkey
506 self._address = address # XXX not final address if eg ('', 0)
507 self._authkey = process.AuthenticationString(authkey)
508 self._state = State()
509 self._state.value = State.INITIAL
510 self._serializer = serializer
511 self._Listener, self._Client = listener_client[serializer]
512 self._ctx = ctx or get_context()
513 self._shutdown_timeout = shutdown_timeout
514
515 def get_server(self):
516 '''
517 Return server object with serve_forever() method and address attribute
518 '''
519 if self._state.value != State.INITIAL:
520 if self._state.value == State.STARTED:
521 raise ProcessError("Already started server")
522 elif self._state.value == State.SHUTDOWN:
523 raise ProcessError("Manager has shut down")
524 else:
525 raise ProcessError(
526 "Unknown state {!r}".format(self._state.value))
527 return Server(self._registry, self._address,
528 self._authkey, self._serializer)
529
530 def connect(self):
531 '''
532 Connect manager object to the server process
533 '''
534 Listener, Client = listener_client[self._serializer]
535 conn = Client(self._address, authkey=self._authkey)
536 dispatch(conn, None, 'dummy')
537 self._state.value = State.STARTED
538
539 def start(self, initializer=None, initargs=()):
540 '''
541 Spawn a server process for this manager object
542 '''
543 if self._state.value != State.INITIAL:
544 if self._state.value == State.STARTED:
545 raise ProcessError("Already started server")
546 elif self._state.value == State.SHUTDOWN:
547 raise ProcessError("Manager has shut down")
548 else:
549 raise ProcessError(
550 "Unknown state {!r}".format(self._state.value))
551
552 if initializer is not None and not callable(initializer):
553 raise TypeError('initializer must be a callable')
554
555 # pipe over which we will retrieve address of server
556 reader, writer = connection.Pipe(duplex=False)
557
558 # spawn process which runs a server
559 self._process = self._ctx.Process(
560 target=type(self)._run_server,
561 args=(self._registry, self._address, self._authkey,
562 self._serializer, writer, initializer, initargs),
563 )
564 ident = ':'.join(str(i) for i in self._process._identity)
565 self._process.name = type(self).__name__ + '-' + ident
566 self._process.start()
567
568 # get address of server
569 writer.close()
570 self._address = reader.recv()
571 reader.close()
572
573 # register a finalizer
574 self._state.value = State.STARTED
575 self.shutdown = util.Finalize(
576 self, type(self)._finalize_manager,
577 args=(self._process, self._address, self._authkey, self._state,
578 self._Client, self._shutdown_timeout),
579 exitpriority=0
580 )
581
582 @classmethod
583 def _run_server(cls, registry, address, authkey, serializer, writer,
584 initializer=None, initargs=()):
585 '''
586 Create a server, report its address and run it
587 '''
588 # bpo-36368: protect server process from KeyboardInterrupt signals
589 signal.signal(signal.SIGINT, signal.SIG_IGN)
590
591 if initializer is not None:
592 initializer(*initargs)
593
594 # create server
595 server = cls._Server(registry, address, authkey, serializer)
596
597 # inform parent process of the server's address
598 writer.send(server.address)
599 writer.close()
600
601 # run the manager
602 util.info('manager serving at %r', server.address)
603 server.serve_forever()
604
605 def _create(self, typeid, /, *args, **kwds):
606 '''
607 Create a new shared object; return the token and exposed tuple
608 '''
609 assert self._state.value == State.STARTED, 'server not yet started'
610 conn = self._Client(self._address, authkey=self._authkey)
611 try:
612 id, exposed = dispatch(conn, None, 'create', (typeid,)+args, kwds)
613 finally:
614 conn.close()
615 return Token(typeid, self._address, id), exposed
616
617 def join(self, timeout=None):
618 '''
619 Join the manager process (if it has been spawned)
620 '''
621 if self._process is not None:
622 self._process.join(timeout)
623 if not self._process.is_alive():
624 self._process = None
625
626 def _debug_info(self):
627 '''
628 Return some info about the servers shared objects and connections
629 '''
630 conn = self._Client(self._address, authkey=self._authkey)
631 try:
632 return dispatch(conn, None, 'debug_info')
633 finally:
634 conn.close()
635
636 def _number_of_objects(self):
637 '''
638 Return the number of shared objects
639 '''
640 conn = self._Client(self._address, authkey=self._authkey)
641 try:
642 return dispatch(conn, None, 'number_of_objects')
643 finally:
644 conn.close()
645
646 def __enter__(self):
647 if self._state.value == State.INITIAL:
648 self.start()
649 if self._state.value != State.STARTED:
650 if self._state.value == State.INITIAL:
651 raise ProcessError("Unable to start server")
652 elif self._state.value == State.SHUTDOWN:
653 raise ProcessError("Manager has shut down")
654 else:
655 raise ProcessError(
656 "Unknown state {!r}".format(self._state.value))
657 return self
658
659 def __exit__(self, exc_type, exc_val, exc_tb):
660 self.shutdown()
661
662 @staticmethod
663 def _finalize_manager(process, address, authkey, state, _Client,
664 shutdown_timeout):
665 '''
666 Shutdown the manager process; will be registered as a finalizer
667 '''
668 if process.is_alive():
669 util.info('sending shutdown message to manager')
670 try:
671 conn = _Client(address, authkey=authkey)
672 try:
673 dispatch(conn, None, 'shutdown')
674 finally:
675 conn.close()
676 except Exception:
677 pass
678
679 process.join(timeout=shutdown_timeout)
680 if process.is_alive():
681 util.info('manager still alive')
682 if hasattr(process, 'terminate'):
683 util.info('trying to `terminate()` manager process')
684 process.terminate()
685 process.join(timeout=shutdown_timeout)
686 if process.is_alive():
687 util.info('manager still alive after terminate')
688 process.kill()
689 process.join()
690
691 state.value = State.SHUTDOWN
692 try:
693 del BaseProxy._address_to_local[address]
694 except KeyError:
695 pass
696
697 @property
698 def address(self):
699 return self._address
700
701 @classmethod
702 def register(cls, typeid, callable=None, proxytype=None, exposed=None,
703 method_to_typeid=None, create_method=True):
704 '''
705 Register a typeid with the manager type
706 '''
707 if '_registry' not in cls.__dict__:
708 cls._registry = cls._registry.copy()
709
710 if proxytype is None:
711 proxytype = AutoProxy
712
713 exposed = exposed or getattr(proxytype, '_exposed_', None)
714
715 method_to_typeid = method_to_typeid or \
716 getattr(proxytype, '_method_to_typeid_', None)
717
718 if method_to_typeid:
719 for key, value in list(method_to_typeid.items()): # isinstance?
720 assert type(key) is str, '%r is not a string' % key
721 assert type(value) is str, '%r is not a string' % value
722
723 cls._registry[typeid] = (
724 callable, exposed, method_to_typeid, proxytype
725 )
726
727 if create_method:
728 def temp(self, /, *args, **kwds):
729 util.debug('requesting creation of a shared %r object', typeid)
730 token, exp = self._create(typeid, *args, **kwds)
731 proxy = proxytype(
732 token, self._serializer, manager=self,
733 authkey=self._authkey, exposed=exp
734 )
735 conn = self._Client(token.address, authkey=self._authkey)
736 dispatch(conn, None, 'decref', (token.id,))
737 return proxy
738 temp.__name__ = typeid
739 setattr(cls, typeid, temp)
740
741 #
742 # Subclass of set which get cleared after a fork
743 #
744
745 class ProcessLocalSet(set):
746 def __init__(self):
747 util.register_after_fork(self, lambda obj: obj.clear())
748 def __reduce__(self):
749 return type(self), ()
750
751 #
752 # Definition of BaseProxy
753 #
754
755 class BaseProxy(object):
756 '''
757 A base for proxies of shared objects
758 '''
759 _address_to_local = {}
760 _mutex = util.ForkAwareThreadLock()
761
762 # Each instance gets a `_serial` number. Unlike `id(...)`, this number
763 # is never reused.
764 _next_serial = 1
765
766 def __init__(self, token, serializer, manager=None,
767 authkey=None, exposed=None, incref=True, manager_owned=False):
768 with BaseProxy._mutex:
769 tls_serials = BaseProxy._address_to_local.get(token.address, None)
770 if tls_serials is None:
771 tls_serials = util.ForkAwareLocal(), ProcessLocalSet()
772 BaseProxy._address_to_local[token.address] = tls_serials
773
774 self._serial = BaseProxy._next_serial
775 BaseProxy._next_serial += 1
776
777 # self._tls is used to record the connection used by this
778 # thread to communicate with the manager at token.address
779 self._tls = tls_serials[0]
780
781 # self._all_serials is a set used to record the identities of all
782 # shared objects for which the current process owns references and
783 # which are in the manager at token.address
784 self._all_serials = tls_serials[1]
785
786 self._token = token
787 self._id = self._token.id
788 self._manager = manager
789 self._serializer = serializer
790 self._Client = listener_client[serializer][1]
791
792 # Should be set to True only when a proxy object is being created
793 # on the manager server; primary use case: nested proxy objects.
794 # RebuildProxy detects when a proxy is being created on the manager
795 # and sets this value appropriately.
796 self._owned_by_manager = manager_owned
797
798 if authkey is not None:
799 self._authkey = process.AuthenticationString(authkey)
800 elif self._manager is not None:
801 self._authkey = self._manager._authkey
802 else:
803 self._authkey = process.current_process().authkey
804
805 if incref:
806 self._incref()
807
808 util.register_after_fork(self, BaseProxy._after_fork)
809
810 def _connect(self):
811 util.debug('making connection to manager')
812 name = process.current_process().name
813 if threading.current_thread().name != 'MainThread':
814 name += '|' + threading.current_thread().name
815 conn = self._Client(self._token.address, authkey=self._authkey)
816 dispatch(conn, None, 'accept_connection', (name,))
817 self._tls.connection = conn
818
819 def _callmethod(self, methodname, args=(), kwds={}):
820 '''
821 Try to call a method of the referent and return a copy of the result
822 '''
823 try:
824 conn = self._tls.connection
825 except AttributeError:
826 util.debug('thread %r does not own a connection',
827 threading.current_thread().name)
828 self._connect()
829 conn = self._tls.connection
830
831 conn.send((self._id, methodname, args, kwds))
832 kind, result = conn.recv()
833
834 if kind == '#RETURN':
835 return result
836 elif kind == '#PROXY':
837 exposed, token = result
838 proxytype = self._manager._registry[token.typeid][-1]
839 token.address = self._token.address
840 proxy = proxytype(
841 token, self._serializer, manager=self._manager,
842 authkey=self._authkey, exposed=exposed
843 )
844 conn = self._Client(token.address, authkey=self._authkey)
845 dispatch(conn, None, 'decref', (token.id,))
846 return proxy
847 try:
848 raise convert_to_error(kind, result)
849 finally:
850 del result # break reference cycle
851
852 def _getvalue(self):
853 '''
854 Get a copy of the value of the referent
855 '''
856 return self._callmethod('#GETVALUE')
857
858 def _incref(self):
859 if self._owned_by_manager:
860 util.debug('owned_by_manager skipped INCREF of %r', self._token.id)
861 return
862
863 conn = self._Client(self._token.address, authkey=self._authkey)
864 dispatch(conn, None, 'incref', (self._id,))
865 util.debug('INCREF %r', self._token.id)
866
867 self._all_serials.add(self._serial)
868
869 state = self._manager and self._manager._state
870
871 self._close = util.Finalize(
872 self, BaseProxy._decref,
873 args=(self._token, self._serial, self._authkey, state,
874 self._tls, self._all_serials, self._Client),
875 exitpriority=10
876 )
877
878 @staticmethod
879 def _decref(token, serial, authkey, state, tls, idset, _Client):
880 idset.discard(serial)
881
882 # check whether manager is still alive
883 if state is None or state.value == State.STARTED:
884 # tell manager this process no longer cares about referent
885 try:
886 util.debug('DECREF %r', token.id)
887 conn = _Client(token.address, authkey=authkey)
888 dispatch(conn, None, 'decref', (token.id,))
889 except Exception as e:
890 util.debug('... decref failed %s', e)
891
892 else:
893 util.debug('DECREF %r -- manager already shutdown', token.id)
894
895 # check whether we can close this thread's connection because
896 # the process owns no more references to objects for this manager
897 if not idset and hasattr(tls, 'connection'):
898 util.debug('thread %r has no more proxies so closing conn',
899 threading.current_thread().name)
900 tls.connection.close()
901 del tls.connection
902
903 def _after_fork(self):
904 self._manager = None
905 try:
906 self._incref()
907 except Exception as e:
908 # the proxy may just be for a manager which has shutdown
909 util.info('incref failed: %s' % e)
910
911 def __reduce__(self):
912 kwds = {}
913 if get_spawning_popen() is not None:
914 kwds['authkey'] = self._authkey
915
916 if getattr(self, '_isauto', False):
917 kwds['exposed'] = self._exposed_
918 return (RebuildProxy,
919 (AutoProxy, self._token, self._serializer, kwds))
920 else:
921 return (RebuildProxy,
922 (type(self), self._token, self._serializer, kwds))
923
924 def __deepcopy__(self, memo):
925 return self._getvalue()
926
927 def __repr__(self):
928 return '<%s object, typeid %r at %#x>' % \
929 (type(self).__name__, self._token.typeid, id(self))
930
931 def __str__(self):
932 '''
933 Return representation of the referent (or a fall-back if that fails)
934 '''
935 try:
936 return self._callmethod('__repr__')
937 except Exception:
938 return repr(self)[:-1] + "; '__str__()' failed>"
939
940 #
941 # Function used for unpickling
942 #
943
944 def RebuildProxy(func, token, serializer, kwds):
945 '''
946 Function used for unpickling proxy objects.
947 '''
948 server = getattr(process.current_process(), '_manager_server', None)
949 if server and server.address == token.address:
950 util.debug('Rebuild a proxy owned by manager, token=%r', token)
951 kwds['manager_owned'] = True
952 if token.id not in server.id_to_local_proxy_obj:
953 server.id_to_local_proxy_obj[token.id] = \
954 server.id_to_obj[token.id]
955 incref = (
956 kwds.pop('incref', True) and
957 not getattr(process.current_process(), '_inheriting', False)
958 )
959 return func(token, serializer, incref=incref, **kwds)
960
961 #
962 # Functions to create proxies and proxy types
963 #
964
965 def MakeProxyType(name, exposed, _cache={}):
966 '''
967 Return a proxy type whose methods are given by `exposed`
968 '''
969 exposed = tuple(exposed)
970 try:
971 return _cache[(name, exposed)]
972 except KeyError:
973 pass
974
975 dic = {}
976
977 for meth in exposed:
978 exec('''def %s(self, /, *args, **kwds):
979 return self._callmethod(%r, args, kwds)''' % (meth, meth), dic)
980
981 ProxyType = type(name, (BaseProxy,), dic)
982 ProxyType._exposed_ = exposed
983 _cache[(name, exposed)] = ProxyType
984 return ProxyType
985
986
987 def AutoProxy(token, serializer, manager=None, authkey=None,
988 exposed=None, incref=True, manager_owned=False):
989 '''
990 Return an auto-proxy for `token`
991 '''
992 _Client = listener_client[serializer][1]
993
994 if exposed is None:
995 conn = _Client(token.address, authkey=authkey)
996 try:
997 exposed = dispatch(conn, None, 'get_methods', (token,))
998 finally:
999 conn.close()
1000
1001 if authkey is None and manager is not None:
1002 authkey = manager._authkey
1003 if authkey is None:
1004 authkey = process.current_process().authkey
1005
1006 ProxyType = MakeProxyType('AutoProxy[%s]' % token.typeid, exposed)
1007 proxy = ProxyType(token, serializer, manager=manager, authkey=authkey,
1008 incref=incref, manager_owned=manager_owned)
1009 proxy._isauto = True
1010 return proxy
1011
1012 #
1013 # Types/callables which we will register with SyncManager
1014 #
1015
1016 class Namespace(object):
1017 def __init__(self, /, **kwds):
1018 self.__dict__.update(kwds)
1019 def __repr__(self):
1020 items = list(self.__dict__.items())
1021 temp = []
1022 for name, value in items:
1023 if not name.startswith('_'):
1024 temp.append('%s=%r' % (name, value))
1025 temp.sort()
1026 return '%s(%s)' % (self.__class__.__name__, ', '.join(temp))
1027
1028 class Value(object):
1029 def __init__(self, typecode, value, lock=True):
1030 self._typecode = typecode
1031 self._value = value
1032 def get(self):
1033 return self._value
1034 def set(self, value):
1035 self._value = value
1036 def __repr__(self):
1037 return '%s(%r, %r)'%(type(self).__name__, self._typecode, self._value)
1038 value = property(get, set)
1039
1040 def Array(typecode, sequence, lock=True):
1041 return array.array(typecode, sequence)
1042
1043 #
1044 # Proxy types used by SyncManager
1045 #
1046
1047 class IteratorProxy(BaseProxy):
1048 _exposed_ = ('__next__', 'send', 'throw', 'close')
1049 def __iter__(self):
1050 return self
1051 def __next__(self, *args):
1052 return self._callmethod('__next__', args)
1053 def send(self, *args):
1054 return self._callmethod('send', args)
1055 def throw(self, *args):
1056 return self._callmethod('throw', args)
1057 def close(self, *args):
1058 return self._callmethod('close', args)
1059
1060
1061 class AcquirerProxy(BaseProxy):
1062 _exposed_ = ('acquire', 'release', 'locked')
1063 def acquire(self, blocking=True, timeout=None):
1064 args = (blocking,) if timeout is None else (blocking, timeout)
1065 return self._callmethod('acquire', args)
1066 def release(self):
1067 return self._callmethod('release')
1068 def locked(self):
1069 return self._callmethod('locked')
1070 def __enter__(self):
1071 return self._callmethod('acquire')
1072 def __exit__(self, exc_type, exc_val, exc_tb):
1073 return self._callmethod('release')
1074
1075
1076 class ConditionProxy(AcquirerProxy):
1077 _exposed_ = ('acquire', 'release', 'locked', 'wait', 'notify', 'notify_all')
1078 def wait(self, timeout=None):
1079 return self._callmethod('wait', (timeout,))
1080 def notify(self, n=1):
1081 return self._callmethod('notify', (n,))
1082 def notify_all(self):
1083 return self._callmethod('notify_all')
1084 def wait_for(self, predicate, timeout=None):
1085 result = predicate()
1086 if result:
1087 return result
1088 if timeout is not None:
1089 endtime = time.monotonic() + timeout
1090 else:
1091 endtime = None
1092 waittime = None
1093 while not result:
1094 if endtime is not None:
1095 waittime = endtime - time.monotonic()
1096 if waittime <= 0:
1097 break
1098 self.wait(waittime)
1099 result = predicate()
1100 return result
1101
1102
1103 class EventProxy(BaseProxy):
1104 _exposed_ = ('is_set', 'set', 'clear', 'wait')
1105 def is_set(self):
1106 return self._callmethod('is_set')
1107 def set(self):
1108 return self._callmethod('set')
1109 def clear(self):
1110 return self._callmethod('clear')
1111 def wait(self, timeout=None):
1112 return self._callmethod('wait', (timeout,))
1113
1114
1115 class BarrierProxy(BaseProxy):
1116 _exposed_ = ('__getattribute__', 'wait', 'abort', 'reset')
1117 def wait(self, timeout=None):
1118 return self._callmethod('wait', (timeout,))
1119 def abort(self):
1120 return self._callmethod('abort')
1121 def reset(self):
1122 return self._callmethod('reset')
1123 @property
1124 def parties(self):
1125 return self._callmethod('__getattribute__', ('parties',))
1126 @property
1127 def n_waiting(self):
1128 return self._callmethod('__getattribute__', ('n_waiting',))
1129 @property
1130 def broken(self):
1131 return self._callmethod('__getattribute__', ('broken',))
1132
1133
1134 class NamespaceProxy(BaseProxy):
1135 _exposed_ = ('__getattribute__', '__setattr__', '__delattr__')
1136 def __getattr__(self, key):
1137 if key[0] == '_':
1138 return object.__getattribute__(self, key)
1139 callmethod = object.__getattribute__(self, '_callmethod')
1140 return callmethod('__getattribute__', (key,))
1141 def __setattr__(self, key, value):
1142 if key[0] == '_':
1143 return object.__setattr__(self, key, value)
1144 callmethod = object.__getattribute__(self, '_callmethod')
1145 return callmethod('__setattr__', (key, value))
1146 def __delattr__(self, key):
1147 if key[0] == '_':
1148 return object.__delattr__(self, key)
1149 callmethod = object.__getattribute__(self, '_callmethod')
1150 return callmethod('__delattr__', (key,))
1151
1152
1153 class ValueProxy(BaseProxy):
1154 _exposed_ = ('get', 'set')
1155 def get(self):
1156 return self._callmethod('get')
1157 def set(self, value):
1158 return self._callmethod('set', (value,))
1159 value = property(get, set)
1160
1161 __class_getitem__ = classmethod(types.GenericAlias)
1162
1163
1164 BaseListProxy = MakeProxyType('BaseListProxy', (
1165 '__add__', '__contains__', '__delitem__', '__getitem__', '__imul__',
1166 '__len__', '__mul__', '__reversed__', '__rmul__', '__setitem__',
1167 'append', 'clear', 'copy', 'count', 'extend', 'index', 'insert', 'pop',
1168 'remove', 'reverse', 'sort',
1169 ))
1170 class ListProxy(BaseListProxy):
1171 def __iadd__(self, value):
1172 self._callmethod('extend', (value,))
1173 return self
1174 def __imul__(self, value):
1175 self._callmethod('__imul__', (value,))
1176 return self
1177
1178 __class_getitem__ = classmethod(types.GenericAlias)
1179
1180 collections.abc.MutableSequence.register(BaseListProxy)
1181
1182 _BaseDictProxy = MakeProxyType('_BaseDictProxy', (
1183 '__contains__', '__delitem__', '__getitem__', '__ior__', '__iter__',
1184 '__len__', '__or__', '__reversed__', '__ror__',
1185 '__setitem__', 'clear', 'copy', 'fromkeys', 'get', 'items',
1186 'keys', 'pop', 'popitem', 'setdefault', 'update', 'values'
1187 ))
1188 _BaseDictProxy._method_to_typeid_ = {
1189 '__iter__': 'Iterator',
1190 }
1191 class DictProxy(_BaseDictProxy):
1192 def __ior__(self, value):
1193 self._callmethod('__ior__', (value,))
1194 return self
1195
1196 __class_getitem__ = classmethod(types.GenericAlias)
1197
1198 collections.abc.MutableMapping.register(_BaseDictProxy)
1199
1200 _BaseSetProxy = MakeProxyType("_BaseSetProxy", (
1201 '__and__', '__class_getitem__', '__contains__', '__iand__', '__ior__',
1202 '__isub__', '__iter__', '__ixor__', '__len__', '__or__', '__rand__',
1203 '__ror__', '__rsub__', '__rxor__', '__sub__', '__xor__',
1204 '__ge__', '__gt__', '__le__', '__lt__',
1205 'add', 'clear', 'copy', 'difference', 'difference_update', 'discard',
1206 'intersection', 'intersection_update', 'isdisjoint', 'issubset',
1207 'issuperset', 'pop', 'remove', 'symmetric_difference',
1208 'symmetric_difference_update', 'union', 'update',
1209 ))
1210
1211 class SetProxy(_BaseSetProxy):
1212 def __ior__(self, value):
1213 self._callmethod('__ior__', (value,))
1214 return self
1215 def __iand__(self, value):
1216 self._callmethod('__iand__', (value,))
1217 return self
1218 def __ixor__(self, value):
1219 self._callmethod('__ixor__', (value,))
1220 return self
1221 def __isub__(self, value):
1222 self._callmethod('__isub__', (value,))
1223 return self
1224
1225 __class_getitem__ = classmethod(types.GenericAlias)
1226
1227 collections.abc.MutableMapping.register(_BaseSetProxy)
1228
1229
1230 ArrayProxy = MakeProxyType('ArrayProxy', (
1231 '__len__', '__getitem__', '__setitem__'
1232 ))
1233
1234
1235 BasePoolProxy = MakeProxyType('PoolProxy', (
1236 'apply', 'apply_async', 'close', 'imap', 'imap_unordered', 'join',
1237 'map', 'map_async', 'starmap', 'starmap_async', 'terminate',
1238 ))
1239 BasePoolProxy._method_to_typeid_ = {
1240 'apply_async': 'AsyncResult',
1241 'map_async': 'AsyncResult',
1242 'starmap_async': 'AsyncResult',
1243 'imap': 'Iterator',
1244 'imap_unordered': 'Iterator'
1245 }
1246 class PoolProxy(BasePoolProxy):
1247 def __enter__(self):
1248 return self
1249 def __exit__(self, exc_type, exc_val, exc_tb):
1250 self.terminate()
1251
1252 #
1253 # Definition of SyncManager
1254 #
1255
1256 class SyncManager(BaseManager):
1257 '''
1258 Subclass of `BaseManager` which supports a number of shared object types.
1259
1260 The types registered are those intended for the synchronization
1261 of threads, plus `dict`, `list` and `Namespace`.
1262
1263 The `multiprocessing.Manager()` function creates started instances of
1264 this class.
1265 '''
1266
1267 SyncManager.register('Queue', queue.Queue)
1268 SyncManager.register('JoinableQueue', queue.Queue)
1269 SyncManager.register('Event', threading.Event, EventProxy)
1270 SyncManager.register('Lock', threading.Lock, AcquirerProxy)
1271 SyncManager.register('RLock', threading.RLock, AcquirerProxy)
1272 SyncManager.register('Semaphore', threading.Semaphore, AcquirerProxy)
1273 SyncManager.register('BoundedSemaphore', threading.BoundedSemaphore,
1274 AcquirerProxy)
1275 SyncManager.register('Condition', threading.Condition, ConditionProxy)
1276 SyncManager.register('Barrier', threading.Barrier, BarrierProxy)
1277 SyncManager.register('Pool', pool.Pool, PoolProxy)
1278 SyncManager.register('list', list, ListProxy)
1279 SyncManager.register('dict', dict, DictProxy)
1280 SyncManager.register('set', set, SetProxy)
1281 SyncManager.register('Value', Value, ValueProxy)
1282 SyncManager.register('Array', Array, ArrayProxy)
1283 SyncManager.register('Namespace', Namespace, NamespaceProxy)
1284
1285 # types returned by methods of PoolProxy
1286 SyncManager.register('Iterator', proxytype=IteratorProxy, create_method=False)
1287 SyncManager.register('AsyncResult', create_method=False)
1288
1289 #
1290 # Definition of SharedMemoryManager and SharedMemoryServer
1291 #
1292
1293 if HAS_SHMEM:
1294 class _SharedMemoryTracker:
1295 "Manages one or more shared memory segments."
1296
1297 def __init__(self, name, segment_names=[]):
1298 self.shared_memory_context_name = name
1299 self.segment_names = segment_names
1300
1301 def register_segment(self, segment_name):
1302 "Adds the supplied shared memory block name to tracker."
1303 util.debug(f"Register segment {segment_name!r} in pid {getpid()}")
1304 self.segment_names.append(segment_name)
1305
1306 def destroy_segment(self, segment_name):
1307 """Calls unlink() on the shared memory block with the supplied name
1308 and removes it from the list of blocks being tracked."""
1309 util.debug(f"Destroy segment {segment_name!r} in pid {getpid()}")
1310 self.segment_names.remove(segment_name)
1311 segment = shared_memory.SharedMemory(segment_name)
1312 segment.close()
1313 segment.unlink()
1314
1315 def unlink(self):
1316 "Calls destroy_segment() on all tracked shared memory blocks."
1317 for segment_name in self.segment_names[:]:
1318 self.destroy_segment(segment_name)
1319
1320 def __del__(self):
1321 util.debug(f"Call {self.__class__.__name__}.__del__ in {getpid()}")
1322 self.unlink()
1323
1324 def __getstate__(self):
1325 return (self.shared_memory_context_name, self.segment_names)
1326
1327 def __setstate__(self, state):
1328 self.__init__(*state)
1329
1330
1331 class SharedMemoryServer(Server):
1332
1333 public = Server.public + \
1334 ['track_segment', 'release_segment', 'list_segments']
1335
1336 def __init__(self, *args, **kwargs):
1337 Server.__init__(self, *args, **kwargs)
1338 address = self.address
1339 # The address of Linux abstract namespaces can be bytes
1340 if isinstance(address, bytes):
1341 address = os.fsdecode(address)
1342 self.shared_memory_context = \
1343 _SharedMemoryTracker(f"shm_{address}_{getpid()}")
1344 util.debug(f"SharedMemoryServer started by pid {getpid()}")
1345
1346 def create(self, c, typeid, /, *args, **kwargs):
1347 """Create a new distributed-shared object (not backed by a shared
1348 memory block) and return its id to be used in a Proxy Object."""
1349 # Unless set up as a shared proxy, don't make shared_memory_context
1350 # a standard part of kwargs. This makes things easier for supplying
1351 # simple functions.
1352 if hasattr(self.registry[typeid][-1], "_shared_memory_proxy"):
1353 kwargs['shared_memory_context'] = self.shared_memory_context
1354 return Server.create(self, c, typeid, *args, **kwargs)
1355
1356 def shutdown(self, c):
1357 "Call unlink() on all tracked shared memory, terminate the Server."
1358 self.shared_memory_context.unlink()
1359 return Server.shutdown(self, c)
1360
1361 def track_segment(self, c, segment_name):
1362 "Adds the supplied shared memory block name to Server's tracker."
1363 self.shared_memory_context.register_segment(segment_name)
1364
1365 def release_segment(self, c, segment_name):
1366 """Calls unlink() on the shared memory block with the supplied name
1367 and removes it from the tracker instance inside the Server."""
1368 self.shared_memory_context.destroy_segment(segment_name)
1369
1370 def list_segments(self, c):
1371 """Returns a list of names of shared memory blocks that the Server
1372 is currently tracking."""
1373 return self.shared_memory_context.segment_names
1374
1375
1376 class SharedMemoryManager(BaseManager):
1377 """Like SyncManager but uses SharedMemoryServer instead of Server.
1378
1379 It provides methods for creating and returning SharedMemory instances
1380 and for creating a list-like object (ShareableList) backed by shared
1381 memory. It also provides methods that create and return Proxy Objects
1382 that support synchronization across processes (i.e. multi-process-safe
1383 locks and semaphores).
1384 """
1385
1386 _Server = SharedMemoryServer
1387
1388 def __init__(self, *args, **kwargs):
1389 if os.name == "posix":
1390 # bpo-36867: Ensure the resource_tracker is running before
1391 # launching the manager process, so that concurrent
1392 # shared_memory manipulation both in the manager and in the
1393 # current process does not create two resource_tracker
1394 # processes.
1395 from . import resource_tracker
1396 resource_tracker.ensure_running()
1397 BaseManager.__init__(self, *args, **kwargs)
1398 util.debug(f"{self.__class__.__name__} created by pid {getpid()}")
1399
1400 def __del__(self):
1401 util.debug(f"{self.__class__.__name__}.__del__ by pid {getpid()}")
1402
1403 def get_server(self):
1404 'Better than monkeypatching for now; merge into Server ultimately'
1405 if self._state.value != State.INITIAL:
1406 if self._state.value == State.STARTED:
1407 raise ProcessError("Already started SharedMemoryServer")
1408 elif self._state.value == State.SHUTDOWN:
1409 raise ProcessError("SharedMemoryManager has shut down")
1410 else:
1411 raise ProcessError(
1412 "Unknown state {!r}".format(self._state.value))
1413 return self._Server(self._registry, self._address,
1414 self._authkey, self._serializer)
1415
1416 def SharedMemory(self, size):
1417 """Returns a new SharedMemory instance with the specified size in
1418 bytes, to be tracked by the manager."""
1419 with self._Client(self._address, authkey=self._authkey) as conn:
1420 sms = shared_memory.SharedMemory(None, create=True, size=size)
1421 try:
1422 dispatch(conn, None, 'track_segment', (sms.name,))
1423 except BaseException as e:
1424 sms.unlink()
1425 raise e
1426 return sms
1427
1428 def ShareableList(self, sequence):
1429 """Returns a new ShareableList instance populated with the values
1430 from the input sequence, to be tracked by the manager."""
1431 with self._Client(self._address, authkey=self._authkey) as conn:
1432 sl = shared_memory.ShareableList(sequence)
1433 try:
1434 dispatch(conn, None, 'track_segment', (sl.shm.name,))
1435 except BaseException as e:
1436 sl.shm.unlink()
1437 raise e
1438 return sl