Debian Codegraph @

Lib/multiprocessing/context.py

raw
1 import os
2 import sys
3 import threading
4
5 from . import process
6 from . import reduction
7
8 __all__ = ()
9
10 #
11 # Exceptions
12 #
13
14 class ProcessError(Exception):
15 pass
16
17 class BufferTooShort(ProcessError):
18 pass
19
20 class TimeoutError(ProcessError):
21 pass
22
23 class AuthenticationError(ProcessError):
24 pass
25
26 #
27 # Base type for contexts. Bound methods of an instance of this type are included in __all__ of __init__.py
28 #
29
30 class BaseContext(object):
31
32 ProcessError = ProcessError
33 BufferTooShort = BufferTooShort
34 TimeoutError = TimeoutError
35 AuthenticationError = AuthenticationError
36
37 current_process = staticmethod(process.current_process)
38 parent_process = staticmethod(process.parent_process)
39 active_children = staticmethod(process.active_children)
40
41 def cpu_count(self):
42 '''Returns the number of CPUs in the system'''
43 num = os.cpu_count()
44 if num is None:
45 raise NotImplementedError('cannot determine number of cpus')
46 else:
47 return num
48
49 def Manager(self):
50 '''Returns a manager associated with a running server process
51
52 The managers methods such as `Lock()`, `Condition()` and `Queue()`
53 can be used to create shared objects.
54 '''
55 from .managers import SyncManager
56 m = SyncManager(ctx=self.get_context())
57 m.start()
58 return m
59
60 def Pipe(self, duplex=True):
61 '''Returns two connection object connected by a pipe'''
62 from .connection import Pipe
63 return Pipe(duplex)
64
65 def Lock(self):
66 '''Returns a non-recursive lock object'''
67 from .synchronize import Lock
68 return Lock(ctx=self.get_context())
69
70 def RLock(self):
71 '''Returns a recursive lock object'''
72 from .synchronize import RLock
73 return RLock(ctx=self.get_context())
74
75 def Condition(self, lock=None):
76 '''Returns a condition object'''
77 from .synchronize import Condition
78 return Condition(lock, ctx=self.get_context())
79
80 def Semaphore(self, value=1):
81 '''Returns a semaphore object'''
82 from .synchronize import Semaphore
83 return Semaphore(value, ctx=self.get_context())
84
85 def BoundedSemaphore(self, value=1):
86 '''Returns a bounded semaphore object'''
87 from .synchronize import BoundedSemaphore
88 return BoundedSemaphore(value, ctx=self.get_context())
89
90 def Event(self):
91 '''Returns an event object'''
92 from .synchronize import Event
93 return Event(ctx=self.get_context())
94
95 def Barrier(self, parties, action=None, timeout=None):
96 '''Returns a barrier object'''
97 from .synchronize import Barrier
98 return Barrier(parties, action, timeout, ctx=self.get_context())
99
100 def Queue(self, maxsize=0):
101 '''Returns a queue object'''
102 from .queues import Queue
103 return Queue(maxsize, ctx=self.get_context())
104
105 def JoinableQueue(self, maxsize=0):
106 '''Returns a queue object'''
107 from .queues import JoinableQueue
108 return JoinableQueue(maxsize, ctx=self.get_context())
109
110 def SimpleQueue(self):
111 '''Returns a queue object'''
112 from .queues import SimpleQueue
113 return SimpleQueue(ctx=self.get_context())
114
115 def Pool(self, processes=None, initializer=None, initargs=(),
116 maxtasksperchild=None):
117 '''Returns a process pool object'''
118 from .pool import Pool
119 return Pool(processes, initializer, initargs, maxtasksperchild,
120 context=self.get_context())
121
122 def RawValue(self, typecode_or_type, *args):
123 '''Returns a shared object'''
124 from .sharedctypes import RawValue
125 return RawValue(typecode_or_type, *args)
126
127 def RawArray(self, typecode_or_type, size_or_initializer):
128 '''Returns a shared array'''
129 from .sharedctypes import RawArray
130 return RawArray(typecode_or_type, size_or_initializer)
131
132 def Value(self, typecode_or_type, *args, lock=True):
133 '''Returns a synchronized shared object'''
134 from .sharedctypes import Value
135 return Value(typecode_or_type, *args, lock=lock,
136 ctx=self.get_context())
137
138 def Array(self, typecode_or_type, size_or_initializer, *, lock=True):
139 '''Returns a synchronized shared array'''
140 from .sharedctypes import Array
141 return Array(typecode_or_type, size_or_initializer, lock=lock,
142 ctx=self.get_context())
143
144 def freeze_support(self):
145 '''Check whether this is a fake forked process in a frozen executable.
146 If so then run code specified by commandline and exit.
147 '''
148 # gh-140814: allow_none=True avoids locking in the default start
149 # method, which would cause a later set_start_method() to fail.
150 # None is safe to pass through: spawn.freeze_support()
151 # independently detects whether this process is a spawned
152 # child, so the start method check here is only an optimization.
153 if (getattr(sys, 'frozen', False)
154 and self.get_start_method(allow_none=True) in ('spawn', None)):
155 from .spawn import freeze_support
156 freeze_support()
157
158 def get_logger(self):
159 '''Return package logger -- if it does not already exist then
160 it is created.
161 '''
162 from .util import get_logger
163 return get_logger()
164
165 def log_to_stderr(self, level=None):
166 '''Turn on logging and add a handler which prints to stderr'''
167 from .util import log_to_stderr
168 return log_to_stderr(level)
169
170 def allow_connection_pickling(self):
171 '''Install support for sending connections and sockets
172 between processes
173 '''
174 # This is undocumented. In previous versions of multiprocessing
175 # its only effect was to make socket objects inheritable on Windows.
176 from . import connection # noqa: F401
177
178 def set_executable(self, executable):
179 '''Sets the path to a python.exe or pythonw.exe binary used to run
180 child processes instead of sys.executable when using the 'spawn'
181 start method. Useful for people embedding Python.
182 '''
183 from .spawn import set_executable
184 set_executable(executable)
185
186 def set_forkserver_preload(self, module_names):
187 '''Set list of module names to try to load in forkserver process.
188 This is really just a hint.
189 '''
190 from .forkserver import set_forkserver_preload
191 set_forkserver_preload(module_names)
192
193 def get_context(self, method=None):
194 if method is None:
195 return self
196 try:
197 ctx = _concrete_contexts[method]
198 except KeyError:
199 raise ValueError('cannot find context for %r' % method) from None
200 ctx._check_available()
201 return ctx
202
203 def get_start_method(self, allow_none=False):
204 return self._name
205
206 def set_start_method(self, method, force=False):
207 raise ValueError('cannot set start method of concrete context')
208
209 @property
210 def reducer(self):
211 '''Controls how objects will be reduced to a form that can be
212 shared with other processes.'''
213 return globals().get('reduction')
214
215 @reducer.setter
216 def reducer(self, reduction):
217 globals()['reduction'] = reduction
218
219 def _check_available(self):
220 pass
221
222 #
223 # Type of default context -- underlying context can be set at most once
224 #
225
226 class Process(process.BaseProcess):
227 _start_method = None
228 @staticmethod
229 def _Popen(process_obj):
230 return _default_context.get_context().Process._Popen(process_obj)
231
232 @staticmethod
233 def _after_fork():
234 return _default_context.get_context().Process._after_fork()
235
236 class DefaultContext(BaseContext):
237 Process = Process
238
239 def __init__(self, context):
240 self._default_context = context
241 self._actual_context = None
242
243 def get_context(self, method=None):
244 if method is None:
245 if self._actual_context is None:
246 self._actual_context = self._default_context
247 return self._actual_context
248 else:
249 return super().get_context(method)
250
251 def set_start_method(self, method, force=False):
252 if self._actual_context is not None and not force:
253 raise RuntimeError('context has already been set')
254 if method is None and force:
255 self._actual_context = None
256 return
257 self._actual_context = self.get_context(method)
258
259 def get_start_method(self, allow_none=False):
260 if self._actual_context is None:
261 if allow_none:
262 return None
263 self._actual_context = self._default_context
264 return self._actual_context._name
265
266 def get_all_start_methods(self):
267 """Returns a list of the supported start methods, default first."""
268 default = self._default_context.get_start_method()
269 start_method_names = [default]
270 start_method_names.extend(
271 name for name in _concrete_contexts if name != default
272 )
273 return start_method_names
274
275
276 #
277 # Context types for fixed start method
278 #
279
280 if sys.platform != 'win32':
281
282 class ForkProcess(process.BaseProcess):
283 _start_method = 'fork'
284 @staticmethod
285 def _Popen(process_obj):
286 from .popen_fork import Popen
287 return Popen(process_obj)
288
289 class SpawnProcess(process.BaseProcess):
290 _start_method = 'spawn'
291 @staticmethod
292 def _Popen(process_obj):
293 from .popen_spawn_posix import Popen
294 return Popen(process_obj)
295
296 @staticmethod
297 def _after_fork():
298 # process is spawned, nothing to do
299 pass
300
301 class ForkServerProcess(process.BaseProcess):
302 _start_method = 'forkserver'
303 @staticmethod
304 def _Popen(process_obj):
305 from .popen_forkserver import Popen
306 return Popen(process_obj)
307
308 class ForkContext(BaseContext):
309 _name = 'fork'
310 Process = ForkProcess
311
312 class SpawnContext(BaseContext):
313 _name = 'spawn'
314 Process = SpawnProcess
315
316 class ForkServerContext(BaseContext):
317 _name = 'forkserver'
318 Process = ForkServerProcess
319 def _check_available(self):
320 if not reduction.HAVE_SEND_HANDLE:
321 raise ValueError('forkserver start method not available')
322
323 _concrete_contexts = {
324 'fork': ForkContext(),
325 'spawn': SpawnContext(),
326 'forkserver': ForkServerContext(),
327 }
328 # bpo-33725: running arbitrary code after fork() is no longer reliable
329 # on macOS since macOS 10.14 (Mojave). Use spawn by default instead.
330 # gh-84559: We changed everyones default to a thread safeish one in 3.14.
331 if reduction.HAVE_SEND_HANDLE and sys.platform != 'darwin':
332 _default_context = DefaultContext(_concrete_contexts['forkserver'])
333 else:
334 _default_context = DefaultContext(_concrete_contexts['spawn'])
335
336 else: # Windows
337
338 class SpawnProcess(process.BaseProcess):
339 _start_method = 'spawn'
340 @staticmethod
341 def _Popen(process_obj):
342 from .popen_spawn_win32 import Popen
343 return Popen(process_obj)
344
345 @staticmethod
346 def _after_fork():
347 # process is spawned, nothing to do
348 pass
349
350 class SpawnContext(BaseContext):
351 _name = 'spawn'
352 Process = SpawnProcess
353
354 _concrete_contexts = {
355 'spawn': SpawnContext(),
356 }
357 _default_context = DefaultContext(_concrete_contexts['spawn'])
358
359 #
360 # Force the start method
361 #
362
363 def _force_start_method(method):
364 _default_context._actual_context = _concrete_contexts[method]
365
366 #
367 # Check that the current thread is spawning a child process
368 #
369
370 _tls = threading.local()
371
372 def get_spawning_popen():
373 return getattr(_tls, 'spawning_popen', None)
374
375 def set_spawning_popen(popen):
376 _tls.spawning_popen = popen
377
378 def assert_spawning(obj):
379 if get_spawning_popen() is None:
380 raise RuntimeError(
381 '%s objects should only be shared between processes'
382 ' through inheritance' % type(obj).__name__
383 )