| 1 |
import os |
| 2 |
import sys |
| 3 |
import threading |
| 4 |
|
| 5 |
from . import process |
| 6 |
from . import reduction |
| 7 |
|
| 8 |
__all__ = () |
| 9 |
|
| 10 |
# |
| 11 |
# Exceptions |
| 12 |
# |
| 13 |
|
| 14 |
class ProcessError(Exception): |
| 15 |
pass |
| 16 |
|
| 17 |
class BufferTooShort(ProcessError): |
| 18 |
pass |
| 19 |
|
| 20 |
class TimeoutError(ProcessError): |
| 21 |
pass |
| 22 |
|
| 23 |
class AuthenticationError(ProcessError): |
| 24 |
pass |
| 25 |
|
| 26 |
# |
| 27 |
# Base type for contexts. Bound methods of an instance of this type are included in __all__ of __init__.py |
| 28 |
# |
| 29 |
|
| 30 |
class BaseContext(object): |
| 31 |
|
| 32 |
ProcessError = ProcessError |
| 33 |
BufferTooShort = BufferTooShort |
| 34 |
TimeoutError = TimeoutError |
| 35 |
AuthenticationError = AuthenticationError |
| 36 |
|
| 37 |
current_process = staticmethod(process.current_process) |
| 38 |
parent_process = staticmethod(process.parent_process) |
| 39 |
active_children = staticmethod(process.active_children) |
| 40 |
|
| 41 |
def cpu_count(self): |
| 42 |
'''Returns the number of CPUs in the system''' |
| 43 |
num = os.cpu_count() |
| 44 |
if num is None: |
| 45 |
raise NotImplementedError('cannot determine number of cpus') |
| 46 |
else: |
| 47 |
return num |
| 48 |
|
| 49 |
def Manager(self): |
| 50 |
'''Returns a manager associated with a running server process |
| 51 |
|
| 52 |
The managers methods such as `Lock()`, `Condition()` and `Queue()` |
| 53 |
can be used to create shared objects. |
| 54 |
''' |
| 55 |
from .managers import SyncManager |
| 56 |
m = SyncManager(ctx=self.get_context()) |
| 57 |
m.start() |
| 58 |
return m |
| 59 |
|
| 60 |
def Pipe(self, duplex=True): |
| 61 |
'''Returns two connection object connected by a pipe''' |
| 62 |
from .connection import Pipe |
| 63 |
return Pipe(duplex) |
| 64 |
|
| 65 |
def Lock(self): |
| 66 |
'''Returns a non-recursive lock object''' |
| 67 |
from .synchronize import Lock |
| 68 |
return Lock(ctx=self.get_context()) |
| 69 |
|
| 70 |
def RLock(self): |
| 71 |
'''Returns a recursive lock object''' |
| 72 |
from .synchronize import RLock |
| 73 |
return RLock(ctx=self.get_context()) |
| 74 |
|
| 75 |
def Condition(self, lock=None): |
| 76 |
'''Returns a condition object''' |
| 77 |
from .synchronize import Condition |
| 78 |
return Condition(lock, ctx=self.get_context()) |
| 79 |
|
| 80 |
def Semaphore(self, value=1): |
| 81 |
'''Returns a semaphore object''' |
| 82 |
from .synchronize import Semaphore |
| 83 |
return Semaphore(value, ctx=self.get_context()) |
| 84 |
|
| 85 |
def BoundedSemaphore(self, value=1): |
| 86 |
'''Returns a bounded semaphore object''' |
| 87 |
from .synchronize import BoundedSemaphore |
| 88 |
return BoundedSemaphore(value, ctx=self.get_context()) |
| 89 |
|
| 90 |
def Event(self): |
| 91 |
'''Returns an event object''' |
| 92 |
from .synchronize import Event |
| 93 |
return Event(ctx=self.get_context()) |
| 94 |
|
| 95 |
def Barrier(self, parties, action=None, timeout=None): |
| 96 |
'''Returns a barrier object''' |
| 97 |
from .synchronize import Barrier |
| 98 |
return Barrier(parties, action, timeout, ctx=self.get_context()) |
| 99 |
|
| 100 |
def Queue(self, maxsize=0): |
| 101 |
'''Returns a queue object''' |
| 102 |
from .queues import Queue |
| 103 |
return Queue(maxsize, ctx=self.get_context()) |
| 104 |
|
| 105 |
def JoinableQueue(self, maxsize=0): |
| 106 |
'''Returns a queue object''' |
| 107 |
from .queues import JoinableQueue |
| 108 |
return JoinableQueue(maxsize, ctx=self.get_context()) |
| 109 |
|
| 110 |
def SimpleQueue(self): |
| 111 |
'''Returns a queue object''' |
| 112 |
from .queues import SimpleQueue |
| 113 |
return SimpleQueue(ctx=self.get_context()) |
| 114 |
|
| 115 |
def Pool(self, processes=None, initializer=None, initargs=(), |
| 116 |
maxtasksperchild=None): |
| 117 |
'''Returns a process pool object''' |
| 118 |
from .pool import Pool |
| 119 |
return Pool(processes, initializer, initargs, maxtasksperchild, |
| 120 |
context=self.get_context()) |
| 121 |
|
| 122 |
def RawValue(self, typecode_or_type, *args): |
| 123 |
'''Returns a shared object''' |
| 124 |
from .sharedctypes import RawValue |
| 125 |
return RawValue(typecode_or_type, *args) |
| 126 |
|
| 127 |
def RawArray(self, typecode_or_type, size_or_initializer): |
| 128 |
'''Returns a shared array''' |
| 129 |
from .sharedctypes import RawArray |
| 130 |
return RawArray(typecode_or_type, size_or_initializer) |
| 131 |
|
| 132 |
def Value(self, typecode_or_type, *args, lock=True): |
| 133 |
'''Returns a synchronized shared object''' |
| 134 |
from .sharedctypes import Value |
| 135 |
return Value(typecode_or_type, *args, lock=lock, |
| 136 |
ctx=self.get_context()) |
| 137 |
|
| 138 |
def Array(self, typecode_or_type, size_or_initializer, *, lock=True): |
| 139 |
'''Returns a synchronized shared array''' |
| 140 |
from .sharedctypes import Array |
| 141 |
return Array(typecode_or_type, size_or_initializer, lock=lock, |
| 142 |
ctx=self.get_context()) |
| 143 |
|
| 144 |
def freeze_support(self): |
| 145 |
'''Check whether this is a fake forked process in a frozen executable. |
| 146 |
If so then run code specified by commandline and exit. |
| 147 |
''' |
| 148 |
# gh-140814: allow_none=True avoids locking in the default start |
| 149 |
# method, which would cause a later set_start_method() to fail. |
| 150 |
# None is safe to pass through: spawn.freeze_support() |
| 151 |
# independently detects whether this process is a spawned |
| 152 |
# child, so the start method check here is only an optimization. |
| 153 |
if (getattr(sys, 'frozen', False) |
| 154 |
and self.get_start_method(allow_none=True) in ('spawn', None)): |
| 155 |
from .spawn import freeze_support |
| 156 |
freeze_support() |
| 157 |
|
| 158 |
def get_logger(self): |
| 159 |
'''Return package logger -- if it does not already exist then |
| 160 |
it is created. |
| 161 |
''' |
| 162 |
from .util import get_logger |
| 163 |
return get_logger() |
| 164 |
|
| 165 |
def log_to_stderr(self, level=None): |
| 166 |
'''Turn on logging and add a handler which prints to stderr''' |
| 167 |
from .util import log_to_stderr |
| 168 |
return log_to_stderr(level) |
| 169 |
|
| 170 |
def allow_connection_pickling(self): |
| 171 |
'''Install support for sending connections and sockets |
| 172 |
between processes |
| 173 |
''' |
| 174 |
# This is undocumented. In previous versions of multiprocessing |
| 175 |
# its only effect was to make socket objects inheritable on Windows. |
| 176 |
from . import connection # noqa: F401 |
| 177 |
|
| 178 |
def set_executable(self, executable): |
| 179 |
'''Sets the path to a python.exe or pythonw.exe binary used to run |
| 180 |
child processes instead of sys.executable when using the 'spawn' |
| 181 |
start method. Useful for people embedding Python. |
| 182 |
''' |
| 183 |
from .spawn import set_executable |
| 184 |
set_executable(executable) |
| 185 |
|
| 186 |
def set_forkserver_preload(self, module_names): |
| 187 |
'''Set list of module names to try to load in forkserver process. |
| 188 |
This is really just a hint. |
| 189 |
''' |
| 190 |
from .forkserver import set_forkserver_preload |
| 191 |
set_forkserver_preload(module_names) |
| 192 |
|
| 193 |
def get_context(self, method=None): |
| 194 |
if method is None: |
| 195 |
return self |
| 196 |
try: |
| 197 |
ctx = _concrete_contexts[method] |
| 198 |
except KeyError: |
| 199 |
raise ValueError('cannot find context for %r' % method) from None |
| 200 |
ctx._check_available() |
| 201 |
return ctx |
| 202 |
|
| 203 |
def get_start_method(self, allow_none=False): |
| 204 |
return self._name |
| 205 |
|
| 206 |
def set_start_method(self, method, force=False): |
| 207 |
raise ValueError('cannot set start method of concrete context') |
| 208 |
|
| 209 |
@property |
| 210 |
def reducer(self): |
| 211 |
'''Controls how objects will be reduced to a form that can be |
| 212 |
shared with other processes.''' |
| 213 |
return globals().get('reduction') |
| 214 |
|
| 215 |
@reducer.setter |
| 216 |
def reducer(self, reduction): |
| 217 |
globals()['reduction'] = reduction |
| 218 |
|
| 219 |
def _check_available(self): |
| 220 |
pass |
| 221 |
|
| 222 |
# |
| 223 |
# Type of default context -- underlying context can be set at most once |
| 224 |
# |
| 225 |
|
| 226 |
class Process(process.BaseProcess): |
| 227 |
_start_method = None |
| 228 |
@staticmethod |
| 229 |
def _Popen(process_obj): |
| 230 |
return _default_context.get_context().Process._Popen(process_obj) |
| 231 |
|
| 232 |
@staticmethod |
| 233 |
def _after_fork(): |
| 234 |
return _default_context.get_context().Process._after_fork() |
| 235 |
|
| 236 |
class DefaultContext(BaseContext): |
| 237 |
Process = Process |
| 238 |
|
| 239 |
def __init__(self, context): |
| 240 |
self._default_context = context |
| 241 |
self._actual_context = None |
| 242 |
|
| 243 |
def get_context(self, method=None): |
| 244 |
if method is None: |
| 245 |
if self._actual_context is None: |
| 246 |
self._actual_context = self._default_context |
| 247 |
return self._actual_context |
| 248 |
else: |
| 249 |
return super().get_context(method) |
| 250 |
|
| 251 |
def set_start_method(self, method, force=False): |
| 252 |
if self._actual_context is not None and not force: |
| 253 |
raise RuntimeError('context has already been set') |
| 254 |
if method is None and force: |
| 255 |
self._actual_context = None |
| 256 |
return |
| 257 |
self._actual_context = self.get_context(method) |
| 258 |
|
| 259 |
def get_start_method(self, allow_none=False): |
| 260 |
if self._actual_context is None: |
| 261 |
if allow_none: |
| 262 |
return None |
| 263 |
self._actual_context = self._default_context |
| 264 |
return self._actual_context._name |
| 265 |
|
| 266 |
def get_all_start_methods(self): |
| 267 |
"""Returns a list of the supported start methods, default first.""" |
| 268 |
default = self._default_context.get_start_method() |
| 269 |
start_method_names = [default] |
| 270 |
start_method_names.extend( |
| 271 |
name for name in _concrete_contexts if name != default |
| 272 |
) |
| 273 |
return start_method_names |
| 274 |
|
| 275 |
|
| 276 |
# |
| 277 |
# Context types for fixed start method |
| 278 |
# |
| 279 |
|
| 280 |
if sys.platform != 'win32': |
| 281 |
|
| 282 |
class ForkProcess(process.BaseProcess): |
| 283 |
_start_method = 'fork' |
| 284 |
@staticmethod |
| 285 |
def _Popen(process_obj): |
| 286 |
from .popen_fork import Popen |
| 287 |
return Popen(process_obj) |
| 288 |
|
| 289 |
class SpawnProcess(process.BaseProcess): |
| 290 |
_start_method = 'spawn' |
| 291 |
@staticmethod |
| 292 |
def _Popen(process_obj): |
| 293 |
from .popen_spawn_posix import Popen |
| 294 |
return Popen(process_obj) |
| 295 |
|
| 296 |
@staticmethod |
| 297 |
def _after_fork(): |
| 298 |
# process is spawned, nothing to do |
| 299 |
pass |
| 300 |
|
| 301 |
class ForkServerProcess(process.BaseProcess): |
| 302 |
_start_method = 'forkserver' |
| 303 |
@staticmethod |
| 304 |
def _Popen(process_obj): |
| 305 |
from .popen_forkserver import Popen |
| 306 |
return Popen(process_obj) |
| 307 |
|
| 308 |
class ForkContext(BaseContext): |
| 309 |
_name = 'fork' |
| 310 |
Process = ForkProcess |
| 311 |
|
| 312 |
class SpawnContext(BaseContext): |
| 313 |
_name = 'spawn' |
| 314 |
Process = SpawnProcess |
| 315 |
|
| 316 |
class ForkServerContext(BaseContext): |
| 317 |
_name = 'forkserver' |
| 318 |
Process = ForkServerProcess |
| 319 |
def _check_available(self): |
| 320 |
if not reduction.HAVE_SEND_HANDLE: |
| 321 |
raise ValueError('forkserver start method not available') |
| 322 |
|
| 323 |
_concrete_contexts = { |
| 324 |
'fork': ForkContext(), |
| 325 |
'spawn': SpawnContext(), |
| 326 |
'forkserver': ForkServerContext(), |
| 327 |
} |
| 328 |
# bpo-33725: running arbitrary code after fork() is no longer reliable |
| 329 |
# on macOS since macOS 10.14 (Mojave). Use spawn by default instead. |
| 330 |
# gh-84559: We changed everyones default to a thread safeish one in 3.14. |
| 331 |
if reduction.HAVE_SEND_HANDLE and sys.platform != 'darwin': |
| 332 |
_default_context = DefaultContext(_concrete_contexts['forkserver']) |
| 333 |
else: |
| 334 |
_default_context = DefaultContext(_concrete_contexts['spawn']) |
| 335 |
|
| 336 |
else: # Windows |
| 337 |
|
| 338 |
class SpawnProcess(process.BaseProcess): |
| 339 |
_start_method = 'spawn' |
| 340 |
@staticmethod |
| 341 |
def _Popen(process_obj): |
| 342 |
from .popen_spawn_win32 import Popen |
| 343 |
return Popen(process_obj) |
| 344 |
|
| 345 |
@staticmethod |
| 346 |
def _after_fork(): |
| 347 |
# process is spawned, nothing to do |
| 348 |
pass |
| 349 |
|
| 350 |
class SpawnContext(BaseContext): |
| 351 |
_name = 'spawn' |
| 352 |
Process = SpawnProcess |
| 353 |
|
| 354 |
_concrete_contexts = { |
| 355 |
'spawn': SpawnContext(), |
| 356 |
} |
| 357 |
_default_context = DefaultContext(_concrete_contexts['spawn']) |
| 358 |
|
| 359 |
# |
| 360 |
# Force the start method |
| 361 |
# |
| 362 |
|
| 363 |
def _force_start_method(method): |
| 364 |
_default_context._actual_context = _concrete_contexts[method] |
| 365 |
|
| 366 |
# |
| 367 |
# Check that the current thread is spawning a child process |
| 368 |
# |
| 369 |
|
| 370 |
_tls = threading.local() |
| 371 |
|
| 372 |
def get_spawning_popen(): |
| 373 |
return getattr(_tls, 'spawning_popen', None) |
| 374 |
|
| 375 |
def set_spawning_popen(popen): |
| 376 |
_tls.spawning_popen = popen |
| 377 |
|
| 378 |
def assert_spawning(obj): |
| 379 |
if get_spawning_popen() is None: |
| 380 |
raise RuntimeError( |
| 381 |
'%s objects should only be shared between processes' |
| 382 |
' through inheritance' % type(obj).__name__ |
| 383 |
) |