| 1 |
# |
| 2 |
# Module providing various facilities to other parts of the package |
| 3 |
# |
| 4 |
# multiprocessing/util.py |
| 5 |
# |
| 6 |
# Copyright (c) 2006-2008, R Oudkerk |
| 7 |
# Licensed to PSF under a Contributor Agreement. |
| 8 |
# |
| 9 |
|
| 10 |
import os |
| 11 |
import itertools |
| 12 |
import sys |
| 13 |
import weakref |
| 14 |
import atexit |
| 15 |
import threading # we want threading to install it's |
| 16 |
# cleanup function before multiprocessing does |
| 17 |
from subprocess import _args_from_interpreter_flags # noqa: F401 |
| 18 |
|
| 19 |
from . import process |
| 20 |
|
| 21 |
__all__ = [ |
| 22 |
'sub_debug', 'debug', 'info', 'sub_warning', 'warn', 'get_logger', |
| 23 |
'log_to_stderr', 'get_temp_dir', 'register_after_fork', |
| 24 |
'is_exiting', 'Finalize', 'ForkAwareThreadLock', 'ForkAwareLocal', |
| 25 |
'close_all_fds_except', 'SUBDEBUG', 'SUBWARNING', |
| 26 |
] |
| 27 |
|
| 28 |
# |
| 29 |
# Logging |
| 30 |
# |
| 31 |
|
| 32 |
NOTSET = 0 |
| 33 |
SUBDEBUG = 5 |
| 34 |
DEBUG = 10 |
| 35 |
INFO = 20 |
| 36 |
SUBWARNING = 25 |
| 37 |
WARNING = 30 |
| 38 |
|
| 39 |
LOGGER_NAME = 'multiprocessing' |
| 40 |
DEFAULT_LOGGING_FORMAT = '[%(levelname)s/%(processName)s] %(message)s' |
| 41 |
|
| 42 |
_logger = None |
| 43 |
_log_to_stderr = False |
| 44 |
|
| 45 |
def sub_debug(msg, *args): |
| 46 |
if _logger: |
| 47 |
_logger.log(SUBDEBUG, msg, *args, stacklevel=2) |
| 48 |
|
| 49 |
def debug(msg, *args): |
| 50 |
if _logger: |
| 51 |
_logger.log(DEBUG, msg, *args, stacklevel=2) |
| 52 |
|
| 53 |
def info(msg, *args): |
| 54 |
if _logger: |
| 55 |
_logger.log(INFO, msg, *args, stacklevel=2) |
| 56 |
|
| 57 |
def warn(msg, *args): |
| 58 |
if _logger: |
| 59 |
_logger.log(WARNING, msg, *args, stacklevel=2) |
| 60 |
|
| 61 |
def sub_warning(msg, *args): |
| 62 |
if _logger: |
| 63 |
_logger.log(SUBWARNING, msg, *args, stacklevel=2) |
| 64 |
|
| 65 |
def get_logger(): |
| 66 |
''' |
| 67 |
Returns logger used by multiprocessing |
| 68 |
''' |
| 69 |
global _logger |
| 70 |
import logging |
| 71 |
|
| 72 |
with logging._lock: |
| 73 |
if not _logger: |
| 74 |
|
| 75 |
_logger = logging.getLogger(LOGGER_NAME) |
| 76 |
_logger.propagate = 0 |
| 77 |
|
| 78 |
# XXX multiprocessing should cleanup before logging |
| 79 |
if hasattr(atexit, 'unregister'): |
| 80 |
atexit.unregister(_exit_function) |
| 81 |
atexit.register(_exit_function) |
| 82 |
else: |
| 83 |
atexit._exithandlers.remove((_exit_function, (), {})) |
| 84 |
atexit._exithandlers.append((_exit_function, (), {})) |
| 85 |
|
| 86 |
return _logger |
| 87 |
|
| 88 |
def log_to_stderr(level=None): |
| 89 |
''' |
| 90 |
Turn on logging and add a handler which prints to stderr |
| 91 |
''' |
| 92 |
global _log_to_stderr |
| 93 |
import logging |
| 94 |
|
| 95 |
logger = get_logger() |
| 96 |
formatter = logging.Formatter(DEFAULT_LOGGING_FORMAT) |
| 97 |
handler = logging.StreamHandler() |
| 98 |
handler.setFormatter(formatter) |
| 99 |
logger.addHandler(handler) |
| 100 |
|
| 101 |
if level: |
| 102 |
logger.setLevel(level) |
| 103 |
_log_to_stderr = True |
| 104 |
return _logger |
| 105 |
|
| 106 |
|
| 107 |
# Abstract socket support |
| 108 |
|
| 109 |
def _platform_supports_abstract_sockets(): |
| 110 |
return sys.platform in ("linux", "android") |
| 111 |
|
| 112 |
|
| 113 |
def is_abstract_socket_namespace(address): |
| 114 |
if not address: |
| 115 |
return False |
| 116 |
if isinstance(address, bytes): |
| 117 |
return address[0] == 0 |
| 118 |
elif isinstance(address, str): |
| 119 |
return address[0] == "\0" |
| 120 |
raise TypeError(f'address type of {address!r} unrecognized') |
| 121 |
|
| 122 |
|
| 123 |
abstract_sockets_supported = _platform_supports_abstract_sockets() |
| 124 |
|
| 125 |
# |
| 126 |
# Function returning a temp directory which will be removed on exit |
| 127 |
# |
| 128 |
|
| 129 |
# Maximum length of a NULL-terminated [1] socket file path is usually |
| 130 |
# between 92 and 108 [2], but Linux is known to use a size of 108 [3]. |
| 131 |
# BSD-based systems usually use a size of 104 or 108 and Windows does |
| 132 |
# not create AF_UNIX sockets. |
| 133 |
# |
| 134 |
# [1]: https://github.com/python/cpython/issues/140734 |
| 135 |
# [2]: https://pubs.opengroup.org/onlinepubs/9799919799/basedefs/sys_un.h.html |
| 136 |
# [3]: https://man7.org/linux/man-pages/man7/unix.7.html |
| 137 |
|
| 138 |
if sys.platform == 'linux': |
| 139 |
_SUN_PATH_MAX = 108 |
| 140 |
elif sys.platform.startswith(('openbsd', 'freebsd')): |
| 141 |
_SUN_PATH_MAX = 104 |
| 142 |
else: |
| 143 |
# On Windows platforms, we do not create AF_UNIX sockets. |
| 144 |
_SUN_PATH_MAX = None if os.name == 'nt' else 92 |
| 145 |
|
| 146 |
def _remove_temp_dir(rmtree, tempdir): |
| 147 |
rmtree(tempdir) |
| 148 |
|
| 149 |
current_process = process.current_process() |
| 150 |
# current_process() can be None if the finalizer is called |
| 151 |
# late during Python finalization |
| 152 |
if current_process is not None: |
| 153 |
current_process._config['tempdir'] = None |
| 154 |
|
| 155 |
def _get_base_temp_dir(tempfile): |
| 156 |
"""Get a temporary directory where socket files will be created. |
| 157 |
|
| 158 |
To prevent additional imports, pass a pre-imported 'tempfile' module. |
| 159 |
""" |
| 160 |
if os.name == 'nt': |
| 161 |
return None |
| 162 |
# Most of the time, the default temporary directory is /tmp. Thus, |
| 163 |
# listener sockets files "$TMPDIR/pymp-XXXXXXXX/sock-XXXXXXXX" do |
| 164 |
# not have a path length exceeding SUN_PATH_MAX. |
| 165 |
# |
| 166 |
# If users specify their own temporary directory, we may be unable |
| 167 |
# to create those files. Therefore, we fall back to the system-wide |
| 168 |
# temporary directory /tmp, assumed to exist on POSIX systems. |
| 169 |
# |
| 170 |
# See https://github.com/python/cpython/issues/132124. |
| 171 |
base_tempdir = tempfile.gettempdir() |
| 172 |
# Files created in a temporary directory are suffixed by a string |
| 173 |
# generated by tempfile._RandomNameSequence, which, by design, |
| 174 |
# is 8 characters long. |
| 175 |
# |
| 176 |
# Thus, the socket file path length (without NULL terminator) will be: |
| 177 |
# |
| 178 |
# len(base_tempdir + '/pymp-XXXXXXXX' + '/sock-XXXXXXXX') |
| 179 |
sun_path_len = len(base_tempdir) + 14 + 14 |
| 180 |
# Strict inequality to account for the NULL terminator. |
| 181 |
# See https://github.com/python/cpython/issues/140734. |
| 182 |
if sun_path_len < _SUN_PATH_MAX: |
| 183 |
return base_tempdir |
| 184 |
# Fallback to the default system-wide temporary directory. |
| 185 |
# This ignores user-defined environment variables. |
| 186 |
# |
| 187 |
# On POSIX systems, /tmp MUST be writable by any application [1]. |
| 188 |
# We however emit a warning if this is not the case to prevent |
| 189 |
# obscure errors later in the execution. |
| 190 |
# |
| 191 |
# On some legacy systems, /var/tmp and /usr/tmp can be present |
| 192 |
# and will be used instead. |
| 193 |
# |
| 194 |
# [1]: https://refspecs.linuxfoundation.org/FHS_3.0/fhs/ch03s18.html |
| 195 |
dirlist = ['/tmp', '/var/tmp', '/usr/tmp'] |
| 196 |
try: |
| 197 |
base_system_tempdir = tempfile._get_default_tempdir(dirlist) |
| 198 |
except FileNotFoundError: |
| 199 |
warn("Process-wide temporary directory %s will not be usable for " |
| 200 |
"creating socket files and no usable system-wide temporary " |
| 201 |
"directory was found in %s", base_tempdir, dirlist) |
| 202 |
# At this point, the system-wide temporary directory is not usable |
| 203 |
# but we may assume that the user-defined one is, even if we will |
| 204 |
# not be able to write socket files out there. |
| 205 |
return base_tempdir |
| 206 |
warn("Ignoring user-defined temporary directory: %s", base_tempdir) |
| 207 |
# at most max(map(len, dirlist)) + 14 + 14 = 36 characters |
| 208 |
assert len(base_system_tempdir) + 14 + 14 < _SUN_PATH_MAX |
| 209 |
return base_system_tempdir |
| 210 |
|
| 211 |
def get_temp_dir(): |
| 212 |
# get name of a temp directory which will be automatically cleaned up |
| 213 |
tempdir = process.current_process()._config.get('tempdir') |
| 214 |
if tempdir is None: |
| 215 |
import shutil, tempfile |
| 216 |
base_tempdir = _get_base_temp_dir(tempfile) |
| 217 |
tempdir = tempfile.mkdtemp(prefix='pymp-', dir=base_tempdir) |
| 218 |
info('created temp directory %s', tempdir) |
| 219 |
# keep a strong reference to shutil.rmtree(), since the finalizer |
| 220 |
# can be called late during Python shutdown |
| 221 |
Finalize(None, _remove_temp_dir, args=(shutil.rmtree, tempdir), |
| 222 |
exitpriority=-100) |
| 223 |
process.current_process()._config['tempdir'] = tempdir |
| 224 |
return tempdir |
| 225 |
|
| 226 |
# |
| 227 |
# Support for reinitialization of objects when bootstrapping a child process |
| 228 |
# |
| 229 |
|
| 230 |
_afterfork_registry = weakref.WeakValueDictionary() |
| 231 |
_afterfork_counter = itertools.count() |
| 232 |
|
| 233 |
def _run_after_forkers(): |
| 234 |
items = list(_afterfork_registry.items()) |
| 235 |
items.sort() |
| 236 |
for (index, ident, func), obj in items: |
| 237 |
try: |
| 238 |
func(obj) |
| 239 |
except Exception as e: |
| 240 |
info('after forker raised exception %s', e) |
| 241 |
|
| 242 |
def register_after_fork(obj, func): |
| 243 |
_afterfork_registry[(next(_afterfork_counter), id(obj), func)] = obj |
| 244 |
|
| 245 |
# |
| 246 |
# Finalization using weakrefs |
| 247 |
# |
| 248 |
|
| 249 |
_finalizer_registry = {} |
| 250 |
_finalizer_counter = itertools.count() |
| 251 |
|
| 252 |
|
| 253 |
class Finalize(object): |
| 254 |
''' |
| 255 |
Class which supports object finalization using weakrefs |
| 256 |
''' |
| 257 |
def __init__(self, obj, callback, args=(), kwargs=None, exitpriority=None): |
| 258 |
if (exitpriority is not None) and not isinstance(exitpriority,int): |
| 259 |
raise TypeError( |
| 260 |
"Exitpriority ({0!r}) must be None or int, not {1!s}".format( |
| 261 |
exitpriority, type(exitpriority))) |
| 262 |
|
| 263 |
if obj is not None: |
| 264 |
self._weakref = weakref.ref(obj, self) |
| 265 |
elif exitpriority is None: |
| 266 |
raise ValueError("Without object, exitpriority cannot be None") |
| 267 |
|
| 268 |
self._callback = callback |
| 269 |
self._args = args |
| 270 |
self._kwargs = kwargs or {} |
| 271 |
self._key = (exitpriority, next(_finalizer_counter)) |
| 272 |
self._pid = os.getpid() |
| 273 |
|
| 274 |
_finalizer_registry[self._key] = self |
| 275 |
|
| 276 |
def __call__(self, wr=None, |
| 277 |
# Need to bind these locally because the globals can have |
| 278 |
# been cleared at shutdown |
| 279 |
_finalizer_registry=_finalizer_registry, |
| 280 |
sub_debug=sub_debug, getpid=os.getpid): |
| 281 |
''' |
| 282 |
Run the callback unless it has already been called or cancelled |
| 283 |
''' |
| 284 |
try: |
| 285 |
del _finalizer_registry[self._key] |
| 286 |
except KeyError: |
| 287 |
sub_debug('finalizer no longer registered') |
| 288 |
else: |
| 289 |
if self._pid != getpid(): |
| 290 |
sub_debug('finalizer ignored because different process') |
| 291 |
res = None |
| 292 |
else: |
| 293 |
sub_debug('finalizer calling %s with args %s and kwargs %s', |
| 294 |
self._callback, self._args, self._kwargs) |
| 295 |
res = self._callback(*self._args, **self._kwargs) |
| 296 |
self._weakref = self._callback = self._args = \ |
| 297 |
self._kwargs = self._key = None |
| 298 |
return res |
| 299 |
|
| 300 |
def cancel(self): |
| 301 |
''' |
| 302 |
Cancel finalization of the object |
| 303 |
''' |
| 304 |
try: |
| 305 |
del _finalizer_registry[self._key] |
| 306 |
except KeyError: |
| 307 |
pass |
| 308 |
else: |
| 309 |
self._weakref = self._callback = self._args = \ |
| 310 |
self._kwargs = self._key = None |
| 311 |
|
| 312 |
def still_active(self): |
| 313 |
''' |
| 314 |
Return whether this finalizer is still waiting to invoke callback |
| 315 |
''' |
| 316 |
return self._key in _finalizer_registry |
| 317 |
|
| 318 |
def __repr__(self): |
| 319 |
try: |
| 320 |
obj = self._weakref() |
| 321 |
except (AttributeError, TypeError): |
| 322 |
obj = None |
| 323 |
|
| 324 |
if obj is None: |
| 325 |
return '<%s object, dead>' % self.__class__.__name__ |
| 326 |
|
| 327 |
x = '<%s object, callback=%s' % ( |
| 328 |
self.__class__.__name__, |
| 329 |
getattr(self._callback, '__name__', self._callback)) |
| 330 |
if self._args: |
| 331 |
x += ', args=' + str(self._args) |
| 332 |
if self._kwargs: |
| 333 |
x += ', kwargs=' + str(self._kwargs) |
| 334 |
if self._key[0] is not None: |
| 335 |
x += ', exitpriority=' + str(self._key[0]) |
| 336 |
return x + '>' |
| 337 |
|
| 338 |
|
| 339 |
def _run_finalizers(minpriority=None): |
| 340 |
''' |
| 341 |
Run all finalizers whose exit priority is not None and at least minpriority |
| 342 |
|
| 343 |
Finalizers with highest priority are called first; finalizers with |
| 344 |
the same priority will be called in reverse order of creation. |
| 345 |
''' |
| 346 |
if _finalizer_registry is None: |
| 347 |
# This function may be called after this module's globals are |
| 348 |
# destroyed. See the _exit_function function in this module for more |
| 349 |
# notes. |
| 350 |
return |
| 351 |
|
| 352 |
if minpriority is None: |
| 353 |
f = lambda p : p[0] is not None |
| 354 |
else: |
| 355 |
f = lambda p : p[0] is not None and p[0] >= minpriority |
| 356 |
|
| 357 |
# Careful: _finalizer_registry may be mutated while this function |
| 358 |
# is running (either by a GC run or by another thread). |
| 359 |
|
| 360 |
# list(_finalizer_registry) should be atomic, while |
| 361 |
# list(_finalizer_registry.items()) is not. |
| 362 |
keys = [key for key in list(_finalizer_registry) if f(key)] |
| 363 |
keys.sort(reverse=True) |
| 364 |
|
| 365 |
for key in keys: |
| 366 |
finalizer = _finalizer_registry.get(key) |
| 367 |
# key may have been removed from the registry |
| 368 |
if finalizer is not None: |
| 369 |
sub_debug('calling %s', finalizer) |
| 370 |
try: |
| 371 |
finalizer() |
| 372 |
except Exception: |
| 373 |
import traceback |
| 374 |
traceback.print_exc() |
| 375 |
|
| 376 |
if minpriority is None: |
| 377 |
_finalizer_registry.clear() |
| 378 |
|
| 379 |
# |
| 380 |
# Clean up on exit |
| 381 |
# |
| 382 |
|
| 383 |
def is_exiting(): |
| 384 |
''' |
| 385 |
Returns true if the process is shutting down |
| 386 |
''' |
| 387 |
return _exiting or _exiting is None |
| 388 |
|
| 389 |
_exiting = False |
| 390 |
|
| 391 |
def _exit_function(info=info, debug=debug, _run_finalizers=_run_finalizers, |
| 392 |
active_children=process.active_children, |
| 393 |
current_process=process.current_process): |
| 394 |
# We hold on to references to functions in the arglist due to the |
| 395 |
# situation described below, where this function is called after this |
| 396 |
# module's globals are destroyed. |
| 397 |
|
| 398 |
global _exiting |
| 399 |
|
| 400 |
if not _exiting: |
| 401 |
_exiting = True |
| 402 |
|
| 403 |
info('process shutting down') |
| 404 |
debug('running all "atexit" finalizers with priority >= 0') |
| 405 |
_run_finalizers(0) |
| 406 |
|
| 407 |
if current_process() is not None: |
| 408 |
# We check if the current process is None here because if |
| 409 |
# it's None, any call to ``active_children()`` will raise |
| 410 |
# an AttributeError (active_children winds up trying to |
| 411 |
# get attributes from util._current_process). One |
| 412 |
# situation where this can happen is if someone has |
| 413 |
# manipulated sys.modules, causing this module to be |
| 414 |
# garbage collected. The destructor for the module type |
| 415 |
# then replaces all values in the module dict with None. |
| 416 |
# For instance, after setuptools runs a test it replaces |
| 417 |
# sys.modules with a copy created earlier. See issues |
| 418 |
# #9775 and #15881. Also related: #4106, #9205, and |
| 419 |
# #9207. |
| 420 |
|
| 421 |
for p in active_children(): |
| 422 |
if p.daemon: |
| 423 |
info('calling terminate() for daemon %s', p.name) |
| 424 |
p._popen.terminate() |
| 425 |
|
| 426 |
for p in active_children(): |
| 427 |
info('calling join() for process %s', p.name) |
| 428 |
p.join() |
| 429 |
|
| 430 |
debug('running the remaining "atexit" finalizers') |
| 431 |
_run_finalizers() |
| 432 |
|
| 433 |
atexit.register(_exit_function) |
| 434 |
|
| 435 |
# |
| 436 |
# Some fork aware types |
| 437 |
# |
| 438 |
|
| 439 |
class ForkAwareThreadLock(object): |
| 440 |
def __init__(self): |
| 441 |
self._lock = threading.Lock() |
| 442 |
self.acquire = self._lock.acquire |
| 443 |
self.release = self._lock.release |
| 444 |
register_after_fork(self, ForkAwareThreadLock._at_fork_reinit) |
| 445 |
|
| 446 |
def _at_fork_reinit(self): |
| 447 |
self._lock._at_fork_reinit() |
| 448 |
|
| 449 |
def __enter__(self): |
| 450 |
return self._lock.__enter__() |
| 451 |
|
| 452 |
def __exit__(self, *args): |
| 453 |
return self._lock.__exit__(*args) |
| 454 |
|
| 455 |
|
| 456 |
class ForkAwareLocal(threading.local): |
| 457 |
def __init__(self): |
| 458 |
register_after_fork(self, lambda obj : obj.__dict__.clear()) |
| 459 |
def __reduce__(self): |
| 460 |
return type(self), () |
| 461 |
|
| 462 |
# |
| 463 |
# Close fds except those specified |
| 464 |
# |
| 465 |
|
| 466 |
try: |
| 467 |
MAXFD = os.sysconf("SC_OPEN_MAX") |
| 468 |
except Exception: |
| 469 |
MAXFD = 256 |
| 470 |
|
| 471 |
def close_all_fds_except(fds): |
| 472 |
fds = list(fds) + [-1, MAXFD] |
| 473 |
fds.sort() |
| 474 |
assert fds[-1] == MAXFD, 'fd too large' |
| 475 |
for i in range(len(fds) - 1): |
| 476 |
os.closerange(fds[i]+1, fds[i+1]) |
| 477 |
# |
| 478 |
# Close sys.stdin and replace stdin with os.devnull |
| 479 |
# |
| 480 |
|
| 481 |
def _close_stdin(): |
| 482 |
if sys.stdin is None: |
| 483 |
return |
| 484 |
|
| 485 |
try: |
| 486 |
sys.stdin.close() |
| 487 |
except (OSError, ValueError): |
| 488 |
pass |
| 489 |
|
| 490 |
try: |
| 491 |
fd = os.open(os.devnull, os.O_RDONLY) |
| 492 |
try: |
| 493 |
sys.stdin = open(fd, encoding="utf-8", closefd=False) |
| 494 |
except: |
| 495 |
os.close(fd) |
| 496 |
raise |
| 497 |
except (OSError, ValueError): |
| 498 |
pass |
| 499 |
|
| 500 |
# |
| 501 |
# Flush standard streams, if any |
| 502 |
# |
| 503 |
|
| 504 |
def _flush_std_streams(): |
| 505 |
try: |
| 506 |
sys.stdout.flush() |
| 507 |
except (AttributeError, ValueError): |
| 508 |
pass |
| 509 |
try: |
| 510 |
sys.stderr.flush() |
| 511 |
except (AttributeError, ValueError): |
| 512 |
pass |
| 513 |
|
| 514 |
# |
| 515 |
# Start a program with only specified fds kept open |
| 516 |
# |
| 517 |
|
| 518 |
def spawnv_passfds(path, args, passfds): |
| 519 |
import _posixsubprocess |
| 520 |
passfds = tuple(sorted(map(int, passfds))) |
| 521 |
errpipe_read, errpipe_write = os.pipe() |
| 522 |
try: |
| 523 |
return _posixsubprocess.fork_exec( |
| 524 |
args, [path], True, passfds, None, None, |
| 525 |
-1, -1, -1, -1, -1, -1, errpipe_read, errpipe_write, |
| 526 |
False, False, -1, None, None, None, -1, None) |
| 527 |
finally: |
| 528 |
os.close(errpipe_read) |
| 529 |
os.close(errpipe_write) |
| 530 |
|
| 531 |
|
| 532 |
def close_fds(*fds): |
| 533 |
"""Close each file descriptor given as an argument""" |
| 534 |
for fd in fds: |
| 535 |
os.close(fd) |
| 536 |
|
| 537 |
|
| 538 |
def _cleanup_tests(): |
| 539 |
"""Cleanup multiprocessing resources when multiprocessing tests |
| 540 |
completed.""" |
| 541 |
|
| 542 |
from test import support |
| 543 |
|
| 544 |
# cleanup multiprocessing |
| 545 |
process._cleanup() |
| 546 |
|
| 547 |
# Stop the ForkServer process if it's running |
| 548 |
from multiprocessing import forkserver |
| 549 |
forkserver._forkserver._stop() |
| 550 |
|
| 551 |
# Stop the ResourceTracker process if it's running |
| 552 |
from multiprocessing import resource_tracker |
| 553 |
resource_tracker._resource_tracker._stop() |
| 554 |
|
| 555 |
# bpo-37421: Explicitly call _run_finalizers() to remove immediately |
| 556 |
# temporary directories created by multiprocessing.util.get_temp_dir(). |
| 557 |
_run_finalizers() |
| 558 |
support.gc_collect() |
| 559 |
|
| 560 |
support.reap_children() |