Debian Codegraph @

Lib/multiprocessing/util.py

raw
1 #
2 # Module providing various facilities to other parts of the package
3 #
4 # multiprocessing/util.py
5 #
6 # Copyright (c) 2006-2008, R Oudkerk
7 # Licensed to PSF under a Contributor Agreement.
8 #
9
10 import os
11 import itertools
12 import sys
13 import weakref
14 import atexit
15 import threading # we want threading to install it's
16 # cleanup function before multiprocessing does
17 from subprocess import _args_from_interpreter_flags # noqa: F401
18
19 from . import process
20
21 __all__ = [
22 'sub_debug', 'debug', 'info', 'sub_warning', 'warn', 'get_logger',
23 'log_to_stderr', 'get_temp_dir', 'register_after_fork',
24 'is_exiting', 'Finalize', 'ForkAwareThreadLock', 'ForkAwareLocal',
25 'close_all_fds_except', 'SUBDEBUG', 'SUBWARNING',
26 ]
27
28 #
29 # Logging
30 #
31
32 NOTSET = 0
33 SUBDEBUG = 5
34 DEBUG = 10
35 INFO = 20
36 SUBWARNING = 25
37 WARNING = 30
38
39 LOGGER_NAME = 'multiprocessing'
40 DEFAULT_LOGGING_FORMAT = '[%(levelname)s/%(processName)s] %(message)s'
41
42 _logger = None
43 _log_to_stderr = False
44
45 def sub_debug(msg, *args):
46 if _logger:
47 _logger.log(SUBDEBUG, msg, *args, stacklevel=2)
48
49 def debug(msg, *args):
50 if _logger:
51 _logger.log(DEBUG, msg, *args, stacklevel=2)
52
53 def info(msg, *args):
54 if _logger:
55 _logger.log(INFO, msg, *args, stacklevel=2)
56
57 def warn(msg, *args):
58 if _logger:
59 _logger.log(WARNING, msg, *args, stacklevel=2)
60
61 def sub_warning(msg, *args):
62 if _logger:
63 _logger.log(SUBWARNING, msg, *args, stacklevel=2)
64
65 def get_logger():
66 '''
67 Returns logger used by multiprocessing
68 '''
69 global _logger
70 import logging
71
72 with logging._lock:
73 if not _logger:
74
75 _logger = logging.getLogger(LOGGER_NAME)
76 _logger.propagate = 0
77
78 # XXX multiprocessing should cleanup before logging
79 if hasattr(atexit, 'unregister'):
80 atexit.unregister(_exit_function)
81 atexit.register(_exit_function)
82 else:
83 atexit._exithandlers.remove((_exit_function, (), {}))
84 atexit._exithandlers.append((_exit_function, (), {}))
85
86 return _logger
87
88 def log_to_stderr(level=None):
89 '''
90 Turn on logging and add a handler which prints to stderr
91 '''
92 global _log_to_stderr
93 import logging
94
95 logger = get_logger()
96 formatter = logging.Formatter(DEFAULT_LOGGING_FORMAT)
97 handler = logging.StreamHandler()
98 handler.setFormatter(formatter)
99 logger.addHandler(handler)
100
101 if level:
102 logger.setLevel(level)
103 _log_to_stderr = True
104 return _logger
105
106
107 # Abstract socket support
108
109 def _platform_supports_abstract_sockets():
110 return sys.platform in ("linux", "android")
111
112
113 def is_abstract_socket_namespace(address):
114 if not address:
115 return False
116 if isinstance(address, bytes):
117 return address[0] == 0
118 elif isinstance(address, str):
119 return address[0] == "\0"
120 raise TypeError(f'address type of {address!r} unrecognized')
121
122
123 abstract_sockets_supported = _platform_supports_abstract_sockets()
124
125 #
126 # Function returning a temp directory which will be removed on exit
127 #
128
129 # Maximum length of a NULL-terminated [1] socket file path is usually
130 # between 92 and 108 [2], but Linux is known to use a size of 108 [3].
131 # BSD-based systems usually use a size of 104 or 108 and Windows does
132 # not create AF_UNIX sockets.
133 #
134 # [1]: https://github.com/python/cpython/issues/140734
135 # [2]: https://pubs.opengroup.org/onlinepubs/9799919799/basedefs/sys_un.h.html
136 # [3]: https://man7.org/linux/man-pages/man7/unix.7.html
137
138 if sys.platform == 'linux':
139 _SUN_PATH_MAX = 108
140 elif sys.platform.startswith(('openbsd', 'freebsd')):
141 _SUN_PATH_MAX = 104
142 else:
143 # On Windows platforms, we do not create AF_UNIX sockets.
144 _SUN_PATH_MAX = None if os.name == 'nt' else 92
145
146 def _remove_temp_dir(rmtree, tempdir):
147 rmtree(tempdir)
148
149 current_process = process.current_process()
150 # current_process() can be None if the finalizer is called
151 # late during Python finalization
152 if current_process is not None:
153 current_process._config['tempdir'] = None
154
155 def _get_base_temp_dir(tempfile):
156 """Get a temporary directory where socket files will be created.
157
158 To prevent additional imports, pass a pre-imported 'tempfile' module.
159 """
160 if os.name == 'nt':
161 return None
162 # Most of the time, the default temporary directory is /tmp. Thus,
163 # listener sockets files "$TMPDIR/pymp-XXXXXXXX/sock-XXXXXXXX" do
164 # not have a path length exceeding SUN_PATH_MAX.
165 #
166 # If users specify their own temporary directory, we may be unable
167 # to create those files. Therefore, we fall back to the system-wide
168 # temporary directory /tmp, assumed to exist on POSIX systems.
169 #
170 # See https://github.com/python/cpython/issues/132124.
171 base_tempdir = tempfile.gettempdir()
172 # Files created in a temporary directory are suffixed by a string
173 # generated by tempfile._RandomNameSequence, which, by design,
174 # is 8 characters long.
175 #
176 # Thus, the socket file path length (without NULL terminator) will be:
177 #
178 # len(base_tempdir + '/pymp-XXXXXXXX' + '/sock-XXXXXXXX')
179 sun_path_len = len(base_tempdir) + 14 + 14
180 # Strict inequality to account for the NULL terminator.
181 # See https://github.com/python/cpython/issues/140734.
182 if sun_path_len < _SUN_PATH_MAX:
183 return base_tempdir
184 # Fallback to the default system-wide temporary directory.
185 # This ignores user-defined environment variables.
186 #
187 # On POSIX systems, /tmp MUST be writable by any application [1].
188 # We however emit a warning if this is not the case to prevent
189 # obscure errors later in the execution.
190 #
191 # On some legacy systems, /var/tmp and /usr/tmp can be present
192 # and will be used instead.
193 #
194 # [1]: https://refspecs.linuxfoundation.org/FHS_3.0/fhs/ch03s18.html
195 dirlist = ['/tmp', '/var/tmp', '/usr/tmp']
196 try:
197 base_system_tempdir = tempfile._get_default_tempdir(dirlist)
198 except FileNotFoundError:
199 warn("Process-wide temporary directory %s will not be usable for "
200 "creating socket files and no usable system-wide temporary "
201 "directory was found in %s", base_tempdir, dirlist)
202 # At this point, the system-wide temporary directory is not usable
203 # but we may assume that the user-defined one is, even if we will
204 # not be able to write socket files out there.
205 return base_tempdir
206 warn("Ignoring user-defined temporary directory: %s", base_tempdir)
207 # at most max(map(len, dirlist)) + 14 + 14 = 36 characters
208 assert len(base_system_tempdir) + 14 + 14 < _SUN_PATH_MAX
209 return base_system_tempdir
210
211 def get_temp_dir():
212 # get name of a temp directory which will be automatically cleaned up
213 tempdir = process.current_process()._config.get('tempdir')
214 if tempdir is None:
215 import shutil, tempfile
216 base_tempdir = _get_base_temp_dir(tempfile)
217 tempdir = tempfile.mkdtemp(prefix='pymp-', dir=base_tempdir)
218 info('created temp directory %s', tempdir)
219 # keep a strong reference to shutil.rmtree(), since the finalizer
220 # can be called late during Python shutdown
221 Finalize(None, _remove_temp_dir, args=(shutil.rmtree, tempdir),
222 exitpriority=-100)
223 process.current_process()._config['tempdir'] = tempdir
224 return tempdir
225
226 #
227 # Support for reinitialization of objects when bootstrapping a child process
228 #
229
230 _afterfork_registry = weakref.WeakValueDictionary()
231 _afterfork_counter = itertools.count()
232
233 def _run_after_forkers():
234 items = list(_afterfork_registry.items())
235 items.sort()
236 for (index, ident, func), obj in items:
237 try:
238 func(obj)
239 except Exception as e:
240 info('after forker raised exception %s', e)
241
242 def register_after_fork(obj, func):
243 _afterfork_registry[(next(_afterfork_counter), id(obj), func)] = obj
244
245 #
246 # Finalization using weakrefs
247 #
248
249 _finalizer_registry = {}
250 _finalizer_counter = itertools.count()
251
252
253 class Finalize(object):
254 '''
255 Class which supports object finalization using weakrefs
256 '''
257 def __init__(self, obj, callback, args=(), kwargs=None, exitpriority=None):
258 if (exitpriority is not None) and not isinstance(exitpriority,int):
259 raise TypeError(
260 "Exitpriority ({0!r}) must be None or int, not {1!s}".format(
261 exitpriority, type(exitpriority)))
262
263 if obj is not None:
264 self._weakref = weakref.ref(obj, self)
265 elif exitpriority is None:
266 raise ValueError("Without object, exitpriority cannot be None")
267
268 self._callback = callback
269 self._args = args
270 self._kwargs = kwargs or {}
271 self._key = (exitpriority, next(_finalizer_counter))
272 self._pid = os.getpid()
273
274 _finalizer_registry[self._key] = self
275
276 def __call__(self, wr=None,
277 # Need to bind these locally because the globals can have
278 # been cleared at shutdown
279 _finalizer_registry=_finalizer_registry,
280 sub_debug=sub_debug, getpid=os.getpid):
281 '''
282 Run the callback unless it has already been called or cancelled
283 '''
284 try:
285 del _finalizer_registry[self._key]
286 except KeyError:
287 sub_debug('finalizer no longer registered')
288 else:
289 if self._pid != getpid():
290 sub_debug('finalizer ignored because different process')
291 res = None
292 else:
293 sub_debug('finalizer calling %s with args %s and kwargs %s',
294 self._callback, self._args, self._kwargs)
295 res = self._callback(*self._args, **self._kwargs)
296 self._weakref = self._callback = self._args = \
297 self._kwargs = self._key = None
298 return res
299
300 def cancel(self):
301 '''
302 Cancel finalization of the object
303 '''
304 try:
305 del _finalizer_registry[self._key]
306 except KeyError:
307 pass
308 else:
309 self._weakref = self._callback = self._args = \
310 self._kwargs = self._key = None
311
312 def still_active(self):
313 '''
314 Return whether this finalizer is still waiting to invoke callback
315 '''
316 return self._key in _finalizer_registry
317
318 def __repr__(self):
319 try:
320 obj = self._weakref()
321 except (AttributeError, TypeError):
322 obj = None
323
324 if obj is None:
325 return '<%s object, dead>' % self.__class__.__name__
326
327 x = '<%s object, callback=%s' % (
328 self.__class__.__name__,
329 getattr(self._callback, '__name__', self._callback))
330 if self._args:
331 x += ', args=' + str(self._args)
332 if self._kwargs:
333 x += ', kwargs=' + str(self._kwargs)
334 if self._key[0] is not None:
335 x += ', exitpriority=' + str(self._key[0])
336 return x + '>'
337
338
339 def _run_finalizers(minpriority=None):
340 '''
341 Run all finalizers whose exit priority is not None and at least minpriority
342
343 Finalizers with highest priority are called first; finalizers with
344 the same priority will be called in reverse order of creation.
345 '''
346 if _finalizer_registry is None:
347 # This function may be called after this module's globals are
348 # destroyed. See the _exit_function function in this module for more
349 # notes.
350 return
351
352 if minpriority is None:
353 f = lambda p : p[0] is not None
354 else:
355 f = lambda p : p[0] is not None and p[0] >= minpriority
356
357 # Careful: _finalizer_registry may be mutated while this function
358 # is running (either by a GC run or by another thread).
359
360 # list(_finalizer_registry) should be atomic, while
361 # list(_finalizer_registry.items()) is not.
362 keys = [key for key in list(_finalizer_registry) if f(key)]
363 keys.sort(reverse=True)
364
365 for key in keys:
366 finalizer = _finalizer_registry.get(key)
367 # key may have been removed from the registry
368 if finalizer is not None:
369 sub_debug('calling %s', finalizer)
370 try:
371 finalizer()
372 except Exception:
373 import traceback
374 traceback.print_exc()
375
376 if minpriority is None:
377 _finalizer_registry.clear()
378
379 #
380 # Clean up on exit
381 #
382
383 def is_exiting():
384 '''
385 Returns true if the process is shutting down
386 '''
387 return _exiting or _exiting is None
388
389 _exiting = False
390
391 def _exit_function(info=info, debug=debug, _run_finalizers=_run_finalizers,
392 active_children=process.active_children,
393 current_process=process.current_process):
394 # We hold on to references to functions in the arglist due to the
395 # situation described below, where this function is called after this
396 # module's globals are destroyed.
397
398 global _exiting
399
400 if not _exiting:
401 _exiting = True
402
403 info('process shutting down')
404 debug('running all "atexit" finalizers with priority >= 0')
405 _run_finalizers(0)
406
407 if current_process() is not None:
408 # We check if the current process is None here because if
409 # it's None, any call to ``active_children()`` will raise
410 # an AttributeError (active_children winds up trying to
411 # get attributes from util._current_process). One
412 # situation where this can happen is if someone has
413 # manipulated sys.modules, causing this module to be
414 # garbage collected. The destructor for the module type
415 # then replaces all values in the module dict with None.
416 # For instance, after setuptools runs a test it replaces
417 # sys.modules with a copy created earlier. See issues
418 # #9775 and #15881. Also related: #4106, #9205, and
419 # #9207.
420
421 for p in active_children():
422 if p.daemon:
423 info('calling terminate() for daemon %s', p.name)
424 p._popen.terminate()
425
426 for p in active_children():
427 info('calling join() for process %s', p.name)
428 p.join()
429
430 debug('running the remaining "atexit" finalizers')
431 _run_finalizers()
432
433 atexit.register(_exit_function)
434
435 #
436 # Some fork aware types
437 #
438
439 class ForkAwareThreadLock(object):
440 def __init__(self):
441 self._lock = threading.Lock()
442 self.acquire = self._lock.acquire
443 self.release = self._lock.release
444 register_after_fork(self, ForkAwareThreadLock._at_fork_reinit)
445
446 def _at_fork_reinit(self):
447 self._lock._at_fork_reinit()
448
449 def __enter__(self):
450 return self._lock.__enter__()
451
452 def __exit__(self, *args):
453 return self._lock.__exit__(*args)
454
455
456 class ForkAwareLocal(threading.local):
457 def __init__(self):
458 register_after_fork(self, lambda obj : obj.__dict__.clear())
459 def __reduce__(self):
460 return type(self), ()
461
462 #
463 # Close fds except those specified
464 #
465
466 try:
467 MAXFD = os.sysconf("SC_OPEN_MAX")
468 except Exception:
469 MAXFD = 256
470
471 def close_all_fds_except(fds):
472 fds = list(fds) + [-1, MAXFD]
473 fds.sort()
474 assert fds[-1] == MAXFD, 'fd too large'
475 for i in range(len(fds) - 1):
476 os.closerange(fds[i]+1, fds[i+1])
477 #
478 # Close sys.stdin and replace stdin with os.devnull
479 #
480
481 def _close_stdin():
482 if sys.stdin is None:
483 return
484
485 try:
486 sys.stdin.close()
487 except (OSError, ValueError):
488 pass
489
490 try:
491 fd = os.open(os.devnull, os.O_RDONLY)
492 try:
493 sys.stdin = open(fd, encoding="utf-8", closefd=False)
494 except:
495 os.close(fd)
496 raise
497 except (OSError, ValueError):
498 pass
499
500 #
501 # Flush standard streams, if any
502 #
503
504 def _flush_std_streams():
505 try:
506 sys.stdout.flush()
507 except (AttributeError, ValueError):
508 pass
509 try:
510 sys.stderr.flush()
511 except (AttributeError, ValueError):
512 pass
513
514 #
515 # Start a program with only specified fds kept open
516 #
517
518 def spawnv_passfds(path, args, passfds):
519 import _posixsubprocess
520 passfds = tuple(sorted(map(int, passfds)))
521 errpipe_read, errpipe_write = os.pipe()
522 try:
523 return _posixsubprocess.fork_exec(
524 args, [path], True, passfds, None, None,
525 -1, -1, -1, -1, -1, -1, errpipe_read, errpipe_write,
526 False, False, -1, None, None, None, -1, None)
527 finally:
528 os.close(errpipe_read)
529 os.close(errpipe_write)
530
531
532 def close_fds(*fds):
533 """Close each file descriptor given as an argument"""
534 for fd in fds:
535 os.close(fd)
536
537
538 def _cleanup_tests():
539 """Cleanup multiprocessing resources when multiprocessing tests
540 completed."""
541
542 from test import support
543
544 # cleanup multiprocessing
545 process._cleanup()
546
547 # Stop the ForkServer process if it's running
548 from multiprocessing import forkserver
549 forkserver._forkserver._stop()
550
551 # Stop the ResourceTracker process if it's running
552 from multiprocessing import resource_tracker
553 resource_tracker._resource_tracker._stop()
554
555 # bpo-37421: Explicitly call _run_finalizers() to remove immediately
556 # temporary directories created by multiprocessing.util.get_temp_dir().
557 _run_finalizers()
558 support.gc_collect()
559
560 support.reap_children()