Debian Codegraph @

Lib/multiprocessing/resource_tracker.py

raw
1 ###############################################################################
2 # Server process to keep track of unlinked resources (like shared memory
3 # segments, semaphores etc.) and clean them.
4 #
5 # On Unix we run a server process which keeps track of unlinked
6 # resources. The server ignores SIGINT and SIGTERM and reads from a
7 # pipe. Every other process of the program has a copy of the writable
8 # end of the pipe, so we get EOF when all other processes have exited.
9 # Then the server process unlinks any remaining resource names.
10 #
11 # This is important because there may be system limits for such resources: for
12 # instance, the system only supports a limited number of named semaphores, and
13 # shared-memory segments live in the RAM. If a python process leaks such a
14 # resource, this resource will not be removed till the next reboot. Without
15 # this resource tracker process, "killall python" would probably leave unlinked
16 # resources.
17
18 import base64
19 import os
20 import signal
21 import sys
22 import threading
23 import time
24 import warnings
25 from collections import deque
26
27 import json
28
29 from . import spawn
30 from . import util
31
32 __all__ = ['ensure_running', 'register', 'unregister']
33
34 _HAVE_SIGMASK = hasattr(signal, 'pthread_sigmask')
35 _IGNORED_SIGNALS = (signal.SIGINT, signal.SIGTERM)
36
37 def cleanup_noop(name):
38 raise RuntimeError('noop should never be registered or cleaned up')
39
40 _CLEANUP_FUNCS = {
41 'noop': cleanup_noop,
42 'dummy': lambda name: None, # Dummy resource used in tests
43 }
44
45 if os.name == 'posix':
46 import _multiprocessing
47 import _posixshmem
48
49 # Use sem_unlink() to clean up named semaphores.
50 #
51 # sem_unlink() may be missing if the Python build process detected the
52 # absence of POSIX named semaphores. In that case, no named semaphores were
53 # ever opened, so no cleanup would be necessary.
54 if hasattr(_multiprocessing, 'sem_unlink'):
55 _CLEANUP_FUNCS.update({
56 'semaphore': _multiprocessing.sem_unlink,
57 })
58 _CLEANUP_FUNCS.update({
59 'shared_memory': _posixshmem.shm_unlink,
60 })
61
62
63 class ReentrantCallError(RuntimeError):
64 pass
65
66
67 class ResourceTracker(object):
68
69 def __init__(self):
70 self._lock = threading.RLock()
71 self._fd = None
72 self._pid = None
73 self._exitcode = None
74 self._reentrant_messages = deque()
75
76 # True to use colon-separated lines, rather than JSON lines,
77 # for internal communication. (Mainly for testing).
78 # Filenames not supported by the simple format will always be sent
79 # using JSON.
80 # The reader should understand all formats.
81 self._use_simple_format = True
82
83 # Set to True by _stop_locked() if the waitpid polling loop ran to
84 # its timeout without reaping the tracker. Exposed for tests.
85 self._waitpid_timed_out = False
86
87 def _reentrant_call_error(self):
88 # gh-109629: this happens if an explicit call to the ResourceTracker
89 # gets interrupted by a garbage collection, invoking a finalizer (*)
90 # that itself calls back into ResourceTracker.
91 # (*) for example the SemLock finalizer
92 raise ReentrantCallError(
93 "Reentrant call into the multiprocessing resource tracker")
94
95 def __del__(self):
96 # making sure child processess are cleaned before ResourceTracker
97 # gets destructed.
98 # see https://github.com/python/cpython/issues/88887
99 # gh-146313: use a timeout to avoid deadlocking if a forked child
100 # still holds the pipe's write end open.
101 self._stop(use_blocking_lock=False, wait_timeout=1.0)
102
103 def _after_fork_in_child(self):
104 # gh-146313: Called in the child right after os.fork().
105 #
106 # The tracker process is a child of the *parent*, not of us, so we
107 # could never waitpid() it anyway. Clearing _pid means our __del__
108 # becomes a no-op (the early return for _pid is None).
109 #
110 # Whether we keep the inherited _fd depends on who forked us:
111 #
112 # - multiprocessing.Process with the 'fork' start method sets
113 # _fork_intent.preserve_fd before forking. The child keeps the
114 # fd and reuses the parent's tracker (gh-80849). This is safe
115 # because multiprocessing's atexit handler joins all children
116 # before the parent's __del__ runs, so by then the fd copies
117 # are gone and the parent can reap the tracker promptly.
118 #
119 # - A raw os.fork() leaves the flag unset. We close the fd in the child after forking so
120 # the parent's __del__ can reap the tracker without waiting
121 # for the child to exit. If we later need a tracker, ensure_running()
122 # will launch a fresh one.
123 self._lock._at_fork_reinit()
124 self._reentrant_messages.clear()
125 self._pid = None
126 self._exitcode = None
127 if (self._fd is not None and
128 not getattr(_fork_intent, 'preserve_fd', False)):
129 fd = self._fd
130 self._fd = None
131 try:
132 os.close(fd)
133 except OSError:
134 pass
135
136 def _stop(self, use_blocking_lock=True, wait_timeout=None):
137 if use_blocking_lock:
138 with self._lock:
139 self._stop_locked(wait_timeout=wait_timeout)
140 else:
141 acquired = self._lock.acquire(blocking=False)
142 try:
143 self._stop_locked(wait_timeout=wait_timeout)
144 finally:
145 if acquired:
146 self._lock.release()
147
148 def _stop_locked(
149 self,
150 close=os.close,
151 waitpid=os.waitpid,
152 waitstatus_to_exitcode=os.waitstatus_to_exitcode,
153 monotonic=time.monotonic,
154 sleep=time.sleep,
155 WNOHANG=getattr(os, 'WNOHANG', None),
156 wait_timeout=None,
157 ):
158 # This shouldn't happen (it might when called by a finalizer)
159 # so we check for it anyway.
160 if self._lock._recursion_count() > 1:
161 raise self._reentrant_call_error()
162 if self._fd is None:
163 # not running
164 return
165 if self._pid is None:
166 return
167
168 # closing the "alive" file descriptor stops main()
169 close(self._fd)
170 self._fd = None
171
172 try:
173 if wait_timeout is None:
174 _, status = waitpid(self._pid, 0)
175 else:
176 # gh-146313: A forked child may still hold the pipe's write
177 # end open, preventing the tracker from seeing EOF and
178 # exiting. Poll with WNOHANG to avoid blocking forever.
179 deadline = monotonic() + wait_timeout
180 delay = 0.001
181 while True:
182 result_pid, status = waitpid(self._pid, WNOHANG)
183 if result_pid != 0:
184 break
185 remaining = deadline - monotonic()
186 if remaining <= 0:
187 # The tracker is still running; it will be
188 # reparented to PID 1 (or the nearest subreaper)
189 # when we exit, and reaped there once all pipe
190 # holders release their fd.
191 self._pid = None
192 self._exitcode = None
193 self._waitpid_timed_out = True
194 return
195 delay = min(delay * 2, remaining, 0.1)
196 sleep(delay)
197 except ChildProcessError:
198 self._pid = None
199 self._exitcode = None
200 return
201
202 self._pid = None
203
204 try:
205 self._exitcode = waitstatus_to_exitcode(status)
206 except ValueError:
207 # os.waitstatus_to_exitcode may raise an exception for invalid values
208 self._exitcode = None
209
210 def getfd(self):
211 self.ensure_running()
212 return self._fd
213
214 def ensure_running(self):
215 '''Make sure that resource tracker process is running.
216
217 This can be run from any process. Usually a child process will use
218 the resource created by its parent.'''
219 return self._ensure_running_and_write()
220
221 def _teardown_dead_process(self):
222 os.close(self._fd)
223
224 # Clean-up to avoid dangling processes.
225 try:
226 # _pid can be None if this process is a child from another
227 # python process, which has started the resource_tracker.
228 if self._pid is not None:
229 os.waitpid(self._pid, 0)
230 except ChildProcessError:
231 # The resource_tracker has already been terminated.
232 pass
233 self._fd = None
234 self._pid = None
235 self._exitcode = None
236
237 warnings.warn('resource_tracker: process died unexpectedly, '
238 'relaunching. Some resources might leak.')
239
240 def _launch(self):
241 fds_to_pass = []
242 try:
243 fds_to_pass.append(sys.stderr.fileno())
244 except Exception:
245 pass
246 r, w = os.pipe()
247 try:
248 fds_to_pass.append(r)
249 # process will out live us, so no need to wait on pid
250 exe = spawn.get_executable()
251 args = [
252 exe,
253 *util._args_from_interpreter_flags(),
254 '-c',
255 f'from multiprocessing.resource_tracker import main;main({r})',
256 ]
257 # bpo-33613: Register a signal mask that will block the signals.
258 # This signal mask will be inherited by the child that is going
259 # to be spawned and will protect the child from a race condition
260 # that can make the child die before it registers signal handlers
261 # for SIGINT and SIGTERM. The mask is unregistered after spawning
262 # the child.
263 prev_sigmask = None
264 try:
265 if _HAVE_SIGMASK:
266 prev_sigmask = signal.pthread_sigmask(signal.SIG_BLOCK, _IGNORED_SIGNALS)
267 pid = util.spawnv_passfds(exe, args, fds_to_pass)
268 finally:
269 if prev_sigmask is not None:
270 signal.pthread_sigmask(signal.SIG_SETMASK, prev_sigmask)
271 except:
272 os.close(w)
273 raise
274 else:
275 self._fd = w
276 self._pid = pid
277 finally:
278 os.close(r)
279
280 def _make_probe_message(self):
281 """Return a probe message."""
282 if self._use_simple_format:
283 return b'PROBE:0:noop\n'
284 return (
285 json.dumps(
286 {"cmd": "PROBE", "rtype": "noop"},
287 ensure_ascii=True,
288 separators=(",", ":"),
289 )
290 + "\n"
291 ).encode("ascii")
292
293 def _ensure_running_and_write(self, msg=None):
294 with self._lock:
295 if self._lock._recursion_count() > 1:
296 # The code below is certainly not reentrant-safe, so bail out
297 if msg is None:
298 raise self._reentrant_call_error()
299 return self._reentrant_messages.append(msg)
300
301 if self._fd is not None:
302 # resource tracker was launched before, is it still running?
303 if msg is None:
304 to_send = self._make_probe_message()
305 else:
306 to_send = msg
307 try:
308 self._write(to_send)
309 except OSError:
310 self._teardown_dead_process()
311 self._launch()
312
313 msg = None # message was sent in probe
314 else:
315 self._launch()
316
317 while True:
318 try:
319 reentrant_msg = self._reentrant_messages.popleft()
320 except IndexError:
321 break
322 self._write(reentrant_msg)
323 if msg is not None:
324 self._write(msg)
325
326 def _check_alive(self):
327 '''Check that the pipe has not been closed by sending a probe.'''
328 try:
329 # We cannot use send here as it calls ensure_running, creating
330 # a cycle.
331 os.write(self._fd, self._make_probe_message())
332 except OSError:
333 return False
334 else:
335 return True
336
337 def register(self, name, rtype):
338 '''Register name of resource with resource tracker.'''
339 self._send('REGISTER', name, rtype)
340
341 def unregister(self, name, rtype):
342 '''Unregister name of resource with resource tracker.'''
343 self._send('UNREGISTER', name, rtype)
344
345 def _write(self, msg):
346 nbytes = os.write(self._fd, msg)
347 assert nbytes == len(msg), f"{nbytes=} != {len(msg)=}"
348
349 def _send(self, cmd, name, rtype):
350 if self._use_simple_format and '\n' not in name:
351 msg = f"{cmd}:{name}:{rtype}\n".encode("ascii")
352 if len(msg) > 512:
353 # posix guarantees that writes to a pipe of less than PIPE_BUF
354 # bytes are atomic, and that PIPE_BUF >= 512
355 raise ValueError('msg too long')
356 self._ensure_running_and_write(msg)
357 return
358
359 # POSIX guarantees that writes to a pipe of less than PIPE_BUF (512 on Linux)
360 # bytes are atomic. Therefore, we want the message to be shorter than 512 bytes.
361 # POSIX shm_open() and sem_open() require the name, including its leading slash,
362 # to be at most NAME_MAX bytes (255 on Linux)
363 # With json.dump(..., ensure_ascii=True) every non-ASCII byte becomes a 6-char
364 # escape like \uDC80.
365 # As we want the overall message to be kept atomic and therefore smaller than 512,
366 # we encode encode the raw name bytes with URL-safe Base64 - so a 255 long name
367 # will not exceed 340 bytes.
368 b = name.encode('utf-8', 'surrogateescape')
369 if len(b) > 255:
370 raise ValueError('shared memory name too long (max 255 bytes)')
371 b64 = base64.urlsafe_b64encode(b).decode('ascii')
372
373 payload = {"cmd": cmd, "rtype": rtype, "base64_name": b64}
374 msg = (json.dumps(payload, ensure_ascii=True, separators=(",", ":")) + "\n").encode("ascii")
375
376 # The entire JSON message is guaranteed < PIPE_BUF (512 bytes) by construction.
377 assert len(msg) <= 512, f"internal error: message too long ({len(msg)} bytes)"
378 assert msg.startswith(b'{')
379
380 self._ensure_running_and_write(msg)
381
382 # gh-146313: Per-thread flag set by .popen_fork.Popen._launch() just before
383 # os.fork(), telling _after_fork_in_child() to keep the inherited pipe fd so
384 # the child can reuse this tracker (gh-80849). Unset for raw os.fork() calls,
385 # where the child instead closes the fd so the parent's __del__ can reap the
386 # tracker. Using threading.local() keeps multiple threads calling
387 # popen_fork.Popen._launch() at once from clobbering eachothers intent.
388 _fork_intent = threading.local()
389
390 _resource_tracker = ResourceTracker()
391 ensure_running = _resource_tracker.ensure_running
392 register = _resource_tracker.register
393 unregister = _resource_tracker.unregister
394 getfd = _resource_tracker.getfd
395
396 # gh-146313: See _after_fork_in_child docstring.
397 if hasattr(os, 'register_at_fork'):
398 os.register_at_fork(after_in_child=_resource_tracker._after_fork_in_child)
399
400
401 def _decode_message(line):
402 if line.startswith(b'{'):
403 try:
404 obj = json.loads(line.decode('ascii'))
405 except Exception as e:
406 raise ValueError("malformed resource_tracker message: %r" % (line,)) from e
407
408 cmd = obj["cmd"]
409 rtype = obj["rtype"]
410 b64 = obj.get("base64_name", "")
411
412 if not isinstance(cmd, str) or not isinstance(rtype, str) or not isinstance(b64, str):
413 raise ValueError("malformed resource_tracker fields: %r" % (obj,))
414
415 try:
416 name = base64.urlsafe_b64decode(b64).decode('utf-8', 'surrogateescape')
417 except ValueError as e:
418 raise ValueError("malformed resource_tracker base64_name: %r" % (b64,)) from e
419 else:
420 cmd, rest = line.strip().decode('ascii').split(':', maxsplit=1)
421 name, rtype = rest.rsplit(':', maxsplit=1)
422 return cmd, rtype, name
423
424
425 def main(fd):
426 '''Run resource tracker.'''
427 # protect the process from ^C and "killall python" etc
428 signal.signal(signal.SIGINT, signal.SIG_IGN)
429 signal.signal(signal.SIGTERM, signal.SIG_IGN)
430 if _HAVE_SIGMASK:
431 signal.pthread_sigmask(signal.SIG_UNBLOCK, _IGNORED_SIGNALS)
432
433 for f in (sys.stdin, sys.stdout):
434 try:
435 f.close()
436 except Exception:
437 pass
438
439 cache = {rtype: set() for rtype in _CLEANUP_FUNCS.keys()}
440 exit_code = 0
441
442 try:
443 # keep track of registered/unregistered resources
444 with open(fd, 'rb') as f:
445 for line in f:
446 try:
447 cmd, rtype, name = _decode_message(line)
448 cleanup_func = _CLEANUP_FUNCS.get(rtype, None)
449 if cleanup_func is None:
450 raise ValueError(
451 f'Cannot register {name} for automatic cleanup: '
452 f'unknown resource type {rtype}')
453
454 if cmd == 'REGISTER':
455 cache[rtype].add(name)
456 elif cmd == 'UNREGISTER':
457 cache[rtype].remove(name)
458 elif cmd == 'PROBE':
459 pass
460 else:
461 raise RuntimeError('unrecognized command %r' % cmd)
462 except Exception:
463 exit_code = 3
464 try:
465 sys.excepthook(*sys.exc_info())
466 except:
467 pass
468 finally:
469 # all processes have terminated; cleanup any remaining resources
470 for rtype, rtype_cache in cache.items():
471 if rtype_cache:
472 try:
473 exit_code = 1
474 if rtype == 'dummy':
475 # The test 'dummy' resource is expected to leak.
476 # We skip the warning (and *only* the warning) for it.
477 pass
478 else:
479 warnings.warn(
480 f'resource_tracker: There appear to be '
481 f'{len(rtype_cache)} leaked {rtype} objects to '
482 f'clean up at shutdown: {rtype_cache}'
483 )
484 except Exception:
485 pass
486 for name in rtype_cache:
487 # For some reason the process which created and registered this
488 # resource has failed to unregister it. Presumably it has
489 # died. We therefore unlink it.
490 try:
491 try:
492 _CLEANUP_FUNCS[rtype](name)
493 except Exception as e:
494 exit_code = 2
495 warnings.warn('resource_tracker: %r: %s' % (name, e))
496 finally:
497 pass
498
499 sys.exit(exit_code)