Debian Codegraph @

Lib/multiprocessing/resource_tracker.py

raw
1 ###############################################################################
2 # Server process to keep track of unlinked resources (like shared memory
3 # segments, semaphores etc.) and clean them.
4 #
5 # On Unix we run a server process which keeps track of unlinked
6 # resources. The server ignores SIGINT and SIGTERM and reads from a
7 # pipe. Every other process of the program has a copy of the writable
8 # end of the pipe, so we get EOF when all other processes have exited.
9 # Then the server process unlinks any remaining resource names.
10 #
11 # This is important because there may be system limits for such resources: for
12 # instance, the system only supports a limited number of named semaphores, and
13 # shared-memory segments live in the RAM. If a python process leaks such a
14 # resource, this resource will not be removed till the next reboot. Without
15 # this resource tracker process, "killall python" would probably leave unlinked
16 # resources.
17
18 import base64
19 import os
20 import signal
21 import sys
22 import threading
23 import time
24 import warnings
25 from collections import deque
26
27 import json
28
29 from . import spawn
30 from . import util
31
32 __all__ = ['ensure_running', 'register', 'unregister']
33
34 _HAVE_SIGMASK = hasattr(signal, 'pthread_sigmask')
35 _IGNORED_SIGNALS = (signal.SIGINT, signal.SIGTERM)
36
37 def cleanup_noop(name):
38 raise RuntimeError('noop should never be registered or cleaned up')
39
40 _CLEANUP_FUNCS = {
41 'noop': cleanup_noop,
42 'dummy': lambda name: None, # Dummy resource used in tests
43 }
44
45 if os.name == 'posix':
46 import _multiprocessing
47 import _posixshmem
48
49 # Use sem_unlink() to clean up named semaphores.
50 #
51 # sem_unlink() may be missing if the Python build process detected the
52 # absence of POSIX named semaphores. In that case, no named semaphores were
53 # ever opened, so no cleanup would be necessary.
54 if hasattr(_multiprocessing, 'sem_unlink'):
55 _CLEANUP_FUNCS['semaphore'] = _multiprocessing.sem_unlink
56 _CLEANUP_FUNCS['shared_memory'] = _posixshmem.shm_unlink
57
58
59 class ReentrantCallError(RuntimeError):
60 pass
61
62
63 class ResourceTracker(object):
64
65 def __init__(self):
66 self._lock = threading.RLock()
67 self._fd = None
68 self._pid = None
69 self._exitcode = None
70 self._reentrant_messages = deque()
71
72 # True to use colon-separated lines, rather than JSON lines,
73 # for internal communication. (Mainly for testing).
74 # Filenames not supported by the simple format will always be sent
75 # using JSON.
76 # The reader should understand all formats.
77 self._use_simple_format = True
78
79 # Set to True by _stop_locked() if the waitpid polling loop ran to
80 # its timeout without reaping the tracker. Exposed for tests.
81 self._waitpid_timed_out = False
82
83 def _reentrant_call_error(self):
84 # gh-109629: this happens if an explicit call to the ResourceTracker
85 # gets interrupted by a garbage collection, invoking a finalizer (*)
86 # that itself calls back into ResourceTracker.
87 # (*) for example the SemLock finalizer
88 raise ReentrantCallError(
89 "Reentrant call into the multiprocessing resource tracker")
90
91 def __del__(self):
92 # making sure child processess are cleaned before ResourceTracker
93 # gets destructed.
94 # see https://github.com/python/cpython/issues/88887
95 # gh-146313: use a timeout to avoid deadlocking if a forked child
96 # still holds the pipe's write end open.
97 self._stop(use_blocking_lock=False, wait_timeout=1.0)
98
99 def _after_fork_in_child(self):
100 # gh-146313: Called in the child right after os.fork().
101 #
102 # The tracker process is a child of the *parent*, not of us, so we
103 # could never waitpid() it anyway. Clearing _pid means our __del__
104 # becomes a no-op (the early return for _pid is None).
105 #
106 # Whether we keep the inherited _fd depends on who forked us:
107 #
108 # - multiprocessing.Process with the 'fork' start method sets
109 # _fork_intent.preserve_fd before forking. The child keeps the
110 # fd and reuses the parent's tracker (gh-80849). This is safe
111 # because multiprocessing's atexit handler joins all children
112 # before the parent's __del__ runs, so by then the fd copies
113 # are gone and the parent can reap the tracker promptly.
114 #
115 # - A raw os.fork() leaves the flag unset. We close the fd in the child after forking so
116 # the parent's __del__ can reap the tracker without waiting
117 # for the child to exit. If we later need a tracker, ensure_running()
118 # will launch a fresh one.
119 self._lock._at_fork_reinit()
120 self._reentrant_messages.clear()
121 self._pid = None
122 self._exitcode = None
123 if (self._fd is not None and
124 not getattr(_fork_intent, 'preserve_fd', False)):
125 fd = self._fd
126 self._fd = None
127 try:
128 os.close(fd)
129 except OSError:
130 pass
131
132 def _stop(self, use_blocking_lock=True, wait_timeout=None):
133 if use_blocking_lock:
134 with self._lock:
135 self._stop_locked(wait_timeout=wait_timeout)
136 else:
137 acquired = self._lock.acquire(blocking=False)
138 try:
139 self._stop_locked(wait_timeout=wait_timeout)
140 finally:
141 if acquired:
142 self._lock.release()
143
144 def _stop_locked(
145 self,
146 close=os.close,
147 waitpid=os.waitpid,
148 waitstatus_to_exitcode=os.waitstatus_to_exitcode,
149 monotonic=time.monotonic,
150 sleep=time.sleep,
151 WNOHANG=getattr(os, 'WNOHANG', None),
152 wait_timeout=None,
153 ):
154 # This shouldn't happen (it might when called by a finalizer)
155 # so we check for it anyway.
156 if self._lock._recursion_count() > 1:
157 raise self._reentrant_call_error()
158 if self._fd is None:
159 # not running
160 return
161 if self._pid is None:
162 return
163
164 # closing the "alive" file descriptor stops main()
165 close(self._fd)
166 self._fd = None
167
168 try:
169 if wait_timeout is None:
170 _, status = waitpid(self._pid, 0)
171 else:
172 # gh-146313: A forked child may still hold the pipe's write
173 # end open, preventing the tracker from seeing EOF and
174 # exiting. Poll with WNOHANG to avoid blocking forever.
175 deadline = monotonic() + wait_timeout
176 delay = 0.001
177 while True:
178 result_pid, status = waitpid(self._pid, WNOHANG)
179 if result_pid != 0:
180 break
181 remaining = deadline - monotonic()
182 if remaining <= 0:
183 # The tracker is still running; it will be
184 # reparented to PID 1 (or the nearest subreaper)
185 # when we exit, and reaped there once all pipe
186 # holders release their fd.
187 self._pid = None
188 self._exitcode = None
189 self._waitpid_timed_out = True
190 return
191 delay = min(delay * 2, remaining, 0.1)
192 sleep(delay)
193 except ChildProcessError:
194 self._pid = None
195 self._exitcode = None
196 return
197
198 self._pid = None
199
200 try:
201 self._exitcode = waitstatus_to_exitcode(status)
202 except ValueError:
203 # os.waitstatus_to_exitcode may raise an exception for invalid values
204 self._exitcode = None
205
206 def getfd(self):
207 self.ensure_running()
208 return self._fd
209
210 def ensure_running(self):
211 '''Make sure that resource tracker process is running.
212
213 This can be run from any process. Usually a child process will use
214 the resource created by its parent.'''
215 return self._ensure_running_and_write()
216
217 def _teardown_dead_process(self):
218 os.close(self._fd)
219
220 # Clean-up to avoid dangling processes.
221 try:
222 # _pid can be None if this process is a child from another
223 # python process, which has started the resource_tracker.
224 if self._pid is not None:
225 os.waitpid(self._pid, 0)
226 except ChildProcessError:
227 # The resource_tracker has already been terminated.
228 pass
229 self._fd = None
230 self._pid = None
231 self._exitcode = None
232
233 warnings.warn('resource_tracker: process died unexpectedly, '
234 'relaunching. Some resources might leak.')
235
236 def _launch(self):
237 fds_to_pass = []
238 try:
239 fds_to_pass.append(sys.stderr.fileno())
240 except Exception:
241 pass
242 r, w = os.pipe()
243 try:
244 fds_to_pass.append(r)
245 # process will out live us, so no need to wait on pid
246 exe = spawn.get_executable()
247 args = [
248 exe,
249 *util._args_from_interpreter_flags(),
250 '-c',
251 f'from multiprocessing.resource_tracker import main;main({r})',
252 ]
253 # bpo-33613: Register a signal mask that will block the signals.
254 # This signal mask will be inherited by the child that is going
255 # to be spawned and will protect the child from a race condition
256 # that can make the child die before it registers signal handlers
257 # for SIGINT and SIGTERM. The mask is unregistered after spawning
258 # the child.
259 prev_sigmask = None
260 try:
261 if _HAVE_SIGMASK:
262 prev_sigmask = signal.pthread_sigmask(signal.SIG_BLOCK, _IGNORED_SIGNALS)
263 pid = util.spawnv_passfds(exe, args, fds_to_pass)
264 finally:
265 if prev_sigmask is not None:
266 signal.pthread_sigmask(signal.SIG_SETMASK, prev_sigmask)
267 except:
268 os.close(w)
269 raise
270 else:
271 self._fd = w
272 self._pid = pid
273 finally:
274 os.close(r)
275
276 def _make_probe_message(self):
277 """Return a probe message."""
278 if self._use_simple_format:
279 return b'PROBE:0:noop\n'
280 return (
281 json.dumps(
282 {"cmd": "PROBE", "rtype": "noop"},
283 ensure_ascii=True,
284 separators=(",", ":"),
285 )
286 + "\n"
287 ).encode("ascii")
288
289 def _ensure_running_and_write(self, msg=None):
290 with self._lock:
291 if self._lock._recursion_count() > 1:
292 # The code below is certainly not reentrant-safe, so bail out
293 if msg is None:
294 raise self._reentrant_call_error()
295 return self._reentrant_messages.append(msg)
296
297 if self._fd is not None:
298 # resource tracker was launched before, is it still running?
299 if msg is None:
300 to_send = self._make_probe_message()
301 else:
302 to_send = msg
303 try:
304 self._write(to_send)
305 except OSError:
306 self._teardown_dead_process()
307 self._launch()
308
309 msg = None # message was sent in probe
310 else:
311 self._launch()
312
313 while True:
314 try:
315 reentrant_msg = self._reentrant_messages.popleft()
316 except IndexError:
317 break
318 self._write(reentrant_msg)
319 if msg is not None:
320 self._write(msg)
321
322 def _check_alive(self):
323 '''Check that the pipe has not been closed by sending a probe.'''
324 try:
325 # We cannot use send here as it calls ensure_running, creating
326 # a cycle.
327 os.write(self._fd, self._make_probe_message())
328 except OSError:
329 return False
330 else:
331 return True
332
333 def register(self, name, rtype):
334 '''Register name of resource with resource tracker.'''
335 self._send('REGISTER', name, rtype)
336
337 def unregister(self, name, rtype):
338 '''Unregister name of resource with resource tracker.'''
339 self._send('UNREGISTER', name, rtype)
340
341 def _write(self, msg):
342 nbytes = os.write(self._fd, msg)
343 assert nbytes == len(msg), f"{nbytes=} != {len(msg)=}"
344
345 def _send(self, cmd, name, rtype):
346 if self._use_simple_format and '\n' not in name:
347 msg = f"{cmd}:{name}:{rtype}\n".encode("ascii")
348 if len(msg) > 512:
349 # posix guarantees that writes to a pipe of less than PIPE_BUF
350 # bytes are atomic, and that PIPE_BUF >= 512
351 raise ValueError('msg too long')
352 self._ensure_running_and_write(msg)
353 return
354
355 # POSIX guarantees that writes to a pipe of less than PIPE_BUF (512 on Linux)
356 # bytes are atomic. Therefore, we want the message to be shorter than 512 bytes.
357 # POSIX shm_open() and sem_open() require the name, including its leading slash,
358 # to be at most NAME_MAX bytes (255 on Linux)
359 # With json.dump(..., ensure_ascii=True) every non-ASCII byte becomes a 6-char
360 # escape like \uDC80.
361 # As we want the overall message to be kept atomic and therefore smaller than 512,
362 # we encode encode the raw name bytes with URL-safe Base64 - so a 255 long name
363 # will not exceed 340 bytes.
364 b = name.encode('utf-8', 'surrogateescape')
365 if len(b) > 255:
366 raise ValueError('shared memory name too long (max 255 bytes)')
367 b64 = base64.urlsafe_b64encode(b).decode('ascii')
368
369 payload = {"cmd": cmd, "rtype": rtype, "base64_name": b64}
370 msg = (json.dumps(payload, ensure_ascii=True, separators=(",", ":")) + "\n").encode("ascii")
371
372 # The entire JSON message is guaranteed < PIPE_BUF (512 bytes) by construction.
373 assert len(msg) <= 512, f"internal error: message too long ({len(msg)} bytes)"
374 assert msg.startswith(b'{')
375
376 self._ensure_running_and_write(msg)
377
378 # gh-146313: Per-thread flag set by .popen_fork.Popen._launch() just before
379 # os.fork(), telling _after_fork_in_child() to keep the inherited pipe fd so
380 # the child can reuse this tracker (gh-80849). Unset for raw os.fork() calls,
381 # where the child instead closes the fd so the parent's __del__ can reap the
382 # tracker. Using threading.local() keeps multiple threads calling
383 # popen_fork.Popen._launch() at once from clobbering eachothers intent.
384 _fork_intent = threading.local()
385
386 _resource_tracker = ResourceTracker()
387 ensure_running = _resource_tracker.ensure_running
388 register = _resource_tracker.register
389 unregister = _resource_tracker.unregister
390 getfd = _resource_tracker.getfd
391
392 # gh-146313: See _after_fork_in_child docstring.
393 if hasattr(os, 'register_at_fork'):
394 os.register_at_fork(after_in_child=_resource_tracker._after_fork_in_child)
395
396
397 def _decode_message(line):
398 if line.startswith(b'{'):
399 try:
400 obj = json.loads(line.decode('ascii'))
401 except Exception as e:
402 raise ValueError("malformed resource_tracker message: %r" % (line,)) from e
403
404 cmd = obj["cmd"]
405 rtype = obj["rtype"]
406 b64 = obj.get("base64_name", "")
407
408 if not isinstance(cmd, str) or not isinstance(rtype, str) or not isinstance(b64, str):
409 raise ValueError("malformed resource_tracker fields: %r" % (obj,))
410
411 try:
412 name = base64.urlsafe_b64decode(b64).decode('utf-8', 'surrogateescape')
413 except ValueError as e:
414 raise ValueError("malformed resource_tracker base64_name: %r" % (b64,)) from e
415 else:
416 cmd, rest = line.strip().decode('ascii').split(':', maxsplit=1)
417 name, rtype = rest.rsplit(':', maxsplit=1)
418 return cmd, rtype, name
419
420
421 def main(fd):
422 '''Run resource tracker.'''
423 # protect the process from ^C and "killall python" etc
424 signal.signal(signal.SIGINT, signal.SIG_IGN)
425 signal.signal(signal.SIGTERM, signal.SIG_IGN)
426 if _HAVE_SIGMASK:
427 signal.pthread_sigmask(signal.SIG_UNBLOCK, _IGNORED_SIGNALS)
428
429 for f in (sys.stdin, sys.stdout):
430 try:
431 f.close()
432 except Exception:
433 pass
434
435 cache = {rtype: set() for rtype in _CLEANUP_FUNCS.keys()}
436 exit_code = 0
437
438 try:
439 # keep track of registered/unregistered resources
440 with open(fd, 'rb') as f:
441 for line in f:
442 try:
443 cmd, rtype, name = _decode_message(line)
444 cleanup_func = _CLEANUP_FUNCS.get(rtype, None)
445 if cleanup_func is None:
446 raise ValueError(
447 f'Cannot register {name} for automatic cleanup: '
448 f'unknown resource type {rtype}')
449
450 if cmd == 'REGISTER':
451 cache[rtype].add(name)
452 elif cmd == 'UNREGISTER':
453 cache[rtype].remove(name)
454 elif cmd == 'PROBE':
455 pass
456 else:
457 raise RuntimeError('unrecognized command %r' % cmd)
458 except Exception:
459 exit_code = 3
460 try:
461 sys.excepthook(*sys.exc_info())
462 except:
463 pass
464 finally:
465 # all processes have terminated; cleanup any remaining resources
466 for rtype, rtype_cache in cache.items():
467 if rtype_cache:
468 try:
469 exit_code = 1
470 if rtype == 'dummy':
471 # The test 'dummy' resource is expected to leak.
472 # We skip the warning (and *only* the warning) for it.
473 pass
474 else:
475 warnings.warn(
476 f'resource_tracker: There appear to be '
477 f'{len(rtype_cache)} leaked {rtype} objects to '
478 f'clean up at shutdown: {rtype_cache}'
479 )
480 except Exception:
481 pass
482 for name in rtype_cache:
483 # For some reason the process which created and registered this
484 # resource has failed to unregister it. Presumably it has
485 # died. We therefore unlink it.
486 try:
487 try:
488 _CLEANUP_FUNCS[rtype](name)
489 except Exception as e:
490 exit_code = 2
491 warnings.warn('resource_tracker: %r: %s' % (name, e))
492 finally:
493 pass
494
495 sys.exit(exit_code)