Debian Codegraph @

Lib/multiprocessing/forkserver.py

raw
1 import atexit
2 import errno
3 import os
4 import selectors
5 import signal
6 import socket
7 import struct
8 import sys
9 import threading
10 import warnings
11
12 from . import connection
13 from . import process
14 from .context import reduction
15 from . import resource_tracker
16 from . import spawn
17 from . import util
18
19 __all__ = ['ensure_running', 'get_inherited_fds', 'connect_to_new_process',
20 'set_forkserver_preload']
21
22 #
23 #
24 #
25
26 MAXFDS_TO_SEND = 256
27 SIGNED_STRUCT = struct.Struct('q') # large enough for pid_t
28
29 #
30 # Forkserver class
31 #
32
33 class ForkServer(object):
34
35 def __init__(self):
36 self._forkserver_address = None
37 self._forkserver_alive_fd = None
38 self._forkserver_pid = None
39 self._inherited_fds = None
40 self._lock = threading.Lock()
41 self._preload_modules = ['__main__']
42
43 def _stop(self):
44 # Method used by unit tests to stop the server
45 with self._lock:
46 self._stop_unlocked()
47
48 def _stop_unlocked(self):
49 if self._forkserver_pid is None:
50 return
51
52 # close the "alive" file descriptor asks the server to stop
53 os.close(self._forkserver_alive_fd)
54 self._forkserver_alive_fd = None
55
56 os.waitpid(self._forkserver_pid, 0)
57 self._forkserver_pid = None
58
59 if not util.is_abstract_socket_namespace(self._forkserver_address):
60 os.unlink(self._forkserver_address)
61 self._forkserver_address = None
62
63 def set_forkserver_preload(self, modules_names):
64 '''Set list of module names to try to load in forkserver process.'''
65 if not all(type(mod) is str for mod in modules_names):
66 raise TypeError('module_names must be a list of strings')
67 self._preload_modules = modules_names
68
69 def get_inherited_fds(self):
70 '''Return list of fds inherited from parent process.
71
72 This returns None if the current process was not started by fork
73 server.
74 '''
75 return self._inherited_fds
76
77 def connect_to_new_process(self, fds):
78 '''Request forkserver to create a child process.
79
80 Returns a pair of fds (status_r, data_w). The calling process can read
81 the child process's pid and (eventually) its returncode from status_r.
82 The calling process should write to data_w the pickled preparation and
83 process data.
84 '''
85 self.ensure_running()
86 if len(fds) + 4 >= MAXFDS_TO_SEND:
87 raise ValueError('too many fds')
88 with socket.socket(socket.AF_UNIX) as client:
89 client.connect(self._forkserver_address)
90 parent_r, child_w = os.pipe()
91 child_r, parent_w = os.pipe()
92 allfds = [child_r, child_w, self._forkserver_alive_fd,
93 resource_tracker.getfd()]
94 allfds += fds
95 try:
96 reduction.sendfds(client, allfds)
97 return parent_r, parent_w
98 except:
99 os.close(parent_r)
100 os.close(parent_w)
101 raise
102 finally:
103 os.close(child_r)
104 os.close(child_w)
105
106 def ensure_running(self):
107 '''Make sure that a fork server is running.
108
109 This can be called from any process. Note that usually a child
110 process will just reuse the forkserver started by its parent, so
111 ensure_running() will do nothing.
112 '''
113 with self._lock:
114 resource_tracker.ensure_running()
115 if self._forkserver_pid is not None:
116 # forkserver was launched before, is it still running?
117 pid, status = os.waitpid(self._forkserver_pid, os.WNOHANG)
118 if not pid:
119 # still alive
120 return
121 # dead, launch it again
122 os.close(self._forkserver_alive_fd)
123 self._forkserver_address = None
124 self._forkserver_alive_fd = None
125 self._forkserver_pid = None
126
127 # gh-144503: sys_argv is passed as real argv elements after the
128 # ``-c cmd`` rather than repr'd into main_kws so that a large
129 # parent sys.argv cannot push the single ``-c`` command string
130 # over the OS per-argument length limit (MAX_ARG_STRLEN on Linux).
131 # The child sees them as sys.argv[1:].
132 cmd = ('import sys; '
133 'from multiprocessing.forkserver import main; '
134 'main(%d, %d, %r, sys_argv=sys.argv[1:], **%r)')
135
136 main_kws = {}
137 sys_argv = None
138 if self._preload_modules:
139 data = spawn.get_preparation_data('ignore')
140 if 'sys_path' in data:
141 main_kws['sys_path'] = data['sys_path']
142 if 'init_main_from_path' in data:
143 main_kws['main_path'] = data['init_main_from_path']
144 if 'sys_argv' in data:
145 sys_argv = data['sys_argv']
146
147 with socket.socket(socket.AF_UNIX) as listener:
148 address = connection.arbitrary_address('AF_UNIX')
149 listener.bind(address)
150 if not util.is_abstract_socket_namespace(address):
151 os.chmod(address, 0o600)
152 listener.listen()
153
154 # all client processes own the write end of the "alive" pipe;
155 # when they all terminate the read end becomes ready.
156 alive_r, alive_w = os.pipe()
157 try:
158 fds_to_pass = [listener.fileno(), alive_r]
159 cmd %= (listener.fileno(), alive_r, self._preload_modules,
160 main_kws)
161 exe = spawn.get_executable()
162 args = [exe] + util._args_from_interpreter_flags()
163 args += ['-c', cmd]
164 if sys_argv is not None:
165 args += sys_argv
166 pid = util.spawnv_passfds(exe, args, fds_to_pass)
167 except:
168 os.close(alive_w)
169 raise
170 finally:
171 os.close(alive_r)
172 self._forkserver_address = address
173 self._forkserver_alive_fd = alive_w
174 self._forkserver_pid = pid
175
176 #
177 #
178 #
179
180 def main(listener_fd, alive_r, preload, main_path=None, sys_path=None,
181 *, sys_argv=None):
182 '''Run forkserver.'''
183 if preload:
184 if sys_argv is not None:
185 sys.argv[:] = sys_argv
186 if sys_path is not None:
187 sys.path[:] = sys_path
188 if '__main__' in preload and main_path is not None:
189 process.current_process()._inheriting = True
190 try:
191 spawn.import_main_path(main_path)
192 finally:
193 del process.current_process()._inheriting
194 for modname in preload:
195 try:
196 __import__(modname)
197 except ImportError:
198 pass
199
200 # gh-135335: flush stdout/stderr in case any of the preloaded modules
201 # wrote to them, otherwise children might inherit buffered data
202 util._flush_std_streams()
203
204 util._close_stdin()
205
206 sig_r, sig_w = os.pipe()
207 os.set_blocking(sig_r, False)
208 os.set_blocking(sig_w, False)
209
210 def sigchld_handler(*_unused):
211 # Dummy signal handler, doesn't do anything
212 pass
213
214 handlers = {
215 # unblocking SIGCHLD allows the wakeup fd to notify our event loop
216 signal.SIGCHLD: sigchld_handler,
217 # protect the process from ^C
218 signal.SIGINT: signal.SIG_IGN,
219 }
220 old_handlers = {sig: signal.signal(sig, val)
221 for (sig, val) in handlers.items()}
222
223 # calling os.write() in the Python signal handler is racy
224 signal.set_wakeup_fd(sig_w)
225
226 # map child pids to client fds
227 pid_to_fd = {}
228
229 with socket.socket(socket.AF_UNIX, fileno=listener_fd) as listener, \
230 selectors.DefaultSelector() as selector:
231 _forkserver._forkserver_address = listener.getsockname()
232
233 selector.register(listener, selectors.EVENT_READ)
234 selector.register(alive_r, selectors.EVENT_READ)
235 selector.register(sig_r, selectors.EVENT_READ)
236
237 while True:
238 try:
239 while True:
240 rfds = [key.fileobj for (key, events) in selector.select()]
241 if rfds:
242 break
243
244 if alive_r in rfds:
245 # EOF because no more client processes left
246 assert os.read(alive_r, 1) == b'', "Not at EOF?"
247 raise SystemExit
248
249 if sig_r in rfds:
250 # Got SIGCHLD
251 os.read(sig_r, 65536) # exhaust
252 while True:
253 # Scan for child processes
254 try:
255 pid, sts = os.waitpid(-1, os.WNOHANG)
256 except ChildProcessError:
257 break
258 if pid == 0:
259 break
260 child_w = pid_to_fd.pop(pid, None)
261 if child_w is not None:
262 returncode = os.waitstatus_to_exitcode(sts)
263
264 # Send exit code to client process
265 try:
266 write_signed(child_w, returncode)
267 except BrokenPipeError:
268 # client vanished
269 pass
270 os.close(child_w)
271 else:
272 # This shouldn't happen really
273 warnings.warn('forkserver: waitpid returned '
274 'unexpected pid %d' % pid)
275
276 if listener in rfds:
277 # Incoming fork request
278 with listener.accept()[0] as s:
279 # Receive fds from client
280 fds = reduction.recvfds(s, MAXFDS_TO_SEND + 1)
281 if len(fds) > MAXFDS_TO_SEND:
282 raise RuntimeError(
283 "Too many ({0:n}) fds to send".format(
284 len(fds)))
285 child_r, child_w, *fds = fds
286 s.close()
287 pid = os.fork()
288 if pid == 0:
289 # Child
290 code = 1
291 try:
292 listener.close()
293 selector.close()
294 unused_fds = [alive_r, child_w, sig_r, sig_w]
295 unused_fds.extend(pid_to_fd.values())
296 atexit._clear()
297 atexit.register(util._exit_function)
298 code = _serve_one(child_r, fds,
299 unused_fds,
300 old_handlers)
301 except Exception:
302 sys.excepthook(*sys.exc_info())
303 sys.stderr.flush()
304 finally:
305 atexit._run_exitfuncs()
306 os._exit(code)
307 else:
308 # Send pid to client process
309 try:
310 write_signed(child_w, pid)
311 except BrokenPipeError:
312 # client vanished
313 pass
314 pid_to_fd[pid] = child_w
315 os.close(child_r)
316 for fd in fds:
317 os.close(fd)
318
319 except OSError as e:
320 if e.errno != errno.ECONNABORTED:
321 raise
322
323
324 def _serve_one(child_r, fds, unused_fds, handlers):
325 # close unnecessary stuff and reset signal handlers
326 signal.set_wakeup_fd(-1)
327 for sig, val in handlers.items():
328 signal.signal(sig, val)
329 for fd in unused_fds:
330 os.close(fd)
331
332 (_forkserver._forkserver_alive_fd,
333 resource_tracker._resource_tracker._fd,
334 *_forkserver._inherited_fds) = fds
335
336 # Run process object received over pipe
337 parent_sentinel = os.dup(child_r)
338 code = spawn._main(child_r, parent_sentinel)
339
340 return code
341
342
343 #
344 # Read and write signed numbers
345 #
346
347 def read_signed(fd):
348 data = b''
349 length = SIGNED_STRUCT.size
350 while len(data) < length:
351 s = os.read(fd, length - len(data))
352 if not s:
353 raise EOFError('unexpected EOF')
354 data += s
355 return SIGNED_STRUCT.unpack(data)[0]
356
357 def write_signed(fd, n):
358 msg = SIGNED_STRUCT.pack(n)
359 while msg:
360 nbytes = os.write(fd, msg)
361 if nbytes == 0:
362 raise RuntimeError('should not get here')
363 msg = msg[nbytes:]
364
365 #
366 #
367 #
368
369 _forkserver = ForkServer()
370 ensure_running = _forkserver.ensure_running
371 get_inherited_fds = _forkserver.get_inherited_fds
372 connect_to_new_process = _forkserver.connect_to_new_process
373 set_forkserver_preload = _forkserver.set_forkserver_preload