Debian Codegraph @

Lib/multiprocessing/forkserver.py

raw
1 import atexit
2 import errno
3 import os
4 import selectors
5 import signal
6 import socket
7 import struct
8 import sys
9 import threading
10 import warnings
11
12 from . import AuthenticationError
13 from . import connection
14 from . import process
15 from .context import reduction
16 from . import resource_tracker
17 from . import spawn
18 from . import util
19
20 __all__ = ['ensure_running', 'get_inherited_fds', 'connect_to_new_process',
21 'set_forkserver_preload']
22
23 #
24 #
25 #
26
27 MAXFDS_TO_SEND = 256
28 SIGNED_STRUCT = struct.Struct('q') # large enough for pid_t
29 _AUTHKEY_LEN = 32 # <= PIPEBUF so it fits a single write to an empty pipe.
30
31 #
32 # Forkserver class
33 #
34
35 class ForkServer(object):
36
37 def __init__(self):
38 self._forkserver_authkey = None
39 self._forkserver_address = None
40 self._forkserver_alive_fd = None
41 self._forkserver_pid = None
42 self._inherited_fds = None
43 self._lock = threading.Lock()
44 self._preload_modules = ['__main__']
45
46 def _stop(self):
47 # Method used by unit tests to stop the server
48 with self._lock:
49 self._stop_unlocked()
50
51 def _stop_unlocked(self):
52 if self._forkserver_pid is None:
53 return
54
55 # close the "alive" file descriptor asks the server to stop
56 os.close(self._forkserver_alive_fd)
57 self._forkserver_alive_fd = None
58
59 os.waitpid(self._forkserver_pid, 0)
60 self._forkserver_pid = None
61
62 if not util.is_abstract_socket_namespace(self._forkserver_address):
63 os.unlink(self._forkserver_address)
64 self._forkserver_address = None
65 self._forkserver_authkey = None
66
67 def set_forkserver_preload(self, modules_names):
68 '''Set list of module names to try to load in forkserver process.'''
69 if not all(type(mod) is str for mod in modules_names):
70 raise TypeError('module_names must be a list of strings')
71 self._preload_modules = modules_names
72
73 def get_inherited_fds(self):
74 '''Return list of fds inherited from parent process.
75
76 This returns None if the current process was not started by fork
77 server.
78 '''
79 return self._inherited_fds
80
81 def connect_to_new_process(self, fds):
82 '''Request forkserver to create a child process.
83
84 Returns a pair of fds (status_r, data_w). The calling process can read
85 the child process's pid and (eventually) its returncode from status_r.
86 The calling process should write to data_w the pickled preparation and
87 process data.
88 '''
89 self.ensure_running()
90 assert self._forkserver_authkey
91 if len(fds) + 4 >= MAXFDS_TO_SEND:
92 raise ValueError('too many fds')
93 with socket.socket(socket.AF_UNIX) as client:
94 client.connect(self._forkserver_address)
95 parent_r, child_w = os.pipe()
96 child_r, parent_w = os.pipe()
97 allfds = [child_r, child_w, self._forkserver_alive_fd,
98 resource_tracker.getfd()]
99 allfds += fds
100 try:
101 client.setblocking(True)
102 wrapped_client = connection.Connection(client.fileno())
103 # The other side of this exchange happens in the child as
104 # implemented in main().
105 try:
106 connection.answer_challenge(
107 wrapped_client, self._forkserver_authkey)
108 connection.deliver_challenge(
109 wrapped_client, self._forkserver_authkey)
110 finally:
111 wrapped_client._detach()
112 del wrapped_client
113 reduction.sendfds(client, allfds)
114 return parent_r, parent_w
115 except:
116 os.close(parent_r)
117 os.close(parent_w)
118 raise
119 finally:
120 os.close(child_r)
121 os.close(child_w)
122
123 def ensure_running(self):
124 '''Make sure that a fork server is running.
125
126 This can be called from any process. Note that usually a child
127 process will just reuse the forkserver started by its parent, so
128 ensure_running() will do nothing.
129 '''
130 with self._lock:
131 resource_tracker.ensure_running()
132 if self._forkserver_pid is not None:
133 # forkserver was launched before, is it still running?
134 pid, status = os.waitpid(self._forkserver_pid, os.WNOHANG)
135 if not pid:
136 # still alive
137 return
138 # dead, launch it again
139 os.close(self._forkserver_alive_fd)
140 self._forkserver_authkey = None
141 self._forkserver_address = None
142 self._forkserver_alive_fd = None
143 self._forkserver_pid = None
144
145 # gh-144503: sys_argv is passed as real argv elements after the
146 # ``-c cmd`` rather than repr'd into main_kws so that a large
147 # parent sys.argv cannot push the single ``-c`` command string
148 # over the OS per-argument length limit (MAX_ARG_STRLEN on Linux).
149 # The child sees them as sys.argv[1:].
150 cmd = ('import sys; '
151 'from multiprocessing.forkserver import main; '
152 'main(%d, %d, %r, sys_argv=sys.argv[1:], **%r)')
153
154 main_kws = {}
155 sys_argv = None
156 if self._preload_modules:
157 data = spawn.get_preparation_data('ignore')
158 if 'sys_path' in data:
159 main_kws['sys_path'] = data['sys_path']
160 if 'init_main_from_path' in data:
161 main_kws['main_path'] = data['init_main_from_path']
162 if 'sys_argv' in data:
163 sys_argv = data['sys_argv']
164
165 with socket.socket(socket.AF_UNIX) as listener:
166 address = connection.arbitrary_address('AF_UNIX')
167 listener.bind(address)
168 if not util.is_abstract_socket_namespace(address):
169 os.chmod(address, 0o600)
170 listener.listen()
171
172 # all client processes own the write end of the "alive" pipe;
173 # when they all terminate the read end becomes ready.
174 alive_r, alive_w = os.pipe()
175 # A short lived pipe to initialize the forkserver authkey.
176 authkey_r, authkey_w = os.pipe()
177 try:
178 fds_to_pass = [listener.fileno(), alive_r, authkey_r]
179 main_kws['authkey_r'] = authkey_r
180 cmd %= (listener.fileno(), alive_r, self._preload_modules,
181 main_kws)
182 exe = spawn.get_executable()
183 args = [exe] + util._args_from_interpreter_flags()
184 args += ['-c', cmd]
185 if sys_argv is not None:
186 args += sys_argv
187 pid = util.spawnv_passfds(exe, args, fds_to_pass)
188 except:
189 os.close(alive_w)
190 os.close(authkey_w)
191 raise
192 finally:
193 os.close(alive_r)
194 os.close(authkey_r)
195 # Authenticate our control socket to prevent access from
196 # processes we have not shared this key with.
197 try:
198 self._forkserver_authkey = os.urandom(_AUTHKEY_LEN)
199 os.write(authkey_w, self._forkserver_authkey)
200 finally:
201 os.close(authkey_w)
202 self._forkserver_address = address
203 self._forkserver_alive_fd = alive_w
204 self._forkserver_pid = pid
205
206 #
207 #
208 #
209
210 def main(listener_fd, alive_r, preload, main_path=None, sys_path=None,
211 *, sys_argv=None, authkey_r=None):
212 """Run forkserver."""
213 if authkey_r is not None:
214 try:
215 authkey = os.read(authkey_r, _AUTHKEY_LEN)
216 assert len(authkey) == _AUTHKEY_LEN, f'{len(authkey)} < {_AUTHKEY_LEN}'
217 finally:
218 os.close(authkey_r)
219 else:
220 authkey = b''
221
222 if preload:
223 if sys_argv is not None:
224 sys.argv[:] = sys_argv
225 if sys_path is not None:
226 sys.path[:] = sys_path
227 if '__main__' in preload and main_path is not None:
228 process.current_process()._inheriting = True
229 try:
230 spawn.import_main_path(main_path)
231 finally:
232 del process.current_process()._inheriting
233 for modname in preload:
234 try:
235 __import__(modname)
236 except ImportError:
237 pass
238
239 # gh-135335: flush stdout/stderr in case any of the preloaded modules
240 # wrote to them, otherwise children might inherit buffered data
241 util._flush_std_streams()
242
243 util._close_stdin()
244
245 sig_r, sig_w = os.pipe()
246 os.set_blocking(sig_r, False)
247 os.set_blocking(sig_w, False)
248
249 def sigchld_handler(*_unused):
250 # Dummy signal handler, doesn't do anything
251 pass
252
253 handlers = {
254 # unblocking SIGCHLD allows the wakeup fd to notify our event loop
255 signal.SIGCHLD: sigchld_handler,
256 # protect the process from ^C
257 signal.SIGINT: signal.SIG_IGN,
258 }
259 old_handlers = {sig: signal.signal(sig, val)
260 for (sig, val) in handlers.items()}
261
262 # calling os.write() in the Python signal handler is racy
263 signal.set_wakeup_fd(sig_w)
264
265 # map child pids to client fds
266 pid_to_fd = {}
267
268 with socket.socket(socket.AF_UNIX, fileno=listener_fd) as listener, \
269 selectors.DefaultSelector() as selector:
270 _forkserver._forkserver_address = listener.getsockname()
271
272 selector.register(listener, selectors.EVENT_READ)
273 selector.register(alive_r, selectors.EVENT_READ)
274 selector.register(sig_r, selectors.EVENT_READ)
275
276 while True:
277 try:
278 while True:
279 rfds = [key.fileobj for (key, events) in selector.select()]
280 if rfds:
281 break
282
283 if alive_r in rfds:
284 # EOF because no more client processes left
285 assert os.read(alive_r, 1) == b'', "Not at EOF?"
286 raise SystemExit
287
288 if sig_r in rfds:
289 # Got SIGCHLD
290 os.read(sig_r, 65536) # exhaust
291 while True:
292 # Scan for child processes
293 try:
294 pid, sts = os.waitpid(-1, os.WNOHANG)
295 except ChildProcessError:
296 break
297 if pid == 0:
298 break
299 child_w = pid_to_fd.pop(pid, None)
300 if child_w is not None:
301 returncode = os.waitstatus_to_exitcode(sts)
302
303 # Send exit code to client process
304 try:
305 write_signed(child_w, returncode)
306 except BrokenPipeError:
307 # client vanished
308 pass
309 os.close(child_w)
310 else:
311 # This shouldn't happen really
312 warnings.warn('forkserver: waitpid returned '
313 'unexpected pid %d' % pid)
314
315 if listener in rfds:
316 # Incoming fork request
317 with listener.accept()[0] as s:
318 try:
319 if authkey:
320 wrapped_s = connection.Connection(s.fileno())
321 # The other side of this exchange happens in
322 # in connect_to_new_process().
323 try:
324 connection.deliver_challenge(
325 wrapped_s, authkey)
326 connection.answer_challenge(
327 wrapped_s, authkey)
328 finally:
329 wrapped_s._detach()
330 del wrapped_s
331 # Receive fds from client
332 fds = reduction.recvfds(s, MAXFDS_TO_SEND + 1)
333 except (EOFError, BrokenPipeError, AuthenticationError):
334 s.close()
335 continue
336 if len(fds) > MAXFDS_TO_SEND:
337 raise RuntimeError(
338 "Too many ({0:n}) fds to send".format(
339 len(fds)))
340 child_r, child_w, *fds = fds
341 s.close()
342 pid = os.fork()
343 if pid == 0:
344 # Child
345 code = 1
346 try:
347 listener.close()
348 selector.close()
349 unused_fds = [alive_r, child_w, sig_r, sig_w]
350 unused_fds.extend(pid_to_fd.values())
351 atexit._clear()
352 atexit.register(util._exit_function)
353 code = _serve_one(child_r, fds,
354 unused_fds,
355 old_handlers)
356 except Exception:
357 sys.excepthook(*sys.exc_info())
358 sys.stderr.flush()
359 finally:
360 atexit._run_exitfuncs()
361 os._exit(code)
362 else:
363 # Send pid to client process
364 try:
365 write_signed(child_w, pid)
366 except BrokenPipeError:
367 # client vanished
368 pass
369 pid_to_fd[pid] = child_w
370 os.close(child_r)
371 for fd in fds:
372 os.close(fd)
373
374 except OSError as e:
375 if e.errno != errno.ECONNABORTED:
376 raise
377
378
379 def _serve_one(child_r, fds, unused_fds, handlers):
380 # close unnecessary stuff and reset signal handlers
381 signal.set_wakeup_fd(-1)
382 for sig, val in handlers.items():
383 signal.signal(sig, val)
384 for fd in unused_fds:
385 os.close(fd)
386
387 (_forkserver._forkserver_alive_fd,
388 resource_tracker._resource_tracker._fd,
389 *_forkserver._inherited_fds) = fds
390
391 # Run process object received over pipe
392 parent_sentinel = os.dup(child_r)
393 code = spawn._main(child_r, parent_sentinel)
394
395 return code
396
397
398 #
399 # Read and write signed numbers
400 #
401
402 def read_signed(fd):
403 data = bytearray(SIGNED_STRUCT.size)
404 unread = memoryview(data)
405 while unread:
406 count = os.readinto(fd, unread)
407 if count == 0:
408 raise EOFError('unexpected EOF')
409 unread = unread[count:]
410
411 return SIGNED_STRUCT.unpack(data)[0]
412
413 def write_signed(fd, n):
414 msg = SIGNED_STRUCT.pack(n)
415 while msg:
416 nbytes = os.write(fd, msg)
417 if nbytes == 0:
418 raise RuntimeError('should not get here')
419 msg = msg[nbytes:]
420
421 #
422 #
423 #
424
425 _forkserver = ForkServer()
426 ensure_running = _forkserver.ensure_running
427 get_inherited_fds = _forkserver.get_inherited_fds
428 connect_to_new_process = _forkserver.connect_to_new_process
429 set_forkserver_preload = _forkserver.set_forkserver_preload