Debian Codegraph @

loky/backend/popen_loky_posix.py

raw
1 ###############################################################################
2 # Popen for LokyProcess.
3 #
4 # author: Thomas Moreau and Olivier Grisel
5 #
6 import os
7 import sys
8 import signal
9 import pickle
10 from io import BytesIO
11 from multiprocessing import util, process
12 from multiprocessing.connection import wait
13 from multiprocessing.context import set_spawning_popen
14
15 from . import reduction, resource_tracker, spawn
16
17
18 __all__ = ["Popen"]
19
20
21 #
22 # Wrapper for an fd used while launching a process
23 #
24
25
26 class _DupFd:
27 def __init__(self, fd):
28 self.fd = reduction._mk_inheritable(fd)
29
30 def detach(self):
31 return self.fd
32
33
34 #
35 # Start child process using subprocess.Popen
36 #
37
38
39 class Popen:
40 method = "loky"
41 DupFd = _DupFd
42
43 def __init__(self, process_obj):
44 sys.stdout.flush()
45 sys.stderr.flush()
46 self.returncode = None
47 self._fds = []
48 self._launch(process_obj)
49
50 def duplicate_for_child(self, fd):
51 self._fds.append(fd)
52 return reduction._mk_inheritable(fd)
53
54 def poll(self, flag=os.WNOHANG):
55 if self.returncode is None:
56 while True:
57 try:
58 pid, sts = os.waitpid(self.pid, flag)
59 except OSError:
60 # Child process not yet created. See #1731717
61 # e.errno == errno.ECHILD == 10
62 return None
63 else:
64 break
65 if pid == self.pid:
66 if os.WIFSIGNALED(sts):
67 self.returncode = -os.WTERMSIG(sts)
68 else:
69 assert os.WIFEXITED(sts)
70 self.returncode = os.WEXITSTATUS(sts)
71 return self.returncode
72
73 def wait(self, timeout=None):
74 if self.returncode is None:
75 if timeout is not None:
76 if not wait([self.sentinel], timeout):
77 return None
78 # This shouldn't block if wait() returned successfully.
79 return self.poll(os.WNOHANG if timeout == 0.0 else 0)
80 return self.returncode
81
82 def terminate(self):
83 if self.returncode is None:
84 try:
85 os.kill(self.pid, signal.SIGTERM)
86 except ProcessLookupError:
87 pass
88 except OSError:
89 if self.wait(timeout=0.1) is None:
90 raise
91
92 def _launch(self, process_obj):
93
94 tracker_fd = resource_tracker._resource_tracker.getfd()
95
96 fp = BytesIO()
97 set_spawning_popen(self)
98 try:
99 prep_data = spawn.get_preparation_data(
100 process_obj._name,
101 getattr(process_obj, "init_main_module", True),
102 )
103 reduction.dump(prep_data, fp)
104 reduction.dump(process_obj, fp)
105
106 finally:
107 set_spawning_popen(None)
108
109 try:
110 parent_r, child_w = os.pipe()
111 child_r, parent_w = os.pipe()
112 # for fd in self._fds:
113 # _mk_inheritable(fd)
114
115 cmd_python = [sys.executable]
116 cmd_python += ["-m", self.__module__]
117 cmd_python += ["--process-name", str(process_obj.name)]
118 cmd_python += ["--pipe", str(reduction._mk_inheritable(child_r))]
119 reduction._mk_inheritable(child_w)
120 reduction._mk_inheritable(tracker_fd)
121 self._fds += [child_r, child_w, tracker_fd]
122 if os.name == "posix":
123 mp_tracker_fd = prep_data["mp_tracker_fd"]
124 self.duplicate_for_child(mp_tracker_fd)
125
126 from .fork_exec import fork_exec
127
128 pid = fork_exec(cmd_python, self._fds, env=process_obj.env)
129 util.debug(
130 f"launched python with pid {pid} and cmd:\n{cmd_python}"
131 )
132 self.sentinel = parent_r
133
134 method = "getbuffer"
135 if not hasattr(fp, method):
136 method = "getvalue"
137 with os.fdopen(parent_w, "wb") as f:
138 f.write(getattr(fp, method)())
139 self.pid = pid
140 finally:
141 if parent_r is not None:
142 util.Finalize(self, os.close, (parent_r,))
143 for fd in (child_r, child_w):
144 if fd is not None:
145 os.close(fd)
146
147 @staticmethod
148 def thread_is_spawning():
149 return True
150
151
152 if __name__ == "__main__":
153 import argparse
154
155 parser = argparse.ArgumentParser("Command line parser")
156 parser.add_argument(
157 "--pipe", type=int, required=True, help="File handle for the pipe"
158 )
159 parser.add_argument(
160 "--process-name",
161 type=str,
162 default=None,
163 help="Identifier for debugging purpose",
164 )
165
166 args = parser.parse_args()
167
168 info = {}
169 exitcode = 1
170 try:
171 with os.fdopen(args.pipe, "rb") as from_parent:
172 process.current_process()._inheriting = True
173 try:
174 prep_data = pickle.load(from_parent)
175 spawn.prepare(prep_data)
176 process_obj = pickle.load(from_parent)
177 finally:
178 del process.current_process()._inheriting
179
180 exitcode = process_obj._bootstrap()
181 except Exception:
182 print("\n\n" + "-" * 80)
183 print(f"{args.process_name} failed with traceback: ")
184 print("-" * 80)
185 import traceback
186
187 print(traceback.format_exc())
188 print("\n" + "-" * 80)
189 finally:
190 if from_parent is not None:
191 from_parent.close()
192
193 sys.exit(exitcode)