| 1 |
############################################################################### |
| 2 |
# Popen for LokyProcess. |
| 3 |
# |
| 4 |
# author: Thomas Moreau and Olivier Grisel |
| 5 |
# |
| 6 |
import os |
| 7 |
import sys |
| 8 |
import signal |
| 9 |
import pickle |
| 10 |
from io import BytesIO |
| 11 |
from multiprocessing import util, process |
| 12 |
from multiprocessing.connection import wait |
| 13 |
from multiprocessing.context import set_spawning_popen |
| 14 |
|
| 15 |
from . import reduction, resource_tracker, spawn |
| 16 |
|
| 17 |
|
| 18 |
__all__ = ["Popen"] |
| 19 |
|
| 20 |
|
| 21 |
# |
| 22 |
# Wrapper for an fd used while launching a process |
| 23 |
# |
| 24 |
|
| 25 |
|
| 26 |
class _DupFd: |
| 27 |
def __init__(self, fd): |
| 28 |
self.fd = reduction._mk_inheritable(fd) |
| 29 |
|
| 30 |
def detach(self): |
| 31 |
return self.fd |
| 32 |
|
| 33 |
|
| 34 |
# |
| 35 |
# Start child process using subprocess.Popen |
| 36 |
# |
| 37 |
|
| 38 |
|
| 39 |
class Popen: |
| 40 |
method = "loky" |
| 41 |
DupFd = _DupFd |
| 42 |
|
| 43 |
def __init__(self, process_obj): |
| 44 |
sys.stdout.flush() |
| 45 |
sys.stderr.flush() |
| 46 |
self.returncode = None |
| 47 |
self._fds = [] |
| 48 |
self._launch(process_obj) |
| 49 |
|
| 50 |
def duplicate_for_child(self, fd): |
| 51 |
self._fds.append(fd) |
| 52 |
return reduction._mk_inheritable(fd) |
| 53 |
|
| 54 |
def poll(self, flag=os.WNOHANG): |
| 55 |
if self.returncode is None: |
| 56 |
while True: |
| 57 |
try: |
| 58 |
pid, sts = os.waitpid(self.pid, flag) |
| 59 |
except OSError: |
| 60 |
# Child process not yet created. See #1731717 |
| 61 |
# e.errno == errno.ECHILD == 10 |
| 62 |
return None |
| 63 |
else: |
| 64 |
break |
| 65 |
if pid == self.pid: |
| 66 |
if os.WIFSIGNALED(sts): |
| 67 |
self.returncode = -os.WTERMSIG(sts) |
| 68 |
else: |
| 69 |
assert os.WIFEXITED(sts) |
| 70 |
self.returncode = os.WEXITSTATUS(sts) |
| 71 |
return self.returncode |
| 72 |
|
| 73 |
def wait(self, timeout=None): |
| 74 |
if self.returncode is None: |
| 75 |
if timeout is not None: |
| 76 |
if not wait([self.sentinel], timeout): |
| 77 |
return None |
| 78 |
# This shouldn't block if wait() returned successfully. |
| 79 |
return self.poll(os.WNOHANG if timeout == 0.0 else 0) |
| 80 |
return self.returncode |
| 81 |
|
| 82 |
def terminate(self): |
| 83 |
if self.returncode is None: |
| 84 |
try: |
| 85 |
os.kill(self.pid, signal.SIGTERM) |
| 86 |
except ProcessLookupError: |
| 87 |
pass |
| 88 |
except OSError: |
| 89 |
if self.wait(timeout=0.1) is None: |
| 90 |
raise |
| 91 |
|
| 92 |
def _launch(self, process_obj): |
| 93 |
|
| 94 |
tracker_fd = resource_tracker._resource_tracker.getfd() |
| 95 |
|
| 96 |
fp = BytesIO() |
| 97 |
set_spawning_popen(self) |
| 98 |
try: |
| 99 |
prep_data = spawn.get_preparation_data( |
| 100 |
process_obj._name, |
| 101 |
getattr(process_obj, "init_main_module", True), |
| 102 |
) |
| 103 |
reduction.dump(prep_data, fp) |
| 104 |
reduction.dump(process_obj, fp) |
| 105 |
|
| 106 |
finally: |
| 107 |
set_spawning_popen(None) |
| 108 |
|
| 109 |
try: |
| 110 |
parent_r, child_w = os.pipe() |
| 111 |
child_r, parent_w = os.pipe() |
| 112 |
# for fd in self._fds: |
| 113 |
# _mk_inheritable(fd) |
| 114 |
|
| 115 |
cmd_python = [sys.executable] |
| 116 |
cmd_python += ["-m", self.__module__] |
| 117 |
cmd_python += ["--process-name", str(process_obj.name)] |
| 118 |
cmd_python += ["--pipe", str(reduction._mk_inheritable(child_r))] |
| 119 |
reduction._mk_inheritable(child_w) |
| 120 |
reduction._mk_inheritable(tracker_fd) |
| 121 |
self._fds += [child_r, child_w, tracker_fd] |
| 122 |
if os.name == "posix": |
| 123 |
mp_tracker_fd = prep_data["mp_tracker_fd"] |
| 124 |
self.duplicate_for_child(mp_tracker_fd) |
| 125 |
|
| 126 |
from .fork_exec import fork_exec |
| 127 |
|
| 128 |
pid = fork_exec(cmd_python, self._fds, env=process_obj.env) |
| 129 |
util.debug( |
| 130 |
f"launched python with pid {pid} and cmd:\n{cmd_python}" |
| 131 |
) |
| 132 |
self.sentinel = parent_r |
| 133 |
|
| 134 |
method = "getbuffer" |
| 135 |
if not hasattr(fp, method): |
| 136 |
method = "getvalue" |
| 137 |
with os.fdopen(parent_w, "wb") as f: |
| 138 |
f.write(getattr(fp, method)()) |
| 139 |
self.pid = pid |
| 140 |
finally: |
| 141 |
if parent_r is not None: |
| 142 |
util.Finalize(self, os.close, (parent_r,)) |
| 143 |
for fd in (child_r, child_w): |
| 144 |
if fd is not None: |
| 145 |
os.close(fd) |
| 146 |
|
| 147 |
@staticmethod |
| 148 |
def thread_is_spawning(): |
| 149 |
return True |
| 150 |
|
| 151 |
|
| 152 |
if __name__ == "__main__": |
| 153 |
import argparse |
| 154 |
|
| 155 |
parser = argparse.ArgumentParser("Command line parser") |
| 156 |
parser.add_argument( |
| 157 |
"--pipe", type=int, required=True, help="File handle for the pipe" |
| 158 |
) |
| 159 |
parser.add_argument( |
| 160 |
"--process-name", |
| 161 |
type=str, |
| 162 |
default=None, |
| 163 |
help="Identifier for debugging purpose", |
| 164 |
) |
| 165 |
|
| 166 |
args = parser.parse_args() |
| 167 |
|
| 168 |
info = {} |
| 169 |
exitcode = 1 |
| 170 |
try: |
| 171 |
with os.fdopen(args.pipe, "rb") as from_parent: |
| 172 |
process.current_process()._inheriting = True |
| 173 |
try: |
| 174 |
prep_data = pickle.load(from_parent) |
| 175 |
spawn.prepare(prep_data) |
| 176 |
process_obj = pickle.load(from_parent) |
| 177 |
finally: |
| 178 |
del process.current_process()._inheriting |
| 179 |
|
| 180 |
exitcode = process_obj._bootstrap() |
| 181 |
except Exception: |
| 182 |
print("\n\n" + "-" * 80) |
| 183 |
print(f"{args.process_name} failed with traceback: ") |
| 184 |
print("-" * 80) |
| 185 |
import traceback |
| 186 |
|
| 187 |
print(traceback.format_exc()) |
| 188 |
print("\n" + "-" * 80) |
| 189 |
finally: |
| 190 |
if from_parent is not None: |
| 191 |
from_parent.close() |
| 192 |
|
| 193 |
sys.exit(exitcode) |