Debian Codegraph @

loky/backend/popen_loky_win32.py

raw
1 import os
2 import sys
3 import msvcrt
4 import _winapi
5 from pickle import load
6 from multiprocessing import process, util
7 from multiprocessing.context import set_spawning_popen
8 from multiprocessing.popen_spawn_win32 import Popen as _Popen
9
10 from . import reduction, spawn
11
12
13 __all__ = ["Popen"]
14
15 #
16 #
17 #
18
19
20 def _path_eq(p1, p2):
21 return p1 == p2 or os.path.normcase(p1) == os.path.normcase(p2)
22
23
24 WINENV = hasattr(sys, "_base_executable") and not _path_eq(
25 sys.executable, sys._base_executable
26 )
27
28
29 def _close_handles(*handles):
30 for handle in handles:
31 _winapi.CloseHandle(handle)
32
33
34 #
35 # We define a Popen class similar to the one from subprocess, but
36 # whose constructor takes a process object as its argument.
37 #
38
39
40 class Popen(_Popen):
41 """
42 Start a subprocess to run the code of a process object.
43
44 We differ from cpython implementation with the way we handle environment
45 variables, in order to be able to modify then in the child processes before
46 importing any library, in order to control the number of threads in C-level
47 threadpools.
48
49 We also use the loky preparation data, in particular to handle main_module
50 inits and the loky resource tracker.
51 """
52
53 method = "loky"
54
55 def __init__(self, process_obj):
56 prep_data = spawn.get_preparation_data(
57 process_obj._name, getattr(process_obj, "init_main_module", True)
58 )
59
60 # read end of pipe will be duplicated by the child process
61 # -- see spawn_main() in spawn.py.
62 #
63 # bpo-33929: Previously, the read end of pipe was "stolen" by the child
64 # process, but it leaked a handle if the child process had been
65 # terminated before it could steal the handle from the parent process.
66 rhandle, whandle = _winapi.CreatePipe(None, 0)
67 wfd = msvcrt.open_osfhandle(whandle, 0)
68 cmd = get_command_line(parent_pid=os.getpid(), pipe_handle=rhandle)
69
70 python_exe = spawn.get_executable()
71
72 # copy the environment variables to set in the child process
73 child_env = {**os.environ, **process_obj.env}
74
75 # bpo-35797: When running in a venv, we bypass the redirect
76 # executor and launch our base Python.
77 if WINENV and _path_eq(python_exe, sys.executable):
78 cmd[0] = python_exe = sys._base_executable
79 child_env["__PYVENV_LAUNCHER__"] = sys.executable
80
81 cmd = " ".join(f'"{x}"' for x in cmd)
82
83 with open(wfd, "wb") as to_child:
84 # start process
85 try:
86 hp, ht, pid, _ = _winapi.CreateProcess(
87 python_exe,
88 cmd,
89 None,
90 None,
91 False,
92 0,
93 child_env,
94 None,
95 None,
96 )
97 _winapi.CloseHandle(ht)
98 except BaseException:
99 _winapi.CloseHandle(rhandle)
100 raise
101
102 # set attributes of self
103 self.pid = pid
104 self.returncode = None
105 self._handle = hp
106 self.sentinel = int(hp)
107 self.finalizer = util.Finalize(
108 self, _close_handles, (self.sentinel, int(rhandle))
109 )
110
111 # send information to child
112 set_spawning_popen(self)
113 try:
114 reduction.dump(prep_data, to_child)
115 reduction.dump(process_obj, to_child)
116 finally:
117 set_spawning_popen(None)
118
119
120 def get_command_line(pipe_handle, parent_pid, **kwds):
121 """Returns prefix of command line used for spawning a child process."""
122 if getattr(sys, "frozen", False):
123 return [sys.executable, "--multiprocessing-fork", pipe_handle]
124 else:
125 prog = (
126 "from loky.backend.popen_loky_win32 import main; "
127 f"main(pipe_handle={pipe_handle}, parent_pid={parent_pid})"
128 )
129 opts = util._args_from_interpreter_flags()
130 return [
131 spawn.get_executable(),
132 *opts,
133 "-c",
134 prog,
135 "--multiprocessing-fork",
136 ]
137
138
139 def is_forking(argv):
140 """Return whether commandline indicates we are forking."""
141 if len(argv) >= 2 and argv[1] == "--multiprocessing-fork":
142 return True
143 else:
144 return False
145
146
147 def main(pipe_handle, parent_pid=None):
148 """Run code specified by data received over pipe."""
149 assert is_forking(sys.argv), "Not forking"
150
151 if parent_pid is not None:
152 source_process = _winapi.OpenProcess(
153 _winapi.SYNCHRONIZE | _winapi.PROCESS_DUP_HANDLE, False, parent_pid
154 )
155 else:
156 source_process = None
157 new_handle = reduction.duplicate(
158 pipe_handle, source_process=source_process
159 )
160 fd = msvcrt.open_osfhandle(new_handle, os.O_RDONLY)
161 parent_sentinel = source_process
162
163 with os.fdopen(fd, "rb", closefd=True) as from_parent:
164 process.current_process()._inheriting = True
165 try:
166 preparation_data = load(from_parent)
167 spawn.prepare(preparation_data, parent_sentinel)
168 self = load(from_parent)
169 finally:
170 del process.current_process()._inheriting
171
172 exitcode = self._bootstrap(parent_sentinel)
173 sys.exit(exitcode)