| 1 |
# |
| 2 |
# Code used to start processes when using the spawn or forkserver |
| 3 |
# start methods. |
| 4 |
# |
| 5 |
# multiprocessing/spawn.py |
| 6 |
# |
| 7 |
# Copyright (c) 2006-2008, R Oudkerk |
| 8 |
# Licensed to PSF under a Contributor Agreement. |
| 9 |
# |
| 10 |
|
| 11 |
import os |
| 12 |
import sys |
| 13 |
import runpy |
| 14 |
import types |
| 15 |
|
| 16 |
from . import get_start_method, set_start_method |
| 17 |
from . import process |
| 18 |
from .context import reduction |
| 19 |
from . import util |
| 20 |
|
| 21 |
__all__ = ['_main', 'freeze_support', 'set_executable', 'get_executable', |
| 22 |
'get_preparation_data', 'get_command_line', 'import_main_path'] |
| 23 |
|
| 24 |
# |
| 25 |
# _python_exe is the assumed path to the python executable. |
| 26 |
# People embedding Python want to modify it. |
| 27 |
# |
| 28 |
|
| 29 |
if sys.platform != 'win32': |
| 30 |
WINEXE = False |
| 31 |
WINSERVICE = False |
| 32 |
else: |
| 33 |
WINEXE = getattr(sys, 'frozen', False) |
| 34 |
WINSERVICE = sys.executable and sys.executable.lower().endswith("pythonservice.exe") |
| 35 |
|
| 36 |
def set_executable(exe): |
| 37 |
global _python_exe |
| 38 |
if exe is None: |
| 39 |
_python_exe = exe |
| 40 |
elif sys.platform == 'win32': |
| 41 |
_python_exe = os.fsdecode(exe) |
| 42 |
else: |
| 43 |
_python_exe = os.fsencode(exe) |
| 44 |
|
| 45 |
def get_executable(): |
| 46 |
return _python_exe |
| 47 |
|
| 48 |
if WINSERVICE: |
| 49 |
set_executable(os.path.join(sys.exec_prefix, 'python.exe')) |
| 50 |
else: |
| 51 |
set_executable(sys.executable) |
| 52 |
|
| 53 |
# |
| 54 |
# |
| 55 |
# |
| 56 |
|
| 57 |
def is_forking(argv): |
| 58 |
''' |
| 59 |
Return whether commandline indicates we are forking |
| 60 |
''' |
| 61 |
if len(argv) >= 2 and argv[1] == '--multiprocessing-fork': |
| 62 |
return True |
| 63 |
else: |
| 64 |
return False |
| 65 |
|
| 66 |
|
| 67 |
def freeze_support(): |
| 68 |
''' |
| 69 |
Run code for process object if this in not the main process |
| 70 |
''' |
| 71 |
if is_forking(sys.argv): |
| 72 |
kwds = {} |
| 73 |
for arg in sys.argv[2:]: |
| 74 |
name, value = arg.split('=') |
| 75 |
if value == 'None': |
| 76 |
kwds[name] = None |
| 77 |
else: |
| 78 |
kwds[name] = int(value) |
| 79 |
spawn_main(**kwds) |
| 80 |
sys.exit() |
| 81 |
|
| 82 |
|
| 83 |
def get_command_line(**kwds): |
| 84 |
''' |
| 85 |
Returns prefix of command line used for spawning a child process |
| 86 |
''' |
| 87 |
if getattr(sys, 'frozen', False): |
| 88 |
return ([sys.executable, '--multiprocessing-fork'] + |
| 89 |
['%s=%r' % item for item in kwds.items()]) |
| 90 |
else: |
| 91 |
prog = 'from multiprocessing.spawn import spawn_main; spawn_main(%s)' |
| 92 |
prog %= ', '.join('%s=%r' % item for item in kwds.items()) |
| 93 |
opts = util._args_from_interpreter_flags() |
| 94 |
exe = get_executable() |
| 95 |
return [exe] + opts + ['-c', prog, '--multiprocessing-fork'] |
| 96 |
|
| 97 |
|
| 98 |
def spawn_main(pipe_handle, parent_pid=None, tracker_fd=None): |
| 99 |
''' |
| 100 |
Run code specified by data received over pipe |
| 101 |
''' |
| 102 |
assert is_forking(sys.argv), "Not forking" |
| 103 |
if sys.platform == 'win32': |
| 104 |
import msvcrt |
| 105 |
import _winapi |
| 106 |
|
| 107 |
if parent_pid is not None: |
| 108 |
source_process = _winapi.OpenProcess( |
| 109 |
_winapi.SYNCHRONIZE | _winapi.PROCESS_DUP_HANDLE, |
| 110 |
False, parent_pid) |
| 111 |
else: |
| 112 |
source_process = None |
| 113 |
new_handle = reduction.duplicate(pipe_handle, |
| 114 |
source_process=source_process) |
| 115 |
fd = msvcrt.open_osfhandle(new_handle, os.O_RDONLY) |
| 116 |
parent_sentinel = source_process |
| 117 |
else: |
| 118 |
from . import resource_tracker |
| 119 |
resource_tracker._resource_tracker._fd = tracker_fd |
| 120 |
fd = pipe_handle |
| 121 |
parent_sentinel = os.dup(pipe_handle) |
| 122 |
exitcode = _main(fd, parent_sentinel) |
| 123 |
sys.exit(exitcode) |
| 124 |
|
| 125 |
|
| 126 |
def _main(fd, parent_sentinel): |
| 127 |
with os.fdopen(fd, 'rb', closefd=True) as from_parent: |
| 128 |
process.current_process()._inheriting = True |
| 129 |
try: |
| 130 |
preparation_data = reduction.pickle.load(from_parent) |
| 131 |
prepare(preparation_data) |
| 132 |
self = reduction.pickle.load(from_parent) |
| 133 |
finally: |
| 134 |
del process.current_process()._inheriting |
| 135 |
return self._bootstrap(parent_sentinel) |
| 136 |
|
| 137 |
|
| 138 |
def _check_not_importing_main(): |
| 139 |
if getattr(process.current_process(), '_inheriting', False): |
| 140 |
raise RuntimeError(''' |
| 141 |
An attempt has been made to start a new process before the |
| 142 |
current process has finished its bootstrapping phase. |
| 143 |
|
| 144 |
This probably means that you are not using fork to start your |
| 145 |
child processes and you have forgotten to use the proper idiom |
| 146 |
in the main module: |
| 147 |
|
| 148 |
if __name__ == '__main__': |
| 149 |
freeze_support() |
| 150 |
... |
| 151 |
|
| 152 |
The "freeze_support()" line can be omitted if the program |
| 153 |
is not going to be frozen to produce an executable. |
| 154 |
|
| 155 |
To fix this issue, refer to the "Safe importing of main module" |
| 156 |
section in https://docs.python.org/3/library/multiprocessing.html |
| 157 |
''') |
| 158 |
|
| 159 |
|
| 160 |
def get_preparation_data(name): |
| 161 |
''' |
| 162 |
Return info about parent needed by child to unpickle process object |
| 163 |
''' |
| 164 |
_check_not_importing_main() |
| 165 |
d = dict( |
| 166 |
log_to_stderr=util._log_to_stderr, |
| 167 |
authkey=process.current_process().authkey, |
| 168 |
) |
| 169 |
|
| 170 |
if util._logger is not None: |
| 171 |
d['log_level'] = util._logger.getEffectiveLevel() |
| 172 |
|
| 173 |
sys_path=sys.path.copy() |
| 174 |
try: |
| 175 |
i = sys_path.index('') |
| 176 |
except ValueError: |
| 177 |
pass |
| 178 |
else: |
| 179 |
sys_path[i] = process.ORIGINAL_DIR |
| 180 |
|
| 181 |
d.update( |
| 182 |
name=name, |
| 183 |
sys_path=sys_path, |
| 184 |
sys_argv=sys.argv, |
| 185 |
orig_dir=process.ORIGINAL_DIR, |
| 186 |
dir=os.getcwd(), |
| 187 |
start_method=get_start_method(allow_none=True), |
| 188 |
) |
| 189 |
|
| 190 |
# Figure out whether to initialise main in the subprocess as a module |
| 191 |
# or through direct execution (or to leave it alone entirely) |
| 192 |
main_module = sys.modules['__main__'] |
| 193 |
main_mod_name = getattr(main_module.__spec__, "name", None) |
| 194 |
if main_mod_name is not None: |
| 195 |
d['init_main_from_name'] = main_mod_name |
| 196 |
elif sys.platform != 'win32' or (not WINEXE and not WINSERVICE): |
| 197 |
main_path = getattr(main_module, '__file__', None) |
| 198 |
if main_path is not None: |
| 199 |
if (not os.path.isabs(main_path) and |
| 200 |
process.ORIGINAL_DIR is not None): |
| 201 |
main_path = os.path.join(process.ORIGINAL_DIR, main_path) |
| 202 |
d['init_main_from_path'] = os.path.normpath(main_path) |
| 203 |
|
| 204 |
return d |
| 205 |
|
| 206 |
# |
| 207 |
# Prepare current process |
| 208 |
# |
| 209 |
|
| 210 |
old_main_modules = [] |
| 211 |
|
| 212 |
def prepare(data): |
| 213 |
''' |
| 214 |
Try to get current process ready to unpickle process object |
| 215 |
''' |
| 216 |
if 'name' in data: |
| 217 |
process.current_process().name = data['name'] |
| 218 |
|
| 219 |
if 'authkey' in data: |
| 220 |
process.current_process().authkey = data['authkey'] |
| 221 |
|
| 222 |
if 'log_to_stderr' in data and data['log_to_stderr']: |
| 223 |
util.log_to_stderr() |
| 224 |
|
| 225 |
if 'log_level' in data: |
| 226 |
util.get_logger().setLevel(data['log_level']) |
| 227 |
|
| 228 |
if 'sys_path' in data: |
| 229 |
sys.path = data['sys_path'] |
| 230 |
|
| 231 |
if 'sys_argv' in data: |
| 232 |
sys.argv = data['sys_argv'] |
| 233 |
|
| 234 |
if 'dir' in data: |
| 235 |
os.chdir(data['dir']) |
| 236 |
|
| 237 |
if 'orig_dir' in data: |
| 238 |
process.ORIGINAL_DIR = data['orig_dir'] |
| 239 |
|
| 240 |
if 'start_method' in data: |
| 241 |
set_start_method(data['start_method'], force=True) |
| 242 |
|
| 243 |
if 'init_main_from_name' in data: |
| 244 |
_fixup_main_from_name(data['init_main_from_name']) |
| 245 |
elif 'init_main_from_path' in data: |
| 246 |
_fixup_main_from_path(data['init_main_from_path']) |
| 247 |
|
| 248 |
# Multiprocessing module helpers to fix up the main module in |
| 249 |
# spawned subprocesses |
| 250 |
def _fixup_main_from_name(mod_name): |
| 251 |
# __main__.py files for packages, directories, zip archives, etc, run |
| 252 |
# their "main only" code unconditionally, so we don't even try to |
| 253 |
# populate anything in __main__, nor do we make any changes to |
| 254 |
# __main__ attributes |
| 255 |
current_main = sys.modules['__main__'] |
| 256 |
if mod_name == "__main__" or mod_name.endswith(".__main__"): |
| 257 |
return |
| 258 |
|
| 259 |
# If this process was forked, __main__ may already be populated |
| 260 |
if getattr(current_main.__spec__, "name", None) == mod_name: |
| 261 |
return |
| 262 |
|
| 263 |
# Otherwise, __main__ may contain some non-main code where we need to |
| 264 |
# support unpickling it properly. We rerun it as __mp_main__ and make |
| 265 |
# the normal __main__ an alias to that |
| 266 |
old_main_modules.append(current_main) |
| 267 |
main_module = types.ModuleType("__mp_main__") |
| 268 |
main_content = runpy.run_module(mod_name, |
| 269 |
run_name="__mp_main__", |
| 270 |
alter_sys=True) |
| 271 |
main_module.__dict__.update(main_content) |
| 272 |
sys.modules['__main__'] = sys.modules['__mp_main__'] = main_module |
| 273 |
|
| 274 |
|
| 275 |
def _fixup_main_from_path(main_path): |
| 276 |
# If this process was forked, __main__ may already be populated |
| 277 |
current_main = sys.modules['__main__'] |
| 278 |
|
| 279 |
# Unfortunately, the main ipython launch script historically had no |
| 280 |
# "if __name__ == '__main__'" guard, so we work around that |
| 281 |
# by treating it like a __main__.py file |
| 282 |
# See https://github.com/ipython/ipython/issues/4698 |
| 283 |
main_name = os.path.splitext(os.path.basename(main_path))[0] |
| 284 |
if main_name == 'ipython': |
| 285 |
return |
| 286 |
|
| 287 |
# Otherwise, if __file__ already has the setting we expect, |
| 288 |
# there's nothing more to do |
| 289 |
if getattr(current_main, '__file__', None) == main_path: |
| 290 |
return |
| 291 |
|
| 292 |
# If the parent process has sent a path through rather than a module |
| 293 |
# name we assume it is an executable script that may contain |
| 294 |
# non-main code that needs to be executed |
| 295 |
old_main_modules.append(current_main) |
| 296 |
main_module = types.ModuleType("__mp_main__") |
| 297 |
main_content = runpy.run_path(main_path, |
| 298 |
run_name="__mp_main__") |
| 299 |
main_module.__dict__.update(main_content) |
| 300 |
sys.modules['__main__'] = sys.modules['__mp_main__'] = main_module |
| 301 |
|
| 302 |
|
| 303 |
def import_main_path(main_path): |
| 304 |
''' |
| 305 |
Set sys.modules['__main__'] to module at main_path |
| 306 |
''' |
| 307 |
_fixup_main_from_path(main_path) |