| 1 |
############################################################################### |
| 2 |
# Prepares and processes the data to setup the new process environment |
| 3 |
# |
| 4 |
# author: Thomas Moreau and Olivier Grisel |
| 5 |
# |
| 6 |
# adapted from multiprocessing/spawn.py (17/02/2017) |
| 7 |
# * Improve logging data |
| 8 |
# |
| 9 |
import os |
| 10 |
import sys |
| 11 |
import runpy |
| 12 |
import textwrap |
| 13 |
import types |
| 14 |
from multiprocessing import process, util |
| 15 |
|
| 16 |
|
| 17 |
if sys.platform != "win32": |
| 18 |
WINEXE = False |
| 19 |
WINSERVICE = False |
| 20 |
else: |
| 21 |
import msvcrt |
| 22 |
from multiprocessing.reduction import duplicate |
| 23 |
|
| 24 |
WINEXE = sys.platform == "win32" and getattr(sys, "frozen", False) |
| 25 |
WINSERVICE = sys.executable.lower().endswith("pythonservice.exe") |
| 26 |
|
| 27 |
if WINSERVICE: |
| 28 |
_python_exe = os.path.join(sys.exec_prefix, "python.exe") |
| 29 |
else: |
| 30 |
_python_exe = sys.executable |
| 31 |
|
| 32 |
|
| 33 |
def get_executable(): |
| 34 |
return _python_exe |
| 35 |
|
| 36 |
|
| 37 |
def _check_not_importing_main(): |
| 38 |
if getattr(process.current_process(), "_inheriting", False): |
| 39 |
raise RuntimeError( |
| 40 |
textwrap.dedent( |
| 41 |
"""\ |
| 42 |
An attempt has been made to start a new process before the |
| 43 |
current process has finished its bootstrapping phase. |
| 44 |
|
| 45 |
This probably means that you are not using fork to start your |
| 46 |
child processes and you have forgotten to use the proper idiom |
| 47 |
in the main module: |
| 48 |
|
| 49 |
if __name__ == '__main__': |
| 50 |
freeze_support() |
| 51 |
... |
| 52 |
|
| 53 |
The "freeze_support()" line can be omitted if the program |
| 54 |
is not going to be frozen to produce an executable.""" |
| 55 |
) |
| 56 |
) |
| 57 |
|
| 58 |
|
| 59 |
def get_preparation_data(name, init_main_module=True): |
| 60 |
"""Return info about parent needed by child to unpickle process object.""" |
| 61 |
_check_not_importing_main() |
| 62 |
d = dict( |
| 63 |
log_to_stderr=util._log_to_stderr, |
| 64 |
authkey=bytes(process.current_process().authkey), |
| 65 |
name=name, |
| 66 |
sys_argv=sys.argv, |
| 67 |
orig_dir=process.ORIGINAL_DIR, |
| 68 |
dir=os.getcwd(), |
| 69 |
) |
| 70 |
|
| 71 |
# Send sys_path and make sure the current directory will not be changed |
| 72 |
d["sys_path"] = [p if p != "" else process.ORIGINAL_DIR for p in sys.path] |
| 73 |
|
| 74 |
# Make sure to pass the information if the multiprocessing logger is active |
| 75 |
if util._logger is not None: |
| 76 |
d["log_level"] = util._logger.getEffectiveLevel() |
| 77 |
if util._logger.handlers: |
| 78 |
h = util._logger.handlers[0] |
| 79 |
d["log_fmt"] = h.formatter._fmt |
| 80 |
|
| 81 |
# Tell the child how to communicate with the resource_tracker |
| 82 |
from .resource_tracker import _resource_tracker |
| 83 |
|
| 84 |
_resource_tracker.ensure_running() |
| 85 |
if sys.platform == "win32": |
| 86 |
d["tracker_fd"] = msvcrt.get_osfhandle(_resource_tracker._fd) |
| 87 |
else: |
| 88 |
d["tracker_fd"] = _resource_tracker._fd |
| 89 |
|
| 90 |
if os.name == "posix": |
| 91 |
# joblib/loky#242: allow loky processes to retrieve the resource |
| 92 |
# tracker of their parent in case the child processes depickles |
| 93 |
# shared_memory objects, that are still tracked by multiprocessing's |
| 94 |
# resource_tracker by default. |
| 95 |
# XXX: this is a workaround that may be error prone: in the future, it |
| 96 |
# would be better to have loky subclass multiprocessing's shared_memory |
| 97 |
# to force registration of shared_memory segments via loky's |
| 98 |
# resource_tracker. |
| 99 |
from multiprocessing.resource_tracker import ( |
| 100 |
_resource_tracker as mp_resource_tracker, |
| 101 |
) |
| 102 |
|
| 103 |
# multiprocessing's resource_tracker must be running before loky |
| 104 |
# process is created (othewise the child won't be able to use it if it |
| 105 |
# is created later on) |
| 106 |
mp_resource_tracker.ensure_running() |
| 107 |
d["mp_tracker_fd"] = mp_resource_tracker._fd |
| 108 |
|
| 109 |
# Figure out whether to initialise main in the subprocess as a module |
| 110 |
# or through direct execution (or to leave it alone entirely) |
| 111 |
if init_main_module: |
| 112 |
main_module = sys.modules["__main__"] |
| 113 |
try: |
| 114 |
main_mod_name = getattr(main_module.__spec__, "name", None) |
| 115 |
except BaseException: |
| 116 |
main_mod_name = None |
| 117 |
if main_mod_name is not None: |
| 118 |
d["init_main_from_name"] = main_mod_name |
| 119 |
elif sys.platform != "win32" or (not WINEXE and not WINSERVICE): |
| 120 |
main_path = getattr(main_module, "__file__", None) |
| 121 |
if main_path is not None: |
| 122 |
if ( |
| 123 |
not os.path.isabs(main_path) |
| 124 |
and process.ORIGINAL_DIR is not None |
| 125 |
): |
| 126 |
main_path = os.path.join(process.ORIGINAL_DIR, main_path) |
| 127 |
d["init_main_from_path"] = os.path.normpath(main_path) |
| 128 |
|
| 129 |
return d |
| 130 |
|
| 131 |
|
| 132 |
# |
| 133 |
# Prepare current process |
| 134 |
# |
| 135 |
old_main_modules = [] |
| 136 |
|
| 137 |
|
| 138 |
def prepare(data, parent_sentinel=None): |
| 139 |
"""Try to get current process ready to unpickle process object.""" |
| 140 |
if "name" in data: |
| 141 |
process.current_process().name = data["name"] |
| 142 |
|
| 143 |
if "authkey" in data: |
| 144 |
process.current_process().authkey = data["authkey"] |
| 145 |
|
| 146 |
if "log_to_stderr" in data and data["log_to_stderr"]: |
| 147 |
util.log_to_stderr() |
| 148 |
|
| 149 |
if "log_level" in data: |
| 150 |
util.get_logger().setLevel(data["log_level"]) |
| 151 |
|
| 152 |
if "log_fmt" in data: |
| 153 |
import logging |
| 154 |
|
| 155 |
util.get_logger().handlers[0].setFormatter( |
| 156 |
logging.Formatter(data["log_fmt"]) |
| 157 |
) |
| 158 |
|
| 159 |
if "sys_path" in data: |
| 160 |
sys.path = data["sys_path"] |
| 161 |
|
| 162 |
if "sys_argv" in data: |
| 163 |
sys.argv = data["sys_argv"] |
| 164 |
|
| 165 |
if "dir" in data: |
| 166 |
os.chdir(data["dir"]) |
| 167 |
|
| 168 |
if "orig_dir" in data: |
| 169 |
process.ORIGINAL_DIR = data["orig_dir"] |
| 170 |
|
| 171 |
if "mp_tracker_fd" in data: |
| 172 |
from multiprocessing.resource_tracker import ( |
| 173 |
_resource_tracker as mp_resource_tracker, |
| 174 |
) |
| 175 |
|
| 176 |
mp_resource_tracker._fd = data["mp_tracker_fd"] |
| 177 |
if "tracker_fd" in data: |
| 178 |
from .resource_tracker import _resource_tracker |
| 179 |
|
| 180 |
if sys.platform == "win32": |
| 181 |
handle = data["tracker_fd"] |
| 182 |
handle = duplicate(handle, source_process=parent_sentinel) |
| 183 |
_resource_tracker._fd = msvcrt.open_osfhandle(handle, os.O_RDONLY) |
| 184 |
else: |
| 185 |
_resource_tracker._fd = data["tracker_fd"] |
| 186 |
|
| 187 |
if "init_main_from_name" in data: |
| 188 |
_fixup_main_from_name(data["init_main_from_name"]) |
| 189 |
elif "init_main_from_path" in data: |
| 190 |
_fixup_main_from_path(data["init_main_from_path"]) |
| 191 |
|
| 192 |
|
| 193 |
# Multiprocessing module helpers to fix up the main module in |
| 194 |
# spawned subprocesses |
| 195 |
def _fixup_main_from_name(mod_name): |
| 196 |
# __main__.py files for packages, directories, zip archives, etc, run |
| 197 |
# their "main only" code unconditionally, so we don't even try to |
| 198 |
# populate anything in __main__, nor do we make any changes to |
| 199 |
# __main__ attributes |
| 200 |
current_main = sys.modules["__main__"] |
| 201 |
if mod_name == "__main__" or mod_name.endswith(".__main__"): |
| 202 |
return |
| 203 |
|
| 204 |
# If this process was forked, __main__ may already be populated |
| 205 |
if getattr(current_main.__spec__, "name", None) == mod_name: |
| 206 |
return |
| 207 |
|
| 208 |
# Otherwise, __main__ may contain some non-main code where we need to |
| 209 |
# support unpickling it properly. We rerun it as __mp_main__ and make |
| 210 |
# the normal __main__ an alias to that |
| 211 |
old_main_modules.append(current_main) |
| 212 |
main_module = types.ModuleType("__mp_main__") |
| 213 |
main_content = runpy.run_module( |
| 214 |
mod_name, run_name="__mp_main__", alter_sys=True |
| 215 |
) |
| 216 |
main_module.__dict__.update(main_content) |
| 217 |
sys.modules["__main__"] = sys.modules["__mp_main__"] = main_module |
| 218 |
|
| 219 |
|
| 220 |
def _fixup_main_from_path(main_path): |
| 221 |
# If this process was forked, __main__ may already be populated |
| 222 |
current_main = sys.modules["__main__"] |
| 223 |
|
| 224 |
# Unfortunately, the main ipython launch script historically had no |
| 225 |
# "if __name__ == '__main__'" guard, so we work around that |
| 226 |
# by treating it like a __main__.py file |
| 227 |
# See https://github.com/ipython/ipython/issues/4698 |
| 228 |
main_name = os.path.splitext(os.path.basename(main_path))[0] |
| 229 |
if main_name == "ipython": |
| 230 |
return |
| 231 |
|
| 232 |
# Otherwise, if __file__ already has the setting we expect, |
| 233 |
# there's nothing more to do |
| 234 |
if getattr(current_main, "__file__", None) == main_path: |
| 235 |
return |
| 236 |
|
| 237 |
# If the parent process has sent a path through rather than a module |
| 238 |
# name we assume it is an executable script that may contain |
| 239 |
# non-main code that needs to be executed |
| 240 |
old_main_modules.append(current_main) |
| 241 |
main_module = types.ModuleType("__mp_main__") |
| 242 |
main_content = runpy.run_path(main_path, run_name="__mp_main__") |
| 243 |
main_module.__dict__.update(main_content) |
| 244 |
sys.modules["__main__"] = sys.modules["__mp_main__"] = main_module |