| 1 |
"""Implements InterpreterPoolExecutor.""" |
| 2 |
|
| 3 |
from concurrent import interpreters |
| 4 |
import sys |
| 5 |
from . import thread as _thread |
| 6 |
import traceback |
| 7 |
|
| 8 |
|
| 9 |
def do_call(results, func, args, kwargs): |
| 10 |
try: |
| 11 |
return func(*args, **kwargs) |
| 12 |
except BaseException as exc: |
| 13 |
# Send the captured exception out on the results queue, |
| 14 |
# but still leave it unhandled for the interpreter to handle. |
| 15 |
try: |
| 16 |
results.put(exc) |
| 17 |
except interpreters.NotShareableError: |
| 18 |
# The exception is not shareable. |
| 19 |
print('exception is not shareable:', file=sys.stderr) |
| 20 |
traceback.print_exception(exc) |
| 21 |
results.put(None) |
| 22 |
raise # re-raise |
| 23 |
|
| 24 |
|
| 25 |
class WorkerContext(_thread.WorkerContext): |
| 26 |
|
| 27 |
@classmethod |
| 28 |
def prepare(cls, initializer, initargs): |
| 29 |
def resolve_task(fn, args, kwargs): |
| 30 |
if isinstance(fn, str): |
| 31 |
# XXX Circle back to this later. |
| 32 |
raise TypeError('scripts not supported') |
| 33 |
else: |
| 34 |
task = (fn, args, kwargs) |
| 35 |
return task |
| 36 |
|
| 37 |
if initializer is not None: |
| 38 |
try: |
| 39 |
initdata = resolve_task(initializer, initargs, {}) |
| 40 |
except ValueError: |
| 41 |
if isinstance(initializer, str) and initargs: |
| 42 |
raise ValueError(f'an initializer script does not take args, got {initargs!r}') |
| 43 |
raise # re-raise |
| 44 |
else: |
| 45 |
initdata = None |
| 46 |
def create_context(): |
| 47 |
return cls(initdata) |
| 48 |
return create_context, resolve_task |
| 49 |
|
| 50 |
def __init__(self, initdata): |
| 51 |
self.initdata = initdata |
| 52 |
self.interp = None |
| 53 |
self.results = None |
| 54 |
|
| 55 |
def __del__(self): |
| 56 |
if self.interp is not None: |
| 57 |
self.finalize() |
| 58 |
|
| 59 |
def initialize(self): |
| 60 |
assert self.interp is None, self.interp |
| 61 |
self.interp = interpreters.create() |
| 62 |
try: |
| 63 |
maxsize = 0 |
| 64 |
self.results = interpreters.create_queue(maxsize) |
| 65 |
|
| 66 |
if self.initdata: |
| 67 |
self.run(self.initdata) |
| 68 |
except BaseException: |
| 69 |
self.finalize() |
| 70 |
raise # re-raise |
| 71 |
|
| 72 |
def finalize(self): |
| 73 |
interp = self.interp |
| 74 |
results = self.results |
| 75 |
self.results = None |
| 76 |
self.interp = None |
| 77 |
if results is not None: |
| 78 |
del results |
| 79 |
if interp is not None: |
| 80 |
interp.close() |
| 81 |
|
| 82 |
def run(self, task): |
| 83 |
try: |
| 84 |
return self.interp.call(do_call, self.results, *task) |
| 85 |
except interpreters.ExecutionFailed as wrapper: |
| 86 |
# Wait for the exception data to show up. |
| 87 |
exc = self.results.get() |
| 88 |
if exc is None: |
| 89 |
# The exception must have been not shareable. |
| 90 |
raise # re-raise |
| 91 |
raise exc from wrapper |
| 92 |
|
| 93 |
|
| 94 |
class BrokenInterpreterPool(_thread.BrokenThreadPool): |
| 95 |
""" |
| 96 |
Raised when a worker thread in an InterpreterPoolExecutor failed initializing. |
| 97 |
""" |
| 98 |
|
| 99 |
|
| 100 |
class InterpreterPoolExecutor(_thread.ThreadPoolExecutor): |
| 101 |
|
| 102 |
BROKEN = BrokenInterpreterPool |
| 103 |
|
| 104 |
@classmethod |
| 105 |
def prepare_context(cls, initializer, initargs): |
| 106 |
return WorkerContext.prepare(initializer, initargs) |
| 107 |
|
| 108 |
def __init__(self, max_workers=None, thread_name_prefix='', |
| 109 |
initializer=None, initargs=()): |
| 110 |
"""Initializes a new InterpreterPoolExecutor instance. |
| 111 |
|
| 112 |
Args: |
| 113 |
max_workers: The maximum number of interpreters that can be used to |
| 114 |
execute the given calls. |
| 115 |
thread_name_prefix: An optional name prefix to give our threads. |
| 116 |
initializer: A callable or script used to initialize |
| 117 |
each worker interpreter. |
| 118 |
initargs: A tuple of arguments to pass to the initializer. |
| 119 |
""" |
| 120 |
thread_name_prefix = (thread_name_prefix or |
| 121 |
(f"InterpreterPoolExecutor-{self._counter()}")) |
| 122 |
super().__init__(max_workers, thread_name_prefix, |
| 123 |
initializer, initargs) |