| 1 |
# Copyright (C) 2012 Yahoo! Inc. All Rights Reserved. |
| 2 |
# |
| 3 |
# Licensed under the Apache License, Version 2.0 (the "License"); you may |
| 4 |
# not use this file except in compliance with the License. You may obtain |
| 5 |
# a copy of the License at |
| 6 |
# |
| 7 |
# http://www.apache.org/licenses/LICENSE-2.0 |
| 8 |
# |
| 9 |
# Unless required by applicable law or agreed to in writing, software |
| 10 |
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT |
| 11 |
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the |
| 12 |
# License for the specific language governing permissions and limitations |
| 13 |
# under the License. |
| 14 |
|
| 15 |
import collections |
| 16 |
import contextlib |
| 17 |
import itertools |
| 18 |
import threading |
| 19 |
|
| 20 |
from automaton import runners |
| 21 |
from concurrent import futures |
| 22 |
import fasteners |
| 23 |
import functools |
| 24 |
import networkx as nx |
| 25 |
from oslo_utils import excutils |
| 26 |
from oslo_utils import strutils |
| 27 |
from oslo_utils import timeutils |
| 28 |
|
| 29 |
from taskflow.engines.action_engine import builder |
| 30 |
from taskflow.engines.action_engine import compiler |
| 31 |
from taskflow.engines.action_engine import executor |
| 32 |
from taskflow.engines.action_engine import runtime |
| 33 |
from taskflow.engines import base |
| 34 |
from taskflow import exceptions as exc |
| 35 |
from taskflow import logging |
| 36 |
from taskflow import states |
| 37 |
from taskflow import storage |
| 38 |
from taskflow.types import failure |
| 39 |
from taskflow.utils import misc |
| 40 |
|
| 41 |
LOG = logging.getLogger(__name__) |
| 42 |
|
| 43 |
|
| 44 |
@contextlib.contextmanager |
| 45 |
def _start_stop(task_executor, retry_executor): |
| 46 |
# A teenie helper context manager to safely start/stop engine executors... |
| 47 |
task_executor.start() |
| 48 |
try: |
| 49 |
retry_executor.start() |
| 50 |
try: |
| 51 |
yield (task_executor, retry_executor) |
| 52 |
finally: |
| 53 |
retry_executor.stop() |
| 54 |
finally: |
| 55 |
task_executor.stop() |
| 56 |
|
| 57 |
|
| 58 |
def _pre_check(check_compiled=True, check_storage_ensured=True, |
| 59 |
check_validated=True): |
| 60 |
"""Engine state precondition checking decorator.""" |
| 61 |
|
| 62 |
def decorator(meth): |
| 63 |
do_what = meth.__name__ |
| 64 |
|
| 65 |
@functools.wraps(meth) |
| 66 |
def wrapper(self, *args, **kwargs): |
| 67 |
if check_compiled and not self._compiled: |
| 68 |
raise exc.InvalidState("Can not %s an engine which" |
| 69 |
" has not been compiled" % do_what) |
| 70 |
if check_storage_ensured and not self._storage_ensured: |
| 71 |
raise exc.InvalidState("Can not %s an engine" |
| 72 |
" which has not had its storage" |
| 73 |
" populated" % do_what) |
| 74 |
if check_validated and not self._validated: |
| 75 |
raise exc.InvalidState("Can not %s an engine which" |
| 76 |
" has not been validated" % do_what) |
| 77 |
return meth(self, *args, **kwargs) |
| 78 |
|
| 79 |
return wrapper |
| 80 |
|
| 81 |
return decorator |
| 82 |
|
| 83 |
|
| 84 |
class ActionEngine(base.Engine): |
| 85 |
"""Generic action-based engine. |
| 86 |
|
| 87 |
This engine compiles the flow (and any subflows) into a compilation unit |
| 88 |
which contains the full runtime definition to be executed and then uses |
| 89 |
this compilation unit in combination with the executor, runtime, machine |
| 90 |
builder and storage classes to attempt to run your flow (and any |
| 91 |
subflows & contained atoms) to completion. |
| 92 |
|
| 93 |
NOTE(harlowja): during this process it is permissible and valid to have a |
| 94 |
task or multiple tasks in the execution graph fail (at the same time even), |
| 95 |
which will cause the process of reversion or retrying to commence. See the |
| 96 |
valid states in the states module to learn more about what other states |
| 97 |
the tasks and flow being ran can go through. |
| 98 |
|
| 99 |
**Engine options:** |
| 100 |
|
| 101 |
+----------------------+-----------------------+------+------------+ |
| 102 |
| Name/key | Description | Type | Default | |
| 103 |
+======================+=======================+======+============+ |
| 104 |
| ``defer_reverts`` | This option lets you | bool | ``False`` | |
| 105 |
| | safely nest flows | | | |
| 106 |
| | with retries inside | | | |
| 107 |
| | flows without retries | | | |
| 108 |
| | and it still behaves | | | |
| 109 |
| | as a user would | | | |
| 110 |
| | expect (for example | | | |
| 111 |
| | if the retry gets | | | |
| 112 |
| | exhausted it reverts | | | |
| 113 |
| | the outer flow unless | | | |
| 114 |
| | the outer flow has a | | | |
| 115 |
| | has a separate retry | | | |
| 116 |
| | behavior). | | | |
| 117 |
+----------------------+-----------------------+------+------------+ |
| 118 |
| ``never_resolve`` | When true, instead | bool | ``False`` | |
| 119 |
| | of reverting | | | |
| 120 |
| | and trying to resolve | | | |
| 121 |
| | a atom failure the | | | |
| 122 |
| | engine will skip | | | |
| 123 |
| | reverting and abort | | | |
| 124 |
| | instead of reverting | | | |
| 125 |
| | and/or retrying. | | | |
| 126 |
+----------------------+-----------------------+------+------------+ |
| 127 |
| ``inject_transient`` | When true, values | bool | ``True`` | |
| 128 |
| | that are local to | | | |
| 129 |
| | each atoms scope | | | |
| 130 |
| | are injected into | | | |
| 131 |
| | storage into a | | | |
| 132 |
| | transient location | | | |
| 133 |
| | (typically a local | | | |
| 134 |
| | dictionary), when | | | |
| 135 |
| | false those values | | | |
| 136 |
| | are instead persisted | | | |
| 137 |
| | into atom details | | | |
| 138 |
| | (and saved in a non- | | | |
| 139 |
| | transient manner). | | | |
| 140 |
+----------------------+-----------------------+------+------------+ |
| 141 |
""" |
| 142 |
|
| 143 |
NO_RERAISING_STATES = frozenset([states.SUSPENDED, states.SUCCESS]) |
| 144 |
""" |
| 145 |
States that if the engine stops in will **not** cause any potential |
| 146 |
failures to be reraised. States **not** in this list will cause any |
| 147 |
failure/s that were captured (if any) to get reraised. |
| 148 |
""" |
| 149 |
|
| 150 |
IGNORABLE_STATES = frozenset( |
| 151 |
itertools.chain([states.SCHEDULING, states.WAITING, states.RESUMING, |
| 152 |
states.ANALYZING], builder.META_STATES)) |
| 153 |
""" |
| 154 |
Informational states this engines internal machine yields back while |
| 155 |
running, not useful to have the engine record but useful to provide to |
| 156 |
end-users when doing execution iterations via :py:meth:`.run_iter`. |
| 157 |
""" |
| 158 |
|
| 159 |
MAX_MACHINE_STATES_RETAINED = 10 |
| 160 |
""" |
| 161 |
During :py:meth:`~.run_iter` the last X state machine transitions will |
| 162 |
be recorded (typically only useful on failure). |
| 163 |
""" |
| 164 |
|
| 165 |
def __init__(self, flow, flow_detail, backend, options): |
| 166 |
super().__init__(flow, flow_detail, backend, options) |
| 167 |
self._runtime = None |
| 168 |
self._compiled = False |
| 169 |
self._compilation = None |
| 170 |
self._compiler = compiler.PatternCompiler(flow) |
| 171 |
self._lock = threading.RLock() |
| 172 |
self._storage_ensured = False |
| 173 |
self._validated = False |
| 174 |
# Retries are not *currently* executed out of the engines process |
| 175 |
# or thread (this could change in the future if we desire it to). |
| 176 |
self._retry_executor = executor.SerialRetryExecutor() |
| 177 |
self._inject_transient = strutils.bool_from_string( |
| 178 |
self._options.get('inject_transient', True)) |
| 179 |
self._gather_statistics = strutils.bool_from_string( |
| 180 |
self._options.get('gather_statistics', True)) |
| 181 |
self._statistics = {} |
| 182 |
|
| 183 |
@_pre_check(check_compiled=True, |
| 184 |
# NOTE(harlowja): We can alter the state of the |
| 185 |
# flow without ensuring its storage is setup for |
| 186 |
# its atoms (since this state change does not affect |
| 187 |
# those units). |
| 188 |
check_storage_ensured=False, |
| 189 |
check_validated=False) |
| 190 |
def suspend(self): |
| 191 |
self._change_state(states.SUSPENDING) |
| 192 |
|
| 193 |
@property |
| 194 |
def statistics(self): |
| 195 |
return self._statistics |
| 196 |
|
| 197 |
@property |
| 198 |
def compilation(self): |
| 199 |
"""The compilation result. |
| 200 |
|
| 201 |
NOTE(harlowja): Only accessible after compilation has completed (None |
| 202 |
will be returned when this property is accessed before compilation has |
| 203 |
completed successfully). |
| 204 |
""" |
| 205 |
if self._compiled: |
| 206 |
return self._compilation |
| 207 |
else: |
| 208 |
return None |
| 209 |
|
| 210 |
@misc.cachedproperty |
| 211 |
def storage(self): |
| 212 |
"""The storage unit for this engine. |
| 213 |
|
| 214 |
NOTE(harlowja): the atom argument lookup strategy will change for |
| 215 |
this storage unit after |
| 216 |
:py:func:`~taskflow.engines.base.Engine.compile` has |
| 217 |
completed (since **only** after compilation is the actual structure |
| 218 |
known). Before :py:func:`~taskflow.engines.base.Engine.compile` |
| 219 |
has completed the atom argument lookup strategy lookup will be |
| 220 |
restricted to injected arguments **only** (this will **not** reflect |
| 221 |
the actual runtime lookup strategy, which typically will be, but is |
| 222 |
not always different). |
| 223 |
""" |
| 224 |
def _scope_fetcher(atom_name): |
| 225 |
if self._compiled: |
| 226 |
return self._runtime.fetch_scopes_for(atom_name) |
| 227 |
else: |
| 228 |
return None |
| 229 |
return storage.Storage(self._flow_detail, |
| 230 |
backend=self._backend, |
| 231 |
scope_fetcher=_scope_fetcher) |
| 232 |
|
| 233 |
def run(self, timeout=None): |
| 234 |
"""Runs the engine (or die trying). |
| 235 |
|
| 236 |
:param timeout: timeout to wait for any atoms to complete (this timeout |
| 237 |
will be used during the waiting period that occurs when |
| 238 |
unfinished atoms are being waited on). |
| 239 |
""" |
| 240 |
with fasteners.try_lock(self._lock) as was_locked: |
| 241 |
if not was_locked: |
| 242 |
raise exc.ExecutionFailure("Engine currently locked, please" |
| 243 |
" try again later") |
| 244 |
for _state in self.run_iter(timeout=timeout): |
| 245 |
pass |
| 246 |
|
| 247 |
def run_iter(self, timeout=None): |
| 248 |
"""Runs the engine using iteration (or die trying). |
| 249 |
|
| 250 |
:param timeout: timeout to wait for any atoms to complete (this timeout |
| 251 |
will be used during the waiting period that occurs after the |
| 252 |
waiting state is yielded when unfinished atoms are being waited |
| 253 |
on). |
| 254 |
|
| 255 |
Instead of running to completion in a blocking manner, this will |
| 256 |
return a generator which will yield back the various states that the |
| 257 |
engine is going through (and can be used to run multiple engines at |
| 258 |
once using a generator per engine). The iterator returned also |
| 259 |
responds to the ``send()`` method from :pep:`0342` and will attempt to |
| 260 |
suspend itself if a truthy value is sent in (the suspend may be |
| 261 |
delayed until all active atoms have finished). |
| 262 |
|
| 263 |
NOTE(harlowja): using the ``run_iter`` method will **not** retain the |
| 264 |
engine lock while executing so the user should ensure that there is |
| 265 |
only one entity using a returned engine iterator (one per engine) at a |
| 266 |
given time. |
| 267 |
""" |
| 268 |
self.compile() |
| 269 |
self.prepare() |
| 270 |
self.validate() |
| 271 |
# Keep track of the last X state changes, which if a failure happens |
| 272 |
# are quite useful to log (and the performance of tracking this |
| 273 |
# should be negligible). |
| 274 |
last_transitions = collections.deque( |
| 275 |
maxlen=max(1, self.MAX_MACHINE_STATES_RETAINED)) |
| 276 |
with _start_stop(self._task_executor, self._retry_executor): |
| 277 |
self._change_state(states.RUNNING) |
| 278 |
if self._gather_statistics: |
| 279 |
self._statistics.clear() |
| 280 |
w = timeutils.StopWatch() |
| 281 |
w.start() |
| 282 |
else: |
| 283 |
w = None |
| 284 |
try: |
| 285 |
closed = False |
| 286 |
machine, memory = self._runtime.builder.build( |
| 287 |
self._statistics, timeout=timeout, |
| 288 |
gather_statistics=self._gather_statistics) |
| 289 |
r = runners.FiniteRunner(machine) |
| 290 |
for transition in r.run_iter(builder.START): |
| 291 |
last_transitions.append(transition) |
| 292 |
_prior_state, new_state = transition |
| 293 |
# NOTE(harlowja): skip over meta-states |
| 294 |
if new_state in builder.META_STATES: |
| 295 |
continue |
| 296 |
if new_state == states.FAILURE: |
| 297 |
failure.Failure.reraise_if_any(memory.failures) |
| 298 |
if closed: |
| 299 |
continue |
| 300 |
try: |
| 301 |
try_suspend = yield new_state |
| 302 |
except GeneratorExit: |
| 303 |
# The generator was closed, attempt to suspend and |
| 304 |
# continue looping until we have cleanly closed up |
| 305 |
# shop... |
| 306 |
closed = True |
| 307 |
self.suspend() |
| 308 |
except Exception: |
| 309 |
# Capture the failure, and ensure that the |
| 310 |
# machine will notice that something externally |
| 311 |
# has sent an exception in and that it should |
| 312 |
# finish up and reraise. |
| 313 |
memory.failures.append(failure.Failure()) |
| 314 |
closed = True |
| 315 |
else: |
| 316 |
if try_suspend: |
| 317 |
self.suspend() |
| 318 |
except Exception: |
| 319 |
with excutils.save_and_reraise_exception(): |
| 320 |
LOG.exception("Engine execution has failed, something" |
| 321 |
" bad must have happened (last" |
| 322 |
" %s machine transitions were %s)", |
| 323 |
last_transitions.maxlen, |
| 324 |
list(last_transitions)) |
| 325 |
self._change_state(states.FAILURE) |
| 326 |
else: |
| 327 |
if last_transitions: |
| 328 |
_prior_state, new_state = last_transitions[-1] |
| 329 |
if new_state not in self.IGNORABLE_STATES: |
| 330 |
self._change_state(new_state) |
| 331 |
if new_state not in self.NO_RERAISING_STATES: |
| 332 |
e_failures = self.storage.get_execute_failures() |
| 333 |
r_failures = self.storage.get_revert_failures() |
| 334 |
er_failures = itertools.chain( |
| 335 |
e_failures.values(), |
| 336 |
r_failures.values()) |
| 337 |
failure.Failure.reraise_if_any(er_failures) |
| 338 |
finally: |
| 339 |
if w is not None: |
| 340 |
w.stop() |
| 341 |
self._statistics['active_for'] = w.elapsed() |
| 342 |
|
| 343 |
@staticmethod |
| 344 |
def _check_compilation(compilation): |
| 345 |
"""Performs post compilation validation/checks.""" |
| 346 |
seen = set() |
| 347 |
dups = set() |
| 348 |
execution_graph = compilation.execution_graph |
| 349 |
for node, node_attrs in execution_graph.nodes(data=True): |
| 350 |
if node_attrs['kind'] in compiler.ATOMS: |
| 351 |
atom_name = node.name |
| 352 |
if atom_name in seen: |
| 353 |
dups.add(atom_name) |
| 354 |
else: |
| 355 |
seen.add(atom_name) |
| 356 |
if dups: |
| 357 |
raise exc.Duplicate( |
| 358 |
"Atoms with duplicate names found: %s" % (sorted(dups))) |
| 359 |
return compilation |
| 360 |
|
| 361 |
def _change_state(self, state): |
| 362 |
moved, old_state = self.storage.change_flow_state(state) |
| 363 |
if moved: |
| 364 |
details = { |
| 365 |
'engine': self, |
| 366 |
'flow_name': self.storage.flow_name, |
| 367 |
'flow_uuid': self.storage.flow_uuid, |
| 368 |
'old_state': old_state, |
| 369 |
} |
| 370 |
self.notifier.notify(state, details) |
| 371 |
|
| 372 |
def _ensure_storage(self): |
| 373 |
"""Ensure all contained atoms exist in the storage unit.""" |
| 374 |
self.storage.ensure_atoms( |
| 375 |
self._runtime.iterate_nodes(compiler.ATOMS)) |
| 376 |
for atom in self._runtime.iterate_nodes(compiler.ATOMS): |
| 377 |
if atom.inject: |
| 378 |
self.storage.inject_atom_args(atom.name, atom.inject, |
| 379 |
transient=self._inject_transient) |
| 380 |
|
| 381 |
@fasteners.locked |
| 382 |
@_pre_check(check_validated=False) |
| 383 |
def validate(self): |
| 384 |
# At this point we can check to ensure all dependencies are either |
| 385 |
# flow/task provided or storage provided, if there are still missing |
| 386 |
# dependencies then this flow will fail at runtime (which we can avoid |
| 387 |
# by failing at validation time). |
| 388 |
if LOG.isEnabledFor(logging.TRACE): |
| 389 |
execution_graph = self._compilation.execution_graph |
| 390 |
LOG.trace("Validating scoping and argument visibility for" |
| 391 |
" execution graph with %s nodes and %s edges with" |
| 392 |
" density %0.3f", execution_graph.number_of_nodes(), |
| 393 |
execution_graph.number_of_edges(), |
| 394 |
nx.density(execution_graph)) |
| 395 |
missing = set() |
| 396 |
# Attempt to retain a chain of what was missing (so that the final |
| 397 |
# raised exception for the flow has the nodes that had missing |
| 398 |
# dependencies). |
| 399 |
last_cause = None |
| 400 |
last_node = None |
| 401 |
missing_nodes = 0 |
| 402 |
for atom in self._runtime.iterate_nodes(compiler.ATOMS): |
| 403 |
exec_missing = self.storage.fetch_unsatisfied_args( |
| 404 |
atom.name, atom.rebind, optional_args=atom.optional) |
| 405 |
revert_missing = self.storage.fetch_unsatisfied_args( |
| 406 |
atom.name, atom.revert_rebind, |
| 407 |
optional_args=atom.revert_optional) |
| 408 |
atom_missing = (('execute', exec_missing), |
| 409 |
('revert', revert_missing)) |
| 410 |
for method, method_missing in atom_missing: |
| 411 |
if method_missing: |
| 412 |
cause = exc.MissingDependencies(atom, |
| 413 |
sorted(method_missing), |
| 414 |
cause=last_cause, |
| 415 |
method=method) |
| 416 |
last_cause = cause |
| 417 |
last_node = atom |
| 418 |
missing_nodes += 1 |
| 419 |
missing.update(method_missing) |
| 420 |
if missing: |
| 421 |
# For when a task is provided (instead of a flow) and that |
| 422 |
# task is the only item in the graph and its missing deps, avoid |
| 423 |
# re-wrapping it in yet another exception... |
| 424 |
if missing_nodes == 1 and last_node is self._flow: |
| 425 |
raise last_cause |
| 426 |
else: |
| 427 |
raise exc.MissingDependencies(self._flow, |
| 428 |
sorted(missing), |
| 429 |
cause=last_cause) |
| 430 |
self._validated = True |
| 431 |
|
| 432 |
@fasteners.locked |
| 433 |
@_pre_check(check_storage_ensured=False, check_validated=False) |
| 434 |
def prepare(self): |
| 435 |
if not self._storage_ensured: |
| 436 |
# Set our own state to resuming -> (ensure atoms exist |
| 437 |
# in storage) -> suspended in the storage unit and notify any |
| 438 |
# attached listeners of these changes. |
| 439 |
self._change_state(states.RESUMING) |
| 440 |
self._ensure_storage() |
| 441 |
self._change_state(states.SUSPENDED) |
| 442 |
self._storage_ensured = True |
| 443 |
# Reset everything back to pending (if we were previously reverted). |
| 444 |
if self.storage.get_flow_state() == states.REVERTED: |
| 445 |
self.reset() |
| 446 |
|
| 447 |
@fasteners.locked |
| 448 |
@_pre_check(check_validated=False) |
| 449 |
def reset(self): |
| 450 |
# This transitions *all* contained atoms back into the PENDING state |
| 451 |
# with an intention to EXECUTE (or dies trying to do that) and then |
| 452 |
# changes the state of the flow to PENDING so that it can then run... |
| 453 |
self._runtime.reset_all() |
| 454 |
self._change_state(states.PENDING) |
| 455 |
|
| 456 |
@fasteners.locked |
| 457 |
def compile(self): |
| 458 |
if self._compiled: |
| 459 |
return |
| 460 |
self._compilation = self._check_compilation(self._compiler.compile()) |
| 461 |
self._runtime = runtime.Runtime(self._compilation, |
| 462 |
self.storage, |
| 463 |
self.atom_notifier, |
| 464 |
self._task_executor, |
| 465 |
self._retry_executor, |
| 466 |
options=self._options) |
| 467 |
self._runtime.compile() |
| 468 |
self._compiled = True |
| 469 |
|
| 470 |
|
| 471 |
class SerialActionEngine(ActionEngine): |
| 472 |
"""Engine that runs tasks in serial manner.""" |
| 473 |
|
| 474 |
def __init__(self, flow, flow_detail, backend, options): |
| 475 |
super().__init__(flow, flow_detail, backend, options) |
| 476 |
self._task_executor = executor.SerialTaskExecutor() |
| 477 |
|
| 478 |
|
| 479 |
class _ExecutorTypeMatch(collections.namedtuple('_ExecutorTypeMatch', |
| 480 |
['types', 'executor_cls'])): |
| 481 |
def matches(self, executor): |
| 482 |
return isinstance(executor, self.types) |
| 483 |
|
| 484 |
|
| 485 |
class _ExecutorTextMatch(collections.namedtuple('_ExecutorTextMatch', |
| 486 |
['strings', 'executor_cls'])): |
| 487 |
def matches(self, text): |
| 488 |
return text.lower() in self.strings |
| 489 |
|
| 490 |
|
| 491 |
class ParallelActionEngine(ActionEngine): |
| 492 |
"""Engine that runs tasks in parallel manner. |
| 493 |
|
| 494 |
**Additional engine options:** |
| 495 |
|
| 496 |
* ``executor``: a object that implements a :pep:`3148` compatible executor |
| 497 |
interface; it will be used for scheduling tasks. The following |
| 498 |
type are applicable (other unknown types passed will cause a type |
| 499 |
error to be raised). |
| 500 |
|
| 501 |
========================= =============================================== |
| 502 |
Type provided Executor used |
| 503 |
========================= =============================================== |
| 504 |
|cft|.ThreadPoolExecutor :class:`~.executor.ParallelThreadTaskExecutor` |
| 505 |
|cfp|.ProcessPoolExecutor :class:`~.|pe|.ParallelProcessTaskExecutor` |
| 506 |
|cf|._base.Executor :class:`~.executor.ParallelThreadTaskExecutor` |
| 507 |
========================= =============================================== |
| 508 |
|
| 509 |
* ``executor``: a string that will be used to select a :pep:`3148` |
| 510 |
compatible executor; it will be used for scheduling tasks. The following |
| 511 |
string are applicable (other unknown strings passed will cause a value |
| 512 |
error to be raised). |
| 513 |
|
| 514 |
=========================== =============================================== |
| 515 |
String (case insensitive) Executor used |
| 516 |
=========================== =============================================== |
| 517 |
``process`` :class:`~.|pe|.ParallelProcessTaskExecutor` |
| 518 |
``processes`` :class:`~.|pe|.ParallelProcessTaskExecutor` |
| 519 |
``thread`` :class:`~.executor.ParallelThreadTaskExecutor` |
| 520 |
``threaded`` :class:`~.executor.ParallelThreadTaskExecutor` |
| 521 |
``threads`` :class:`~.executor.ParallelThreadTaskExecutor` |
| 522 |
``greenthread`` :class:`~.executor.ParallelThreadTaskExecutor` |
| 523 |
(greened version) |
| 524 |
``greedthreaded`` :class:`~.executor.ParallelThreadTaskExecutor` |
| 525 |
(greened version) |
| 526 |
``greenthreads`` :class:`~.executor.ParallelThreadTaskExecutor` |
| 527 |
(greened version) |
| 528 |
=========================== =============================================== |
| 529 |
|
| 530 |
* ``max_workers``: a integer that will affect the number of parallel |
| 531 |
workers that are used to dispatch tasks into (this number is bounded |
| 532 |
by the maximum parallelization your workflow can support). |
| 533 |
|
| 534 |
* ``wait_timeout``: a float (in seconds) that will affect the |
| 535 |
parallel process task executor (and therefore is **only** applicable when |
| 536 |
the executor provided above is of the process variant). This number |
| 537 |
affects how much time the process task executor waits for messages from |
| 538 |
child processes (typically indicating they have finished or failed). A |
| 539 |
lower number will have high granularity but *currently* involves more |
| 540 |
polling while a higher number will involve less polling but a slower time |
| 541 |
for an engine to notice a task has completed. |
| 542 |
|
| 543 |
.. |cfp| replace:: concurrent.futures.process |
| 544 |
.. |cft| replace:: concurrent.futures.thread |
| 545 |
.. |cf| replace:: concurrent.futures |
| 546 |
""" |
| 547 |
|
| 548 |
# One of these types should match when a object (non-string) is provided |
| 549 |
# for the 'executor' option. |
| 550 |
# |
| 551 |
# NOTE(harlowja): the reason we use the library/built-in futures is to |
| 552 |
# allow for instances of that to be detected and handled correctly, instead |
| 553 |
# of forcing everyone to use our derivatives (futurist or other)... |
| 554 |
_executor_cls_matchers = [ |
| 555 |
_ExecutorTypeMatch((futures.ThreadPoolExecutor,), |
| 556 |
executor.ParallelThreadTaskExecutor), |
| 557 |
_ExecutorTypeMatch((futures.Executor,), |
| 558 |
executor.ParallelThreadTaskExecutor), |
| 559 |
] |
| 560 |
|
| 561 |
# One of these should match when a string/text is provided for the |
| 562 |
# 'executor' option (a mixed case equivalent is allowed since the match |
| 563 |
# will be lower-cased before checking). |
| 564 |
_executor_str_matchers = [ |
| 565 |
_ExecutorTextMatch(frozenset(['thread', 'threads', 'threaded']), |
| 566 |
executor.ParallelThreadTaskExecutor), |
| 567 |
_ExecutorTextMatch(frozenset(['greenthread', 'greenthreads', |
| 568 |
'greenthreaded']), |
| 569 |
executor.ParallelGreenThreadTaskExecutor), |
| 570 |
] |
| 571 |
|
| 572 |
# Used when no executor is provided (either a string or object)... |
| 573 |
_default_executor_cls = executor.ParallelThreadTaskExecutor |
| 574 |
|
| 575 |
def __init__(self, flow, flow_detail, backend, options): |
| 576 |
super().__init__(flow, flow_detail, backend, options) |
| 577 |
# This ensures that any provided executor will be validated before |
| 578 |
# we get to far in the compilation/execution pipeline... |
| 579 |
self._task_executor = self._fetch_task_executor(self._options) |
| 580 |
|
| 581 |
@classmethod |
| 582 |
def _fetch_task_executor(cls, options): |
| 583 |
kwargs = {} |
| 584 |
executor_cls = cls._default_executor_cls |
| 585 |
# Match the desired executor to a class that will work with it... |
| 586 |
desired_executor = options.get('executor') |
| 587 |
if isinstance(desired_executor, str): |
| 588 |
matched_executor_cls = None |
| 589 |
for m in cls._executor_str_matchers: |
| 590 |
if m.matches(desired_executor): |
| 591 |
matched_executor_cls = m.executor_cls |
| 592 |
break |
| 593 |
if matched_executor_cls is None: |
| 594 |
expected = set() |
| 595 |
for m in cls._executor_str_matchers: |
| 596 |
expected.update(m.strings) |
| 597 |
raise ValueError("Unknown executor string '%s' expected" |
| 598 |
" one of %s (or mixed case equivalent)" |
| 599 |
% (desired_executor, list(expected))) |
| 600 |
else: |
| 601 |
executor_cls = matched_executor_cls |
| 602 |
elif desired_executor is not None: |
| 603 |
matched_executor_cls = None |
| 604 |
for m in cls._executor_cls_matchers: |
| 605 |
if m.matches(desired_executor): |
| 606 |
matched_executor_cls = m.executor_cls |
| 607 |
break |
| 608 |
if matched_executor_cls is None: |
| 609 |
expected = set() |
| 610 |
for m in cls._executor_cls_matchers: |
| 611 |
expected.update(m.types) |
| 612 |
raise TypeError("Unknown executor '%s' (%s) expected an" |
| 613 |
" instance of %s" % (desired_executor, |
| 614 |
type(desired_executor), |
| 615 |
list(expected))) |
| 616 |
else: |
| 617 |
executor_cls = matched_executor_cls |
| 618 |
kwargs['executor'] = desired_executor |
| 619 |
try: |
| 620 |
for (k, value_converter) in executor_cls.constructor_options: |
| 621 |
try: |
| 622 |
kwargs[k] = value_converter(options[k]) |
| 623 |
except KeyError: |
| 624 |
pass |
| 625 |
except AttributeError: |
| 626 |
pass |
| 627 |
return executor_cls(**kwargs) |