Debian Codegraph @

taskflow/engines/action_engine/engine.py

raw
1 # Copyright (C) 2012 Yahoo! Inc. All Rights Reserved.
2 #
3 # Licensed under the Apache License, Version 2.0 (the "License"); you may
4 # not use this file except in compliance with the License. You may obtain
5 # a copy of the License at
6 #
7 # http://www.apache.org/licenses/LICENSE-2.0
8 #
9 # Unless required by applicable law or agreed to in writing, software
10 # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
11 # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
12 # License for the specific language governing permissions and limitations
13 # under the License.
14
15 import collections
16 import contextlib
17 import itertools
18 import threading
19
20 from automaton import runners
21 from concurrent import futures
22 import fasteners
23 import functools
24 import networkx as nx
25 from oslo_utils import excutils
26 from oslo_utils import strutils
27 from oslo_utils import timeutils
28
29 from taskflow.engines.action_engine import builder
30 from taskflow.engines.action_engine import compiler
31 from taskflow.engines.action_engine import executor
32 from taskflow.engines.action_engine import runtime
33 from taskflow.engines import base
34 from taskflow import exceptions as exc
35 from taskflow import logging
36 from taskflow import states
37 from taskflow import storage
38 from taskflow.types import failure
39 from taskflow.utils import misc
40
41 LOG = logging.getLogger(__name__)
42
43
44 @contextlib.contextmanager
45 def _start_stop(task_executor, retry_executor):
46 # A teenie helper context manager to safely start/stop engine executors...
47 task_executor.start()
48 try:
49 retry_executor.start()
50 try:
51 yield (task_executor, retry_executor)
52 finally:
53 retry_executor.stop()
54 finally:
55 task_executor.stop()
56
57
58 def _pre_check(check_compiled=True, check_storage_ensured=True,
59 check_validated=True):
60 """Engine state precondition checking decorator."""
61
62 def decorator(meth):
63 do_what = meth.__name__
64
65 @functools.wraps(meth)
66 def wrapper(self, *args, **kwargs):
67 if check_compiled and not self._compiled:
68 raise exc.InvalidState("Can not %s an engine which"
69 " has not been compiled" % do_what)
70 if check_storage_ensured and not self._storage_ensured:
71 raise exc.InvalidState("Can not %s an engine"
72 " which has not had its storage"
73 " populated" % do_what)
74 if check_validated and not self._validated:
75 raise exc.InvalidState("Can not %s an engine which"
76 " has not been validated" % do_what)
77 return meth(self, *args, **kwargs)
78
79 return wrapper
80
81 return decorator
82
83
84 class ActionEngine(base.Engine):
85 """Generic action-based engine.
86
87 This engine compiles the flow (and any subflows) into a compilation unit
88 which contains the full runtime definition to be executed and then uses
89 this compilation unit in combination with the executor, runtime, machine
90 builder and storage classes to attempt to run your flow (and any
91 subflows & contained atoms) to completion.
92
93 NOTE(harlowja): during this process it is permissible and valid to have a
94 task or multiple tasks in the execution graph fail (at the same time even),
95 which will cause the process of reversion or retrying to commence. See the
96 valid states in the states module to learn more about what other states
97 the tasks and flow being ran can go through.
98
99 **Engine options:**
100
101 +----------------------+-----------------------+------+------------+
102 | Name/key | Description | Type | Default |
103 +======================+=======================+======+============+
104 | ``defer_reverts`` | This option lets you | bool | ``False`` |
105 | | safely nest flows | | |
106 | | with retries inside | | |
107 | | flows without retries | | |
108 | | and it still behaves | | |
109 | | as a user would | | |
110 | | expect (for example | | |
111 | | if the retry gets | | |
112 | | exhausted it reverts | | |
113 | | the outer flow unless | | |
114 | | the outer flow has a | | |
115 | | has a separate retry | | |
116 | | behavior). | | |
117 +----------------------+-----------------------+------+------------+
118 | ``never_resolve`` | When true, instead | bool | ``False`` |
119 | | of reverting | | |
120 | | and trying to resolve | | |
121 | | a atom failure the | | |
122 | | engine will skip | | |
123 | | reverting and abort | | |
124 | | instead of reverting | | |
125 | | and/or retrying. | | |
126 +----------------------+-----------------------+------+------------+
127 | ``inject_transient`` | When true, values | bool | ``True`` |
128 | | that are local to | | |
129 | | each atoms scope | | |
130 | | are injected into | | |
131 | | storage into a | | |
132 | | transient location | | |
133 | | (typically a local | | |
134 | | dictionary), when | | |
135 | | false those values | | |
136 | | are instead persisted | | |
137 | | into atom details | | |
138 | | (and saved in a non- | | |
139 | | transient manner). | | |
140 +----------------------+-----------------------+------+------------+
141 """
142
143 NO_RERAISING_STATES = frozenset([states.SUSPENDED, states.SUCCESS])
144 """
145 States that if the engine stops in will **not** cause any potential
146 failures to be reraised. States **not** in this list will cause any
147 failure/s that were captured (if any) to get reraised.
148 """
149
150 IGNORABLE_STATES = frozenset(
151 itertools.chain([states.SCHEDULING, states.WAITING, states.RESUMING,
152 states.ANALYZING], builder.META_STATES))
153 """
154 Informational states this engines internal machine yields back while
155 running, not useful to have the engine record but useful to provide to
156 end-users when doing execution iterations via :py:meth:`.run_iter`.
157 """
158
159 MAX_MACHINE_STATES_RETAINED = 10
160 """
161 During :py:meth:`~.run_iter` the last X state machine transitions will
162 be recorded (typically only useful on failure).
163 """
164
165 def __init__(self, flow, flow_detail, backend, options):
166 super().__init__(flow, flow_detail, backend, options)
167 self._runtime = None
168 self._compiled = False
169 self._compilation = None
170 self._compiler = compiler.PatternCompiler(flow)
171 self._lock = threading.RLock()
172 self._storage_ensured = False
173 self._validated = False
174 # Retries are not *currently* executed out of the engines process
175 # or thread (this could change in the future if we desire it to).
176 self._retry_executor = executor.SerialRetryExecutor()
177 self._inject_transient = strutils.bool_from_string(
178 self._options.get('inject_transient', True))
179 self._gather_statistics = strutils.bool_from_string(
180 self._options.get('gather_statistics', True))
181 self._statistics = {}
182
183 @_pre_check(check_compiled=True,
184 # NOTE(harlowja): We can alter the state of the
185 # flow without ensuring its storage is setup for
186 # its atoms (since this state change does not affect
187 # those units).
188 check_storage_ensured=False,
189 check_validated=False)
190 def suspend(self):
191 self._change_state(states.SUSPENDING)
192
193 @property
194 def statistics(self):
195 return self._statistics
196
197 @property
198 def compilation(self):
199 """The compilation result.
200
201 NOTE(harlowja): Only accessible after compilation has completed (None
202 will be returned when this property is accessed before compilation has
203 completed successfully).
204 """
205 if self._compiled:
206 return self._compilation
207 else:
208 return None
209
210 @misc.cachedproperty
211 def storage(self):
212 """The storage unit for this engine.
213
214 NOTE(harlowja): the atom argument lookup strategy will change for
215 this storage unit after
216 :py:func:`~taskflow.engines.base.Engine.compile` has
217 completed (since **only** after compilation is the actual structure
218 known). Before :py:func:`~taskflow.engines.base.Engine.compile`
219 has completed the atom argument lookup strategy lookup will be
220 restricted to injected arguments **only** (this will **not** reflect
221 the actual runtime lookup strategy, which typically will be, but is
222 not always different).
223 """
224 def _scope_fetcher(atom_name):
225 if self._compiled:
226 return self._runtime.fetch_scopes_for(atom_name)
227 else:
228 return None
229 return storage.Storage(self._flow_detail,
230 backend=self._backend,
231 scope_fetcher=_scope_fetcher)
232
233 def run(self, timeout=None):
234 """Runs the engine (or die trying).
235
236 :param timeout: timeout to wait for any atoms to complete (this timeout
237 will be used during the waiting period that occurs when
238 unfinished atoms are being waited on).
239 """
240 with fasteners.try_lock(self._lock) as was_locked:
241 if not was_locked:
242 raise exc.ExecutionFailure("Engine currently locked, please"
243 " try again later")
244 for _state in self.run_iter(timeout=timeout):
245 pass
246
247 def run_iter(self, timeout=None):
248 """Runs the engine using iteration (or die trying).
249
250 :param timeout: timeout to wait for any atoms to complete (this timeout
251 will be used during the waiting period that occurs after the
252 waiting state is yielded when unfinished atoms are being waited
253 on).
254
255 Instead of running to completion in a blocking manner, this will
256 return a generator which will yield back the various states that the
257 engine is going through (and can be used to run multiple engines at
258 once using a generator per engine). The iterator returned also
259 responds to the ``send()`` method from :pep:`0342` and will attempt to
260 suspend itself if a truthy value is sent in (the suspend may be
261 delayed until all active atoms have finished).
262
263 NOTE(harlowja): using the ``run_iter`` method will **not** retain the
264 engine lock while executing so the user should ensure that there is
265 only one entity using a returned engine iterator (one per engine) at a
266 given time.
267 """
268 self.compile()
269 self.prepare()
270 self.validate()
271 # Keep track of the last X state changes, which if a failure happens
272 # are quite useful to log (and the performance of tracking this
273 # should be negligible).
274 last_transitions = collections.deque(
275 maxlen=max(1, self.MAX_MACHINE_STATES_RETAINED))
276 with _start_stop(self._task_executor, self._retry_executor):
277 self._change_state(states.RUNNING)
278 if self._gather_statistics:
279 self._statistics.clear()
280 w = timeutils.StopWatch()
281 w.start()
282 else:
283 w = None
284 try:
285 closed = False
286 machine, memory = self._runtime.builder.build(
287 self._statistics, timeout=timeout,
288 gather_statistics=self._gather_statistics)
289 r = runners.FiniteRunner(machine)
290 for transition in r.run_iter(builder.START):
291 last_transitions.append(transition)
292 _prior_state, new_state = transition
293 # NOTE(harlowja): skip over meta-states
294 if new_state in builder.META_STATES:
295 continue
296 if new_state == states.FAILURE:
297 failure.Failure.reraise_if_any(memory.failures)
298 if closed:
299 continue
300 try:
301 try_suspend = yield new_state
302 except GeneratorExit:
303 # The generator was closed, attempt to suspend and
304 # continue looping until we have cleanly closed up
305 # shop...
306 closed = True
307 self.suspend()
308 except Exception:
309 # Capture the failure, and ensure that the
310 # machine will notice that something externally
311 # has sent an exception in and that it should
312 # finish up and reraise.
313 memory.failures.append(failure.Failure())
314 closed = True
315 else:
316 if try_suspend:
317 self.suspend()
318 except Exception:
319 with excutils.save_and_reraise_exception():
320 LOG.exception("Engine execution has failed, something"
321 " bad must have happened (last"
322 " %s machine transitions were %s)",
323 last_transitions.maxlen,
324 list(last_transitions))
325 self._change_state(states.FAILURE)
326 else:
327 if last_transitions:
328 _prior_state, new_state = last_transitions[-1]
329 if new_state not in self.IGNORABLE_STATES:
330 self._change_state(new_state)
331 if new_state not in self.NO_RERAISING_STATES:
332 e_failures = self.storage.get_execute_failures()
333 r_failures = self.storage.get_revert_failures()
334 er_failures = itertools.chain(
335 e_failures.values(),
336 r_failures.values())
337 failure.Failure.reraise_if_any(er_failures)
338 finally:
339 if w is not None:
340 w.stop()
341 self._statistics['active_for'] = w.elapsed()
342
343 @staticmethod
344 def _check_compilation(compilation):
345 """Performs post compilation validation/checks."""
346 seen = set()
347 dups = set()
348 execution_graph = compilation.execution_graph
349 for node, node_attrs in execution_graph.nodes(data=True):
350 if node_attrs['kind'] in compiler.ATOMS:
351 atom_name = node.name
352 if atom_name in seen:
353 dups.add(atom_name)
354 else:
355 seen.add(atom_name)
356 if dups:
357 raise exc.Duplicate(
358 "Atoms with duplicate names found: %s" % (sorted(dups)))
359 return compilation
360
361 def _change_state(self, state):
362 moved, old_state = self.storage.change_flow_state(state)
363 if moved:
364 details = {
365 'engine': self,
366 'flow_name': self.storage.flow_name,
367 'flow_uuid': self.storage.flow_uuid,
368 'old_state': old_state,
369 }
370 self.notifier.notify(state, details)
371
372 def _ensure_storage(self):
373 """Ensure all contained atoms exist in the storage unit."""
374 self.storage.ensure_atoms(
375 self._runtime.iterate_nodes(compiler.ATOMS))
376 for atom in self._runtime.iterate_nodes(compiler.ATOMS):
377 if atom.inject:
378 self.storage.inject_atom_args(atom.name, atom.inject,
379 transient=self._inject_transient)
380
381 @fasteners.locked
382 @_pre_check(check_validated=False)
383 def validate(self):
384 # At this point we can check to ensure all dependencies are either
385 # flow/task provided or storage provided, if there are still missing
386 # dependencies then this flow will fail at runtime (which we can avoid
387 # by failing at validation time).
388 if LOG.isEnabledFor(logging.TRACE):
389 execution_graph = self._compilation.execution_graph
390 LOG.trace("Validating scoping and argument visibility for"
391 " execution graph with %s nodes and %s edges with"
392 " density %0.3f", execution_graph.number_of_nodes(),
393 execution_graph.number_of_edges(),
394 nx.density(execution_graph))
395 missing = set()
396 # Attempt to retain a chain of what was missing (so that the final
397 # raised exception for the flow has the nodes that had missing
398 # dependencies).
399 last_cause = None
400 last_node = None
401 missing_nodes = 0
402 for atom in self._runtime.iterate_nodes(compiler.ATOMS):
403 exec_missing = self.storage.fetch_unsatisfied_args(
404 atom.name, atom.rebind, optional_args=atom.optional)
405 revert_missing = self.storage.fetch_unsatisfied_args(
406 atom.name, atom.revert_rebind,
407 optional_args=atom.revert_optional)
408 atom_missing = (('execute', exec_missing),
409 ('revert', revert_missing))
410 for method, method_missing in atom_missing:
411 if method_missing:
412 cause = exc.MissingDependencies(atom,
413 sorted(method_missing),
414 cause=last_cause,
415 method=method)
416 last_cause = cause
417 last_node = atom
418 missing_nodes += 1
419 missing.update(method_missing)
420 if missing:
421 # For when a task is provided (instead of a flow) and that
422 # task is the only item in the graph and its missing deps, avoid
423 # re-wrapping it in yet another exception...
424 if missing_nodes == 1 and last_node is self._flow:
425 raise last_cause
426 else:
427 raise exc.MissingDependencies(self._flow,
428 sorted(missing),
429 cause=last_cause)
430 self._validated = True
431
432 @fasteners.locked
433 @_pre_check(check_storage_ensured=False, check_validated=False)
434 def prepare(self):
435 if not self._storage_ensured:
436 # Set our own state to resuming -> (ensure atoms exist
437 # in storage) -> suspended in the storage unit and notify any
438 # attached listeners of these changes.
439 self._change_state(states.RESUMING)
440 self._ensure_storage()
441 self._change_state(states.SUSPENDED)
442 self._storage_ensured = True
443 # Reset everything back to pending (if we were previously reverted).
444 if self.storage.get_flow_state() == states.REVERTED:
445 self.reset()
446
447 @fasteners.locked
448 @_pre_check(check_validated=False)
449 def reset(self):
450 # This transitions *all* contained atoms back into the PENDING state
451 # with an intention to EXECUTE (or dies trying to do that) and then
452 # changes the state of the flow to PENDING so that it can then run...
453 self._runtime.reset_all()
454 self._change_state(states.PENDING)
455
456 @fasteners.locked
457 def compile(self):
458 if self._compiled:
459 return
460 self._compilation = self._check_compilation(self._compiler.compile())
461 self._runtime = runtime.Runtime(self._compilation,
462 self.storage,
463 self.atom_notifier,
464 self._task_executor,
465 self._retry_executor,
466 options=self._options)
467 self._runtime.compile()
468 self._compiled = True
469
470
471 class SerialActionEngine(ActionEngine):
472 """Engine that runs tasks in serial manner."""
473
474 def __init__(self, flow, flow_detail, backend, options):
475 super().__init__(flow, flow_detail, backend, options)
476 self._task_executor = executor.SerialTaskExecutor()
477
478
479 class _ExecutorTypeMatch(collections.namedtuple('_ExecutorTypeMatch',
480 ['types', 'executor_cls'])):
481 def matches(self, executor):
482 return isinstance(executor, self.types)
483
484
485 class _ExecutorTextMatch(collections.namedtuple('_ExecutorTextMatch',
486 ['strings', 'executor_cls'])):
487 def matches(self, text):
488 return text.lower() in self.strings
489
490
491 class ParallelActionEngine(ActionEngine):
492 """Engine that runs tasks in parallel manner.
493
494 **Additional engine options:**
495
496 * ``executor``: a object that implements a :pep:`3148` compatible executor
497 interface; it will be used for scheduling tasks. The following
498 type are applicable (other unknown types passed will cause a type
499 error to be raised).
500
501 ========================= ===============================================
502 Type provided Executor used
503 ========================= ===============================================
504 |cft|.ThreadPoolExecutor :class:`~.executor.ParallelThreadTaskExecutor`
505 |cfp|.ProcessPoolExecutor :class:`~.|pe|.ParallelProcessTaskExecutor`
506 |cf|._base.Executor :class:`~.executor.ParallelThreadTaskExecutor`
507 ========================= ===============================================
508
509 * ``executor``: a string that will be used to select a :pep:`3148`
510 compatible executor; it will be used for scheduling tasks. The following
511 string are applicable (other unknown strings passed will cause a value
512 error to be raised).
513
514 =========================== ===============================================
515 String (case insensitive) Executor used
516 =========================== ===============================================
517 ``process`` :class:`~.|pe|.ParallelProcessTaskExecutor`
518 ``processes`` :class:`~.|pe|.ParallelProcessTaskExecutor`
519 ``thread`` :class:`~.executor.ParallelThreadTaskExecutor`
520 ``threaded`` :class:`~.executor.ParallelThreadTaskExecutor`
521 ``threads`` :class:`~.executor.ParallelThreadTaskExecutor`
522 ``greenthread`` :class:`~.executor.ParallelThreadTaskExecutor`
523 (greened version)
524 ``greedthreaded`` :class:`~.executor.ParallelThreadTaskExecutor`
525 (greened version)
526 ``greenthreads`` :class:`~.executor.ParallelThreadTaskExecutor`
527 (greened version)
528 =========================== ===============================================
529
530 * ``max_workers``: a integer that will affect the number of parallel
531 workers that are used to dispatch tasks into (this number is bounded
532 by the maximum parallelization your workflow can support).
533
534 * ``wait_timeout``: a float (in seconds) that will affect the
535 parallel process task executor (and therefore is **only** applicable when
536 the executor provided above is of the process variant). This number
537 affects how much time the process task executor waits for messages from
538 child processes (typically indicating they have finished or failed). A
539 lower number will have high granularity but *currently* involves more
540 polling while a higher number will involve less polling but a slower time
541 for an engine to notice a task has completed.
542
543 .. |cfp| replace:: concurrent.futures.process
544 .. |cft| replace:: concurrent.futures.thread
545 .. |cf| replace:: concurrent.futures
546 """
547
548 # One of these types should match when a object (non-string) is provided
549 # for the 'executor' option.
550 #
551 # NOTE(harlowja): the reason we use the library/built-in futures is to
552 # allow for instances of that to be detected and handled correctly, instead
553 # of forcing everyone to use our derivatives (futurist or other)...
554 _executor_cls_matchers = [
555 _ExecutorTypeMatch((futures.ThreadPoolExecutor,),
556 executor.ParallelThreadTaskExecutor),
557 _ExecutorTypeMatch((futures.Executor,),
558 executor.ParallelThreadTaskExecutor),
559 ]
560
561 # One of these should match when a string/text is provided for the
562 # 'executor' option (a mixed case equivalent is allowed since the match
563 # will be lower-cased before checking).
564 _executor_str_matchers = [
565 _ExecutorTextMatch(frozenset(['thread', 'threads', 'threaded']),
566 executor.ParallelThreadTaskExecutor),
567 _ExecutorTextMatch(frozenset(['greenthread', 'greenthreads',
568 'greenthreaded']),
569 executor.ParallelGreenThreadTaskExecutor),
570 ]
571
572 # Used when no executor is provided (either a string or object)...
573 _default_executor_cls = executor.ParallelThreadTaskExecutor
574
575 def __init__(self, flow, flow_detail, backend, options):
576 super().__init__(flow, flow_detail, backend, options)
577 # This ensures that any provided executor will be validated before
578 # we get to far in the compilation/execution pipeline...
579 self._task_executor = self._fetch_task_executor(self._options)
580
581 @classmethod
582 def _fetch_task_executor(cls, options):
583 kwargs = {}
584 executor_cls = cls._default_executor_cls
585 # Match the desired executor to a class that will work with it...
586 desired_executor = options.get('executor')
587 if isinstance(desired_executor, str):
588 matched_executor_cls = None
589 for m in cls._executor_str_matchers:
590 if m.matches(desired_executor):
591 matched_executor_cls = m.executor_cls
592 break
593 if matched_executor_cls is None:
594 expected = set()
595 for m in cls._executor_str_matchers:
596 expected.update(m.strings)
597 raise ValueError("Unknown executor string '%s' expected"
598 " one of %s (or mixed case equivalent)"
599 % (desired_executor, list(expected)))
600 else:
601 executor_cls = matched_executor_cls
602 elif desired_executor is not None:
603 matched_executor_cls = None
604 for m in cls._executor_cls_matchers:
605 if m.matches(desired_executor):
606 matched_executor_cls = m.executor_cls
607 break
608 if matched_executor_cls is None:
609 expected = set()
610 for m in cls._executor_cls_matchers:
611 expected.update(m.types)
612 raise TypeError("Unknown executor '%s' (%s) expected an"
613 " instance of %s" % (desired_executor,
614 type(desired_executor),
615 list(expected)))
616 else:
617 executor_cls = matched_executor_cls
618 kwargs['executor'] = desired_executor
619 try:
620 for (k, value_converter) in executor_cls.constructor_options:
621 try:
622 kwargs[k] = value_converter(options[k])
623 except KeyError:
624 pass
625 except AttributeError:
626 pass
627 return executor_cls(**kwargs)