| 1 |
# Copyright (C) 2014 Yahoo! Inc. All Rights Reserved. |
| 2 |
# |
| 3 |
# Licensed under the Apache License, Version 2.0 (the "License"); you may |
| 4 |
# not use this file except in compliance with the License. You may obtain |
| 5 |
# a copy of the License at |
| 6 |
# |
| 7 |
# http://www.apache.org/licenses/LICENSE-2.0 |
| 8 |
# |
| 9 |
# Unless required by applicable law or agreed to in writing, software |
| 10 |
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT |
| 11 |
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the |
| 12 |
# License for the specific language governing permissions and limitations |
| 13 |
# under the License. |
| 14 |
|
| 15 |
import collections |
| 16 |
import functools |
| 17 |
|
| 18 |
from futurist import waiters |
| 19 |
|
| 20 |
from taskflow import deciders as de |
| 21 |
from taskflow.engines.action_engine.actions import retry as ra |
| 22 |
from taskflow.engines.action_engine.actions import task as ta |
| 23 |
from taskflow.engines.action_engine import builder as bu |
| 24 |
from taskflow.engines.action_engine import compiler as com |
| 25 |
from taskflow.engines.action_engine import completer as co |
| 26 |
from taskflow.engines.action_engine import scheduler as sched |
| 27 |
from taskflow.engines.action_engine import scopes as sc |
| 28 |
from taskflow.engines.action_engine import selector as se |
| 29 |
from taskflow.engines.action_engine import traversal as tr |
| 30 |
from taskflow import exceptions as exc |
| 31 |
from taskflow import logging |
| 32 |
from taskflow import states as st |
| 33 |
from taskflow.utils import misc |
| 34 |
|
| 35 |
from taskflow.flow import (LINK_DECIDER, LINK_DECIDER_DEPTH) # noqa |
| 36 |
|
| 37 |
# Small helper to make the edge decider tuples more easily useable... |
| 38 |
_EdgeDecider = collections.namedtuple('_EdgeDecider', |
| 39 |
'from_node,kind,decider,depth') |
| 40 |
|
| 41 |
LOG = logging.getLogger(__name__) |
| 42 |
|
| 43 |
|
| 44 |
class Runtime: |
| 45 |
"""A aggregate of runtime objects, properties, ... used during execution. |
| 46 |
|
| 47 |
This object contains various utility methods and properties that represent |
| 48 |
the collection of runtime components and functionality needed for an |
| 49 |
action engine to run to completion. |
| 50 |
""" |
| 51 |
|
| 52 |
def __init__(self, compilation, storage, atom_notifier, |
| 53 |
task_executor, retry_executor, |
| 54 |
options=None): |
| 55 |
self._atom_notifier = atom_notifier |
| 56 |
self._task_executor = task_executor |
| 57 |
self._retry_executor = retry_executor |
| 58 |
self._storage = storage |
| 59 |
self._compilation = compilation |
| 60 |
self._atom_cache = {} |
| 61 |
self._options = misc.safe_copy_dict(options) |
| 62 |
|
| 63 |
def _walk_edge_deciders(self, graph, atom): |
| 64 |
"""Iterates through all nodes, deciders that alter atoms execution.""" |
| 65 |
# This is basically a reverse breadth first exploration, with |
| 66 |
# special logic to further traverse down flow nodes as needed... |
| 67 |
predecessors_iter = graph.predecessors |
| 68 |
nodes = collections.deque((u_node, atom) |
| 69 |
for u_node in predecessors_iter(atom)) |
| 70 |
visited = set() |
| 71 |
while nodes: |
| 72 |
u_node, v_node = nodes.popleft() |
| 73 |
u_node_kind = graph.nodes[u_node]['kind'] |
| 74 |
u_v_data = graph.adj[u_node][v_node] |
| 75 |
try: |
| 76 |
decider = u_v_data[LINK_DECIDER] |
| 77 |
decider_depth = u_v_data.get(LINK_DECIDER_DEPTH) |
| 78 |
if decider_depth is None: |
| 79 |
decider_depth = de.Depth.ALL |
| 80 |
yield _EdgeDecider(u_node, u_node_kind, |
| 81 |
decider, decider_depth) |
| 82 |
except KeyError: |
| 83 |
pass |
| 84 |
if u_node_kind == com.FLOW and u_node not in visited: |
| 85 |
# Avoid re-exploring the same flow if we get to this same |
| 86 |
# flow by a different *future* path... |
| 87 |
visited.add(u_node) |
| 88 |
# Since we *currently* jump over flow node(s), we need to make |
| 89 |
# sure that any prior decider that was directed at this flow |
| 90 |
# node also gets used during future decisions about this |
| 91 |
# atom node. |
| 92 |
nodes.extend((u_u_node, u_node) |
| 93 |
for u_u_node in predecessors_iter(u_node)) |
| 94 |
|
| 95 |
def compile(self): |
| 96 |
"""Compiles & caches frequently used execution helper objects. |
| 97 |
|
| 98 |
Build out a cache of commonly used item that are associated |
| 99 |
with the contained atoms (by name), and are useful to have for |
| 100 |
quick lookup on (for example, the change state handler function for |
| 101 |
each atom, the scope walker object for each atom, the task or retry |
| 102 |
specific scheduler and so-on). |
| 103 |
""" |
| 104 |
change_state_handlers = { |
| 105 |
com.TASK: functools.partial(self.task_action.change_state, |
| 106 |
progress=0.0), |
| 107 |
com.RETRY: self.retry_action.change_state, |
| 108 |
} |
| 109 |
schedulers = { |
| 110 |
com.RETRY: self.retry_scheduler, |
| 111 |
com.TASK: self.task_scheduler, |
| 112 |
} |
| 113 |
check_transition_handlers = { |
| 114 |
com.TASK: st.check_task_transition, |
| 115 |
com.RETRY: st.check_retry_transition, |
| 116 |
} |
| 117 |
actions = { |
| 118 |
com.TASK: self.task_action, |
| 119 |
com.RETRY: self.retry_action, |
| 120 |
} |
| 121 |
graph = self._compilation.execution_graph |
| 122 |
for node, node_data in graph.nodes(data=True): |
| 123 |
node_kind = node_data['kind'] |
| 124 |
if node_kind in com.FLOWS: |
| 125 |
continue |
| 126 |
elif node_kind in com.ATOMS: |
| 127 |
check_transition_handler = check_transition_handlers[node_kind] |
| 128 |
change_state_handler = change_state_handlers[node_kind] |
| 129 |
scheduler = schedulers[node_kind] |
| 130 |
action = actions[node_kind] |
| 131 |
else: |
| 132 |
raise exc.CompilationFailure("Unknown node kind '%s'" |
| 133 |
" encountered" % node_kind) |
| 134 |
metadata = {} |
| 135 |
deciders_it = self._walk_edge_deciders(graph, node) |
| 136 |
walker = sc.ScopeWalker(self.compilation, node, names_only=True) |
| 137 |
metadata['scope_walker'] = walker |
| 138 |
metadata['check_transition_handler'] = check_transition_handler |
| 139 |
metadata['change_state_handler'] = change_state_handler |
| 140 |
metadata['scheduler'] = scheduler |
| 141 |
metadata['edge_deciders'] = tuple(deciders_it) |
| 142 |
metadata['action'] = action |
| 143 |
LOG.trace("Compiled %s metadata for node %s (%s)", |
| 144 |
metadata, node.name, node_kind) |
| 145 |
self._atom_cache[node.name] = metadata |
| 146 |
# TODO(harlowja): optimize the different decider depths to avoid |
| 147 |
# repeated full successor searching; this can be done by searching |
| 148 |
# for the widest depth of parent(s), and limiting the search of |
| 149 |
# children by the that depth. |
| 150 |
|
| 151 |
@property |
| 152 |
def compilation(self): |
| 153 |
return self._compilation |
| 154 |
|
| 155 |
@property |
| 156 |
def storage(self): |
| 157 |
return self._storage |
| 158 |
|
| 159 |
@property |
| 160 |
def options(self): |
| 161 |
return self._options |
| 162 |
|
| 163 |
@misc.cachedproperty |
| 164 |
def selector(self): |
| 165 |
return se.Selector(self) |
| 166 |
|
| 167 |
@misc.cachedproperty |
| 168 |
def builder(self): |
| 169 |
return bu.MachineBuilder(self, waiters.wait_for_any) |
| 170 |
|
| 171 |
@misc.cachedproperty |
| 172 |
def completer(self): |
| 173 |
return co.Completer(self) |
| 174 |
|
| 175 |
@misc.cachedproperty |
| 176 |
def scheduler(self): |
| 177 |
return sched.Scheduler(self) |
| 178 |
|
| 179 |
@misc.cachedproperty |
| 180 |
def task_scheduler(self): |
| 181 |
return sched.TaskScheduler(self) |
| 182 |
|
| 183 |
@misc.cachedproperty |
| 184 |
def retry_scheduler(self): |
| 185 |
return sched.RetryScheduler(self) |
| 186 |
|
| 187 |
@misc.cachedproperty |
| 188 |
def retry_action(self): |
| 189 |
return ra.RetryAction(self._storage, |
| 190 |
self._atom_notifier, |
| 191 |
self._retry_executor) |
| 192 |
|
| 193 |
@misc.cachedproperty |
| 194 |
def task_action(self): |
| 195 |
return ta.TaskAction(self._storage, |
| 196 |
self._atom_notifier, |
| 197 |
self._task_executor) |
| 198 |
|
| 199 |
def _fetch_atom_metadata_entry(self, atom_name, metadata_key): |
| 200 |
return self._atom_cache[atom_name][metadata_key] |
| 201 |
|
| 202 |
def check_atom_transition(self, atom, current_state, target_state): |
| 203 |
"""Checks if the atom can transition to the provided target state.""" |
| 204 |
# This does not check if the name exists (since this is only used |
| 205 |
# internally to the engine, and is not exposed to atoms that will |
| 206 |
# not exist and therefore doesn't need to handle that case). |
| 207 |
check_transition_handler = self._fetch_atom_metadata_entry( |
| 208 |
atom.name, 'check_transition_handler') |
| 209 |
return check_transition_handler(current_state, target_state) |
| 210 |
|
| 211 |
def fetch_edge_deciders(self, atom): |
| 212 |
"""Fetches the edge deciders for the given atom.""" |
| 213 |
# This does not check if the name exists (since this is only used |
| 214 |
# internally to the engine, and is not exposed to atoms that will |
| 215 |
# not exist and therefore doesn't need to handle that case). |
| 216 |
return self._fetch_atom_metadata_entry(atom.name, 'edge_deciders') |
| 217 |
|
| 218 |
def fetch_scheduler(self, atom): |
| 219 |
"""Fetches the cached specific scheduler for the given atom.""" |
| 220 |
# This does not check if the name exists (since this is only used |
| 221 |
# internally to the engine, and is not exposed to atoms that will |
| 222 |
# not exist and therefore doesn't need to handle that case). |
| 223 |
return self._fetch_atom_metadata_entry(atom.name, 'scheduler') |
| 224 |
|
| 225 |
def fetch_action(self, atom): |
| 226 |
"""Fetches the cached action handler for the given atom.""" |
| 227 |
metadata = self._atom_cache[atom.name] |
| 228 |
return metadata['action'] |
| 229 |
|
| 230 |
def fetch_scopes_for(self, atom_name): |
| 231 |
"""Fetches a walker of the visible scopes for the given atom.""" |
| 232 |
try: |
| 233 |
return self._fetch_atom_metadata_entry(atom_name, 'scope_walker') |
| 234 |
except KeyError: |
| 235 |
# This signals to the caller that there is no walker for whatever |
| 236 |
# atom name was given that doesn't really have any associated atom |
| 237 |
# known to be named with that name; this is done since the storage |
| 238 |
# layer will call into this layer to fetch a scope for a named |
| 239 |
# atom and users can provide random names that do not actually |
| 240 |
# exist... |
| 241 |
return None |
| 242 |
|
| 243 |
# Various helper methods used by the runtime components; not for public |
| 244 |
# consumption... |
| 245 |
|
| 246 |
def iterate_retries(self, state=None): |
| 247 |
"""Iterates retry atoms that match the provided state. |
| 248 |
|
| 249 |
If no state is provided it will yield back all retry atoms. |
| 250 |
""" |
| 251 |
if state: |
| 252 |
atoms = list(self.iterate_nodes((com.RETRY,))) |
| 253 |
atom_states = self._storage.get_atoms_states(atom.name |
| 254 |
for atom in atoms) |
| 255 |
for atom in atoms: |
| 256 |
atom_state, _atom_intention = atom_states[atom.name] |
| 257 |
if atom_state == state: |
| 258 |
yield atom |
| 259 |
else: |
| 260 |
for atom in self.iterate_nodes((com.RETRY,)): |
| 261 |
yield atom |
| 262 |
|
| 263 |
def iterate_nodes(self, allowed_kinds): |
| 264 |
"""Yields back all nodes of specified kinds in the execution graph.""" |
| 265 |
graph = self._compilation.execution_graph |
| 266 |
for node, node_data in graph.nodes(data=True): |
| 267 |
if node_data['kind'] in allowed_kinds: |
| 268 |
yield node |
| 269 |
|
| 270 |
def is_success(self): |
| 271 |
"""Checks if all atoms in the execution graph are in 'happy' state.""" |
| 272 |
atoms = list(self.iterate_nodes(com.ATOMS)) |
| 273 |
atom_states = self._storage.get_atoms_states(atom.name |
| 274 |
for atom in atoms) |
| 275 |
for atom in atoms: |
| 276 |
atom_state, _atom_intention = atom_states[atom.name] |
| 277 |
if atom_state == st.IGNORE: |
| 278 |
continue |
| 279 |
if atom_state != st.SUCCESS: |
| 280 |
return False |
| 281 |
return True |
| 282 |
|
| 283 |
def find_retry(self, node): |
| 284 |
"""Returns the retry atom associated to the given node (or none).""" |
| 285 |
graph = self._compilation.execution_graph |
| 286 |
return graph.nodes[node].get(com.RETRY) |
| 287 |
|
| 288 |
def reset_atoms(self, atoms, state=st.PENDING, intention=st.EXECUTE): |
| 289 |
"""Resets all the provided atoms to the given state and intention.""" |
| 290 |
tweaked = [] |
| 291 |
for atom in atoms: |
| 292 |
cur_intention = self.storage.get_atom_intention(atom.name) |
| 293 |
# Don't trigger a RETRY if the atom needs to be REVERTED. |
| 294 |
# This is a workaround for a bug when REVERT_ALL is applied to |
| 295 |
# unordered flows |
| 296 |
# (https://bugs.launchpad.net/taskflow/+bug/2043808) |
| 297 |
# A subflow may trigger a REVERT_ALL, all the atoms of all the |
| 298 |
# related subflows are marked as REVERT but a task of a related |
| 299 |
# flow may still be running in another thread. If this task |
| 300 |
# triggers a RETRY, it overrides the previously set REVERT status, |
| 301 |
# breaking the revert path of the flow. |
| 302 |
if cur_intention == st.REVERT and intention == st.RETRY: |
| 303 |
continue |
| 304 |
if state or intention: |
| 305 |
tweaked.append((atom, state, intention)) |
| 306 |
if state: |
| 307 |
change_state_handler = self._fetch_atom_metadata_entry( |
| 308 |
atom.name, 'change_state_handler') |
| 309 |
change_state_handler(atom, state) |
| 310 |
if intention: |
| 311 |
self.storage.set_atom_intention(atom.name, intention) |
| 312 |
return tweaked |
| 313 |
|
| 314 |
def reset_all(self, state=st.PENDING, intention=st.EXECUTE): |
| 315 |
"""Resets all atoms to the given state and intention.""" |
| 316 |
return self.reset_atoms(self.iterate_nodes(com.ATOMS), |
| 317 |
state=state, intention=intention) |
| 318 |
|
| 319 |
def reset_subgraph(self, atom, state=st.PENDING, intention=st.EXECUTE): |
| 320 |
"""Resets a atoms subgraph to the given state and intention. |
| 321 |
|
| 322 |
The subgraph is contained of **all** of the atoms successors. |
| 323 |
""" |
| 324 |
execution_graph = self._compilation.execution_graph |
| 325 |
atoms_it = tr.depth_first_iterate(execution_graph, atom, |
| 326 |
tr.Direction.FORWARD) |
| 327 |
return self.reset_atoms(atoms_it, state=state, intention=intention) |
| 328 |
|
| 329 |
def retry_subflow(self, retry): |
| 330 |
"""Prepares a retrys + its subgraph for execution. |
| 331 |
|
| 332 |
This sets the retrys intention to ``EXECUTE`` and resets all of its |
| 333 |
subgraph (its successors) to the ``PENDING`` state with an ``EXECUTE`` |
| 334 |
intention. |
| 335 |
""" |
| 336 |
tweaked = self.reset_atoms([retry], state=None, intention=st.EXECUTE) |
| 337 |
tweaked.extend(self.reset_subgraph(retry)) |
| 338 |
return tweaked |