Debian Codegraph @

taskflow/engines/action_engine/runtime.py

raw
1 # Copyright (C) 2014 Yahoo! Inc. All Rights Reserved.
2 #
3 # Licensed under the Apache License, Version 2.0 (the "License"); you may
4 # not use this file except in compliance with the License. You may obtain
5 # a copy of the License at
6 #
7 # http://www.apache.org/licenses/LICENSE-2.0
8 #
9 # Unless required by applicable law or agreed to in writing, software
10 # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
11 # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
12 # License for the specific language governing permissions and limitations
13 # under the License.
14
15 import collections
16 import functools
17
18 from futurist import waiters
19
20 from taskflow import deciders as de
21 from taskflow.engines.action_engine.actions import retry as ra
22 from taskflow.engines.action_engine.actions import task as ta
23 from taskflow.engines.action_engine import builder as bu
24 from taskflow.engines.action_engine import compiler as com
25 from taskflow.engines.action_engine import completer as co
26 from taskflow.engines.action_engine import scheduler as sched
27 from taskflow.engines.action_engine import scopes as sc
28 from taskflow.engines.action_engine import selector as se
29 from taskflow.engines.action_engine import traversal as tr
30 from taskflow import exceptions as exc
31 from taskflow import logging
32 from taskflow import states as st
33 from taskflow.utils import misc
34
35 from taskflow.flow import (LINK_DECIDER, LINK_DECIDER_DEPTH) # noqa
36
37 # Small helper to make the edge decider tuples more easily useable...
38 _EdgeDecider = collections.namedtuple('_EdgeDecider',
39 'from_node,kind,decider,depth')
40
41 LOG = logging.getLogger(__name__)
42
43
44 class Runtime:
45 """A aggregate of runtime objects, properties, ... used during execution.
46
47 This object contains various utility methods and properties that represent
48 the collection of runtime components and functionality needed for an
49 action engine to run to completion.
50 """
51
52 def __init__(self, compilation, storage, atom_notifier,
53 task_executor, retry_executor,
54 options=None):
55 self._atom_notifier = atom_notifier
56 self._task_executor = task_executor
57 self._retry_executor = retry_executor
58 self._storage = storage
59 self._compilation = compilation
60 self._atom_cache = {}
61 self._options = misc.safe_copy_dict(options)
62
63 def _walk_edge_deciders(self, graph, atom):
64 """Iterates through all nodes, deciders that alter atoms execution."""
65 # This is basically a reverse breadth first exploration, with
66 # special logic to further traverse down flow nodes as needed...
67 predecessors_iter = graph.predecessors
68 nodes = collections.deque((u_node, atom)
69 for u_node in predecessors_iter(atom))
70 visited = set()
71 while nodes:
72 u_node, v_node = nodes.popleft()
73 u_node_kind = graph.nodes[u_node]['kind']
74 u_v_data = graph.adj[u_node][v_node]
75 try:
76 decider = u_v_data[LINK_DECIDER]
77 decider_depth = u_v_data.get(LINK_DECIDER_DEPTH)
78 if decider_depth is None:
79 decider_depth = de.Depth.ALL
80 yield _EdgeDecider(u_node, u_node_kind,
81 decider, decider_depth)
82 except KeyError:
83 pass
84 if u_node_kind == com.FLOW and u_node not in visited:
85 # Avoid re-exploring the same flow if we get to this same
86 # flow by a different *future* path...
87 visited.add(u_node)
88 # Since we *currently* jump over flow node(s), we need to make
89 # sure that any prior decider that was directed at this flow
90 # node also gets used during future decisions about this
91 # atom node.
92 nodes.extend((u_u_node, u_node)
93 for u_u_node in predecessors_iter(u_node))
94
95 def compile(self):
96 """Compiles & caches frequently used execution helper objects.
97
98 Build out a cache of commonly used item that are associated
99 with the contained atoms (by name), and are useful to have for
100 quick lookup on (for example, the change state handler function for
101 each atom, the scope walker object for each atom, the task or retry
102 specific scheduler and so-on).
103 """
104 change_state_handlers = {
105 com.TASK: functools.partial(self.task_action.change_state,
106 progress=0.0),
107 com.RETRY: self.retry_action.change_state,
108 }
109 schedulers = {
110 com.RETRY: self.retry_scheduler,
111 com.TASK: self.task_scheduler,
112 }
113 check_transition_handlers = {
114 com.TASK: st.check_task_transition,
115 com.RETRY: st.check_retry_transition,
116 }
117 actions = {
118 com.TASK: self.task_action,
119 com.RETRY: self.retry_action,
120 }
121 graph = self._compilation.execution_graph
122 for node, node_data in graph.nodes(data=True):
123 node_kind = node_data['kind']
124 if node_kind in com.FLOWS:
125 continue
126 elif node_kind in com.ATOMS:
127 check_transition_handler = check_transition_handlers[node_kind]
128 change_state_handler = change_state_handlers[node_kind]
129 scheduler = schedulers[node_kind]
130 action = actions[node_kind]
131 else:
132 raise exc.CompilationFailure("Unknown node kind '%s'"
133 " encountered" % node_kind)
134 metadata = {}
135 deciders_it = self._walk_edge_deciders(graph, node)
136 walker = sc.ScopeWalker(self.compilation, node, names_only=True)
137 metadata['scope_walker'] = walker
138 metadata['check_transition_handler'] = check_transition_handler
139 metadata['change_state_handler'] = change_state_handler
140 metadata['scheduler'] = scheduler
141 metadata['edge_deciders'] = tuple(deciders_it)
142 metadata['action'] = action
143 LOG.trace("Compiled %s metadata for node %s (%s)",
144 metadata, node.name, node_kind)
145 self._atom_cache[node.name] = metadata
146 # TODO(harlowja): optimize the different decider depths to avoid
147 # repeated full successor searching; this can be done by searching
148 # for the widest depth of parent(s), and limiting the search of
149 # children by the that depth.
150
151 @property
152 def compilation(self):
153 return self._compilation
154
155 @property
156 def storage(self):
157 return self._storage
158
159 @property
160 def options(self):
161 return self._options
162
163 @misc.cachedproperty
164 def selector(self):
165 return se.Selector(self)
166
167 @misc.cachedproperty
168 def builder(self):
169 return bu.MachineBuilder(self, waiters.wait_for_any)
170
171 @misc.cachedproperty
172 def completer(self):
173 return co.Completer(self)
174
175 @misc.cachedproperty
176 def scheduler(self):
177 return sched.Scheduler(self)
178
179 @misc.cachedproperty
180 def task_scheduler(self):
181 return sched.TaskScheduler(self)
182
183 @misc.cachedproperty
184 def retry_scheduler(self):
185 return sched.RetryScheduler(self)
186
187 @misc.cachedproperty
188 def retry_action(self):
189 return ra.RetryAction(self._storage,
190 self._atom_notifier,
191 self._retry_executor)
192
193 @misc.cachedproperty
194 def task_action(self):
195 return ta.TaskAction(self._storage,
196 self._atom_notifier,
197 self._task_executor)
198
199 def _fetch_atom_metadata_entry(self, atom_name, metadata_key):
200 return self._atom_cache[atom_name][metadata_key]
201
202 def check_atom_transition(self, atom, current_state, target_state):
203 """Checks if the atom can transition to the provided target state."""
204 # This does not check if the name exists (since this is only used
205 # internally to the engine, and is not exposed to atoms that will
206 # not exist and therefore doesn't need to handle that case).
207 check_transition_handler = self._fetch_atom_metadata_entry(
208 atom.name, 'check_transition_handler')
209 return check_transition_handler(current_state, target_state)
210
211 def fetch_edge_deciders(self, atom):
212 """Fetches the edge deciders for the given atom."""
213 # This does not check if the name exists (since this is only used
214 # internally to the engine, and is not exposed to atoms that will
215 # not exist and therefore doesn't need to handle that case).
216 return self._fetch_atom_metadata_entry(atom.name, 'edge_deciders')
217
218 def fetch_scheduler(self, atom):
219 """Fetches the cached specific scheduler for the given atom."""
220 # This does not check if the name exists (since this is only used
221 # internally to the engine, and is not exposed to atoms that will
222 # not exist and therefore doesn't need to handle that case).
223 return self._fetch_atom_metadata_entry(atom.name, 'scheduler')
224
225 def fetch_action(self, atom):
226 """Fetches the cached action handler for the given atom."""
227 metadata = self._atom_cache[atom.name]
228 return metadata['action']
229
230 def fetch_scopes_for(self, atom_name):
231 """Fetches a walker of the visible scopes for the given atom."""
232 try:
233 return self._fetch_atom_metadata_entry(atom_name, 'scope_walker')
234 except KeyError:
235 # This signals to the caller that there is no walker for whatever
236 # atom name was given that doesn't really have any associated atom
237 # known to be named with that name; this is done since the storage
238 # layer will call into this layer to fetch a scope for a named
239 # atom and users can provide random names that do not actually
240 # exist...
241 return None
242
243 # Various helper methods used by the runtime components; not for public
244 # consumption...
245
246 def iterate_retries(self, state=None):
247 """Iterates retry atoms that match the provided state.
248
249 If no state is provided it will yield back all retry atoms.
250 """
251 if state:
252 atoms = list(self.iterate_nodes((com.RETRY,)))
253 atom_states = self._storage.get_atoms_states(atom.name
254 for atom in atoms)
255 for atom in atoms:
256 atom_state, _atom_intention = atom_states[atom.name]
257 if atom_state == state:
258 yield atom
259 else:
260 for atom in self.iterate_nodes((com.RETRY,)):
261 yield atom
262
263 def iterate_nodes(self, allowed_kinds):
264 """Yields back all nodes of specified kinds in the execution graph."""
265 graph = self._compilation.execution_graph
266 for node, node_data in graph.nodes(data=True):
267 if node_data['kind'] in allowed_kinds:
268 yield node
269
270 def is_success(self):
271 """Checks if all atoms in the execution graph are in 'happy' state."""
272 atoms = list(self.iterate_nodes(com.ATOMS))
273 atom_states = self._storage.get_atoms_states(atom.name
274 for atom in atoms)
275 for atom in atoms:
276 atom_state, _atom_intention = atom_states[atom.name]
277 if atom_state == st.IGNORE:
278 continue
279 if atom_state != st.SUCCESS:
280 return False
281 return True
282
283 def find_retry(self, node):
284 """Returns the retry atom associated to the given node (or none)."""
285 graph = self._compilation.execution_graph
286 return graph.nodes[node].get(com.RETRY)
287
288 def reset_atoms(self, atoms, state=st.PENDING, intention=st.EXECUTE):
289 """Resets all the provided atoms to the given state and intention."""
290 tweaked = []
291 for atom in atoms:
292 cur_intention = self.storage.get_atom_intention(atom.name)
293 # Don't trigger a RETRY if the atom needs to be REVERTED.
294 # This is a workaround for a bug when REVERT_ALL is applied to
295 # unordered flows
296 # (https://bugs.launchpad.net/taskflow/+bug/2043808)
297 # A subflow may trigger a REVERT_ALL, all the atoms of all the
298 # related subflows are marked as REVERT but a task of a related
299 # flow may still be running in another thread. If this task
300 # triggers a RETRY, it overrides the previously set REVERT status,
301 # breaking the revert path of the flow.
302 if cur_intention == st.REVERT and intention == st.RETRY:
303 continue
304 if state or intention:
305 tweaked.append((atom, state, intention))
306 if state:
307 change_state_handler = self._fetch_atom_metadata_entry(
308 atom.name, 'change_state_handler')
309 change_state_handler(atom, state)
310 if intention:
311 self.storage.set_atom_intention(atom.name, intention)
312 return tweaked
313
314 def reset_all(self, state=st.PENDING, intention=st.EXECUTE):
315 """Resets all atoms to the given state and intention."""
316 return self.reset_atoms(self.iterate_nodes(com.ATOMS),
317 state=state, intention=intention)
318
319 def reset_subgraph(self, atom, state=st.PENDING, intention=st.EXECUTE):
320 """Resets a atoms subgraph to the given state and intention.
321
322 The subgraph is contained of **all** of the atoms successors.
323 """
324 execution_graph = self._compilation.execution_graph
325 atoms_it = tr.depth_first_iterate(execution_graph, atom,
326 tr.Direction.FORWARD)
327 return self.reset_atoms(atoms_it, state=state, intention=intention)
328
329 def retry_subflow(self, retry):
330 """Prepares a retrys + its subgraph for execution.
331
332 This sets the retrys intention to ``EXECUTE`` and resets all of its
333 subgraph (its successors) to the ``PENDING`` state with an ``EXECUTE``
334 intention.
335 """
336 tweaked = self.reset_atoms([retry], state=None, intention=st.EXECUTE)
337 tweaked.extend(self.reset_subgraph(retry))
338 return tweaked