Debian Codegraph @

taskflow/engines/action_engine/deciders.py

raw
1 # Copyright (C) 2015 Yahoo! Inc. All Rights Reserved.
2 #
3 # Licensed under the Apache License, Version 2.0 (the "License"); you may
4 # not use this file except in compliance with the License. You may obtain
5 # a copy of the License at
6 #
7 # http://www.apache.org/licenses/LICENSE-2.0
8 #
9 # Unless required by applicable law or agreed to in writing, software
10 # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
11 # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
12 # License for the specific language governing permissions and limitations
13 # under the License.
14
15 import abc
16 import itertools
17
18 from taskflow import deciders
19 from taskflow.engines.action_engine import compiler
20 from taskflow.engines.action_engine import traversal
21 from taskflow import logging
22 from taskflow import states
23
24 LOG = logging.getLogger(__name__)
25
26
27 class Decider(metaclass=abc.ABCMeta):
28 """Base class for deciders.
29
30 Provides interface to be implemented by sub-classes.
31
32 Deciders check whether next atom in flow should be executed or not.
33 """
34
35 @abc.abstractmethod
36 def tally(self, runtime):
37 """Tally edge deciders on whether this decider should allow running.
38
39 The returned value is a list of edge deciders that voted
40 'nay' (do not allow running).
41 """
42
43 @abc.abstractmethod
44 def affect(self, runtime, nay_voters):
45 """Affects associated atoms due to at least one 'nay' edge decider.
46
47 This will alter the associated atom + some set of successor atoms by
48 setting there state and intention to ``IGNORE`` so that they are
49 ignored in future runtime activities.
50 """
51
52 def check_and_affect(self, runtime):
53 """Handles :py:func:`~.tally` + :py:func:`~.affect` in right order.
54
55 NOTE(harlowja): If there are zero 'nay' edge deciders then it is
56 assumed this decider should allow running.
57
58 Returns boolean of whether this decider allows for running (or not).
59 """
60 nay_voters = self.tally(runtime)
61 if nay_voters:
62 self.affect(runtime, nay_voters)
63 return False
64 return True
65
66
67 def _affect_all_successors(atom, runtime):
68 execution_graph = runtime.compilation.execution_graph
69 successors_iter = traversal.depth_first_iterate(
70 execution_graph, atom, traversal.Direction.FORWARD)
71 runtime.reset_atoms(itertools.chain([atom], successors_iter),
72 state=states.IGNORE, intention=states.IGNORE)
73
74
75 def _affect_successor_tasks_in_same_flow(atom, runtime):
76 execution_graph = runtime.compilation.execution_graph
77 successors_iter = traversal.depth_first_iterate(
78 execution_graph, atom, traversal.Direction.FORWARD,
79 # Do not go through nested flows but do follow *all* tasks that
80 # are directly connected in this same flow (thus the reason this is
81 # called the same flow decider); retries are direct successors
82 # of flows, so they should also be not traversed through, but
83 # setting this explicitly ensures that.
84 through_flows=False, through_retries=False)
85 runtime.reset_atoms(itertools.chain([atom], successors_iter),
86 state=states.IGNORE, intention=states.IGNORE)
87
88
89 def _affect_atom(atom, runtime):
90 runtime.reset_atoms([atom], state=states.IGNORE, intention=states.IGNORE)
91
92
93 def _affect_direct_task_neighbors(atom, runtime):
94 def _walk_neighbors():
95 execution_graph = runtime.compilation.execution_graph
96 for node in execution_graph.successors(atom):
97 node_data = execution_graph.nodes[node]
98 if node_data['kind'] == compiler.TASK:
99 yield node
100 successors_iter = _walk_neighbors()
101 runtime.reset_atoms(itertools.chain([atom], successors_iter),
102 state=states.IGNORE, intention=states.IGNORE)
103
104
105 class IgnoreDecider(Decider):
106 """Checks any provided edge-deciders and determines if ok to run."""
107
108 _depth_strategies = {
109 deciders.Depth.ALL: _affect_all_successors,
110 deciders.Depth.ATOM: _affect_atom,
111 deciders.Depth.FLOW: _affect_successor_tasks_in_same_flow,
112 deciders.Depth.NEIGHBORS: _affect_direct_task_neighbors,
113 }
114
115 def __init__(self, atom, edge_deciders):
116 self._atom = atom
117 self._edge_deciders = edge_deciders
118
119 def tally(self, runtime):
120 voters = {
121 'run_it': [],
122 'do_not_run_it': [],
123 'ignored': [],
124 }
125 history = {}
126 if self._edge_deciders:
127 # Gather all atoms (the ones that were not ignored) results so
128 # that those results can be used by the decider(s) that are
129 # making a decision as to pass or not pass...
130 states_intentions = runtime.storage.get_atoms_states(
131 ed.from_node.name for ed in self._edge_deciders
132 if ed.kind in compiler.ATOMS)
133 for atom_name in states_intentions.keys():
134 atom_state, _atom_intention = states_intentions[atom_name]
135 if atom_state != states.IGNORE:
136 history[atom_name] = runtime.storage.get(atom_name)
137 for ed in self._edge_deciders:
138 if (ed.kind in compiler.ATOMS and
139 # It was an ignored atom (not included in history and
140 # the only way that is possible is via above loop
141 # skipping it...)
142 ed.from_node.name not in history):
143 voters['ignored'].append(ed)
144 continue
145 if not ed.decider(history=history):
146 voters['do_not_run_it'].append(ed)
147 else:
148 voters['run_it'].append(ed)
149 if LOG.isEnabledFor(logging.TRACE):
150 LOG.trace("Out of %s deciders there were %s 'do no run it'"
151 " voters, %s 'do run it' voters and %s 'ignored'"
152 " voters for transition to atom '%s' given history %s",
153 sum(len(eds) for eds in voters.values()),
154 list(ed.from_node.name
155 for ed in voters['do_not_run_it']),
156 list(ed.from_node.name for ed in voters['run_it']),
157 list(ed.from_node.name for ed in voters['ignored']),
158 self._atom.name, history)
159 return voters['do_not_run_it']
160
161 def affect(self, runtime, nay_voters):
162 # If there were many 'nay' edge deciders that were targeted
163 # at this atom, then we need to pick the one which has the widest
164 # impact and respect that one as the decider depth that will
165 # actually affect things.
166 widest_depth = deciders.pick_widest(ed.depth for ed in nay_voters)
167 affector = self._depth_strategies[widest_depth]
168 return affector(self._atom, runtime)
169
170
171 class NoOpDecider(Decider):
172 """No-op decider that says it is always ok to run & has no effect(s)."""
173
174 def tally(self, runtime):
175 """Always good to go."""
176 return []
177
178 def affect(self, runtime, nay_voters):
179 """Does nothing."""