Debian Codegraph @

taskflow/engines/action_engine/builder.py

raw
1 # Copyright (C) 2012 Yahoo! Inc. All Rights Reserved.
2 #
3 # Licensed under the Apache License, Version 2.0 (the "License"); you may
4 # not use this file except in compliance with the License. You may obtain
5 # a copy of the License at
6 #
7 # http://www.apache.org/licenses/LICENSE-2.0
8 #
9 # Unless required by applicable law or agreed to in writing, software
10 # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
11 # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
12 # License for the specific language governing permissions and limitations
13 # under the License.
14
15 from concurrent import futures
16 import weakref
17
18 from automaton import machines
19 from oslo_utils import timeutils
20
21 from taskflow import logging
22 from taskflow import states as st
23 from taskflow.types import failure
24 from taskflow.utils import iter_utils
25
26 # Default waiting state timeout (in seconds).
27 WAITING_TIMEOUT = 60
28
29 # Meta states the state machine uses.
30 UNDEFINED = 'UNDEFINED'
31 GAME_OVER = 'GAME_OVER'
32 META_STATES = (GAME_OVER, UNDEFINED)
33
34 # Event name constants the state machine uses.
35 SCHEDULE = 'schedule_next'
36 WAIT = 'wait_finished'
37 ANALYZE = 'examine_finished'
38 FINISH = 'completed'
39 FAILED = 'failed'
40 SUSPENDED = 'suspended'
41 SUCCESS = 'success'
42 REVERTED = 'reverted'
43 START = 'start'
44
45 # Internal enums used to denote how/if a atom was completed."""
46 FAILED_COMPLETING = 'failed_completing'
47 WAS_CANCELLED = 'was_cancelled'
48 SUCCESSFULLY_COMPLETED = 'successfully_completed'
49
50
51 # For these states we will gather how long (in seconds) the
52 # state was in-progress (cumulatively if the state is entered multiple
53 # times)
54 TIMED_STATES = (st.ANALYZING, st.RESUMING, st.SCHEDULING, st.WAITING)
55
56 LOG = logging.getLogger(__name__)
57
58
59 class MachineMemory:
60 """State machine memory."""
61
62 def __init__(self):
63 self.next_up = set()
64 self.not_done = set()
65 self.failures = []
66 self.done = set()
67
68 def cancel_futures(self):
69 """Attempts to cancel any not done futures."""
70 for fut in self.not_done:
71 fut.cancel()
72
73
74 class MachineBuilder:
75 """State machine *builder* that powers the engine components.
76
77 NOTE(harlowja): the machine (states and events that will trigger
78 transitions) that this builds is represented by the following
79 table::
80
81 +--------------+------------------+------------+----------+---------+
82 | Start | Event | End | On Enter | On Exit |
83 +--------------+------------------+------------+----------+---------+
84 | ANALYZING | completed | GAME_OVER | . | . |
85 | ANALYZING | schedule_next | SCHEDULING | . | . |
86 | ANALYZING | wait_finished | WAITING | . | . |
87 | FAILURE[$] | . | . | . | . |
88 | GAME_OVER | failed | FAILURE | . | . |
89 | GAME_OVER | reverted | REVERTED | . | . |
90 | GAME_OVER | success | SUCCESS | . | . |
91 | GAME_OVER | suspended | SUSPENDED | . | . |
92 | RESUMING | schedule_next | SCHEDULING | . | . |
93 | REVERTED[$] | . | . | . | . |
94 | SCHEDULING | wait_finished | WAITING | . | . |
95 | SUCCESS[$] | . | . | . | . |
96 | SUSPENDED[$] | . | . | . | . |
97 | UNDEFINED[^] | start | RESUMING | . | . |
98 | WAITING | examine_finished | ANALYZING | . | . |
99 +--------------+------------------+------------+----------+---------+
100
101 Between any of these yielded states (minus ``GAME_OVER`` and ``UNDEFINED``)
102 if the engine has been suspended or the engine has failed (due to a
103 non-resolveable task failure or scheduling failure) the machine will stop
104 executing new tasks (currently running tasks will be allowed to complete)
105 and this machines run loop will be broken.
106
107 NOTE(harlowja): If the runtimes scheduler component is able to schedule
108 tasks in parallel, this enables parallel running and/or reversion.
109 """
110
111 def __init__(self, runtime, waiter):
112 self._runtime = weakref.proxy(runtime)
113 self._selector = runtime.selector
114 self._completer = runtime.completer
115 self._scheduler = runtime.scheduler
116 self._storage = runtime.storage
117 self._waiter = waiter
118
119 def build(self, statistics, timeout=None, gather_statistics=True):
120 """Builds a state-machine (that is used during running)."""
121 if gather_statistics:
122 watches = {}
123 state_statistics = {}
124 statistics['seconds_per_state'] = state_statistics
125 watches = {}
126 for timed_state in TIMED_STATES:
127 state_statistics[timed_state.lower()] = 0.0
128 watches[timed_state] = timeutils.StopWatch()
129 statistics['discarded_failures'] = 0
130 statistics['awaiting'] = 0
131 statistics['completed'] = 0
132 statistics['incomplete'] = 0
133
134 memory = MachineMemory()
135 if timeout is None:
136 timeout = WAITING_TIMEOUT
137
138 # Cache some local functions/methods...
139 do_complete = self._completer.complete
140 do_complete_failure = self._completer.complete_failure
141 get_atom_intention = self._storage.get_atom_intention
142
143 def do_schedule(next_nodes):
144 with self._storage.lock.write_lock():
145 return self._scheduler.schedule(
146 sorted(next_nodes,
147 key=lambda node: getattr(node, 'priority', 0),
148 reverse=True))
149
150 def iter_next_atoms(atom=None, apply_deciders=True):
151 # Yields and filters and tweaks the next atoms to run...
152 maybe_atoms_it = self._selector.iter_next_atoms(atom=atom)
153 for atom, late_decider in maybe_atoms_it:
154 if apply_deciders:
155 proceed = late_decider.check_and_affect(self._runtime)
156 if proceed:
157 yield atom
158 else:
159 yield atom
160
161 def resume(old_state, new_state, event):
162 # This reaction function just updates the state machines memory
163 # to include any nodes that need to be executed (from a previous
164 # attempt, which may be empty if never ran before) and any nodes
165 # that are now ready to be ran.
166 with self._storage.lock.write_lock():
167 memory.next_up.update(
168 iter_utils.unique_seen((self._completer.resume(),
169 iter_next_atoms())))
170 return SCHEDULE
171
172 def game_over(old_state, new_state, event):
173 # This reaction function is mainly a intermediary delegation
174 # function that analyzes the current memory and transitions to
175 # the appropriate handler that will deal with the memory values,
176 # it is *always* called before the final state is entered.
177 if memory.failures:
178 return FAILED
179 with self._storage.lock.read_lock():
180 leftover_atoms = iter_utils.count(
181 # Avoid activating the deciders, since at this point
182 # the engine is finishing and there will be no more further
183 # work done anyway...
184 iter_next_atoms(apply_deciders=False))
185 if leftover_atoms:
186 # Ok we didn't finish (either reverting or executing...) so
187 # that means we must of been stopped at some point...
188 LOG.trace("Suspension determined to have been reacted to"
189 " since (at least) %s atoms have been left in an"
190 " unfinished state", leftover_atoms)
191 return SUSPENDED
192 elif self._runtime.is_success():
193 return SUCCESS
194 else:
195 return REVERTED
196
197 def schedule(old_state, new_state, event):
198 # This reaction function starts to schedule the memory's next
199 # nodes (iff the engine is still runnable, which it may not be
200 # if the user of this engine has requested the engine/storage
201 # that holds this information to stop or suspend); handles failures
202 # that occur during this process safely...
203 with self._storage.lock.write_lock():
204 current_flow_state = self._storage.get_flow_state()
205 if current_flow_state == st.RUNNING and memory.next_up:
206 not_done, failures = do_schedule(memory.next_up)
207 if not_done:
208 memory.not_done.update(not_done)
209 if failures:
210 memory.failures.extend(failures)
211 memory.next_up.intersection_update(not_done)
212 elif current_flow_state == st.SUSPENDING and memory.not_done:
213 # Try to force anything not cancelled to now be cancelled
214 # so that the executor that gets it does not continue to
215 # try to work on it (if the future execution is still in
216 # its backlog, if it's already being executed, this will
217 # do nothing).
218 memory.cancel_futures()
219 return WAIT
220
221 def complete_an_atom(fut):
222 # This completes a single atom saving its result in
223 # storage and preparing whatever predecessors or successors will
224 # now be ready to execute (or revert or retry...); it also
225 # handles failures that occur during this process safely...
226 atom = fut.atom
227 try:
228 outcome, result = fut.result()
229 do_complete(atom, outcome, result)
230 if isinstance(result, failure.Failure):
231 retain = do_complete_failure(atom, outcome, result)
232 if retain:
233 memory.failures.append(result)
234 else:
235 # NOTE(harlowja): avoid making any intention request
236 # to storage unless we are sure we are in DEBUG
237 # enabled logging (otherwise we will call this all
238 # the time even when DEBUG is not enabled, which
239 # would suck...)
240 if LOG.isEnabledFor(logging.DEBUG):
241 intention = get_atom_intention(atom.name)
242 LOG.debug("Discarding failure '%s' (in response"
243 " to outcome '%s') under completion"
244 " units request during completion of"
245 " atom '%s' (intention is to %s)",
246 result, outcome, atom, intention)
247 if gather_statistics:
248 statistics['discarded_failures'] += 1
249 if gather_statistics:
250 statistics['completed'] += 1
251 except futures.CancelledError:
252 # Well it got cancelled, skip doing anything
253 # and move on; at a further time it will be resumed
254 # and something should be done with it to get it
255 # going again.
256 return WAS_CANCELLED
257 except Exception:
258 memory.failures.append(failure.Failure())
259 LOG.exception("Engine '%s' atom post-completion"
260 " failed", atom)
261 return FAILED_COMPLETING
262 else:
263 return SUCCESSFULLY_COMPLETED
264
265 def wait(old_state, new_state, event):
266 # TODO(harlowja): maybe we should start doing 'yield from' this
267 # call sometime in the future, or equivalent that will work in
268 # py2 and py3.
269 if memory.not_done:
270 done, not_done = self._waiter(memory.not_done, timeout=timeout)
271 memory.done.update(done)
272 memory.not_done = not_done
273 return ANALYZE
274
275 def analyze(old_state, new_state, event):
276 # This reaction function is responsible for analyzing all nodes
277 # that have finished executing/reverting and figuring
278 # out what nodes are now ready to be ran (and then triggering those
279 # nodes to be scheduled in the future); handles failures that
280 # occur during this process safely...
281 next_up = set()
282 with self._storage.lock.write_lock():
283 while memory.done:
284 fut = memory.done.pop()
285 # Force it to be completed so that we can ensure that
286 # before we iterate over any successors or predecessors
287 # that we know it has been completed and saved and so on...
288 completion_status = complete_an_atom(fut)
289 if (not memory.failures
290 and completion_status != WAS_CANCELLED):
291 atom = fut.atom
292 try:
293 more_work = set(iter_next_atoms(atom=atom))
294 except Exception:
295 memory.failures.append(failure.Failure())
296 LOG.exception(
297 "Engine '%s' atom post-completion"
298 " next atom searching failed", atom)
299 else:
300 next_up.update(more_work)
301 current_flow_state = self._storage.get_flow_state()
302 if (current_flow_state == st.RUNNING
303 and next_up and not memory.failures):
304 memory.next_up.update(next_up)
305 return SCHEDULE
306 elif memory.not_done:
307 if current_flow_state == st.SUSPENDING:
308 memory.cancel_futures()
309 return WAIT
310 else:
311 return FINISH
312
313 def on_exit(old_state, event):
314 LOG.trace("Exiting old state '%s' in response to event '%s'",
315 old_state, event)
316 if gather_statistics:
317 if old_state in watches:
318 w = watches[old_state]
319 w.stop()
320 state_statistics[old_state.lower()] += w.elapsed()
321 if old_state in (st.SCHEDULING, st.WAITING):
322 statistics['incomplete'] = len(memory.not_done)
323 if old_state in (st.ANALYZING, st.SCHEDULING):
324 statistics['awaiting'] = len(memory.next_up)
325
326 def on_enter(new_state, event):
327 LOG.trace("Entering new state '%s' in response to event '%s'",
328 new_state, event)
329 if gather_statistics and new_state in watches:
330 watches[new_state].restart()
331
332 state_kwargs = {
333 'on_exit': on_exit,
334 'on_enter': on_enter,
335 }
336 m = machines.FiniteMachine()
337 m.add_state(GAME_OVER, **state_kwargs)
338 m.add_state(UNDEFINED, **state_kwargs)
339 m.add_state(st.ANALYZING, **state_kwargs)
340 m.add_state(st.RESUMING, **state_kwargs)
341 m.add_state(st.REVERTED, terminal=True, **state_kwargs)
342 m.add_state(st.SCHEDULING, **state_kwargs)
343 m.add_state(st.SUCCESS, terminal=True, **state_kwargs)
344 m.add_state(st.SUSPENDED, terminal=True, **state_kwargs)
345 m.add_state(st.WAITING, **state_kwargs)
346 m.add_state(st.FAILURE, terminal=True, **state_kwargs)
347 m.default_start_state = UNDEFINED
348
349 m.add_transition(GAME_OVER, st.REVERTED, REVERTED)
350 m.add_transition(GAME_OVER, st.SUCCESS, SUCCESS)
351 m.add_transition(GAME_OVER, st.SUSPENDED, SUSPENDED)
352 m.add_transition(GAME_OVER, st.FAILURE, FAILED)
353 m.add_transition(UNDEFINED, st.RESUMING, START)
354 m.add_transition(st.ANALYZING, GAME_OVER, FINISH)
355 m.add_transition(st.ANALYZING, st.SCHEDULING, SCHEDULE)
356 m.add_transition(st.ANALYZING, st.WAITING, WAIT)
357 m.add_transition(st.RESUMING, st.SCHEDULING, SCHEDULE)
358 m.add_transition(st.SCHEDULING, st.WAITING, WAIT)
359 m.add_transition(st.WAITING, st.ANALYZING, ANALYZE)
360
361 m.add_reaction(GAME_OVER, FINISH, game_over)
362 m.add_reaction(st.ANALYZING, ANALYZE, analyze)
363 m.add_reaction(st.RESUMING, START, resume)
364 m.add_reaction(st.SCHEDULING, SCHEDULE, schedule)
365 m.add_reaction(st.WAITING, WAIT, wait)
366
367 m.freeze()
368 return (m, memory)