| 1 |
# Copyright (C) 2014 Yahoo! Inc. All Rights Reserved. |
| 2 |
# |
| 3 |
# Licensed under the Apache License, Version 2.0 (the "License"); you may |
| 4 |
# not use this file except in compliance with the License. You may obtain |
| 5 |
# a copy of the License at |
| 6 |
# |
| 7 |
# http://www.apache.org/licenses/LICENSE-2.0 |
| 8 |
# |
| 9 |
# Unless required by applicable law or agreed to in writing, software |
| 10 |
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT |
| 11 |
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the |
| 12 |
# License for the specific language governing permissions and limitations |
| 13 |
# under the License. |
| 14 |
|
| 15 |
import abc |
| 16 |
import weakref |
| 17 |
|
| 18 |
from oslo_utils import reflection |
| 19 |
from oslo_utils import strutils |
| 20 |
|
| 21 |
from taskflow.engines.action_engine import compiler as co |
| 22 |
from taskflow.engines.action_engine import executor as ex |
| 23 |
from taskflow import logging |
| 24 |
from taskflow import retry as retry_atom |
| 25 |
from taskflow import states as st |
| 26 |
|
| 27 |
LOG = logging.getLogger(__name__) |
| 28 |
|
| 29 |
|
| 30 |
class Strategy(metaclass=abc.ABCMeta): |
| 31 |
"""Failure resolution strategy base class.""" |
| 32 |
|
| 33 |
strategy = None |
| 34 |
|
| 35 |
def __init__(self, runtime): |
| 36 |
self._runtime = runtime |
| 37 |
|
| 38 |
@abc.abstractmethod |
| 39 |
def apply(self): |
| 40 |
"""Applies some algorithm to resolve some detected failure.""" |
| 41 |
|
| 42 |
def __str__(self): |
| 43 |
base = reflection.get_class_name(self, fully_qualified=False) |
| 44 |
if self.strategy is not None: |
| 45 |
strategy_name = self.strategy.name |
| 46 |
else: |
| 47 |
strategy_name = "???" |
| 48 |
return base + "(strategy=%s)" % (strategy_name) |
| 49 |
|
| 50 |
|
| 51 |
class RevertAndRetry(Strategy): |
| 52 |
"""Sets the *associated* subflow for revert to be later retried.""" |
| 53 |
|
| 54 |
strategy = retry_atom.RETRY |
| 55 |
|
| 56 |
def __init__(self, runtime, retry): |
| 57 |
super().__init__(runtime) |
| 58 |
self._retry = retry |
| 59 |
|
| 60 |
def apply(self): |
| 61 |
tweaked = self._runtime.reset_atoms([self._retry], state=None, |
| 62 |
intention=st.RETRY) |
| 63 |
tweaked.extend(self._runtime.reset_subgraph(self._retry, state=None, |
| 64 |
intention=st.REVERT)) |
| 65 |
return tweaked |
| 66 |
|
| 67 |
|
| 68 |
class RevertAll(Strategy): |
| 69 |
"""Sets *all* nodes/atoms to the ``REVERT`` intention.""" |
| 70 |
|
| 71 |
strategy = retry_atom.REVERT_ALL |
| 72 |
|
| 73 |
def __init__(self, runtime): |
| 74 |
super().__init__(runtime) |
| 75 |
|
| 76 |
def apply(self): |
| 77 |
return self._runtime.reset_atoms( |
| 78 |
self._runtime.iterate_nodes(co.ATOMS), |
| 79 |
state=None, intention=st.REVERT) |
| 80 |
|
| 81 |
|
| 82 |
class Revert(Strategy): |
| 83 |
"""Sets atom and *associated* nodes to the ``REVERT`` intention.""" |
| 84 |
|
| 85 |
strategy = retry_atom.REVERT |
| 86 |
|
| 87 |
def __init__(self, runtime, atom): |
| 88 |
super().__init__(runtime) |
| 89 |
self._atom = atom |
| 90 |
|
| 91 |
def apply(self): |
| 92 |
tweaked = self._runtime.reset_atoms([self._atom], state=None, |
| 93 |
intention=st.REVERT) |
| 94 |
tweaked.extend(self._runtime.reset_subgraph(self._atom, state=None, |
| 95 |
intention=st.REVERT)) |
| 96 |
return tweaked |
| 97 |
|
| 98 |
|
| 99 |
class Completer: |
| 100 |
"""Completes atoms using actions to complete them.""" |
| 101 |
|
| 102 |
def __init__(self, runtime): |
| 103 |
self._runtime = weakref.proxy(runtime) |
| 104 |
self._storage = runtime.storage |
| 105 |
self._undefined_resolver = RevertAll(self._runtime) |
| 106 |
self._defer_reverts = strutils.bool_from_string( |
| 107 |
self._runtime.options.get('defer_reverts', False)) |
| 108 |
self._resolve = not strutils.bool_from_string( |
| 109 |
self._runtime.options.get('never_resolve', False)) |
| 110 |
|
| 111 |
def resume(self): |
| 112 |
"""Resumes atoms in the contained graph. |
| 113 |
|
| 114 |
This is done to allow any previously completed or failed atoms to |
| 115 |
be analyzed, there results processed and any potential atoms affected |
| 116 |
to be adjusted as needed. |
| 117 |
|
| 118 |
This should return a set of atoms which should be the initial set of |
| 119 |
atoms that were previously not finished (due to a RUNNING or REVERTING |
| 120 |
attempt not previously finishing). |
| 121 |
""" |
| 122 |
atoms = list(self._runtime.iterate_nodes(co.ATOMS)) |
| 123 |
atom_states = self._storage.get_atoms_states(atom.name |
| 124 |
for atom in atoms) |
| 125 |
if self._resolve: |
| 126 |
for atom in atoms: |
| 127 |
atom_state, _atom_intention = atom_states[atom.name] |
| 128 |
if atom_state == st.FAILURE: |
| 129 |
self._process_atom_failure( |
| 130 |
atom, self._storage.get(atom.name)) |
| 131 |
for retry in self._runtime.iterate_retries(st.RETRYING): |
| 132 |
retry_affected_atoms_it = self._runtime.retry_subflow(retry) |
| 133 |
for atom, state, intention in retry_affected_atoms_it: |
| 134 |
if state: |
| 135 |
atom_states[atom.name] = (state, intention) |
| 136 |
unfinished_atoms = set() |
| 137 |
for atom in atoms: |
| 138 |
atom_state, _atom_intention = atom_states[atom.name] |
| 139 |
if atom_state in (st.RUNNING, st.REVERTING): |
| 140 |
unfinished_atoms.add(atom) |
| 141 |
LOG.trace("Resuming atom '%s' since it was left in" |
| 142 |
" state %s", atom, atom_state) |
| 143 |
return unfinished_atoms |
| 144 |
|
| 145 |
def complete_failure(self, node, outcome, failure): |
| 146 |
"""Performs post-execution completion of a nodes failure. |
| 147 |
|
| 148 |
Returns whether the result should be saved into an accumulator of |
| 149 |
failures or whether this should not be done. |
| 150 |
""" |
| 151 |
if outcome == ex.EXECUTED and self._resolve: |
| 152 |
self._process_atom_failure(node, failure) |
| 153 |
# We resolved something, carry on... |
| 154 |
return False |
| 155 |
else: |
| 156 |
# Reverting failed (or resolving was turned off), always |
| 157 |
# retain the failure... |
| 158 |
return True |
| 159 |
|
| 160 |
def complete(self, node, outcome, result): |
| 161 |
"""Performs post-execution completion of a node result.""" |
| 162 |
handler = self._runtime.fetch_action(node) |
| 163 |
if outcome == ex.EXECUTED: |
| 164 |
handler.complete_execution(node, result) |
| 165 |
else: |
| 166 |
handler.complete_reversion(node, result) |
| 167 |
|
| 168 |
def _determine_resolution(self, atom, failure): |
| 169 |
"""Determines which resolution strategy to activate/apply.""" |
| 170 |
retry = self._runtime.find_retry(atom) |
| 171 |
if retry is not None: |
| 172 |
# Ask retry controller what to do in case of failure. |
| 173 |
handler = self._runtime.fetch_action(retry) |
| 174 |
strategy = handler.on_failure(retry, atom, failure) |
| 175 |
if strategy == retry_atom.RETRY: |
| 176 |
return RevertAndRetry(self._runtime, retry) |
| 177 |
elif strategy == retry_atom.REVERT: |
| 178 |
# Ask parent retry and figure out what to do... |
| 179 |
parent_resolver = self._determine_resolution(retry, failure) |
| 180 |
# In the future, this will be the only behavior. REVERT |
| 181 |
# should defer to the parent retry if it exists, or use the |
| 182 |
# default REVERT_ALL if it doesn't. |
| 183 |
if self._defer_reverts: |
| 184 |
return parent_resolver |
| 185 |
# Ok if the parent resolver says something not REVERT, and |
| 186 |
# it isn't just using the undefined resolver, assume the |
| 187 |
# parent knows best. |
| 188 |
if parent_resolver is not self._undefined_resolver: |
| 189 |
if parent_resolver.strategy != retry_atom.REVERT: |
| 190 |
return parent_resolver |
| 191 |
return Revert(self._runtime, retry) |
| 192 |
elif strategy == retry_atom.REVERT_ALL: |
| 193 |
return RevertAll(self._runtime) |
| 194 |
else: |
| 195 |
raise ValueError("Unknown atom failure resolution" |
| 196 |
" action/strategy '%s'" % strategy) |
| 197 |
else: |
| 198 |
return self._undefined_resolver |
| 199 |
|
| 200 |
def _process_atom_failure(self, atom, failure): |
| 201 |
"""Processes atom failure & applies resolution strategies. |
| 202 |
|
| 203 |
On atom failure this will find the atoms associated retry controller |
| 204 |
and ask that controller for the strategy to perform to resolve that |
| 205 |
failure. After getting a resolution strategy decision this method will |
| 206 |
then adjust the needed other atoms intentions, and states, ... so that |
| 207 |
the failure can be worked around. |
| 208 |
""" |
| 209 |
resolver = self._determine_resolution(atom, failure) |
| 210 |
LOG.debug("Applying resolver '%s' to resolve failure '%s'" |
| 211 |
" of atom '%s'", resolver, failure, atom) |
| 212 |
tweaked = resolver.apply() |
| 213 |
# Only show the tweaked node list when trace is on, otherwise |
| 214 |
# just show the amount/count of nodes tweaks... |
| 215 |
if LOG.isEnabledFor(logging.TRACE): |
| 216 |
LOG.trace("Modified/tweaked %s nodes while applying" |
| 217 |
" resolver '%s'", tweaked, resolver) |
| 218 |
else: |
| 219 |
LOG.debug("Modified/tweaked %s nodes while applying" |
| 220 |
" resolver '%s'", len(tweaked), resolver) |