| 1 |
# Copyright (C) 2012 Yahoo! Inc. All Rights Reserved. |
| 2 |
# Copyright (C) 2013 Rackspace Hosting All Rights Reserved. |
| 3 |
# |
| 4 |
# Licensed under the Apache License, Version 2.0 (the "License"); you may |
| 5 |
# not use this file except in compliance with the License. You may obtain |
| 6 |
# a copy of the License at |
| 7 |
# |
| 8 |
# http://www.apache.org/licenses/LICENSE-2.0 |
| 9 |
# |
| 10 |
# Unless required by applicable law or agreed to in writing, software |
| 11 |
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT |
| 12 |
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the |
| 13 |
# License for the specific language governing permissions and limitations |
| 14 |
# under the License. |
| 15 |
|
| 16 |
import abc |
| 17 |
import copy |
| 18 |
import os |
| 19 |
|
| 20 |
from oslo_utils import timeutils |
| 21 |
from oslo_utils import uuidutils |
| 22 |
|
| 23 |
from taskflow import exceptions as exc |
| 24 |
from taskflow import states |
| 25 |
from taskflow.types import failure as ft |
| 26 |
from taskflow.utils import misc |
| 27 |
|
| 28 |
|
| 29 |
# Internal helpers... |
| 30 |
|
| 31 |
|
| 32 |
def _format_meta(metadata, indent): |
| 33 |
"""Format the common metadata dictionary in the same manner.""" |
| 34 |
if not metadata: |
| 35 |
return [] |
| 36 |
lines = [ |
| 37 |
'%s- metadata:' % (" " * indent), |
| 38 |
] |
| 39 |
for (k, v) in metadata.items(): |
| 40 |
# Progress for now is a special snowflake and will be formatted |
| 41 |
# in percent format. |
| 42 |
if k == 'progress' and isinstance(v, misc.NUMERIC_TYPES): |
| 43 |
v = "%0.2f%%" % (v * 100.0) |
| 44 |
lines.append("{}+ {} = {}".format(" " * (indent + 2), k, v)) |
| 45 |
return lines |
| 46 |
|
| 47 |
|
| 48 |
def _format_shared(obj, indent): |
| 49 |
"""Format the common shared attributes in the same manner.""" |
| 50 |
if obj is None: |
| 51 |
return [] |
| 52 |
lines = [] |
| 53 |
for attr_name in ("uuid", "state"): |
| 54 |
if not hasattr(obj, attr_name): |
| 55 |
continue |
| 56 |
lines.append("{}- {} = {}".format(" " * indent, attr_name, |
| 57 |
getattr(obj, attr_name))) |
| 58 |
return lines |
| 59 |
|
| 60 |
|
| 61 |
def _is_all_none(arg, *args): |
| 62 |
if arg is not None: |
| 63 |
return False |
| 64 |
for more_arg in args: |
| 65 |
if more_arg is not None: |
| 66 |
return False |
| 67 |
return True |
| 68 |
|
| 69 |
|
| 70 |
def _copy_function(deep_copy): |
| 71 |
if deep_copy: |
| 72 |
return copy.deepcopy |
| 73 |
else: |
| 74 |
return lambda x: x |
| 75 |
|
| 76 |
|
| 77 |
def _safe_marshal_time(when): |
| 78 |
if not when: |
| 79 |
return None |
| 80 |
return timeutils.marshall_now(now=when) |
| 81 |
|
| 82 |
|
| 83 |
def _safe_unmarshal_time(when): |
| 84 |
if not when: |
| 85 |
return None |
| 86 |
return timeutils.unmarshall_time(when) |
| 87 |
|
| 88 |
|
| 89 |
def _fix_meta(data): |
| 90 |
# Handle the case where older schemas allowed this to be non-dict by |
| 91 |
# correcting this case by replacing it with a dictionary when a non-dict |
| 92 |
# is found. |
| 93 |
meta = data.get('meta') |
| 94 |
if not isinstance(meta, dict): |
| 95 |
meta = {} |
| 96 |
return meta |
| 97 |
|
| 98 |
|
| 99 |
class LogBook: |
| 100 |
"""A collection of flow details and associated metadata. |
| 101 |
|
| 102 |
Typically this class contains a collection of flow detail entries |
| 103 |
for a given engine (or job) so that those entities can track what 'work' |
| 104 |
has been completed for resumption, reverting and miscellaneous tracking |
| 105 |
purposes. |
| 106 |
|
| 107 |
The data contained within this class need **not** be persisted to the |
| 108 |
backend storage in real time. The data in this class will only be |
| 109 |
guaranteed to be persisted when a save occurs via some backend |
| 110 |
connection. |
| 111 |
|
| 112 |
NOTE(harlowja): the naming of this class is analogous to a ship's log or a |
| 113 |
similar type of record used in detailing work that has been completed (or |
| 114 |
work that has not been completed). |
| 115 |
|
| 116 |
:ivar created_at: A ``datetime.datetime`` object of when this logbook |
| 117 |
was created. |
| 118 |
:ivar updated_at: A ``datetime.datetime`` object of when this logbook |
| 119 |
was last updated at. |
| 120 |
:ivar meta: A dictionary of meta-data associated with this logbook. |
| 121 |
""" |
| 122 |
def __init__(self, name, uuid=None): |
| 123 |
if uuid: |
| 124 |
self._uuid = uuid |
| 125 |
else: |
| 126 |
self._uuid = uuidutils.generate_uuid() |
| 127 |
self._name = name |
| 128 |
self._flowdetails_by_id = {} |
| 129 |
self.created_at = timeutils.utcnow() |
| 130 |
self.updated_at = None |
| 131 |
self.meta = {} |
| 132 |
|
| 133 |
def pformat(self, indent=0, linesep=os.linesep): |
| 134 |
"""Pretty formats this logbook into a string. |
| 135 |
|
| 136 |
>>> from taskflow.persistence import models |
| 137 |
>>> tmp = models.LogBook("example") |
| 138 |
>>> print(tmp.pformat()) |
| 139 |
LogBook: 'example' |
| 140 |
- uuid = ... |
| 141 |
- created_at = ... |
| 142 |
""" |
| 143 |
cls_name = self.__class__.__name__ |
| 144 |
lines = ["{}{}: '{}'".format(" " * indent, cls_name, self.name)] |
| 145 |
lines.extend(_format_shared(self, indent=indent + 1)) |
| 146 |
lines.extend(_format_meta(self.meta, indent=indent + 1)) |
| 147 |
if self.created_at is not None: |
| 148 |
lines.append("%s- created_at = %s" |
| 149 |
% (" " * (indent + 1), |
| 150 |
self.created_at.isoformat())) |
| 151 |
if self.updated_at is not None: |
| 152 |
lines.append("%s- updated_at = %s" |
| 153 |
% (" " * (indent + 1), |
| 154 |
self.updated_at.isoformat())) |
| 155 |
for flow_detail in self: |
| 156 |
lines.append(flow_detail.pformat(indent=indent + 1, |
| 157 |
linesep=linesep)) |
| 158 |
return linesep.join(lines) |
| 159 |
|
| 160 |
def add(self, fd): |
| 161 |
"""Adds a new flow detail into this logbook. |
| 162 |
|
| 163 |
NOTE(harlowja): if an existing flow detail exists with the same |
| 164 |
uuid the existing one will be overwritten with the newly provided |
| 165 |
one. |
| 166 |
|
| 167 |
Does not *guarantee* that the details will be immediately saved. |
| 168 |
""" |
| 169 |
self._flowdetails_by_id[fd.uuid] = fd |
| 170 |
self.updated_at = timeutils.utcnow() |
| 171 |
|
| 172 |
def find(self, flow_uuid): |
| 173 |
"""Locate the flow detail corresponding to the given uuid. |
| 174 |
|
| 175 |
:returns: the flow detail with that uuid |
| 176 |
:rtype: :py:class:`.FlowDetail` (or ``None`` if not found) |
| 177 |
""" |
| 178 |
return self._flowdetails_by_id.get(flow_uuid, None) |
| 179 |
|
| 180 |
def merge(self, lb, deep_copy=False): |
| 181 |
"""Merges the current object state with the given ones state. |
| 182 |
|
| 183 |
If ``deep_copy`` is provided as truthy then the |
| 184 |
local object will use ``copy.deepcopy`` to replace this objects |
| 185 |
local attributes with the provided objects attributes (**only** if |
| 186 |
there is a difference between this objects attributes and the |
| 187 |
provided attributes). If ``deep_copy`` is falsey (the default) then a |
| 188 |
reference copy will occur instead when a difference is detected. |
| 189 |
|
| 190 |
NOTE(harlowja): If the provided object is this object itself |
| 191 |
then **no** merging is done. Also note that this does **not** merge |
| 192 |
the flow details contained in either. |
| 193 |
|
| 194 |
:returns: this logbook (freshly merged with the incoming object) |
| 195 |
:rtype: :py:class:`.LogBook` |
| 196 |
""" |
| 197 |
if lb is self: |
| 198 |
return self |
| 199 |
copy_fn = _copy_function(deep_copy) |
| 200 |
if self.meta != lb.meta: |
| 201 |
self.meta = copy_fn(lb.meta) |
| 202 |
if lb.created_at != self.created_at: |
| 203 |
self.created_at = copy_fn(lb.created_at) |
| 204 |
if lb.updated_at != self.updated_at: |
| 205 |
self.updated_at = copy_fn(lb.updated_at) |
| 206 |
return self |
| 207 |
|
| 208 |
def to_dict(self, marshal_time=False): |
| 209 |
"""Translates the internal state of this object to a ``dict``. |
| 210 |
|
| 211 |
NOTE(harlowja): The returned ``dict`` does **not** include any |
| 212 |
contained flow details. |
| 213 |
|
| 214 |
:returns: this logbook in ``dict`` form |
| 215 |
""" |
| 216 |
if not marshal_time: |
| 217 |
marshal_fn = lambda x: x |
| 218 |
else: |
| 219 |
marshal_fn = _safe_marshal_time |
| 220 |
return { |
| 221 |
'name': self.name, |
| 222 |
'meta': self.meta, |
| 223 |
'uuid': self.uuid, |
| 224 |
'updated_at': marshal_fn(self.updated_at), |
| 225 |
'created_at': marshal_fn(self.created_at), |
| 226 |
} |
| 227 |
|
| 228 |
@classmethod |
| 229 |
def from_dict(cls, data, unmarshal_time=False): |
| 230 |
"""Translates the given ``dict`` into an instance of this class. |
| 231 |
|
| 232 |
NOTE(harlowja): the ``dict`` provided should come from a prior |
| 233 |
call to :meth:`.to_dict`. |
| 234 |
|
| 235 |
:returns: a new logbook |
| 236 |
:rtype: :py:class:`.LogBook` |
| 237 |
""" |
| 238 |
if not unmarshal_time: |
| 239 |
unmarshal_fn = lambda x: x |
| 240 |
else: |
| 241 |
unmarshal_fn = _safe_unmarshal_time |
| 242 |
obj = cls(data['name'], uuid=data['uuid']) |
| 243 |
obj.updated_at = unmarshal_fn(data['updated_at']) |
| 244 |
obj.created_at = unmarshal_fn(data['created_at']) |
| 245 |
obj.meta = _fix_meta(data) |
| 246 |
return obj |
| 247 |
|
| 248 |
@property |
| 249 |
def uuid(self): |
| 250 |
"""The unique identifer of this logbook.""" |
| 251 |
return self._uuid |
| 252 |
|
| 253 |
@property |
| 254 |
def name(self): |
| 255 |
"""The name of this logbook.""" |
| 256 |
return self._name |
| 257 |
|
| 258 |
def __iter__(self): |
| 259 |
yield from self._flowdetails_by_id.values() |
| 260 |
|
| 261 |
def __len__(self): |
| 262 |
return len(self._flowdetails_by_id) |
| 263 |
|
| 264 |
def copy(self, retain_contents=True): |
| 265 |
"""Copies this logbook. |
| 266 |
|
| 267 |
Creates a shallow copy of this logbook. If this logbook contains |
| 268 |
flow details and ``retain_contents`` is truthy (the default) then |
| 269 |
the flow details container will be shallow copied (the flow details |
| 270 |
contained there-in will **not** be copied). If ``retain_contents`` is |
| 271 |
falsey then the copied logbook will have **no** contained flow |
| 272 |
details (but it will have the rest of the local objects attributes |
| 273 |
copied). |
| 274 |
|
| 275 |
:returns: a new logbook |
| 276 |
:rtype: :py:class:`.LogBook` |
| 277 |
""" |
| 278 |
clone = copy.copy(self) |
| 279 |
if not retain_contents: |
| 280 |
clone._flowdetails_by_id = {} |
| 281 |
else: |
| 282 |
clone._flowdetails_by_id = self._flowdetails_by_id.copy() |
| 283 |
if self.meta: |
| 284 |
clone.meta = self.meta.copy() |
| 285 |
return clone |
| 286 |
|
| 287 |
|
| 288 |
class FlowDetail: |
| 289 |
"""A collection of atom details and associated metadata. |
| 290 |
|
| 291 |
Typically this class contains a collection of atom detail entries that |
| 292 |
represent the atoms in a given flow structure (along with any other needed |
| 293 |
metadata relevant to that flow). |
| 294 |
|
| 295 |
The data contained within this class need **not** be persisted to the |
| 296 |
backend storage in real time. The data in this class will only be |
| 297 |
guaranteed to be persisted when a save (or update) occurs via some backend |
| 298 |
connection. |
| 299 |
|
| 300 |
:ivar meta: A dictionary of meta-data associated with this flow detail. |
| 301 |
""" |
| 302 |
def __init__(self, name, uuid): |
| 303 |
self._uuid = uuid |
| 304 |
self._name = name |
| 305 |
self._atomdetails_by_id = {} |
| 306 |
# TODO(bnemec): This should be documented as an ivar, but can't be due |
| 307 |
# to https://github.com/sphinx-doc/sphinx/issues/2549 |
| 308 |
#: The state of the flow associated with this flow detail. |
| 309 |
self.state = None |
| 310 |
self.meta = {} |
| 311 |
|
| 312 |
def update(self, fd): |
| 313 |
"""Updates the objects state to be the same as the given one. |
| 314 |
|
| 315 |
This will assign the private and public attributes of the given |
| 316 |
flow detail directly to this object (replacing any existing |
| 317 |
attributes in this object; even if they are the **same**). |
| 318 |
|
| 319 |
NOTE(harlowja): If the provided object is this object itself |
| 320 |
then **no** update is done. |
| 321 |
|
| 322 |
:returns: this flow detail |
| 323 |
:rtype: :py:class:`.FlowDetail` |
| 324 |
""" |
| 325 |
if fd is self: |
| 326 |
return self |
| 327 |
self._atomdetails_by_id = fd._atomdetails_by_id |
| 328 |
self.state = fd.state |
| 329 |
self.meta = fd.meta |
| 330 |
return self |
| 331 |
|
| 332 |
def pformat(self, indent=0, linesep=os.linesep): |
| 333 |
"""Pretty formats this flow detail into a string. |
| 334 |
|
| 335 |
>>> from oslo_utils import uuidutils |
| 336 |
>>> from taskflow.persistence import models |
| 337 |
>>> flow_detail = models.FlowDetail("example", |
| 338 |
... uuid=uuidutils.generate_uuid()) |
| 339 |
>>> print(flow_detail.pformat()) |
| 340 |
FlowDetail: 'example' |
| 341 |
- uuid = ... |
| 342 |
- state = ... |
| 343 |
""" |
| 344 |
cls_name = self.__class__.__name__ |
| 345 |
lines = ["{}{}: '{}'".format(" " * indent, cls_name, self.name)] |
| 346 |
lines.extend(_format_shared(self, indent=indent + 1)) |
| 347 |
lines.extend(_format_meta(self.meta, indent=indent + 1)) |
| 348 |
for atom_detail in self: |
| 349 |
lines.append(atom_detail.pformat(indent=indent + 1, |
| 350 |
linesep=linesep)) |
| 351 |
return linesep.join(lines) |
| 352 |
|
| 353 |
def merge(self, fd, deep_copy=False): |
| 354 |
"""Merges the current object state with the given one's state. |
| 355 |
|
| 356 |
If ``deep_copy`` is provided as truthy then the |
| 357 |
local object will use ``copy.deepcopy`` to replace this objects |
| 358 |
local attributes with the provided objects attributes (**only** if |
| 359 |
there is a difference between this objects attributes and the |
| 360 |
provided attributes). If ``deep_copy`` is falsey (the default) then a |
| 361 |
reference copy will occur instead when a difference is detected. |
| 362 |
|
| 363 |
NOTE(harlowja): If the provided object is this object itself |
| 364 |
then **no** merging is done. Also this does **not** merge the atom |
| 365 |
details contained in either. |
| 366 |
|
| 367 |
:returns: this flow detail (freshly merged with the incoming object) |
| 368 |
:rtype: :py:class:`.FlowDetail` |
| 369 |
""" |
| 370 |
if fd is self: |
| 371 |
return self |
| 372 |
copy_fn = _copy_function(deep_copy) |
| 373 |
if self.meta != fd.meta: |
| 374 |
self.meta = copy_fn(fd.meta) |
| 375 |
if self.state != fd.state: |
| 376 |
# NOTE(imelnikov): states are just strings, no need to copy. |
| 377 |
self.state = fd.state |
| 378 |
return self |
| 379 |
|
| 380 |
def copy(self, retain_contents=True): |
| 381 |
"""Copies this flow detail. |
| 382 |
|
| 383 |
Creates a shallow copy of this flow detail. If this detail contains |
| 384 |
flow details and ``retain_contents`` is truthy (the default) then |
| 385 |
the atom details container will be shallow copied (the atom details |
| 386 |
contained there-in will **not** be copied). If ``retain_contents`` is |
| 387 |
falsey then the copied flow detail will have **no** contained atom |
| 388 |
details (but it will have the rest of the local objects attributes |
| 389 |
copied). |
| 390 |
|
| 391 |
:returns: a new flow detail |
| 392 |
:rtype: :py:class:`.FlowDetail` |
| 393 |
""" |
| 394 |
clone = copy.copy(self) |
| 395 |
if not retain_contents: |
| 396 |
clone._atomdetails_by_id = {} |
| 397 |
else: |
| 398 |
clone._atomdetails_by_id = self._atomdetails_by_id.copy() |
| 399 |
if self.meta: |
| 400 |
clone.meta = self.meta.copy() |
| 401 |
return clone |
| 402 |
|
| 403 |
def to_dict(self): |
| 404 |
"""Translates the internal state of this object to a ``dict``. |
| 405 |
|
| 406 |
NOTE(harlowja): The returned ``dict`` does **not** include any |
| 407 |
contained atom details. |
| 408 |
|
| 409 |
:returns: this flow detail in ``dict`` form |
| 410 |
""" |
| 411 |
return { |
| 412 |
'name': self.name, |
| 413 |
'meta': self.meta, |
| 414 |
'state': self.state, |
| 415 |
'uuid': self.uuid, |
| 416 |
} |
| 417 |
|
| 418 |
@classmethod |
| 419 |
def from_dict(cls, data): |
| 420 |
"""Translates the given ``dict`` into an instance of this class. |
| 421 |
|
| 422 |
NOTE(harlowja): the ``dict`` provided should come from a prior |
| 423 |
call to :meth:`.to_dict`. |
| 424 |
|
| 425 |
:returns: a new flow detail |
| 426 |
:rtype: :py:class:`.FlowDetail` |
| 427 |
""" |
| 428 |
obj = cls(data['name'], data['uuid']) |
| 429 |
obj.state = data.get('state') |
| 430 |
obj.meta = _fix_meta(data) |
| 431 |
return obj |
| 432 |
|
| 433 |
def add(self, ad): |
| 434 |
"""Adds a new atom detail into this flow detail. |
| 435 |
|
| 436 |
NOTE(harlowja): if an existing atom detail exists with the same |
| 437 |
uuid the existing one will be overwritten with the newly provided |
| 438 |
one. |
| 439 |
|
| 440 |
Does not *guarantee* that the details will be immediately saved. |
| 441 |
""" |
| 442 |
self._atomdetails_by_id[ad.uuid] = ad |
| 443 |
|
| 444 |
def find(self, ad_uuid): |
| 445 |
"""Locate the atom detail corresponding to the given uuid. |
| 446 |
|
| 447 |
:returns: the atom detail with that uuid |
| 448 |
:rtype: :py:class:`.AtomDetail` (or ``None`` if not found) |
| 449 |
""" |
| 450 |
return self._atomdetails_by_id.get(ad_uuid) |
| 451 |
|
| 452 |
@property |
| 453 |
def uuid(self): |
| 454 |
"""The unique identifer of this flow detail.""" |
| 455 |
return self._uuid |
| 456 |
|
| 457 |
@property |
| 458 |
def name(self): |
| 459 |
"""The name of this flow detail.""" |
| 460 |
return self._name |
| 461 |
|
| 462 |
def __iter__(self): |
| 463 |
yield from self._atomdetails_by_id.values() |
| 464 |
|
| 465 |
def __len__(self): |
| 466 |
return len(self._atomdetails_by_id) |
| 467 |
|
| 468 |
|
| 469 |
class AtomDetail(metaclass=abc.ABCMeta): |
| 470 |
"""A collection of atom specific runtime information and metadata. |
| 471 |
|
| 472 |
This is a base **abstract** class that contains attributes that are used |
| 473 |
to connect a atom to the persistence layer before, during, or after it is |
| 474 |
running. It includes any results it may have produced, any state that it |
| 475 |
may be in (for example ``FAILURE``), any exception that occurred when |
| 476 |
running, and any associated stacktrace that may have occurring during an |
| 477 |
exception being thrown. It may also contain any other metadata that |
| 478 |
should also be stored along-side the details about the connected atom. |
| 479 |
|
| 480 |
The data contained within this class need **not** be persisted to the |
| 481 |
backend storage in real time. The data in this class will only be |
| 482 |
guaranteed to be persisted when a save (or update) occurs via some backend |
| 483 |
connection. |
| 484 |
|
| 485 |
:ivar intention: The execution strategy of the atom associated |
| 486 |
with this atom detail (used by an engine/others to |
| 487 |
determine if the associated atom needs to be |
| 488 |
executed, reverted, retried and so-on). |
| 489 |
:ivar meta: A dictionary of meta-data associated with this atom detail. |
| 490 |
:ivar version: A version tuple or string that represents the |
| 491 |
atom version this atom detail is associated with (typically |
| 492 |
used for introspection and any data migration |
| 493 |
strategies). |
| 494 |
:ivar results: Any results the atom produced from either its |
| 495 |
``execute`` method or from other sources. |
| 496 |
:ivar revert_results: Any results the atom produced from either its |
| 497 |
``revert`` method or from other sources. |
| 498 |
:ivar AtomDetail.failure: If the atom failed (due to its ``execute`` method |
| 499 |
raising) this will be a |
| 500 |
:py:class:`~taskflow.types.failure.Failure` object that |
| 501 |
represents that failure (if there was no failure this |
| 502 |
will be set to none). |
| 503 |
:ivar revert_failure: If the atom failed (possibly due to its ``revert`` |
| 504 |
method raising) this will be a |
| 505 |
:py:class:`~taskflow.types.failure.Failure` object |
| 506 |
that represents that failure (if there was no |
| 507 |
failure this will be set to none). |
| 508 |
""" |
| 509 |
|
| 510 |
def __init__(self, name, uuid): |
| 511 |
self._uuid = uuid |
| 512 |
self._name = name |
| 513 |
# TODO(bnemec): This should be documented as an ivar, but can't be due |
| 514 |
# to https://github.com/sphinx-doc/sphinx/issues/2549 |
| 515 |
#: The state of the atom associated with this atom detail. |
| 516 |
self.state = None |
| 517 |
self.intention = states.EXECUTE |
| 518 |
self.results = None |
| 519 |
self.failure = None |
| 520 |
self.revert_results = None |
| 521 |
self.revert_failure = None |
| 522 |
self.meta = {} |
| 523 |
self.version = None |
| 524 |
|
| 525 |
@property |
| 526 |
def last_results(self): |
| 527 |
"""Gets the atoms last result. |
| 528 |
|
| 529 |
If the atom has produced many results (for example if it has been |
| 530 |
retried, reverted, executed and ...) this returns the last one of |
| 531 |
many results. |
| 532 |
""" |
| 533 |
return self.results |
| 534 |
|
| 535 |
def update(self, ad): |
| 536 |
"""Updates the object's state to be the same as the given one. |
| 537 |
|
| 538 |
This will assign the private and public attributes of the given |
| 539 |
atom detail directly to this object (replacing any existing |
| 540 |
attributes in this object; even if they are the **same**). |
| 541 |
|
| 542 |
NOTE(harlowja): If the provided object is this object itself |
| 543 |
then **no** update is done. |
| 544 |
|
| 545 |
:returns: this atom detail |
| 546 |
:rtype: :py:class:`.AtomDetail` |
| 547 |
""" |
| 548 |
if ad is self: |
| 549 |
return self |
| 550 |
self.state = ad.state |
| 551 |
self.intention = ad.intention |
| 552 |
self.meta = ad.meta |
| 553 |
self.failure = ad.failure |
| 554 |
self.results = ad.results |
| 555 |
self.revert_results = ad.revert_results |
| 556 |
self.revert_failure = ad.revert_failure |
| 557 |
self.version = ad.version |
| 558 |
return self |
| 559 |
|
| 560 |
@abc.abstractmethod |
| 561 |
def merge(self, other, deep_copy=False): |
| 562 |
"""Merges the current object state with the given ones state. |
| 563 |
|
| 564 |
If ``deep_copy`` is provided as truthy then the |
| 565 |
local object will use ``copy.deepcopy`` to replace this objects |
| 566 |
local attributes with the provided objects attributes (**only** if |
| 567 |
there is a difference between this objects attributes and the |
| 568 |
provided attributes). If ``deep_copy`` is falsey (the default) then a |
| 569 |
reference copy will occur instead when a difference is detected. |
| 570 |
|
| 571 |
NOTE(harlowja): If the provided object is this object itself |
| 572 |
then **no** merging is done. Do note that **no** results are merged |
| 573 |
in this method. That operation **must** to be the responsibilty of |
| 574 |
subclasses to implement and override this abstract method |
| 575 |
and provide that merging themselves as they see fit. |
| 576 |
|
| 577 |
:returns: this atom detail (freshly merged with the incoming object) |
| 578 |
:rtype: :py:class:`.AtomDetail` |
| 579 |
""" |
| 580 |
copy_fn = _copy_function(deep_copy) |
| 581 |
# NOTE(imelnikov): states and intentions are just strings, |
| 582 |
# so there is no need to copy them (strings are immutable in python). |
| 583 |
self.state = other.state |
| 584 |
self.intention = other.intention |
| 585 |
if self.failure != other.failure: |
| 586 |
# NOTE(imelnikov): we can't just deep copy Failures, as they |
| 587 |
# contain tracebacks, which are not copyable. |
| 588 |
if other.failure: |
| 589 |
if deep_copy: |
| 590 |
self.failure = other.failure.copy() |
| 591 |
else: |
| 592 |
self.failure = other.failure |
| 593 |
else: |
| 594 |
self.failure = None |
| 595 |
if self.revert_failure != other.revert_failure: |
| 596 |
# NOTE(imelnikov): we can't just deep copy Failures, as they |
| 597 |
# contain tracebacks, which are not copyable. |
| 598 |
if other.revert_failure: |
| 599 |
if deep_copy: |
| 600 |
self.revert_failure = other.revert_failure.copy() |
| 601 |
else: |
| 602 |
self.revert_failure = other.revert_failure |
| 603 |
else: |
| 604 |
self.revert_failure = None |
| 605 |
if self.meta != other.meta: |
| 606 |
self.meta = copy_fn(other.meta) |
| 607 |
if self.version != other.version: |
| 608 |
self.version = copy_fn(other.version) |
| 609 |
return self |
| 610 |
|
| 611 |
@abc.abstractmethod |
| 612 |
def put(self, state, result): |
| 613 |
"""Puts a result (acquired in the given state) into this detail.""" |
| 614 |
|
| 615 |
def to_dict(self): |
| 616 |
"""Translates the internal state of this object to a ``dict``. |
| 617 |
|
| 618 |
:returns: this atom detail in ``dict`` form |
| 619 |
""" |
| 620 |
if self.failure: |
| 621 |
failure = self.failure.to_dict() |
| 622 |
else: |
| 623 |
failure = None |
| 624 |
if self.revert_failure: |
| 625 |
revert_failure = self.revert_failure.to_dict() |
| 626 |
else: |
| 627 |
revert_failure = None |
| 628 |
return { |
| 629 |
'failure': failure, |
| 630 |
'revert_failure': revert_failure, |
| 631 |
'meta': self.meta, |
| 632 |
'name': self.name, |
| 633 |
'results': self.results, |
| 634 |
'revert_results': self.revert_results, |
| 635 |
'state': self.state, |
| 636 |
'version': self.version, |
| 637 |
'intention': self.intention, |
| 638 |
'uuid': self.uuid, |
| 639 |
} |
| 640 |
|
| 641 |
@classmethod |
| 642 |
def from_dict(cls, data): |
| 643 |
"""Translates the given ``dict`` into an instance of this class. |
| 644 |
|
| 645 |
NOTE(harlowja): the ``dict`` provided should come from a prior |
| 646 |
call to :meth:`.to_dict`. |
| 647 |
|
| 648 |
:returns: a new atom detail |
| 649 |
:rtype: :py:class:`.AtomDetail` |
| 650 |
""" |
| 651 |
obj = cls(data['name'], data['uuid']) |
| 652 |
obj.state = data.get('state') |
| 653 |
obj.intention = data.get('intention') |
| 654 |
obj.results = data.get('results') |
| 655 |
obj.revert_results = data.get('revert_results') |
| 656 |
obj.version = data.get('version') |
| 657 |
obj.meta = _fix_meta(data) |
| 658 |
failure = data.get('failure') |
| 659 |
if failure: |
| 660 |
obj.failure = ft.Failure.from_dict(failure) |
| 661 |
revert_failure = data.get('revert_failure') |
| 662 |
if revert_failure: |
| 663 |
obj.revert_failure = ft.Failure.from_dict(revert_failure) |
| 664 |
return obj |
| 665 |
|
| 666 |
@property |
| 667 |
def uuid(self): |
| 668 |
"""The unique identifer of this atom detail.""" |
| 669 |
return self._uuid |
| 670 |
|
| 671 |
@property |
| 672 |
def name(self): |
| 673 |
"""The name of this atom detail.""" |
| 674 |
return self._name |
| 675 |
|
| 676 |
@abc.abstractmethod |
| 677 |
def reset(self, state): |
| 678 |
"""Resets this atom detail and sets ``state`` attribute value.""" |
| 679 |
|
| 680 |
@abc.abstractmethod |
| 681 |
def copy(self): |
| 682 |
"""Copies this atom detail.""" |
| 683 |
|
| 684 |
def pformat(self, indent=0, linesep=os.linesep): |
| 685 |
"""Pretty formats this atom detail into a string.""" |
| 686 |
cls_name = self.__class__.__name__ |
| 687 |
lines = ["{}{}: '{}'".format(" " * (indent), cls_name, self.name)] |
| 688 |
lines.extend(_format_shared(self, indent=indent + 1)) |
| 689 |
lines.append("%s- version = %s" |
| 690 |
% (" " * (indent + 1), misc.get_version_string(self))) |
| 691 |
lines.append("%s- results = %s" |
| 692 |
% (" " * (indent + 1), self.results)) |
| 693 |
lines.append("{}- failure = {}".format(" " * (indent + 1), |
| 694 |
bool(self.failure))) |
| 695 |
lines.extend(_format_meta(self.meta, indent=indent + 1)) |
| 696 |
return linesep.join(lines) |
| 697 |
|
| 698 |
|
| 699 |
class TaskDetail(AtomDetail): |
| 700 |
"""A task detail (an atom detail typically associated with a |tt| atom). |
| 701 |
|
| 702 |
.. |tt| replace:: :py:class:`~taskflow.task.Task` |
| 703 |
""" |
| 704 |
|
| 705 |
def reset(self, state): |
| 706 |
"""Resets this task detail and sets ``state`` attribute value. |
| 707 |
|
| 708 |
This sets any previously set ``results``, ``failure``, |
| 709 |
and ``revert_results`` attributes back to ``None`` and sets the |
| 710 |
state to the provided one, as well as setting this task |
| 711 |
details ``intention`` attribute to ``EXECUTE``. |
| 712 |
""" |
| 713 |
self.results = None |
| 714 |
self.failure = None |
| 715 |
self.revert_results = None |
| 716 |
self.revert_failure = None |
| 717 |
self.state = state |
| 718 |
self.intention = states.EXECUTE |
| 719 |
|
| 720 |
def put(self, state, result): |
| 721 |
"""Puts a result (acquired in the given state) into this detail. |
| 722 |
|
| 723 |
Returns whether this object was modified (or whether it was not). |
| 724 |
""" |
| 725 |
was_altered = False |
| 726 |
if state != self.state: |
| 727 |
self.state = state |
| 728 |
was_altered = True |
| 729 |
if state == states.REVERT_FAILURE: |
| 730 |
if self.revert_failure != result: |
| 731 |
self.revert_failure = result |
| 732 |
was_altered = True |
| 733 |
if not _is_all_none(self.results, self.revert_results): |
| 734 |
self.results = None |
| 735 |
self.revert_results = None |
| 736 |
was_altered = True |
| 737 |
elif state == states.FAILURE: |
| 738 |
if self.failure != result: |
| 739 |
self.failure = result |
| 740 |
was_altered = True |
| 741 |
if not _is_all_none(self.results, self.revert_results, |
| 742 |
self.revert_failure): |
| 743 |
self.results = None |
| 744 |
self.revert_results = None |
| 745 |
self.revert_failure = None |
| 746 |
was_altered = True |
| 747 |
elif state == states.SUCCESS: |
| 748 |
if not _is_all_none(self.revert_results, self.revert_failure, |
| 749 |
self.failure): |
| 750 |
self.revert_results = None |
| 751 |
self.revert_failure = None |
| 752 |
self.failure = None |
| 753 |
was_altered = True |
| 754 |
# We don't really have the ability to determine equality of |
| 755 |
# task (user) results at the current time, without making |
| 756 |
# potentially bad guesses, so assume the task detail always needs |
| 757 |
# to be saved if they are not exactly equivalent... |
| 758 |
if result is not self.results: |
| 759 |
self.results = result |
| 760 |
was_altered = True |
| 761 |
elif state == states.REVERTED: |
| 762 |
if not _is_all_none(self.revert_failure): |
| 763 |
self.revert_failure = None |
| 764 |
was_altered = True |
| 765 |
if result is not self.revert_results: |
| 766 |
self.revert_results = result |
| 767 |
was_altered = True |
| 768 |
return was_altered |
| 769 |
|
| 770 |
def merge(self, other, deep_copy=False): |
| 771 |
"""Merges the current task detail with the given one. |
| 772 |
|
| 773 |
NOTE(harlowja): This merge does **not** copy and replace |
| 774 |
the ``results`` or ``revert_results`` if it differs. Instead the |
| 775 |
current objects ``results`` and ``revert_results`` attributes directly |
| 776 |
becomes (via assignment) the other objects attributes. Also note that |
| 777 |
if the provided object is this object itself then **no** merging is |
| 778 |
done. |
| 779 |
|
| 780 |
See: https://bugs.launchpad.net/taskflow/+bug/1452978 for |
| 781 |
what happens if this is copied at a deeper level (for example by |
| 782 |
using ``copy.deepcopy`` or by using ``copy.copy``). |
| 783 |
|
| 784 |
:returns: this task detail (freshly merged with the incoming object) |
| 785 |
:rtype: :py:class:`.TaskDetail` |
| 786 |
""" |
| 787 |
if not isinstance(other, TaskDetail): |
| 788 |
raise exc.NotImplementedError("Can only merge with other" |
| 789 |
" task details") |
| 790 |
if other is self: |
| 791 |
return self |
| 792 |
super().merge(other, deep_copy=deep_copy) |
| 793 |
self.results = other.results |
| 794 |
self.revert_results = other.revert_results |
| 795 |
return self |
| 796 |
|
| 797 |
def copy(self): |
| 798 |
"""Copies this task detail. |
| 799 |
|
| 800 |
Creates a shallow copy of this task detail (any meta-data and |
| 801 |
version information that this object maintains is shallow |
| 802 |
copied via ``copy.copy``). |
| 803 |
|
| 804 |
NOTE(harlowja): This copy does **not** copy and replace |
| 805 |
the ``results`` or ``revert_results`` attribute if it differs. Instead |
| 806 |
the current objects ``results`` and ``revert_results`` attributes |
| 807 |
directly becomes (via assignment) the cloned objects attributes. |
| 808 |
|
| 809 |
See: https://bugs.launchpad.net/taskflow/+bug/1452978 for |
| 810 |
what happens if this is copied at a deeper level (for example by |
| 811 |
using ``copy.deepcopy`` or by using ``copy.copy``). |
| 812 |
|
| 813 |
:returns: a new task detail |
| 814 |
:rtype: :py:class:`.TaskDetail` |
| 815 |
""" |
| 816 |
clone = copy.copy(self) |
| 817 |
clone.results = self.results |
| 818 |
clone.revert_results = self.revert_results |
| 819 |
if self.meta: |
| 820 |
clone.meta = self.meta.copy() |
| 821 |
if self.version: |
| 822 |
clone.version = copy.copy(self.version) |
| 823 |
return clone |
| 824 |
|
| 825 |
|
| 826 |
class RetryDetail(AtomDetail): |
| 827 |
"""A retry detail (an atom detail typically associated with a |rt| atom). |
| 828 |
|
| 829 |
.. |rt| replace:: :py:class:`~taskflow.retry.Retry` |
| 830 |
""" |
| 831 |
|
| 832 |
def __init__(self, name, uuid): |
| 833 |
super().__init__(name, uuid) |
| 834 |
self.results = [] |
| 835 |
|
| 836 |
def reset(self, state): |
| 837 |
"""Resets this retry detail and sets ``state`` attribute value. |
| 838 |
|
| 839 |
This sets any previously added ``results`` back to an empty list |
| 840 |
and resets the ``failure`` and ``revert_failure`` and |
| 841 |
``revert_results`` attributes back to ``None`` and sets the state |
| 842 |
to the provided one, as well as setting this retry |
| 843 |
details ``intention`` attribute to ``EXECUTE``. |
| 844 |
""" |
| 845 |
self.results = [] |
| 846 |
self.revert_results = None |
| 847 |
self.failure = None |
| 848 |
self.revert_failure = None |
| 849 |
self.state = state |
| 850 |
self.intention = states.EXECUTE |
| 851 |
|
| 852 |
def copy(self): |
| 853 |
"""Copies this retry detail. |
| 854 |
|
| 855 |
Creates a shallow copy of this retry detail (any meta-data and |
| 856 |
version information that this object maintains is shallow |
| 857 |
copied via ``copy.copy``). |
| 858 |
|
| 859 |
NOTE(harlowja): This copy does **not** copy |
| 860 |
the incoming objects ``results`` or ``revert_results`` attributes. |
| 861 |
Instead this objects ``results`` attribute list is iterated over and |
| 862 |
a new list is constructed with each ``(data, failures)`` element in |
| 863 |
that list having its ``failures`` (a dictionary of each named |
| 864 |
:py:class:`~taskflow.types.failure.Failure` object that |
| 865 |
occured) copied but its ``data`` is left untouched. After |
| 866 |
this is done that new list becomes (via assignment) the cloned |
| 867 |
objects ``results`` attribute. The ``revert_results`` is directly |
| 868 |
assigned to the cloned objects ``revert_results`` attribute. |
| 869 |
|
| 870 |
See: https://bugs.launchpad.net/taskflow/+bug/1452978 for |
| 871 |
what happens if the ``data`` in ``results`` is copied at a |
| 872 |
deeper level (for example by using ``copy.deepcopy`` or by |
| 873 |
using ``copy.copy``). |
| 874 |
|
| 875 |
:returns: a new retry detail |
| 876 |
:rtype: :py:class:`.RetryDetail` |
| 877 |
""" |
| 878 |
clone = copy.copy(self) |
| 879 |
results = [] |
| 880 |
# NOTE(imelnikov): we can't just deep copy Failures, as they |
| 881 |
# contain tracebacks, which are not copyable. |
| 882 |
for (data, failures) in self.results: |
| 883 |
copied_failures = {} |
| 884 |
for (key, failure) in failures.items(): |
| 885 |
copied_failures[key] = failure |
| 886 |
results.append((data, copied_failures)) |
| 887 |
clone.results = results |
| 888 |
clone.revert_results = self.revert_results |
| 889 |
if self.meta: |
| 890 |
clone.meta = self.meta.copy() |
| 891 |
if self.version: |
| 892 |
clone.version = copy.copy(self.version) |
| 893 |
return clone |
| 894 |
|
| 895 |
@property |
| 896 |
def last_results(self): |
| 897 |
"""The last result that was produced.""" |
| 898 |
try: |
| 899 |
return self.results[-1][0] |
| 900 |
except IndexError: |
| 901 |
exc.raise_with_cause(exc.NotFound, "Last results not found") |
| 902 |
|
| 903 |
@property |
| 904 |
def last_failures(self): |
| 905 |
"""The last failure dictionary that was produced. |
| 906 |
|
| 907 |
NOTE(harlowja): This is **not** the same as the |
| 908 |
local ``failure`` attribute as the obtained failure dictionary in |
| 909 |
the ``results`` attribute (which is what this returns) is from |
| 910 |
associated atom failures (which is different from the directly |
| 911 |
related failure of the retry unit associated with this |
| 912 |
atom detail). |
| 913 |
""" |
| 914 |
try: |
| 915 |
return self.results[-1][1] |
| 916 |
except IndexError: |
| 917 |
exc.raise_with_cause(exc.NotFound, "Last failures not found") |
| 918 |
|
| 919 |
def put(self, state, result): |
| 920 |
"""Puts a result (acquired in the given state) into this detail. |
| 921 |
|
| 922 |
Returns whether this object was modified (or whether it was not). |
| 923 |
""" |
| 924 |
# Do not clean retry history (only on reset does this happen). |
| 925 |
was_altered = False |
| 926 |
if state != self.state: |
| 927 |
self.state = state |
| 928 |
was_altered = True |
| 929 |
if state == states.REVERT_FAILURE: |
| 930 |
if result != self.revert_failure: |
| 931 |
self.revert_failure = result |
| 932 |
was_altered = True |
| 933 |
if not _is_all_none(self.revert_results): |
| 934 |
self.revert_results = None |
| 935 |
was_altered = True |
| 936 |
elif state == states.FAILURE: |
| 937 |
if result != self.failure: |
| 938 |
self.failure = result |
| 939 |
was_altered = True |
| 940 |
if not _is_all_none(self.revert_results, self.revert_failure): |
| 941 |
self.revert_results = None |
| 942 |
self.revert_failure = None |
| 943 |
was_altered = True |
| 944 |
elif state == states.SUCCESS: |
| 945 |
if not _is_all_none(self.failure, self.revert_failure, |
| 946 |
self.revert_results): |
| 947 |
self.failure = None |
| 948 |
self.revert_failure = None |
| 949 |
self.revert_results = None |
| 950 |
# Track what we produced, so that we can examine it (or avoid |
| 951 |
# using it again). |
| 952 |
self.results.append((result, {})) |
| 953 |
was_altered = True |
| 954 |
elif state == states.REVERTED: |
| 955 |
# We don't really have the ability to determine equality of |
| 956 |
# task (user) results at the current time, without making |
| 957 |
# potentially bad guesses, so assume the retry detail always needs |
| 958 |
# to be saved if they are not exactly equivalent... |
| 959 |
if result is not self.revert_results: |
| 960 |
self.revert_results = result |
| 961 |
was_altered = True |
| 962 |
if not _is_all_none(self.revert_failure): |
| 963 |
self.revert_failure = None |
| 964 |
was_altered = True |
| 965 |
return was_altered |
| 966 |
|
| 967 |
@classmethod |
| 968 |
def from_dict(cls, data): |
| 969 |
"""Translates the given ``dict`` into an instance of this class.""" |
| 970 |
|
| 971 |
def decode_results(results): |
| 972 |
if not results: |
| 973 |
return [] |
| 974 |
new_results = [] |
| 975 |
for (data, failures) in results: |
| 976 |
new_failures = {} |
| 977 |
for (key, data) in failures.items(): |
| 978 |
new_failures[key] = ft.Failure.from_dict(data) |
| 979 |
new_results.append((data, new_failures)) |
| 980 |
return new_results |
| 981 |
|
| 982 |
obj = super().from_dict(data) |
| 983 |
obj.results = decode_results(obj.results) |
| 984 |
return obj |
| 985 |
|
| 986 |
def to_dict(self): |
| 987 |
"""Translates the internal state of this object to a ``dict``.""" |
| 988 |
|
| 989 |
def encode_results(results): |
| 990 |
if not results: |
| 991 |
return [] |
| 992 |
new_results = [] |
| 993 |
for (data, failures) in results: |
| 994 |
new_failures = {} |
| 995 |
for (key, failure) in failures.items(): |
| 996 |
new_failures[key] = failure.to_dict() |
| 997 |
new_results.append((data, new_failures)) |
| 998 |
return new_results |
| 999 |
|
| 1000 |
base = super().to_dict() |
| 1001 |
base['results'] = encode_results(base.get('results')) |
| 1002 |
return base |
| 1003 |
|
| 1004 |
def merge(self, other, deep_copy=False): |
| 1005 |
"""Merges the current retry detail with the given one. |
| 1006 |
|
| 1007 |
NOTE(harlowja): This merge does **not** deep copy |
| 1008 |
the incoming objects ``results`` attribute (if it differs). Instead |
| 1009 |
the incoming objects ``results`` attribute list is **always** iterated |
| 1010 |
over and a new list is constructed with |
| 1011 |
each ``(data, failures)`` element in that list having |
| 1012 |
its ``failures`` (a dictionary of each named |
| 1013 |
:py:class:`~taskflow.types.failure.Failure` objects that |
| 1014 |
occurred) copied but its ``data`` is left untouched. After |
| 1015 |
this is done that new list becomes (via assignment) this |
| 1016 |
objects ``results`` attribute. Also note that if the provided object |
| 1017 |
is this object itself then **no** merging is done. |
| 1018 |
|
| 1019 |
See: https://bugs.launchpad.net/taskflow/+bug/1452978 for |
| 1020 |
what happens if the ``data`` in ``results`` is copied at a |
| 1021 |
deeper level (for example by using ``copy.deepcopy`` or by |
| 1022 |
using ``copy.copy``). |
| 1023 |
|
| 1024 |
:returns: this retry detail (freshly merged with the incoming object) |
| 1025 |
:rtype: :py:class:`.RetryDetail` |
| 1026 |
""" |
| 1027 |
if not isinstance(other, RetryDetail): |
| 1028 |
raise exc.NotImplementedError("Can only merge with other" |
| 1029 |
" retry details") |
| 1030 |
if other is self: |
| 1031 |
return self |
| 1032 |
super().merge(other, deep_copy=deep_copy) |
| 1033 |
results = [] |
| 1034 |
# NOTE(imelnikov): we can't just deep copy Failures, as they |
| 1035 |
# contain tracebacks, which are not copyable. |
| 1036 |
for (data, failures) in other.results: |
| 1037 |
copied_failures = {} |
| 1038 |
for (key, failure) in failures.items(): |
| 1039 |
if deep_copy: |
| 1040 |
copied_failures[key] = failure.copy() |
| 1041 |
else: |
| 1042 |
copied_failures[key] = failure |
| 1043 |
results.append((data, copied_failures)) |
| 1044 |
self.results = results |
| 1045 |
return self |
| 1046 |
|
| 1047 |
|
| 1048 |
_DETAIL_TO_NAME = { |
| 1049 |
RetryDetail: 'RETRY_DETAIL', |
| 1050 |
TaskDetail: 'TASK_DETAIL', |
| 1051 |
} |
| 1052 |
_NAME_TO_DETAIL = {name: cls for (cls, name) in _DETAIL_TO_NAME.items()} |
| 1053 |
ATOM_TYPES = list(_NAME_TO_DETAIL.keys()) |
| 1054 |
|
| 1055 |
|
| 1056 |
def atom_detail_class(atom_type): |
| 1057 |
try: |
| 1058 |
return _NAME_TO_DETAIL[atom_type] |
| 1059 |
except KeyError: |
| 1060 |
raise TypeError("Unknown atom type '%s'" % (atom_type)) |
| 1061 |
|
| 1062 |
|
| 1063 |
def atom_detail_type(atom_detail): |
| 1064 |
try: |
| 1065 |
return _DETAIL_TO_NAME[type(atom_detail)] |
| 1066 |
except KeyError: |
| 1067 |
raise TypeError("Unknown atom '%s' (%s)" |
| 1068 |
% (atom_detail, type(atom_detail))) |