Debian Codegraph @

taskflow/persistence/path_based.py

raw
1 # Copyright (C) 2015 Rackspace Hosting All Rights Reserved.
2 #
3 # Licensed under the Apache License, Version 2.0 (the "License"); you may
4 # not use this file except in compliance with the License. You may obtain
5 # a copy of the License at
6 #
7 # http://www.apache.org/licenses/LICENSE-2.0
8 #
9 # Unless required by applicable law or agreed to in writing, software
10 # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
11 # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
12 # License for the specific language governing permissions and limitations
13 # under the License.
14
15 import abc
16
17 from taskflow import exceptions as exc
18 from taskflow.persistence import base
19 from taskflow.persistence import models
20
21
22 class PathBasedBackend(base.Backend, metaclass=abc.ABCMeta):
23 """Base class for persistence backends that address data by path
24
25 Subclasses of this backend write logbooks, flow details, and atom details
26 to a provided base path in some filesystem-like storage. They will create
27 and store those objects in three key directories (one for logbooks, one
28 for flow details and one for atom details). They create those associated
29 directories and then create files inside those directories that represent
30 the contents of those objects for later reading and writing.
31 """
32
33 #: Default path used when none is provided.
34 DEFAULT_PATH = None
35
36 def __init__(self, conf):
37 super().__init__(conf)
38 self._path = self._conf.get('path', None)
39 if not self._path:
40 self._path = self.DEFAULT_PATH
41
42 @property
43 def path(self):
44 return self._path
45
46
47 class PathBasedConnection(base.Connection, metaclass=abc.ABCMeta):
48 """Base class for path based backend connections."""
49
50 def __init__(self, backend):
51 self._backend = backend
52 self._book_path = self._join_path(backend.path, "books")
53 self._flow_path = self._join_path(backend.path, "flow_details")
54 self._atom_path = self._join_path(backend.path, "atom_details")
55
56 @staticmethod
57 def _serialize(obj):
58 if isinstance(obj, models.LogBook):
59 return obj.to_dict(marshal_time=True)
60 elif isinstance(obj, models.FlowDetail):
61 return obj.to_dict()
62 elif isinstance(obj, models.AtomDetail):
63 return base._format_atom(obj)
64 else:
65 raise exc.StorageFailure("Invalid storage class %s" % type(obj))
66
67 @staticmethod
68 def _deserialize(cls, data):
69 if issubclass(cls, models.LogBook):
70 return cls.from_dict(data, unmarshal_time=True)
71 elif issubclass(cls, models.FlowDetail):
72 return cls.from_dict(data)
73 elif issubclass(cls, models.AtomDetail):
74 atom_class = models.atom_detail_class(data['type'])
75 return atom_class.from_dict(data['atom'])
76 else:
77 raise exc.StorageFailure("Invalid storage class %s" % cls)
78
79 @property
80 def backend(self):
81 return self._backend
82
83 @property
84 def book_path(self):
85 return self._book_path
86
87 @property
88 def flow_path(self):
89 return self._flow_path
90
91 @property
92 def atom_path(self):
93 return self._atom_path
94
95 @abc.abstractmethod
96 def _join_path(self, *parts):
97 """Accept path parts, and return a joined path"""
98
99 @abc.abstractmethod
100 def _get_item(self, path):
101 """Fetch a single item from the backend"""
102
103 @abc.abstractmethod
104 def _set_item(self, path, value, transaction):
105 """Write a single item to the backend"""
106
107 @abc.abstractmethod
108 def _del_tree(self, path, transaction):
109 """Recursively deletes a folder from the backend."""
110
111 @abc.abstractmethod
112 def _get_children(self, path):
113 """Get a list of child items of a path"""
114
115 @abc.abstractmethod
116 def _ensure_path(self, path):
117 """Recursively ensure that a path (folder) in the backend exists"""
118
119 @abc.abstractmethod
120 def _create_link(self, src_path, dest_path, transaction):
121 """Create a symlink-like link between two paths"""
122
123 @abc.abstractmethod
124 def _transaction(self):
125 """Context manager that yields a transaction"""
126
127 def _get_obj_path(self, obj):
128 if isinstance(obj, models.LogBook):
129 path = self.book_path
130 elif isinstance(obj, models.FlowDetail):
131 path = self.flow_path
132 elif isinstance(obj, models.AtomDetail):
133 path = self.atom_path
134 else:
135 raise exc.StorageFailure("Invalid storage class %s" % type(obj))
136 return self._join_path(path, obj.uuid)
137
138 def _update_object(self, obj, transaction, ignore_missing=False):
139 path = self._get_obj_path(obj)
140 try:
141 item_data = self._get_item(path)
142 existing_obj = self._deserialize(type(obj), item_data)
143 obj = existing_obj.merge(obj)
144 except exc.NotFound:
145 if not ignore_missing:
146 raise
147 self._set_item(path, self._serialize(obj), transaction)
148 return obj
149
150 def get_logbooks(self, lazy=False):
151 for book_uuid in self._get_children(self.book_path):
152 yield self.get_logbook(book_uuid, lazy=lazy)
153
154 def get_logbook(self, book_uuid, lazy=False):
155 book_path = self._join_path(self.book_path, book_uuid)
156 book_data = self._get_item(book_path)
157 book = self._deserialize(models.LogBook, book_data)
158 if not lazy:
159 for flow_details in self.get_flows_for_book(book_uuid):
160 book.add(flow_details)
161 return book
162
163 def save_logbook(self, book):
164 book_path = self._get_obj_path(book)
165 with self._transaction() as transaction:
166 self._update_object(book, transaction, ignore_missing=True)
167 for flow_details in book:
168 flow_path = self._get_obj_path(flow_details)
169 link_path = self._join_path(book_path, flow_details.uuid)
170 self._do_update_flow_details(flow_details, transaction,
171 ignore_missing=True)
172 self._create_link(flow_path, link_path, transaction)
173 return book
174
175 def get_flows_for_book(self, book_uuid, lazy=False):
176 book_path = self._join_path(self.book_path, book_uuid)
177 for flow_uuid in self._get_children(book_path):
178 yield self.get_flow_details(flow_uuid, lazy)
179
180 def get_flow_details(self, flow_uuid, lazy=False):
181 flow_path = self._join_path(self.flow_path, flow_uuid)
182 flow_data = self._get_item(flow_path)
183 flow_details = self._deserialize(models.FlowDetail, flow_data)
184 if not lazy:
185 for atom_details in self.get_atoms_for_flow(flow_uuid):
186 flow_details.add(atom_details)
187 return flow_details
188
189 def _do_update_flow_details(self, flow_detail, transaction,
190 ignore_missing=False):
191 flow_path = self._get_obj_path(flow_detail)
192 self._update_object(flow_detail, transaction,
193 ignore_missing=ignore_missing)
194 for atom_details in flow_detail:
195 atom_path = self._get_obj_path(atom_details)
196 link_path = self._join_path(flow_path, atom_details.uuid)
197 self._create_link(atom_path, link_path, transaction)
198 self._update_object(atom_details, transaction, ignore_missing=True)
199 return flow_detail
200
201 def update_flow_details(self, flow_detail, ignore_missing=False):
202 with self._transaction() as transaction:
203 return self._do_update_flow_details(flow_detail, transaction,
204 ignore_missing=ignore_missing)
205
206 def get_atoms_for_flow(self, flow_uuid):
207 flow_path = self._join_path(self.flow_path, flow_uuid)
208 for atom_uuid in self._get_children(flow_path):
209 yield self.get_atom_details(atom_uuid)
210
211 def get_atom_details(self, atom_uuid):
212 atom_path = self._join_path(self.atom_path, atom_uuid)
213 atom_data = self._get_item(atom_path)
214 return self._deserialize(models.AtomDetail, atom_data)
215
216 def update_atom_details(self, atom_detail, ignore_missing=False):
217 with self._transaction() as transaction:
218 return self._update_object(atom_detail, transaction,
219 ignore_missing=ignore_missing)
220
221 def _do_destroy_logbook(self, book_uuid, transaction):
222 book_path = self._join_path(self.book_path, book_uuid)
223 for flow_uuid in self._get_children(book_path):
224 flow_path = self._join_path(self.flow_path, flow_uuid)
225 for atom_uuid in self._get_children(flow_path):
226 atom_path = self._join_path(self.atom_path, atom_uuid)
227 self._del_tree(atom_path, transaction)
228 self._del_tree(flow_path, transaction)
229 self._del_tree(book_path, transaction)
230
231 def destroy_logbook(self, book_uuid):
232 with self._transaction() as transaction:
233 return self._do_destroy_logbook(book_uuid, transaction)
234
235 def clear_all(self):
236 with self._transaction() as transaction:
237 for path in (self.book_path, self.flow_path, self.atom_path):
238 self._del_tree(path, transaction)
239
240 def upgrade(self):
241 for path in (self.book_path, self.flow_path, self.atom_path):
242 self._ensure_path(path)
243
244 def close(self):
245 pass