Debian Codegraph @

taskflow/persistence/backends/impl_memory.py

raw
1 # Copyright (C) 2012 Yahoo! Inc. All Rights Reserved.
2 # Copyright (C) 2013 Rackspace Hosting All Rights Reserved.
3 #
4 # Licensed under the Apache License, Version 2.0 (the "License"); you may
5 # not use this file except in compliance with the License. You may obtain
6 # a copy of the License at
7 #
8 # http://www.apache.org/licenses/LICENSE-2.0
9 #
10 # Unless required by applicable law or agreed to in writing, software
11 # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
12 # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
13 # License for the specific language governing permissions and limitations
14 # under the License.
15
16 import contextlib
17 import copy
18 import itertools
19 import posixpath as pp
20
21 import fasteners
22
23 from taskflow import exceptions as exc
24 from taskflow.persistence import path_based
25 from taskflow.types import tree
26
27
28 class FakeInode(tree.Node):
29 """A in-memory filesystem inode-like object."""
30
31 def __init__(self, item, path, value=None):
32 super().__init__(item, path=path, value=value)
33
34
35 class FakeFilesystem:
36 """An in-memory filesystem-like structure.
37
38 This filesystem uses posix style paths **only** so users must be careful
39 to use the ``posixpath`` module instead of the ``os.path`` one which will
40 vary depending on the operating system which the active python is running
41 in (the decision to use ``posixpath`` was to avoid the path variations
42 which are not relevant in an implementation of a in-memory fake
43 filesystem).
44
45 **Not** thread-safe when a single filesystem is mutated at the same
46 time by multiple threads. For example having multiple threads call into
47 :meth:`~taskflow.persistence.backends.impl_memory.FakeFilesystem.clear`
48 at the same time could potentially end badly. It is thread-safe when only
49 :meth:`~taskflow.persistence.backends.impl_memory.FakeFilesystem.get`
50 or other read-only actions (like calling into
51 :meth:`~taskflow.persistence.backends.impl_memory.FakeFilesystem.ls`)
52 are occurring at the same time.
53
54 Example usage:
55
56 >>> from taskflow.persistence.backends import impl_memory
57 >>> fs = impl_memory.FakeFilesystem()
58 >>> fs.ensure_path('/a/b/c')
59 >>> fs['/a/b/c'] = 'd'
60 >>> print(fs['/a/b/c'])
61 d
62 >>> del fs['/a/b/c']
63 >>> fs.ls("/a/b")
64 []
65 >>> fs.get("/a/b/c", 'blob')
66 'blob'
67 """
68
69 #: Root path of the in-memory filesystem.
70 root_path = pp.sep
71
72 @classmethod
73 def normpath(cls, path):
74 """Return a normalized absolutized version of the pathname path."""
75 if not path:
76 raise ValueError("This filesystem can only normalize paths"
77 " that are not empty")
78 if not path.startswith(cls.root_path):
79 raise ValueError("This filesystem can only normalize"
80 " paths that start with %s: '%s' is not"
81 " valid" % (cls.root_path, path))
82 return pp.normpath(path)
83
84 #: Split a pathname into a tuple of ``(head, tail)``.
85 split = staticmethod(pp.split)
86
87 @staticmethod
88 def join(*pieces):
89 """Join many path segments together."""
90 return pp.sep.join(pieces)
91
92 def __init__(self, deep_copy=True):
93 self._root = FakeInode(self.root_path, self.root_path)
94 self._reverse_mapping = {
95 self.root_path: self._root,
96 }
97 if deep_copy:
98 self._copier = copy.deepcopy
99 else:
100 self._copier = copy.copy
101
102 def ensure_path(self, path):
103 """Ensure the path (and parents) exists."""
104 path = self.normpath(path)
105 # Ignore the root path as we already checked for that; and it
106 # will always exist/can't be removed anyway...
107 if path == self._root.item:
108 return
109 node = self._root
110 for piece in self._iter_pieces(path):
111 child_node = node.find(piece, only_direct=True,
112 include_self=False)
113 if child_node is None:
114 child_node = self._insert_child(node, piece)
115 node = child_node
116
117 def _insert_child(self, parent_node, basename, value=None):
118 child_path = self.join(parent_node.metadata['path'], basename)
119 # This avoids getting '//a/b' (duplicated sep at start)...
120 #
121 # Which can happen easily if something like the following is given.
122 # >>> x = ['/', 'b']
123 # >>> pp.sep.join(x)
124 # '//b'
125 if child_path.startswith(pp.sep * 2):
126 child_path = child_path[1:]
127 child_node = FakeInode(basename, child_path, value=value)
128 parent_node.add(child_node)
129 self._reverse_mapping[child_path] = child_node
130 return child_node
131
132 def _fetch_node(self, path, normalized=False):
133 if not normalized:
134 normed_path = self.normpath(path)
135 else:
136 normed_path = path
137 try:
138 return self._reverse_mapping[normed_path]
139 except KeyError:
140 raise exc.NotFound("Path '%s' not found" % path)
141
142 def get(self, path, default=None):
143 """Fetch the value of given path (and return default if not found)."""
144 try:
145 return self._get_item(self.normpath(path))
146 except exc.NotFound:
147 return default
148
149 def _get_item(self, path, links=None):
150 node = self._fetch_node(path, normalized=True)
151 if 'target' in node.metadata:
152 # Follow the link (and watch out for loops)...
153 path = node.metadata['target']
154 if links is None:
155 links = []
156 if path in links:
157 raise ValueError("Recursive link following not"
158 " allowed (loop %s detected)"
159 % (links + [path]))
160 else:
161 links.append(path)
162 return self._get_item(path, links=links)
163 else:
164 return self._copier(node.metadata['value'])
165
166 def _up_to_root_selector(self, root_node, child_node):
167 # Build the path from the child to the root and stop at the
168 # root, and then form a path string...
169 path_pieces = [child_node.item]
170 for parent_node in child_node.path_iter(include_self=False):
171 if parent_node is root_node:
172 break
173 path_pieces.append(parent_node.item)
174 if len(path_pieces) > 1:
175 path_pieces.reverse()
176 return self.join(*path_pieces)
177
178 @staticmethod
179 def _metadata_path_selector(root_node, child_node):
180 return child_node.metadata['path']
181
182 def ls_r(self, path, absolute=False):
183 """Return list of all children of the given path (recursively)."""
184 node = self._fetch_node(path)
185 if absolute:
186 selector_func = self._metadata_path_selector
187 else:
188 selector_func = self._up_to_root_selector
189 return [selector_func(node, child_node)
190 for child_node in node.bfs_iter()]
191
192 def ls(self, path, absolute=False):
193 """Return list of all children of the given path (not recursive)."""
194 node = self._fetch_node(path)
195 if absolute:
196 selector_func = self._metadata_path_selector
197 else:
198 selector_func = self._up_to_root_selector
199 child_node_it = iter(node)
200 return [selector_func(node, child_node)
201 for child_node in child_node_it]
202
203 def clear(self):
204 """Remove all nodes (except the root) from this filesystem."""
205 self._reverse_mapping = {
206 self.root_path: self._root,
207 }
208 for node in list(self._root.reverse_iter()):
209 node.disassociate()
210
211 def delete(self, path, recursive=False):
212 """Deletes a node (optionally its children) from this filesystem."""
213 path = self.normpath(path)
214 node = self._fetch_node(path, normalized=True)
215 if node is self._root and not recursive:
216 raise ValueError("Can not delete '%s'" % self._root.item)
217 if recursive:
218 child_paths = (child.metadata['path'] for child in node.bfs_iter())
219 else:
220 node_child_count = node.child_count()
221 if node_child_count:
222 raise ValueError("Can not delete '%s', it has %s children"
223 % (path, node_child_count))
224 child_paths = []
225 if node is self._root:
226 # Don't drop/pop the root...
227 paths = child_paths
228 drop_nodes = []
229 else:
230 paths = itertools.chain([path], child_paths)
231 drop_nodes = [node]
232 for path in paths:
233 self._reverse_mapping.pop(path, None)
234 for node in drop_nodes:
235 node.disassociate()
236
237 def _iter_pieces(self, path, include_root=False):
238 if path == self._root.item:
239 # Check for this directly as the following doesn't work with
240 # split correctly:
241 #
242 # >>> path = "/"
243 # path.split(pp.sep)
244 # ['', '']
245 parts = []
246 else:
247 parts = path.split(pp.sep)[1:]
248 if include_root:
249 parts.insert(0, self._root.item)
250 yield from parts
251
252 def __delitem__(self, path):
253 self.delete(path, recursive=True)
254
255 @staticmethod
256 def _stringify_node(node):
257 if 'target' in node.metadata:
258 return "{} (link to {})".format(node.item, node.metadata['target'])
259 else:
260 return str(node.item)
261
262 def pformat(self):
263 """Pretty format this in-memory filesystem."""
264 return self._root.pformat(stringify_node=self._stringify_node)
265
266 def symlink(self, src_path, dest_path):
267 """Link the destionation path to the source path."""
268 dest_path = self.normpath(dest_path)
269 src_path = self.normpath(src_path)
270 try:
271 dest_node = self._fetch_node(dest_path, normalized=True)
272 except exc.NotFound:
273 parent_path, basename = self.split(dest_path)
274 parent_node = self._fetch_node(parent_path, normalized=True)
275 dest_node = self._insert_child(parent_node, basename)
276 dest_node.metadata['target'] = src_path
277
278 def __getitem__(self, path):
279 return self._get_item(self.normpath(path))
280
281 def __setitem__(self, path, value):
282 path = self.normpath(path)
283 value = self._copier(value)
284 try:
285 node = self._fetch_node(path, normalized=True)
286 node.metadata.update(value=value)
287 except exc.NotFound:
288 parent_path, basename = self.split(path)
289 parent_node = self._fetch_node(parent_path, normalized=True)
290 self._insert_child(parent_node, basename, value=value)
291
292
293 class MemoryBackend(path_based.PathBasedBackend):
294 """A in-memory (non-persistent) backend.
295
296 This backend writes logbooks, flow details, and atom details to a
297 in-memory filesystem-like structure (rooted by the ``memory``
298 instance variable).
299
300 This backend does *not* provide true transactional semantics. It does
301 guarantee that there will be no inter-thread race conditions when
302 writing and reading by using a read/write locks.
303 """
304
305 #: Default path used when none is provided.
306 DEFAULT_PATH = pp.sep
307
308 def __init__(self, conf=None):
309 super().__init__(conf)
310 self.memory = FakeFilesystem(deep_copy=self._conf.get('deep_copy',
311 True))
312 self.lock = fasteners.ReaderWriterLock()
313
314 def get_connection(self):
315 return Connection(self)
316
317 def close(self):
318 pass
319
320
321 class Connection(path_based.PathBasedConnection):
322 def __init__(self, backend):
323 super().__init__(backend)
324 self.upgrade()
325
326 @contextlib.contextmanager
327 def _memory_lock(self, write=False):
328 if write:
329 lock = self.backend.lock.write_lock
330 else:
331 lock = self.backend.lock.read_lock
332 with lock():
333 try:
334 yield
335 except exc.TaskFlowException:
336 raise
337 except Exception:
338 exc.raise_with_cause(exc.StorageFailure,
339 "Storage backend internal error")
340
341 def _join_path(self, *parts):
342 return pp.join(*parts)
343
344 def _get_item(self, path):
345 with self._memory_lock():
346 return self.backend.memory[path]
347
348 def _set_item(self, path, value, transaction):
349 self.backend.memory[path] = value
350
351 def _del_tree(self, path, transaction):
352 del self.backend.memory[path]
353
354 def _get_children(self, path):
355 with self._memory_lock():
356 return self.backend.memory.ls(path)
357
358 def _ensure_path(self, path):
359 with self._memory_lock(write=True):
360 self.backend.memory.ensure_path(path)
361
362 def _create_link(self, src_path, dest_path, transaction):
363 self.backend.memory.symlink(src_path, dest_path)
364
365 @contextlib.contextmanager
366 def _transaction(self):
367 """This just wraps a global write-lock."""
368 with self._memory_lock(write=True):
369 yield
370
371 def validate(self):
372 pass