Debian Codegraph @

taskflow/persistence/backends/impl_dir.py

raw
1 # Copyright (C) 2012 Yahoo! Inc. All Rights Reserved.
2 # Copyright (C) 2013 Rackspace Hosting All Rights Reserved.
3 #
4 # Licensed under the Apache License, Version 2.0 (the "License"); you may
5 # not use this file except in compliance with the License. You may obtain
6 # a copy of the License at
7 #
8 # http://www.apache.org/licenses/LICENSE-2.0
9 #
10 # Unless required by applicable law or agreed to in writing, software
11 # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
12 # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
13 # License for the specific language governing permissions and limitations
14 # under the License.
15
16 import contextlib
17 import errno
18 import os
19 import shutil
20
21 import cachetools
22 import fasteners
23 from oslo_serialization import jsonutils
24 from oslo_utils import fileutils
25
26 from taskflow import exceptions as exc
27 from taskflow.persistence import path_based
28 from taskflow.utils import misc
29
30
31 @contextlib.contextmanager
32 def _storagefailure_wrapper():
33 try:
34 yield
35 except exc.TaskFlowException:
36 raise
37 except Exception as e:
38 if isinstance(e, (IOError, OSError)) and e.errno == errno.ENOENT:
39 exc.raise_with_cause(exc.NotFound,
40 'Item not found: %s' % e.filename,
41 cause=e)
42 else:
43 exc.raise_with_cause(exc.StorageFailure,
44 "Storage backend internal error", cause=e)
45
46
47 class DirBackend(path_based.PathBasedBackend):
48 """A directory and file based backend.
49
50 This backend does *not* provide true transactional semantics. It does
51 guarantee that there will be no interprocess race conditions when
52 writing and reading by using a consistent hierarchy of file based locks.
53
54 Example configuration::
55
56 conf = {
57 "path": "/tmp/taskflow", # save data to this root directory
58 "max_cache_size": 1024, # keep up-to 1024 entries in memory
59 }
60 """
61
62 DEFAULT_FILE_ENCODING = 'utf-8'
63 """
64 Default encoding used when decoding or encoding files into or from
65 text/unicode into binary or binary into text/unicode.
66 """
67
68 def __init__(self, conf):
69 super().__init__(conf)
70 max_cache_size = self._conf.get('max_cache_size')
71 if max_cache_size is not None:
72 max_cache_size = int(max_cache_size)
73 if max_cache_size < 1:
74 raise ValueError("Maximum cache size must be greater than"
75 " or equal to one")
76 self.file_cache = cachetools.LRUCache(max_cache_size)
77 else:
78 self.file_cache = {}
79 self.encoding = self._conf.get('encoding', self.DEFAULT_FILE_ENCODING)
80 if not self._path:
81 raise ValueError("Empty path is disallowed")
82 self._path = os.path.abspath(self._path)
83 self.lock = fasteners.ReaderWriterLock()
84
85 def get_connection(self):
86 return Connection(self)
87
88 def close(self):
89 pass
90
91
92 class Connection(path_based.PathBasedConnection):
93 def _read_from(self, filename):
94 # This is very similar to the oslo-incubator fileutils module, but
95 # tweaked to not depend on a global cache, as well as tweaked to not
96 # pull-in the oslo logging module (which is a huge pile of code).
97 mtime = os.path.getmtime(filename)
98 cache_info = self.backend.file_cache.setdefault(filename, {})
99 if not cache_info or mtime > cache_info.get('mtime', 0):
100 with open(filename, encoding=self.backend.encoding) as fp:
101 cache_info['data'] = fp.read()
102 cache_info['mtime'] = mtime
103 return cache_info['data']
104
105 def _write_to(self, filename, contents):
106 contents = misc.binary_encode(contents,
107 encoding=self.backend.encoding)
108 with open(filename, 'wb') as fp:
109 fp.write(contents)
110 self.backend.file_cache.pop(filename, None)
111
112 @contextlib.contextmanager
113 def _path_lock(self, path):
114 lockfile = self._join_path(path, 'lock')
115 with fasteners.InterProcessLock(lockfile) as lock:
116 with _storagefailure_wrapper():
117 yield lock
118
119 def _join_path(self, *parts):
120 return os.path.join(*parts)
121
122 def _get_item(self, path):
123 with self._path_lock(path):
124 item_path = self._join_path(path, 'metadata')
125 return misc.decode_json(self._read_from(item_path))
126
127 def _set_item(self, path, value, transaction):
128 with self._path_lock(path):
129 item_path = self._join_path(path, 'metadata')
130 self._write_to(item_path, jsonutils.dumps(value))
131
132 def _del_tree(self, path, transaction):
133 with self._path_lock(path):
134 shutil.rmtree(path)
135
136 def _get_children(self, path):
137 if path == self.book_path:
138 filter_func = os.path.isdir
139 else:
140 filter_func = os.path.islink
141 with _storagefailure_wrapper():
142 return [child for child in os.listdir(path)
143 if filter_func(self._join_path(path, child))]
144
145 def _ensure_path(self, path):
146 with _storagefailure_wrapper():
147 fileutils.ensure_tree(path)
148
149 def _create_link(self, src_path, dest_path, transaction):
150 with _storagefailure_wrapper():
151 try:
152 os.symlink(src_path, dest_path)
153 except OSError as e:
154 if e.errno != errno.EEXIST:
155 raise
156
157 @contextlib.contextmanager
158 def _transaction(self):
159 """This just wraps a global write-lock."""
160 lock = self.backend.lock.write_lock
161 with lock():
162 yield
163
164 def validate(self):
165 with _storagefailure_wrapper():
166 for p in (self.flow_path, self.atom_path, self.book_path):
167 if not os.path.isdir(p):
168 raise RuntimeError("Missing required directory: %s" % (p))