| 1 |
# Copyright (C) 2012 Yahoo! Inc. All Rights Reserved. |
| 2 |
# Copyright (C) 2013 Rackspace Hosting All Rights Reserved. |
| 3 |
# |
| 4 |
# Licensed under the Apache License, Version 2.0 (the "License"); you may |
| 5 |
# not use this file except in compliance with the License. You may obtain |
| 6 |
# a copy of the License at |
| 7 |
# |
| 8 |
# http://www.apache.org/licenses/LICENSE-2.0 |
| 9 |
# |
| 10 |
# Unless required by applicable law or agreed to in writing, software |
| 11 |
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT |
| 12 |
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the |
| 13 |
# License for the specific language governing permissions and limitations |
| 14 |
# under the License. |
| 15 |
|
| 16 |
import contextlib |
| 17 |
import errno |
| 18 |
import os |
| 19 |
import shutil |
| 20 |
|
| 21 |
import cachetools |
| 22 |
import fasteners |
| 23 |
from oslo_serialization import jsonutils |
| 24 |
from oslo_utils import fileutils |
| 25 |
|
| 26 |
from taskflow import exceptions as exc |
| 27 |
from taskflow.persistence import path_based |
| 28 |
from taskflow.utils import misc |
| 29 |
|
| 30 |
|
| 31 |
@contextlib.contextmanager |
| 32 |
def _storagefailure_wrapper(): |
| 33 |
try: |
| 34 |
yield |
| 35 |
except exc.TaskFlowException: |
| 36 |
raise |
| 37 |
except Exception as e: |
| 38 |
if isinstance(e, (IOError, OSError)) and e.errno == errno.ENOENT: |
| 39 |
exc.raise_with_cause(exc.NotFound, |
| 40 |
'Item not found: %s' % e.filename, |
| 41 |
cause=e) |
| 42 |
else: |
| 43 |
exc.raise_with_cause(exc.StorageFailure, |
| 44 |
"Storage backend internal error", cause=e) |
| 45 |
|
| 46 |
|
| 47 |
class DirBackend(path_based.PathBasedBackend): |
| 48 |
"""A directory and file based backend. |
| 49 |
|
| 50 |
This backend does *not* provide true transactional semantics. It does |
| 51 |
guarantee that there will be no interprocess race conditions when |
| 52 |
writing and reading by using a consistent hierarchy of file based locks. |
| 53 |
|
| 54 |
Example configuration:: |
| 55 |
|
| 56 |
conf = { |
| 57 |
"path": "/tmp/taskflow", # save data to this root directory |
| 58 |
"max_cache_size": 1024, # keep up-to 1024 entries in memory |
| 59 |
} |
| 60 |
""" |
| 61 |
|
| 62 |
DEFAULT_FILE_ENCODING = 'utf-8' |
| 63 |
""" |
| 64 |
Default encoding used when decoding or encoding files into or from |
| 65 |
text/unicode into binary or binary into text/unicode. |
| 66 |
""" |
| 67 |
|
| 68 |
def __init__(self, conf): |
| 69 |
super().__init__(conf) |
| 70 |
max_cache_size = self._conf.get('max_cache_size') |
| 71 |
if max_cache_size is not None: |
| 72 |
max_cache_size = int(max_cache_size) |
| 73 |
if max_cache_size < 1: |
| 74 |
raise ValueError("Maximum cache size must be greater than" |
| 75 |
" or equal to one") |
| 76 |
self.file_cache = cachetools.LRUCache(max_cache_size) |
| 77 |
else: |
| 78 |
self.file_cache = {} |
| 79 |
self.encoding = self._conf.get('encoding', self.DEFAULT_FILE_ENCODING) |
| 80 |
if not self._path: |
| 81 |
raise ValueError("Empty path is disallowed") |
| 82 |
self._path = os.path.abspath(self._path) |
| 83 |
self.lock = fasteners.ReaderWriterLock() |
| 84 |
|
| 85 |
def get_connection(self): |
| 86 |
return Connection(self) |
| 87 |
|
| 88 |
def close(self): |
| 89 |
pass |
| 90 |
|
| 91 |
|
| 92 |
class Connection(path_based.PathBasedConnection): |
| 93 |
def _read_from(self, filename): |
| 94 |
# This is very similar to the oslo-incubator fileutils module, but |
| 95 |
# tweaked to not depend on a global cache, as well as tweaked to not |
| 96 |
# pull-in the oslo logging module (which is a huge pile of code). |
| 97 |
mtime = os.path.getmtime(filename) |
| 98 |
cache_info = self.backend.file_cache.setdefault(filename, {}) |
| 99 |
if not cache_info or mtime > cache_info.get('mtime', 0): |
| 100 |
with open(filename, encoding=self.backend.encoding) as fp: |
| 101 |
cache_info['data'] = fp.read() |
| 102 |
cache_info['mtime'] = mtime |
| 103 |
return cache_info['data'] |
| 104 |
|
| 105 |
def _write_to(self, filename, contents): |
| 106 |
contents = misc.binary_encode(contents, |
| 107 |
encoding=self.backend.encoding) |
| 108 |
with open(filename, 'wb') as fp: |
| 109 |
fp.write(contents) |
| 110 |
self.backend.file_cache.pop(filename, None) |
| 111 |
|
| 112 |
@contextlib.contextmanager |
| 113 |
def _path_lock(self, path): |
| 114 |
lockfile = self._join_path(path, 'lock') |
| 115 |
with fasteners.InterProcessLock(lockfile) as lock: |
| 116 |
with _storagefailure_wrapper(): |
| 117 |
yield lock |
| 118 |
|
| 119 |
def _join_path(self, *parts): |
| 120 |
return os.path.join(*parts) |
| 121 |
|
| 122 |
def _get_item(self, path): |
| 123 |
with self._path_lock(path): |
| 124 |
item_path = self._join_path(path, 'metadata') |
| 125 |
return misc.decode_json(self._read_from(item_path)) |
| 126 |
|
| 127 |
def _set_item(self, path, value, transaction): |
| 128 |
with self._path_lock(path): |
| 129 |
item_path = self._join_path(path, 'metadata') |
| 130 |
self._write_to(item_path, jsonutils.dumps(value)) |
| 131 |
|
| 132 |
def _del_tree(self, path, transaction): |
| 133 |
with self._path_lock(path): |
| 134 |
shutil.rmtree(path) |
| 135 |
|
| 136 |
def _get_children(self, path): |
| 137 |
if path == self.book_path: |
| 138 |
filter_func = os.path.isdir |
| 139 |
else: |
| 140 |
filter_func = os.path.islink |
| 141 |
with _storagefailure_wrapper(): |
| 142 |
return [child for child in os.listdir(path) |
| 143 |
if filter_func(self._join_path(path, child))] |
| 144 |
|
| 145 |
def _ensure_path(self, path): |
| 146 |
with _storagefailure_wrapper(): |
| 147 |
fileutils.ensure_tree(path) |
| 148 |
|
| 149 |
def _create_link(self, src_path, dest_path, transaction): |
| 150 |
with _storagefailure_wrapper(): |
| 151 |
try: |
| 152 |
os.symlink(src_path, dest_path) |
| 153 |
except OSError as e: |
| 154 |
if e.errno != errno.EEXIST: |
| 155 |
raise |
| 156 |
|
| 157 |
@contextlib.contextmanager |
| 158 |
def _transaction(self): |
| 159 |
"""This just wraps a global write-lock.""" |
| 160 |
lock = self.backend.lock.write_lock |
| 161 |
with lock(): |
| 162 |
yield |
| 163 |
|
| 164 |
def validate(self): |
| 165 |
with _storagefailure_wrapper(): |
| 166 |
for p in (self.flow_path, self.atom_path, self.book_path): |
| 167 |
if not os.path.isdir(p): |
| 168 |
raise RuntimeError("Missing required directory: %s" % (p)) |