| 1 |
"""A dumb and slow but simple dbm clone. |
| 2 |
|
| 3 |
For database spam, spam.dir contains the index (a text file), |
| 4 |
spam.bak *may* contain a backup of the index (also a text file), |
| 5 |
while spam.dat contains the data (a binary file). |
| 6 |
|
| 7 |
XXX TO DO: |
| 8 |
|
| 9 |
- seems to contain a bug when updating... |
| 10 |
|
| 11 |
- reclaim free space (currently, space once occupied by deleted or expanded |
| 12 |
items is never reused) |
| 13 |
|
| 14 |
- support concurrent access (currently, if two processes take turns making |
| 15 |
updates, they can mess up the index) |
| 16 |
|
| 17 |
- support efficient access to large databases (currently, the whole index |
| 18 |
is read when the database is opened, and some updates rewrite the whole index) |
| 19 |
|
| 20 |
- support opening for read-only (flag = 'm') |
| 21 |
|
| 22 |
""" |
| 23 |
|
| 24 |
import ast as _ast |
| 25 |
import io as _io |
| 26 |
import os as _os |
| 27 |
import collections.abc |
| 28 |
|
| 29 |
__all__ = ["error", "open"] |
| 30 |
|
| 31 |
_BLOCKSIZE = 512 |
| 32 |
|
| 33 |
error = OSError |
| 34 |
|
| 35 |
class _Database(collections.abc.MutableMapping): |
| 36 |
|
| 37 |
# The on-disk directory and data files can remain in mutually |
| 38 |
# inconsistent states for an arbitrarily long time (see comments |
| 39 |
# at the end of __setitem__). This is only repaired when _commit() |
| 40 |
# gets called. One place _commit() gets called is from __del__(), |
| 41 |
# and if that occurs at program shutdown time, module globals may |
| 42 |
# already have gotten rebound to None. Since it's crucial that |
| 43 |
# _commit() finish successfully, we can't ignore shutdown races |
| 44 |
# here, and _commit() must not reference any globals. |
| 45 |
_os = _os # for _commit() |
| 46 |
_io = _io # for _commit() |
| 47 |
|
| 48 |
def __init__(self, filebasename, mode, flag='c'): |
| 49 |
filebasename = self._os.fsencode(filebasename) |
| 50 |
self._mode = mode |
| 51 |
self._readonly = (flag == 'r') |
| 52 |
|
| 53 |
# The directory file is a text file. Each line looks like |
| 54 |
# "%r, (%d, %d)\n" % (key, pos, siz) |
| 55 |
# where key is the string key, pos is the offset into the dat |
| 56 |
# file of the associated value's first byte, and siz is the number |
| 57 |
# of bytes in the associated value. |
| 58 |
self._dirfile = filebasename + b'.dir' |
| 59 |
|
| 60 |
# The data file is a binary file pointed into by the directory |
| 61 |
# file, and holds the values associated with keys. Each value |
| 62 |
# begins at a _BLOCKSIZE-aligned byte offset, and is a raw |
| 63 |
# binary 8-bit string value. |
| 64 |
self._datfile = filebasename + b'.dat' |
| 65 |
self._bakfile = filebasename + b'.bak' |
| 66 |
|
| 67 |
# The index is an in-memory dict, mirroring the directory file. |
| 68 |
self._index = None # maps keys to (pos, siz) pairs |
| 69 |
|
| 70 |
# Handle the creation |
| 71 |
self._create(flag) |
| 72 |
self._update(flag) |
| 73 |
|
| 74 |
def _create(self, flag): |
| 75 |
if flag == 'n': |
| 76 |
for filename in (self._datfile, self._bakfile, self._dirfile): |
| 77 |
try: |
| 78 |
_os.remove(filename) |
| 79 |
except OSError: |
| 80 |
pass |
| 81 |
# Mod by Jack: create data file if needed |
| 82 |
try: |
| 83 |
f = _io.open(self._datfile, 'r', encoding="Latin-1") |
| 84 |
except OSError: |
| 85 |
if flag not in ('c', 'n'): |
| 86 |
raise |
| 87 |
with _io.open(self._datfile, 'w', encoding="Latin-1") as f: |
| 88 |
self._chmod(self._datfile) |
| 89 |
else: |
| 90 |
f.close() |
| 91 |
|
| 92 |
# Read directory file into the in-memory index dict. |
| 93 |
def _update(self, flag): |
| 94 |
self._modified = False |
| 95 |
self._index = {} |
| 96 |
try: |
| 97 |
f = _io.open(self._dirfile, 'r', encoding="Latin-1") |
| 98 |
except OSError: |
| 99 |
if flag not in ('c', 'n'): |
| 100 |
raise |
| 101 |
with self._io.open(self._dirfile, 'w', encoding="Latin-1") as f: |
| 102 |
self._chmod(self._dirfile) |
| 103 |
else: |
| 104 |
with f: |
| 105 |
for line in f: |
| 106 |
line = line.rstrip() |
| 107 |
key, pos_and_siz_pair = _ast.literal_eval(line) |
| 108 |
key = key.encode('Latin-1') |
| 109 |
self._index[key] = pos_and_siz_pair |
| 110 |
|
| 111 |
# Write the index dict to the directory file. The original directory |
| 112 |
# file (if any) is renamed with a .bak extension first. If a .bak |
| 113 |
# file currently exists, it's deleted. |
| 114 |
def _commit(self): |
| 115 |
# CAUTION: It's vital that _commit() succeed, and _commit() can |
| 116 |
# be called from __del__(). Therefore we must never reference a |
| 117 |
# global in this routine. |
| 118 |
if self._index is None or not self._modified: |
| 119 |
return # nothing to do |
| 120 |
|
| 121 |
try: |
| 122 |
self._os.unlink(self._bakfile) |
| 123 |
except OSError: |
| 124 |
pass |
| 125 |
|
| 126 |
try: |
| 127 |
self._os.rename(self._dirfile, self._bakfile) |
| 128 |
except OSError: |
| 129 |
pass |
| 130 |
|
| 131 |
with self._io.open(self._dirfile, 'w', encoding="Latin-1") as f: |
| 132 |
self._chmod(self._dirfile) |
| 133 |
for key, pos_and_siz_pair in self._index.items(): |
| 134 |
# Use Latin-1 since it has no qualms with any value in any |
| 135 |
# position; UTF-8, though, does care sometimes. |
| 136 |
entry = "%r, %r\n" % (key.decode('Latin-1'), pos_and_siz_pair) |
| 137 |
f.write(entry) |
| 138 |
self._modified = False |
| 139 |
|
| 140 |
sync = _commit |
| 141 |
|
| 142 |
def _verify_open(self): |
| 143 |
if self._index is None: |
| 144 |
raise error('DBM object has already been closed') |
| 145 |
|
| 146 |
def __getitem__(self, key): |
| 147 |
if isinstance(key, str): |
| 148 |
key = key.encode('utf-8') |
| 149 |
self._verify_open() |
| 150 |
pos, siz = self._index[key] # may raise KeyError |
| 151 |
with _io.open(self._datfile, 'rb') as f: |
| 152 |
f.seek(pos) |
| 153 |
dat = f.read(siz) |
| 154 |
return dat |
| 155 |
|
| 156 |
# Append val to the data file, starting at a _BLOCKSIZE-aligned |
| 157 |
# offset. The data file is first padded with NUL bytes (if needed) |
| 158 |
# to get to an aligned offset. Return pair |
| 159 |
# (starting offset of val, len(val)) |
| 160 |
def _addval(self, val): |
| 161 |
with _io.open(self._datfile, 'rb+') as f: |
| 162 |
f.seek(0, 2) |
| 163 |
pos = int(f.tell()) |
| 164 |
npos = ((pos + _BLOCKSIZE - 1) // _BLOCKSIZE) * _BLOCKSIZE |
| 165 |
f.write(b'\0'*(npos-pos)) |
| 166 |
pos = npos |
| 167 |
f.write(val) |
| 168 |
return (pos, len(val)) |
| 169 |
|
| 170 |
# Write val to the data file, starting at offset pos. The caller |
| 171 |
# is responsible for ensuring that there's enough room starting at |
| 172 |
# pos to hold val, without overwriting some other value. Return |
| 173 |
# pair (pos, len(val)). |
| 174 |
def _setval(self, pos, val): |
| 175 |
with _io.open(self._datfile, 'rb+') as f: |
| 176 |
f.seek(pos) |
| 177 |
f.write(val) |
| 178 |
return (pos, len(val)) |
| 179 |
|
| 180 |
# key is a new key whose associated value starts in the data file |
| 181 |
# at offset pos and with length siz. Add an index record to |
| 182 |
# the in-memory index dict, and append one to the directory file. |
| 183 |
def _addkey(self, key, pos_and_siz_pair): |
| 184 |
self._index[key] = pos_and_siz_pair |
| 185 |
with _io.open(self._dirfile, 'a', encoding="Latin-1") as f: |
| 186 |
self._chmod(self._dirfile) |
| 187 |
f.write("%r, %r\n" % (key.decode("Latin-1"), pos_and_siz_pair)) |
| 188 |
|
| 189 |
def __setitem__(self, key, val): |
| 190 |
if self._readonly: |
| 191 |
raise error('The database is opened for reading only') |
| 192 |
if isinstance(key, str): |
| 193 |
key = key.encode('utf-8') |
| 194 |
elif not isinstance(key, (bytes, bytearray)): |
| 195 |
raise TypeError("keys must be bytes or strings") |
| 196 |
if isinstance(val, str): |
| 197 |
val = val.encode('utf-8') |
| 198 |
elif not isinstance(val, (bytes, bytearray)): |
| 199 |
raise TypeError("values must be bytes or strings") |
| 200 |
self._verify_open() |
| 201 |
self._modified = True |
| 202 |
if key not in self._index: |
| 203 |
self._addkey(key, self._addval(val)) |
| 204 |
else: |
| 205 |
# See whether the new value is small enough to fit in the |
| 206 |
# (padded) space currently occupied by the old value. |
| 207 |
pos, siz = self._index[key] |
| 208 |
oldblocks = (siz + _BLOCKSIZE - 1) // _BLOCKSIZE |
| 209 |
newblocks = (len(val) + _BLOCKSIZE - 1) // _BLOCKSIZE |
| 210 |
if newblocks <= oldblocks: |
| 211 |
self._index[key] = self._setval(pos, val) |
| 212 |
else: |
| 213 |
# The new value doesn't fit in the (padded) space used |
| 214 |
# by the old value. The blocks used by the old value are |
| 215 |
# forever lost. |
| 216 |
self._index[key] = self._addval(val) |
| 217 |
|
| 218 |
# Note that _index may be out of synch with the directory |
| 219 |
# file now: _setval() and _addval() don't update the directory |
| 220 |
# file. This also means that the on-disk directory and data |
| 221 |
# files are in a mutually inconsistent state, and they'll |
| 222 |
# remain that way until _commit() is called. Note that this |
| 223 |
# is a disaster (for the database) if the program crashes |
| 224 |
# (so that _commit() never gets called). |
| 225 |
|
| 226 |
def __delitem__(self, key): |
| 227 |
if self._readonly: |
| 228 |
raise error('The database is opened for reading only') |
| 229 |
if isinstance(key, str): |
| 230 |
key = key.encode('utf-8') |
| 231 |
self._verify_open() |
| 232 |
self._modified = True |
| 233 |
# The blocks used by the associated value are lost. |
| 234 |
del self._index[key] |
| 235 |
# XXX It's unclear why we do a _commit() here (the code always |
| 236 |
# XXX has, so I'm not changing it). __setitem__ doesn't try to |
| 237 |
# XXX keep the directory file in synch. Why should we? Or |
| 238 |
# XXX why shouldn't __setitem__? |
| 239 |
self._commit() |
| 240 |
|
| 241 |
def keys(self): |
| 242 |
try: |
| 243 |
return list(self._index) |
| 244 |
except TypeError: |
| 245 |
raise error('DBM object has already been closed') from None |
| 246 |
|
| 247 |
def items(self): |
| 248 |
self._verify_open() |
| 249 |
return [(key, self[key]) for key in self._index.keys()] |
| 250 |
|
| 251 |
def __contains__(self, key): |
| 252 |
if isinstance(key, str): |
| 253 |
key = key.encode('utf-8') |
| 254 |
try: |
| 255 |
return key in self._index |
| 256 |
except TypeError: |
| 257 |
if self._index is None: |
| 258 |
raise error('DBM object has already been closed') from None |
| 259 |
else: |
| 260 |
raise |
| 261 |
|
| 262 |
def iterkeys(self): |
| 263 |
try: |
| 264 |
return iter(self._index) |
| 265 |
except TypeError: |
| 266 |
raise error('DBM object has already been closed') from None |
| 267 |
__iter__ = iterkeys |
| 268 |
|
| 269 |
def __len__(self): |
| 270 |
try: |
| 271 |
return len(self._index) |
| 272 |
except TypeError: |
| 273 |
raise error('DBM object has already been closed') from None |
| 274 |
|
| 275 |
def close(self): |
| 276 |
try: |
| 277 |
self._commit() |
| 278 |
finally: |
| 279 |
self._index = self._datfile = self._dirfile = self._bakfile = None |
| 280 |
|
| 281 |
__del__ = close |
| 282 |
|
| 283 |
def _chmod(self, file): |
| 284 |
self._os.chmod(file, self._mode) |
| 285 |
|
| 286 |
def __enter__(self): |
| 287 |
return self |
| 288 |
|
| 289 |
def __exit__(self, *args): |
| 290 |
self.close() |
| 291 |
|
| 292 |
|
| 293 |
def open(file, flag='c', mode=0o666): |
| 294 |
"""Open the database file, filename, and return corresponding object. |
| 295 |
|
| 296 |
The flag argument, used to control how the database is opened in the |
| 297 |
other DBM implementations, supports only the semantics of 'c' and 'n' |
| 298 |
values. Other values will default to the semantics of 'c' value: |
| 299 |
the database will always opened for update and will be created if it |
| 300 |
does not exist. |
| 301 |
|
| 302 |
The optional mode argument is the UNIX mode of the file, used only when |
| 303 |
the database has to be created. It defaults to octal code 0o666 (and |
| 304 |
will be modified by the prevailing umask). |
| 305 |
|
| 306 |
""" |
| 307 |
|
| 308 |
# Modify mode depending on the umask |
| 309 |
try: |
| 310 |
um = _os.umask(0) |
| 311 |
_os.umask(um) |
| 312 |
except AttributeError: |
| 313 |
pass |
| 314 |
else: |
| 315 |
# Turn off any bits that are set in the umask |
| 316 |
mode = mode & (~um) |
| 317 |
if flag not in ('r', 'w', 'c', 'n'): |
| 318 |
raise ValueError("Flag must be one of 'r', 'w', 'c', or 'n'") |
| 319 |
return _Database(file, mode, flag=flag) |