| 1 |
#!/usr/bin/env python |
| 2 |
# License: GPLv3 Copyright: 2013, Kovid Goyal <kovid at kovidgoyal.net> |
| 3 |
|
| 4 |
import os |
| 5 |
import time |
| 6 |
import unittest |
| 7 |
from contextlib import closing |
| 8 |
from io import BytesIO |
| 9 |
|
| 10 |
from calibre.constants import iswindows |
| 11 |
from calibre.db.tests.base import BaseTest |
| 12 |
from calibre.ptempfile import TemporaryDirectory |
| 13 |
|
| 14 |
|
| 15 |
def read(x, mode='r'): |
| 16 |
with open(x, mode) as f: |
| 17 |
return f.read() |
| 18 |
|
| 19 |
|
| 20 |
class FilesystemTest(BaseTest): |
| 21 |
def get_filesystem_data(self, cache, book_id): |
| 22 |
fmts = cache.field_for('formats', book_id) |
| 23 |
ans = {} |
| 24 |
for fmt in fmts: |
| 25 |
buf = BytesIO() |
| 26 |
if cache.copy_format_to(book_id, fmt, buf): |
| 27 |
ans[fmt] = buf.getvalue() |
| 28 |
buf = BytesIO() |
| 29 |
if cache.copy_cover_to(book_id, buf): |
| 30 |
ans['cover'] = buf.getvalue() |
| 31 |
return ans |
| 32 |
|
| 33 |
def test_metadata_move(self): |
| 34 |
"Test the moving of files when title/author change" |
| 35 |
cl = self.cloned_library |
| 36 |
cache = self.init_cache(cl) |
| 37 |
ae, af, sf = self.assertEqual, self.assertFalse, cache.set_field |
| 38 |
|
| 39 |
# Test that changing metadata on a book with no formats/cover works |
| 40 |
ae(sf('title', {3: 'moved1'}), {3}) |
| 41 |
ae(sf('authors', {3: 'moved1'}), {3}) |
| 42 |
ae(sf('title', {3: 'Moved1'}), {3}) |
| 43 |
ae(sf('authors', {3: 'Moved1'}), {3}) |
| 44 |
ae(cache.field_for('title', 3), 'Moved1') |
| 45 |
ae(cache.field_for('authors', 3), ('Moved1',)) |
| 46 |
|
| 47 |
# Now try with a book that has covers and formats |
| 48 |
orig_data = self.get_filesystem_data(cache, 1) |
| 49 |
orig_fpath = cache.format_abspath(1, 'FMT1') |
| 50 |
ae(sf('title', {1: 'moved'}), {1}) |
| 51 |
ae(sf('authors', {1: 'moved'}), {1}) |
| 52 |
ae(sf('title', {1: 'Moved'}), {1}) |
| 53 |
ae(sf('authors', {1: 'Moved'}), {1}) |
| 54 |
ae(cache.field_for('title', 1), 'Moved') |
| 55 |
ae(cache.field_for('authors', 1), ('Moved',)) |
| 56 |
cache2 = self.init_cache(cl) |
| 57 |
for c in (cache, cache2): |
| 58 |
data = self.get_filesystem_data(c, 1) |
| 59 |
ae(set(orig_data), set(data)) |
| 60 |
ae(orig_data, data, 'Filesystem data does not match') |
| 61 |
ae(c.field_for('path', 1), 'Moved/Moved (1)') |
| 62 |
ae(c.field_for('path', 3), 'Moved1/Moved1 (3)') |
| 63 |
fpath = c.format_abspath(1, 'FMT1').replace(os.sep, '/').split('/') |
| 64 |
ae(fpath[-3:], ['Moved', 'Moved (1)', 'Moved - Moved.fmt1']) |
| 65 |
af(os.path.exists(os.path.dirname(orig_fpath)), 'Original book folder still exists') |
| 66 |
# Check that the filesystem reflects fpath (especially on |
| 67 |
# case-insensitive systems). |
| 68 |
for x in range(1, 4): |
| 69 |
base = os.sep.join(fpath[:-x]) |
| 70 |
part = fpath[-x:][0] |
| 71 |
self.assertIn(part, os.listdir(base)) |
| 72 |
|
| 73 |
initial_side_data = {} |
| 74 |
|
| 75 |
def init_cache(): |
| 76 |
nonlocal cache, initial_side_data |
| 77 |
cache = self.init_cache(self.cloned_library) |
| 78 |
bookdir = os.path.dirname(cache.format_abspath(1, '__COVER_INTERNAL__')) |
| 79 |
with open(os.path.join(bookdir, 'a.side'), 'w') as f: |
| 80 |
f.write('a.side') |
| 81 |
os.mkdir(os.path.join(bookdir, 'subdir')) |
| 82 |
with open(os.path.join(bookdir, 'subdir', 'a.fmt1'), 'w') as f: |
| 83 |
f.write('a.fmt1') |
| 84 |
initial_side_data = side_data() |
| 85 |
|
| 86 |
def side_data(book_id=1): |
| 87 |
bookdir = os.path.dirname(cache.format_abspath(book_id, '__COVER_INTERNAL__')) |
| 88 |
return { |
| 89 |
'a.side': read(os.path.join(bookdir, 'a.side')), |
| 90 |
'a.fmt1': read(os.path.join(bookdir, 'subdir', 'a.fmt1')), |
| 91 |
} |
| 92 |
|
| 93 |
def check_that_filesystem_and_db_entries_match(book_id): |
| 94 |
bookdir = os.path.dirname(cache.format_abspath(book_id, '__COVER_INTERNAL__')) |
| 95 |
if iswindows: |
| 96 |
from calibre_extensions import winutil |
| 97 |
|
| 98 |
bookdir = winutil.get_long_path_name(bookdir) |
| 99 |
bookdir_contents = set(os.listdir(bookdir)) |
| 100 |
expected_contents = {'cover.jpg', 'a.side', 'subdir'} |
| 101 |
for fmt, fname in cache.fields['formats'].table.fname_map[book_id].items(): |
| 102 |
expected_contents.add(fname + '.' + fmt.lower()) |
| 103 |
ae(expected_contents, bookdir_contents) |
| 104 |
fs_path = bookdir.split(os.sep)[-2:] |
| 105 |
db_path = cache.get_book_path(book_id, sep='/').split('/') |
| 106 |
ae(db_path, fs_path) |
| 107 |
ae(initial_side_data, side_data(book_id)) |
| 108 |
|
| 109 |
# test only formats being changed |
| 110 |
init_cache() |
| 111 |
ef = set() |
| 112 |
for efx in cache.list_extra_files(1): |
| 113 |
ef.add(efx.relpath) |
| 114 |
self.assertTrue(os.path.exists(efx.file_path)) |
| 115 |
self.assertEqual(ef, {'a.side', 'subdir/a.fmt1'}) |
| 116 |
fname = cache.fields['formats'].table.fname_map[1]['FMT1'] |
| 117 |
cache.fields['formats'].table.fname_map[1]['FMT1'] = 'some thing else' |
| 118 |
cache.fields['formats'].table.fname_map[1]['FMT2'] = fname.upper() |
| 119 |
cache.backend.update_path( |
| 120 |
1, |
| 121 |
cache.field_for('title', 1), |
| 122 |
cache.field_for('authors', 1)[0], |
| 123 |
cache.fields['path'], |
| 124 |
cache.fields['formats'], |
| 125 |
) |
| 126 |
check_that_filesystem_and_db_entries_match(1) |
| 127 |
|
| 128 |
# test a case only change |
| 129 |
init_cache() |
| 130 |
title = cache.field_for('title', 1) |
| 131 |
self.assertNotEqual(title, title.upper()) |
| 132 |
cache.set_field('title', {1: title.upper()}) |
| 133 |
check_that_filesystem_and_db_entries_match(1) |
| 134 |
|
| 135 |
# test a title change |
| 136 |
init_cache() |
| 137 |
cache.set_field('title', {1: 'new changed title'}) |
| 138 |
check_that_filesystem_and_db_entries_match(1) |
| 139 |
# test an author change |
| 140 |
cache.set_field('authors', {1: ('new changed author',)}) |
| 141 |
check_that_filesystem_and_db_entries_match(1) |
| 142 |
# test a double change |
| 143 |
from calibre.ebooks.metadata.book.base import Metadata |
| 144 |
|
| 145 |
cache.set_metadata(1, Metadata('t1', ('a1', 'a2'))) |
| 146 |
check_that_filesystem_and_db_entries_match(1) |
| 147 |
# check that empty author folders are removed |
| 148 |
for x in os.scandir(cache.backend.library_path): |
| 149 |
if x.is_dir(): |
| 150 |
self.assertTrue(os.listdir(x.path)) |
| 151 |
|
| 152 |
def test_rename_of_extra_files(self): |
| 153 |
cl = self.cloned_library |
| 154 |
cache = self.init_cache(cl) |
| 155 |
cache.add_extra_files(1, {'a': BytesIO(b'aaa'), 'b': BytesIO(b'bbb')}) |
| 156 |
|
| 157 |
def relpaths(): |
| 158 |
return {e.relpath for e in cache.list_extra_files(1)} |
| 159 |
|
| 160 |
self.assertEqual(relpaths(), {'a', 'b'}) |
| 161 |
self.assertEqual(cache.rename_extra_files(1, {'a': 'data/c'}), {'a'}) |
| 162 |
self.assertEqual(relpaths(), {'data/c', 'b'}) |
| 163 |
self.assertEqual(cache.rename_extra_files(1, {'b': 'B'}), {'b'}) |
| 164 |
self.assertEqual(relpaths(), {'data/c', 'B'}) |
| 165 |
self.assertEqual(cache.rename_extra_files(1, {'B': 'data/c'}), set()) |
| 166 |
self.assertEqual(cache.rename_extra_files(1, {'B': 'data/c'}, replace=True), {'B'}) |
| 167 |
|
| 168 |
def test_extra_file_paths_are_confined_to_book_dir(self): |
| 169 |
cl = self.cloned_library |
| 170 |
cache = self.init_cache(cl) |
| 171 |
bookdir = os.path.dirname(cache.format_abspath(1, 'FMT1')) |
| 172 |
sibling = bookdir + ' sibling' |
| 173 |
os.makedirs(sibling) |
| 174 |
bad_relpath = '../' + os.path.basename(sibling) + '/victim' |
| 175 |
victim = os.path.join(sibling, 'victim') |
| 176 |
with open(victim, 'wb') as f: |
| 177 |
f.write(b'outside') |
| 178 |
|
| 179 |
self.assertEqual(cache.add_extra_files(1, {bad_relpath: BytesIO(b'bad')}), {bad_relpath: False}) |
| 180 |
self.assertEqual(read(victim, 'rb'), b'outside') |
| 181 |
|
| 182 |
buf = BytesIO() |
| 183 |
with self.assertRaises(FileNotFoundError): |
| 184 |
cache.copy_extra_file_to(1, bad_relpath, buf) |
| 185 |
self.assertEqual(buf.getvalue(), b'') |
| 186 |
|
| 187 |
cache.add_extra_files(1, {'safe': BytesIO(b'safe'), 'data/inside': BytesIO(b'inside')}) |
| 188 |
self.assertEqual(cache.rename_extra_files(1, {bad_relpath: 'moved'}), set()) |
| 189 |
self.assertEqual(cache.rename_extra_files(1, {'safe': bad_relpath}), set()) |
| 190 |
self.assertEqual(read(victim, 'rb'), b'outside') |
| 191 |
|
| 192 |
def listed(pattern): |
| 193 |
return {e.relpath for e in cache.list_extra_files(1, use_cache=False, pattern=pattern)} |
| 194 |
|
| 195 |
self.assertEqual(listed('data/**/*'), {'data/inside'}) |
| 196 |
sibling_pattern = '../' + os.path.basename(sibling) + '/*' |
| 197 |
for pattern in ( |
| 198 |
sibling_pattern, |
| 199 |
'data/../../' + os.path.basename(sibling) + '/*', |
| 200 |
sibling_pattern.replace('/', '\\'), |
| 201 |
os.path.join(sibling, '*'), |
| 202 |
): |
| 203 |
self.assertEqual(listed(pattern), set()) |
| 204 |
self.assertEqual({e.relpath for e in cache.list_extra_files(1)}, {'safe', 'data/inside'}) |
| 205 |
|
| 206 |
self.assertEqual(cache.remove_extra_files(1, [bad_relpath], permanent=True), {}) |
| 207 |
self.assertEqual(read(victim, 'rb'), b'outside') |
| 208 |
|
| 209 |
@unittest.skipUnless(iswindows, 'Windows only') |
| 210 |
def test_windows_atomic_move(self): |
| 211 |
"Test book file open in another process when changing metadata" |
| 212 |
cl = self.cloned_library |
| 213 |
cache = self.init_cache(cl) |
| 214 |
fpath = cache.format_abspath(1, 'FMT1') |
| 215 |
with open(fpath, 'rb') as f: |
| 216 |
with self.assertRaises(IOError): |
| 217 |
cache.set_field('title', {1: 'Moved'}) |
| 218 |
with self.assertRaises(IOError): |
| 219 |
cache.remove_books({1}) |
| 220 |
self.assertNotEqual(cache.field_for('title', 1), 'Moved', 'Title was changed despite file lock') |
| 221 |
|
| 222 |
# Test on folder with hardlinks |
| 223 |
from calibre.ptempfile import TemporaryDirectory |
| 224 |
from calibre.utils.filenames import WindowsAtomicFolderMove, hardlink_file |
| 225 |
|
| 226 |
raw = b'xxx' |
| 227 |
with TemporaryDirectory() as tdir1, TemporaryDirectory() as tdir2: |
| 228 |
a, b = os.path.join(tdir1, 'a'), os.path.join(tdir1, 'b') |
| 229 |
a = os.path.join(tdir1, 'a') |
| 230 |
with open(a, 'wb') as f: |
| 231 |
f.write(raw) |
| 232 |
hardlink_file(a, b) |
| 233 |
wam = WindowsAtomicFolderMove(tdir1) |
| 234 |
wam.copy_path_to(a, os.path.join(tdir2, 'a')) |
| 235 |
wam.copy_path_to(b, os.path.join(tdir2, 'b')) |
| 236 |
wam.delete_originals() |
| 237 |
self.assertEqual([], os.listdir(tdir1)) |
| 238 |
self.assertEqual({'a', 'b'}, set(os.listdir(tdir2))) |
| 239 |
self.assertEqual(raw, read(os.path.join(tdir2, 'a'), 'rb')) |
| 240 |
self.assertEqual(raw, read(os.path.join(tdir2, 'b'), 'rb')) |
| 241 |
|
| 242 |
def test_library_move(self): |
| 243 |
"Test moving of library" |
| 244 |
from calibre.ptempfile import TemporaryDirectory |
| 245 |
|
| 246 |
cache = self.init_cache() |
| 247 |
self.assertIn('metadata.db', cache.get_top_level_move_items()[0]) |
| 248 |
all_ids = cache.all_book_ids() |
| 249 |
fmt1 = cache.format(1, 'FMT1') |
| 250 |
cov = cache.cover(1) |
| 251 |
odir = cache.backend.library_path |
| 252 |
with TemporaryDirectory('moved_lib') as tdir: |
| 253 |
cache.move_library_to(tdir) |
| 254 |
self.assertIn('moved_lib', cache.backend.library_path) |
| 255 |
self.assertIn('moved_lib', cache.backend.dbpath) |
| 256 |
self.assertEqual(fmt1, cache.format(1, 'FMT1')) |
| 257 |
self.assertEqual(cov, cache.cover(1)) |
| 258 |
cache.reload_from_db() |
| 259 |
self.assertEqual(all_ids, cache.all_book_ids()) |
| 260 |
cache.backend.close() |
| 261 |
self.assertFalse(os.path.exists(odir)) |
| 262 |
os.mkdir(odir) # needed otherwise tearDown() fails |
| 263 |
|
| 264 |
def test_long_filenames(self): |
| 265 |
"Test long file names" |
| 266 |
cache = self.init_cache() |
| 267 |
cache.set_field('title', {1: 'a' * 10000}) |
| 268 |
self.assertLessEqual(len(cache.field_for('path', 1)), cache.backend.PATH_LIMIT * 2) |
| 269 |
cache.set_field('authors', {1: 'b' * 10000}) |
| 270 |
self.assertLessEqual(len(cache.field_for('path', 1)), cache.backend.PATH_LIMIT * 2) |
| 271 |
fpath = cache.format_abspath(1, cache.formats(1)[0]) |
| 272 |
self.assertLessEqual(len(fpath), len(cache.backend.library_path) + cache.backend.PATH_LIMIT * 4) |
| 273 |
|
| 274 |
def test_reserved_names(self): |
| 275 |
"Test that folders are not created with a windows reserve name" |
| 276 |
cache = self.init_cache() |
| 277 |
cache.set_field('authors', {1: 'con'}) |
| 278 |
p = cache.field_for('path', 1).replace(os.sep, '/').split('/') |
| 279 |
self.assertNotIn('con', p) |
| 280 |
|
| 281 |
def test_fname_change(self): |
| 282 |
"Test the changing of the filename but not the folder name" |
| 283 |
cache = self.init_cache() |
| 284 |
title = 'a' * 30 + 'bbb' |
| 285 |
cache.backend.PATH_LIMIT = 100 |
| 286 |
cache.set_field('title', {3: title}) |
| 287 |
cache.add_format(3, 'TXT', BytesIO(b'xxx')) |
| 288 |
cache.backend.PATH_LIMIT = 40 |
| 289 |
cache.set_field('title', {3: title}) |
| 290 |
fpath = cache.format_abspath(3, 'TXT') |
| 291 |
self.assertEqual(sorted([os.path.basename(fpath)]), sorted(os.listdir(os.path.dirname(fpath)))) |
| 292 |
|
| 293 |
def test_export_import(self): |
| 294 |
from calibre.db.cache import import_library |
| 295 |
from calibre.utils.exim import Exporter, Importer |
| 296 |
|
| 297 |
with TemporaryDirectory('export_lib') as tdir: |
| 298 |
for part_size in (8, 1, 1024): |
| 299 |
exporter = Exporter(tdir, part_size=part_size + Exporter.tail_size()) |
| 300 |
files = { |
| 301 |
'a': b'a' * 7, |
| 302 |
'b': b'b' * 7, |
| 303 |
'c': b'c' * 2, |
| 304 |
'd': b'd' * 9, |
| 305 |
'e': b'e' * 3, |
| 306 |
} |
| 307 |
for key, data in files.items(): |
| 308 |
exporter.add_file(BytesIO(data), key) |
| 309 |
exporter.commit() |
| 310 |
importer = Importer(tdir) |
| 311 |
for key, expected in files.items(): |
| 312 |
with importer.start_file(key, key) as f: |
| 313 |
actual = f.read() |
| 314 |
self.assertEqual(expected, actual, key) |
| 315 |
self.assertFalse(importer.corrupted_files) |
| 316 |
cache = self.init_cache() |
| 317 |
bookdir = os.path.dirname(cache.format_abspath(1, '__COVER_INTERNAL__')) |
| 318 |
with open(os.path.join(bookdir, 'exf'), 'w') as f: |
| 319 |
f.write('exf') |
| 320 |
os.mkdir(os.path.join(bookdir, 'sub')) |
| 321 |
with open(os.path.join(bookdir, 'sub', 'recurse'), 'w') as f: |
| 322 |
f.write('recurse') |
| 323 |
self.assertEqual({ef.relpath for ef in cache.list_extra_files(1, pattern='sub/**/*')}, {'sub/recurse'}) |
| 324 |
self.assertEqual({ef.relpath for ef in cache.list_extra_files(1)}, {'exf', 'sub/recurse'}) |
| 325 |
for part_size in (512, 1027, None): |
| 326 |
with TemporaryDirectory('export_lib') as tdir, TemporaryDirectory('import_lib') as idir: |
| 327 |
exporter = Exporter(tdir, part_size=part_size if part_size is None else (part_size + Exporter.tail_size())) |
| 328 |
cache.export_library('l', exporter) |
| 329 |
exporter.commit() |
| 330 |
importer = Importer(tdir) |
| 331 |
ic = import_library('l', importer, idir) |
| 332 |
self.assertFalse(importer.corrupted_files) |
| 333 |
self.assertEqual(cache.all_book_ids(), ic.all_book_ids()) |
| 334 |
for book_id in cache.all_book_ids(): |
| 335 |
self.assertEqual(cache.cover(book_id), ic.cover(book_id), f'Covers not identical for book: {book_id}') |
| 336 |
for fmt in cache.formats(book_id): |
| 337 |
self.assertEqual(cache.format(book_id, fmt), ic.format(book_id, fmt)) |
| 338 |
self.assertEqual(cache.format_metadata(book_id, fmt)['mtime'], cache.format_metadata(book_id, fmt)['mtime']) |
| 339 |
bookdir = os.path.dirname(ic.format_abspath(1, '__COVER_INTERNAL__')) |
| 340 |
self.assertEqual('exf', read(os.path.join(bookdir, 'exf'))) |
| 341 |
self.assertEqual('recurse', read(os.path.join(bookdir, 'sub', 'recurse'))) |
| 342 |
r1 = cache.add_notes_resource(b'res1', 'res.jpg', mtime=time.time() - 113) |
| 343 |
r2 = cache.add_notes_resource(b'res2', 'res.jpg', mtime=time.time() - 1115) |
| 344 |
cache.set_notes_for('authors', 2, 'some notes', resource_hashes=(r1, r2)) |
| 345 |
cache.add_format(1, 'TXT', BytesIO(b'testing exim')) |
| 346 |
cache.fts_indexing_sleep_time = 0.001 |
| 347 |
cache.enable_fts() |
| 348 |
cache.set_fts_num_of_workers(4) |
| 349 |
st = time.monotonic() |
| •350 |
while cache.fts_indexing_left > 0 and time.monotonic() - st < 300: |
| 351 |
time.sleep(0.05) |
| 352 |
if cache.fts_indexing_left > 0: |
| 353 |
raise ValueError('FTS indexing did not complete') |
| 354 |
self.assertEqual(cache.fts_search('exim')[0]['id'], 1) |
| 355 |
with TemporaryDirectory('export_lib') as tdir, TemporaryDirectory('import_lib') as idir: |
| 356 |
exporter = Exporter(tdir) |
| 357 |
cache.export_library('l', exporter) |
| 358 |
exporter.commit() |
| 359 |
importer = Importer(tdir) |
| 360 |
ic = import_library('l', importer, idir) |
| 361 |
self.assertFalse(importer.corrupted_files) |
| 362 |
self.assertEqual(ic.fts_search('exim')[0]['id'], 1) |
| 363 |
self.assertEqual(cache.notes_for('authors', 2), ic.notes_for('authors', 2)) |
| 364 |
a, b = cache.get_notes_resource(r1), ic.get_notes_resource(r1) |
| 365 |
( |
| 366 |
at, |
| 367 |
bt, |
| 368 |
) = a.pop('mtime'), b.pop('mtime') |
| 369 |
self.assertEqual(a, b) |
| 370 |
self.assertLess(abs(at - bt), 2) |
| 371 |
|
| 372 |
def test_backup_database_retries_transient_ioerror(self): |
| 373 |
import apsw |
| 374 |
|
| 375 |
cache = self.init_cache() |
| 376 |
backend = cache.backend |
| 377 |
step_calls = {'count': 0} |
| 378 |
orig_connection = apsw.Connection |
| 379 |
|
| 380 |
class FlakyBackup: |
| 381 |
def __init__(self, real): |
| 382 |
self.real = real |
| 383 |
|
| 384 |
def __enter__(self): |
| 385 |
self.real.__enter__() |
| 386 |
return self |
| 387 |
|
| 388 |
def __exit__(self, *a): |
| 389 |
return self.real.__exit__(*a) |
| 390 |
|
| 391 |
@property |
| 392 |
def done(self): |
| 393 |
return self.real.done |
| 394 |
|
| 395 |
def step(self, npages): |
| 396 |
step_calls['count'] += 1 |
| 397 |
if step_calls['count'] == 1: |
| 398 |
e = apsw.IOError('not an error') |
| 399 |
e.extendedresult = apsw.SQLITE_IOERR_SHORT_READ |
| 400 |
raise e |
| 401 |
return self.real.step(npages) |
| 402 |
|
| 403 |
class FlakyConnection(orig_connection): |
| 404 |
def backup(self, *a, **kw): |
| 405 |
return FlakyBackup(super().backup(*a, **kw)) |
| 406 |
|
| 407 |
with TemporaryDirectory('backup_db') as tdir: |
| 408 |
path = os.path.join(tdir, 'backup.db') |
| 409 |
apsw.Connection = FlakyConnection |
| 410 |
try: |
| 411 |
backend.backup_database(path) |
| 412 |
finally: |
| 413 |
apsw.Connection = orig_connection |
| 414 |
self.assertGreater(step_calls['count'], 1) |
| 415 |
with closing(orig_connection(path)) as conn: |
| 416 |
self.assertTrue(conn.cursor().execute('SELECT id FROM books').fetchall()) |
| 417 |
|
| 418 |
# A non-transient I/O error must not be retried |
| 419 |
class FatalBackup(FlakyBackup): |
| 420 |
def step(self, npages): |
| 421 |
step_calls['count'] += 1 |
| 422 |
e = apsw.IOError('disk I/O error') |
| 423 |
e.extendedresult = apsw.SQLITE_IOERR_WRITE |
| 424 |
raise e |
| 425 |
|
| 426 |
class FatalConnection(orig_connection): |
| 427 |
def backup(self, *a, **kw): |
| 428 |
return FatalBackup(super().backup(*a, **kw)) |
| 429 |
|
| 430 |
with TemporaryDirectory('backup_db') as tdir: |
| 431 |
path = os.path.join(tdir, 'backup.db') |
| 432 |
step_calls['count'] = 0 |
| 433 |
apsw.Connection = FatalConnection |
| 434 |
try: |
| 435 |
self.assertRaises(apsw.IOError, backend.backup_database, path) |
| 436 |
finally: |
| 437 |
apsw.Connection = orig_connection |
| 438 |
self.assertEqual(step_calls['count'], 1) |
| 439 |
|
| 440 |
def test_find_books_in_directory(self): |
| 441 |
from calibre.db.adding import compile_rule, find_books_in_directory |
| 442 |
|
| 443 |
def strip(files): |
| 444 |
return frozenset({os.path.basename(x) for x in files}) |
| 445 |
|
| 446 |
def q(one, two): |
| 447 |
one, two = {strip(a) for a in one}, {strip(b) for b in two} |
| 448 |
self.assertEqual(one, two) |
| 449 |
|
| 450 |
def r(action='ignore', match_type='startswith', query=''): |
| 451 |
return {'action': action, 'match_type': match_type, 'query': query} |
| 452 |
|
| 453 |
def c(*rules): |
| 454 |
return tuple(map(compile_rule, rules)) |
| 455 |
|
| 456 |
files = ['added.epub', 'ignored.md', 'non-book.other'] |
| 457 |
q(['added.epub ignored.md'.split()], find_books_in_directory('', True, listdir_impl=lambda x: files)) |
| 458 |
q([['added.epub'], ['ignored.md']], find_books_in_directory('', False, listdir_impl=lambda x, **k: files)) |
| 459 |
for rules in ( |
| 460 |
c(r(query='ignored.'), r(action='add', match_type='endswith', query='.OTHER')), |
| 461 |
c(r(match_type='glob', query='*.md'), r(action='add', match_type='matches', query=r'.+\.other$')), |
| 462 |
c(r(match_type='not_startswith', query='IGnored.', action='add'), r(query='ignored.md')), |
| 463 |
): |
| 464 |
q( |
| 465 |
['added.epub non-book.other'.split()], |
| 466 |
find_books_in_directory('', True, compiled_rules=rules, listdir_impl=lambda x: files), |
| 467 |
) |