Debian Codegraph @

src/calibre/db/tests/filesystem.py

raw ◀ ▶
1 Debian patch touch this file
1 #!/usr/bin/env python
2 # License: GPLv3 Copyright: 2013, Kovid Goyal <kovid at kovidgoyal.net>
3
4 import os
5 import time
6 import unittest
7 from contextlib import closing
8 from io import BytesIO
9
10 from calibre.constants import iswindows
11 from calibre.db.tests.base import BaseTest
12 from calibre.ptempfile import TemporaryDirectory
13
14
15 def read(x, mode='r'):
16 with open(x, mode) as f:
17 return f.read()
18
19
20 class FilesystemTest(BaseTest):
21 def get_filesystem_data(self, cache, book_id):
22 fmts = cache.field_for('formats', book_id)
23 ans = {}
24 for fmt in fmts:
25 buf = BytesIO()
26 if cache.copy_format_to(book_id, fmt, buf):
27 ans[fmt] = buf.getvalue()
28 buf = BytesIO()
29 if cache.copy_cover_to(book_id, buf):
30 ans['cover'] = buf.getvalue()
31 return ans
32
33 def test_metadata_move(self):
34 "Test the moving of files when title/author change"
35 cl = self.cloned_library
36 cache = self.init_cache(cl)
37 ae, af, sf = self.assertEqual, self.assertFalse, cache.set_field
38
39 # Test that changing metadata on a book with no formats/cover works
40 ae(sf('title', {3: 'moved1'}), {3})
41 ae(sf('authors', {3: 'moved1'}), {3})
42 ae(sf('title', {3: 'Moved1'}), {3})
43 ae(sf('authors', {3: 'Moved1'}), {3})
44 ae(cache.field_for('title', 3), 'Moved1')
45 ae(cache.field_for('authors', 3), ('Moved1',))
46
47 # Now try with a book that has covers and formats
48 orig_data = self.get_filesystem_data(cache, 1)
49 orig_fpath = cache.format_abspath(1, 'FMT1')
50 ae(sf('title', {1: 'moved'}), {1})
51 ae(sf('authors', {1: 'moved'}), {1})
52 ae(sf('title', {1: 'Moved'}), {1})
53 ae(sf('authors', {1: 'Moved'}), {1})
54 ae(cache.field_for('title', 1), 'Moved')
55 ae(cache.field_for('authors', 1), ('Moved',))
56 cache2 = self.init_cache(cl)
57 for c in (cache, cache2):
58 data = self.get_filesystem_data(c, 1)
59 ae(set(orig_data), set(data))
60 ae(orig_data, data, 'Filesystem data does not match')
61 ae(c.field_for('path', 1), 'Moved/Moved (1)')
62 ae(c.field_for('path', 3), 'Moved1/Moved1 (3)')
63 fpath = c.format_abspath(1, 'FMT1').replace(os.sep, '/').split('/')
64 ae(fpath[-3:], ['Moved', 'Moved (1)', 'Moved - Moved.fmt1'])
65 af(os.path.exists(os.path.dirname(orig_fpath)), 'Original book folder still exists')
66 # Check that the filesystem reflects fpath (especially on
67 # case-insensitive systems).
68 for x in range(1, 4):
69 base = os.sep.join(fpath[:-x])
70 part = fpath[-x:][0]
71 self.assertIn(part, os.listdir(base))
72
73 initial_side_data = {}
74
75 def init_cache():
76 nonlocal cache, initial_side_data
77 cache = self.init_cache(self.cloned_library)
78 bookdir = os.path.dirname(cache.format_abspath(1, '__COVER_INTERNAL__'))
79 with open(os.path.join(bookdir, 'a.side'), 'w') as f:
80 f.write('a.side')
81 os.mkdir(os.path.join(bookdir, 'subdir'))
82 with open(os.path.join(bookdir, 'subdir', 'a.fmt1'), 'w') as f:
83 f.write('a.fmt1')
84 initial_side_data = side_data()
85
86 def side_data(book_id=1):
87 bookdir = os.path.dirname(cache.format_abspath(book_id, '__COVER_INTERNAL__'))
88 return {
89 'a.side': read(os.path.join(bookdir, 'a.side')),
90 'a.fmt1': read(os.path.join(bookdir, 'subdir', 'a.fmt1')),
91 }
92
93 def check_that_filesystem_and_db_entries_match(book_id):
94 bookdir = os.path.dirname(cache.format_abspath(book_id, '__COVER_INTERNAL__'))
95 if iswindows:
96 from calibre_extensions import winutil
97
98 bookdir = winutil.get_long_path_name(bookdir)
99 bookdir_contents = set(os.listdir(bookdir))
100 expected_contents = {'cover.jpg', 'a.side', 'subdir'}
101 for fmt, fname in cache.fields['formats'].table.fname_map[book_id].items():
102 expected_contents.add(fname + '.' + fmt.lower())
103 ae(expected_contents, bookdir_contents)
104 fs_path = bookdir.split(os.sep)[-2:]
105 db_path = cache.get_book_path(book_id, sep='/').split('/')
106 ae(db_path, fs_path)
107 ae(initial_side_data, side_data(book_id))
108
109 # test only formats being changed
110 init_cache()
111 ef = set()
112 for efx in cache.list_extra_files(1):
113 ef.add(efx.relpath)
114 self.assertTrue(os.path.exists(efx.file_path))
115 self.assertEqual(ef, {'a.side', 'subdir/a.fmt1'})
116 fname = cache.fields['formats'].table.fname_map[1]['FMT1']
117 cache.fields['formats'].table.fname_map[1]['FMT1'] = 'some thing else'
118 cache.fields['formats'].table.fname_map[1]['FMT2'] = fname.upper()
119 cache.backend.update_path(
120 1,
121 cache.field_for('title', 1),
122 cache.field_for('authors', 1)[0],
123 cache.fields['path'],
124 cache.fields['formats'],
125 )
126 check_that_filesystem_and_db_entries_match(1)
127
128 # test a case only change
129 init_cache()
130 title = cache.field_for('title', 1)
131 self.assertNotEqual(title, title.upper())
132 cache.set_field('title', {1: title.upper()})
133 check_that_filesystem_and_db_entries_match(1)
134
135 # test a title change
136 init_cache()
137 cache.set_field('title', {1: 'new changed title'})
138 check_that_filesystem_and_db_entries_match(1)
139 # test an author change
140 cache.set_field('authors', {1: ('new changed author',)})
141 check_that_filesystem_and_db_entries_match(1)
142 # test a double change
143 from calibre.ebooks.metadata.book.base import Metadata
144
145 cache.set_metadata(1, Metadata('t1', ('a1', 'a2')))
146 check_that_filesystem_and_db_entries_match(1)
147 # check that empty author folders are removed
148 for x in os.scandir(cache.backend.library_path):
149 if x.is_dir():
150 self.assertTrue(os.listdir(x.path))
151
152 def test_rename_of_extra_files(self):
153 cl = self.cloned_library
154 cache = self.init_cache(cl)
155 cache.add_extra_files(1, {'a': BytesIO(b'aaa'), 'b': BytesIO(b'bbb')})
156
157 def relpaths():
158 return {e.relpath for e in cache.list_extra_files(1)}
159
160 self.assertEqual(relpaths(), {'a', 'b'})
161 self.assertEqual(cache.rename_extra_files(1, {'a': 'data/c'}), {'a'})
162 self.assertEqual(relpaths(), {'data/c', 'b'})
163 self.assertEqual(cache.rename_extra_files(1, {'b': 'B'}), {'b'})
164 self.assertEqual(relpaths(), {'data/c', 'B'})
165 self.assertEqual(cache.rename_extra_files(1, {'B': 'data/c'}), set())
166 self.assertEqual(cache.rename_extra_files(1, {'B': 'data/c'}, replace=True), {'B'})
167
168 def test_extra_file_paths_are_confined_to_book_dir(self):
169 cl = self.cloned_library
170 cache = self.init_cache(cl)
171 bookdir = os.path.dirname(cache.format_abspath(1, 'FMT1'))
172 sibling = bookdir + ' sibling'
173 os.makedirs(sibling)
174 bad_relpath = '../' + os.path.basename(sibling) + '/victim'
175 victim = os.path.join(sibling, 'victim')
176 with open(victim, 'wb') as f:
177 f.write(b'outside')
178
179 self.assertEqual(cache.add_extra_files(1, {bad_relpath: BytesIO(b'bad')}), {bad_relpath: False})
180 self.assertEqual(read(victim, 'rb'), b'outside')
181
182 buf = BytesIO()
183 with self.assertRaises(FileNotFoundError):
184 cache.copy_extra_file_to(1, bad_relpath, buf)
185 self.assertEqual(buf.getvalue(), b'')
186
187 cache.add_extra_files(1, {'safe': BytesIO(b'safe'), 'data/inside': BytesIO(b'inside')})
188 self.assertEqual(cache.rename_extra_files(1, {bad_relpath: 'moved'}), set())
189 self.assertEqual(cache.rename_extra_files(1, {'safe': bad_relpath}), set())
190 self.assertEqual(read(victim, 'rb'), b'outside')
191
192 def listed(pattern):
193 return {e.relpath for e in cache.list_extra_files(1, use_cache=False, pattern=pattern)}
194
195 self.assertEqual(listed('data/**/*'), {'data/inside'})
196 sibling_pattern = '../' + os.path.basename(sibling) + '/*'
197 for pattern in (
198 sibling_pattern,
199 'data/../../' + os.path.basename(sibling) + '/*',
200 sibling_pattern.replace('/', '\\'),
201 os.path.join(sibling, '*'),
202 ):
203 self.assertEqual(listed(pattern), set())
204 self.assertEqual({e.relpath for e in cache.list_extra_files(1)}, {'safe', 'data/inside'})
205
206 self.assertEqual(cache.remove_extra_files(1, [bad_relpath], permanent=True), {})
207 self.assertEqual(read(victim, 'rb'), b'outside')
208
209 @unittest.skipUnless(iswindows, 'Windows only')
210 def test_windows_atomic_move(self):
211 "Test book file open in another process when changing metadata"
212 cl = self.cloned_library
213 cache = self.init_cache(cl)
214 fpath = cache.format_abspath(1, 'FMT1')
215 with open(fpath, 'rb') as f:
216 with self.assertRaises(IOError):
217 cache.set_field('title', {1: 'Moved'})
218 with self.assertRaises(IOError):
219 cache.remove_books({1})
220 self.assertNotEqual(cache.field_for('title', 1), 'Moved', 'Title was changed despite file lock')
221
222 # Test on folder with hardlinks
223 from calibre.ptempfile import TemporaryDirectory
224 from calibre.utils.filenames import WindowsAtomicFolderMove, hardlink_file
225
226 raw = b'xxx'
227 with TemporaryDirectory() as tdir1, TemporaryDirectory() as tdir2:
228 a, b = os.path.join(tdir1, 'a'), os.path.join(tdir1, 'b')
229 a = os.path.join(tdir1, 'a')
230 with open(a, 'wb') as f:
231 f.write(raw)
232 hardlink_file(a, b)
233 wam = WindowsAtomicFolderMove(tdir1)
234 wam.copy_path_to(a, os.path.join(tdir2, 'a'))
235 wam.copy_path_to(b, os.path.join(tdir2, 'b'))
236 wam.delete_originals()
237 self.assertEqual([], os.listdir(tdir1))
238 self.assertEqual({'a', 'b'}, set(os.listdir(tdir2)))
239 self.assertEqual(raw, read(os.path.join(tdir2, 'a'), 'rb'))
240 self.assertEqual(raw, read(os.path.join(tdir2, 'b'), 'rb'))
241
242 def test_library_move(self):
243 "Test moving of library"
244 from calibre.ptempfile import TemporaryDirectory
245
246 cache = self.init_cache()
247 self.assertIn('metadata.db', cache.get_top_level_move_items()[0])
248 all_ids = cache.all_book_ids()
249 fmt1 = cache.format(1, 'FMT1')
250 cov = cache.cover(1)
251 odir = cache.backend.library_path
252 with TemporaryDirectory('moved_lib') as tdir:
253 cache.move_library_to(tdir)
254 self.assertIn('moved_lib', cache.backend.library_path)
255 self.assertIn('moved_lib', cache.backend.dbpath)
256 self.assertEqual(fmt1, cache.format(1, 'FMT1'))
257 self.assertEqual(cov, cache.cover(1))
258 cache.reload_from_db()
259 self.assertEqual(all_ids, cache.all_book_ids())
260 cache.backend.close()
261 self.assertFalse(os.path.exists(odir))
262 os.mkdir(odir) # needed otherwise tearDown() fails
263
264 def test_long_filenames(self):
265 "Test long file names"
266 cache = self.init_cache()
267 cache.set_field('title', {1: 'a' * 10000})
268 self.assertLessEqual(len(cache.field_for('path', 1)), cache.backend.PATH_LIMIT * 2)
269 cache.set_field('authors', {1: 'b' * 10000})
270 self.assertLessEqual(len(cache.field_for('path', 1)), cache.backend.PATH_LIMIT * 2)
271 fpath = cache.format_abspath(1, cache.formats(1)[0])
272 self.assertLessEqual(len(fpath), len(cache.backend.library_path) + cache.backend.PATH_LIMIT * 4)
273
274 def test_reserved_names(self):
275 "Test that folders are not created with a windows reserve name"
276 cache = self.init_cache()
277 cache.set_field('authors', {1: 'con'})
278 p = cache.field_for('path', 1).replace(os.sep, '/').split('/')
279 self.assertNotIn('con', p)
280
281 def test_fname_change(self):
282 "Test the changing of the filename but not the folder name"
283 cache = self.init_cache()
284 title = 'a' * 30 + 'bbb'
285 cache.backend.PATH_LIMIT = 100
286 cache.set_field('title', {3: title})
287 cache.add_format(3, 'TXT', BytesIO(b'xxx'))
288 cache.backend.PATH_LIMIT = 40
289 cache.set_field('title', {3: title})
290 fpath = cache.format_abspath(3, 'TXT')
291 self.assertEqual(sorted([os.path.basename(fpath)]), sorted(os.listdir(os.path.dirname(fpath))))
292
293 def test_export_import(self):
294 from calibre.db.cache import import_library
295 from calibre.utils.exim import Exporter, Importer
296
297 with TemporaryDirectory('export_lib') as tdir:
298 for part_size in (8, 1, 1024):
299 exporter = Exporter(tdir, part_size=part_size + Exporter.tail_size())
300 files = {
301 'a': b'a' * 7,
302 'b': b'b' * 7,
303 'c': b'c' * 2,
304 'd': b'd' * 9,
305 'e': b'e' * 3,
306 }
307 for key, data in files.items():
308 exporter.add_file(BytesIO(data), key)
309 exporter.commit()
310 importer = Importer(tdir)
311 for key, expected in files.items():
312 with importer.start_file(key, key) as f:
313 actual = f.read()
314 self.assertEqual(expected, actual, key)
315 self.assertFalse(importer.corrupted_files)
316 cache = self.init_cache()
317 bookdir = os.path.dirname(cache.format_abspath(1, '__COVER_INTERNAL__'))
318 with open(os.path.join(bookdir, 'exf'), 'w') as f:
319 f.write('exf')
320 os.mkdir(os.path.join(bookdir, 'sub'))
321 with open(os.path.join(bookdir, 'sub', 'recurse'), 'w') as f:
322 f.write('recurse')
323 self.assertEqual({ef.relpath for ef in cache.list_extra_files(1, pattern='sub/**/*')}, {'sub/recurse'})
324 self.assertEqual({ef.relpath for ef in cache.list_extra_files(1)}, {'exf', 'sub/recurse'})
325 for part_size in (512, 1027, None):
326 with TemporaryDirectory('export_lib') as tdir, TemporaryDirectory('import_lib') as idir:
327 exporter = Exporter(tdir, part_size=part_size if part_size is None else (part_size + Exporter.tail_size()))
328 cache.export_library('l', exporter)
329 exporter.commit()
330 importer = Importer(tdir)
331 ic = import_library('l', importer, idir)
332 self.assertFalse(importer.corrupted_files)
333 self.assertEqual(cache.all_book_ids(), ic.all_book_ids())
334 for book_id in cache.all_book_ids():
335 self.assertEqual(cache.cover(book_id), ic.cover(book_id), f'Covers not identical for book: {book_id}')
336 for fmt in cache.formats(book_id):
337 self.assertEqual(cache.format(book_id, fmt), ic.format(book_id, fmt))
338 self.assertEqual(cache.format_metadata(book_id, fmt)['mtime'], cache.format_metadata(book_id, fmt)['mtime'])
339 bookdir = os.path.dirname(ic.format_abspath(1, '__COVER_INTERNAL__'))
340 self.assertEqual('exf', read(os.path.join(bookdir, 'exf')))
341 self.assertEqual('recurse', read(os.path.join(bookdir, 'sub', 'recurse')))
342 r1 = cache.add_notes_resource(b'res1', 'res.jpg', mtime=time.time() - 113)
343 r2 = cache.add_notes_resource(b'res2', 'res.jpg', mtime=time.time() - 1115)
344 cache.set_notes_for('authors', 2, 'some notes', resource_hashes=(r1, r2))
345 cache.add_format(1, 'TXT', BytesIO(b'testing exim'))
346 cache.fts_indexing_sleep_time = 0.001
347 cache.enable_fts()
348 cache.set_fts_num_of_workers(4)
349 st = time.monotonic()
•350 while cache.fts_indexing_left > 0 and time.monotonic() - st < 300:
351 time.sleep(0.05)
352 if cache.fts_indexing_left > 0:
353 raise ValueError('FTS indexing did not complete')
354 self.assertEqual(cache.fts_search('exim')[0]['id'], 1)
355 with TemporaryDirectory('export_lib') as tdir, TemporaryDirectory('import_lib') as idir:
356 exporter = Exporter(tdir)
357 cache.export_library('l', exporter)
358 exporter.commit()
359 importer = Importer(tdir)
360 ic = import_library('l', importer, idir)
361 self.assertFalse(importer.corrupted_files)
362 self.assertEqual(ic.fts_search('exim')[0]['id'], 1)
363 self.assertEqual(cache.notes_for('authors', 2), ic.notes_for('authors', 2))
364 a, b = cache.get_notes_resource(r1), ic.get_notes_resource(r1)
365 (
366 at,
367 bt,
368 ) = a.pop('mtime'), b.pop('mtime')
369 self.assertEqual(a, b)
370 self.assertLess(abs(at - bt), 2)
371
372 def test_backup_database_retries_transient_ioerror(self):
373 import apsw
374
375 cache = self.init_cache()
376 backend = cache.backend
377 step_calls = {'count': 0}
378 orig_connection = apsw.Connection
379
380 class FlakyBackup:
381 def __init__(self, real):
382 self.real = real
383
384 def __enter__(self):
385 self.real.__enter__()
386 return self
387
388 def __exit__(self, *a):
389 return self.real.__exit__(*a)
390
391 @property
392 def done(self):
393 return self.real.done
394
395 def step(self, npages):
396 step_calls['count'] += 1
397 if step_calls['count'] == 1:
398 e = apsw.IOError('not an error')
399 e.extendedresult = apsw.SQLITE_IOERR_SHORT_READ
400 raise e
401 return self.real.step(npages)
402
403 class FlakyConnection(orig_connection):
404 def backup(self, *a, **kw):
405 return FlakyBackup(super().backup(*a, **kw))
406
407 with TemporaryDirectory('backup_db') as tdir:
408 path = os.path.join(tdir, 'backup.db')
409 apsw.Connection = FlakyConnection
410 try:
411 backend.backup_database(path)
412 finally:
413 apsw.Connection = orig_connection
414 self.assertGreater(step_calls['count'], 1)
415 with closing(orig_connection(path)) as conn:
416 self.assertTrue(conn.cursor().execute('SELECT id FROM books').fetchall())
417
418 # A non-transient I/O error must not be retried
419 class FatalBackup(FlakyBackup):
420 def step(self, npages):
421 step_calls['count'] += 1
422 e = apsw.IOError('disk I/O error')
423 e.extendedresult = apsw.SQLITE_IOERR_WRITE
424 raise e
425
426 class FatalConnection(orig_connection):
427 def backup(self, *a, **kw):
428 return FatalBackup(super().backup(*a, **kw))
429
430 with TemporaryDirectory('backup_db') as tdir:
431 path = os.path.join(tdir, 'backup.db')
432 step_calls['count'] = 0
433 apsw.Connection = FatalConnection
434 try:
435 self.assertRaises(apsw.IOError, backend.backup_database, path)
436 finally:
437 apsw.Connection = orig_connection
438 self.assertEqual(step_calls['count'], 1)
439
440 def test_find_books_in_directory(self):
441 from calibre.db.adding import compile_rule, find_books_in_directory
442
443 def strip(files):
444 return frozenset({os.path.basename(x) for x in files})
445
446 def q(one, two):
447 one, two = {strip(a) for a in one}, {strip(b) for b in two}
448 self.assertEqual(one, two)
449
450 def r(action='ignore', match_type='startswith', query=''):
451 return {'action': action, 'match_type': match_type, 'query': query}
452
453 def c(*rules):
454 return tuple(map(compile_rule, rules))
455
456 files = ['added.epub', 'ignored.md', 'non-book.other']
457 q(['added.epub ignored.md'.split()], find_books_in_directory('', True, listdir_impl=lambda x: files))
458 q([['added.epub'], ['ignored.md']], find_books_in_directory('', False, listdir_impl=lambda x, **k: files))
459 for rules in (
460 c(r(query='ignored.'), r(action='add', match_type='endswith', query='.OTHER')),
461 c(r(match_type='glob', query='*.md'), r(action='add', match_type='matches', query=r'.+\.other$')),
462 c(r(match_type='not_startswith', query='IGnored.', action='add'), r(query='ignored.md')),
463 ):
464 q(
465 ['added.epub non-book.other'.split()],
466 find_books_in_directory('', True, compiled_rules=rules, listdir_impl=lambda x: files),
467 )