Debian Codegraph @

src/calibre/db/cache.py

raw
1 #!/usr/bin/env python
2 # License: GPLv3 Copyright: 2011, Kovid Goyal <[email protected]>
3
4 import hashlib
5 import operator
6 import os
7 import random
8 import shutil
9 import sys
10 import traceback
11 import types
12 import weakref
13 from collections import defaultdict
14 from collections.abc import Callable, Iterable, Iterator, MutableSet, Set
15 from contextlib import contextmanager
16 from datetime import datetime
17 from functools import partial, wraps
18 from io import DEFAULT_BUFFER_SIZE, BytesIO
19 from queue import Queue, ShutDown
20 from threading import Lock
21 from time import mktime, monotonic, time
22 from typing import Any, NamedTuple
23
24 from calibre import as_unicode, detect_ncpus
25 from calibre.constants import iswindows, preferred_encoding
26 from calibre.customize.ui import run_plugins_on_import, run_plugins_on_postadd, run_plugins_on_postdelete, run_plugins_on_postimport
27 from calibre.db import SPOOL_SIZE, _get_next_series_num_for_list
28 from calibre.db.annotations import merge_annotations
29 from calibre.db.categories import get_categories
30 from calibre.db.constants import COVER_FILE_NAME, DATA_DIR_NAME, NOTES_DIR_NAME, Pages
31 from calibre.db.errors import NoSuchBook, NoSuchFormat
32 from calibre.db.fields import IDENTITY, InvalidLinkTable, create_field
33 from calibre.db.lazy import FormatMetadata, FormatsList, ProxyMetadata
34 from calibre.db.listeners import EventDispatcher, EventType
35 from calibre.db.locking import DowngradeLockError, LockingError, RWLockWrapper, SafeReadLock, create_locks, try_lock
36 from calibre.db.notes.connect import copy_marked_up_text
37 from calibre.db.page_count import MaintainPageCounts
38 from calibre.db.search import Search
39 from calibre.db.tables import VirtualTable
40 from calibre.db.utils import type_safe_sort_key_function
41 from calibre.db.write import get_series_values, sqlite_datetime, uniq
42 from calibre.ebooks import check_ebook_format
43 from calibre.ebooks.metadata import author_to_author_sort, string_to_authors, title_sort
44 from calibre.ebooks.metadata.book.base import Metadata
45 from calibre.ebooks.metadata.opf2 import metadata_to_opf
46 from calibre.ptempfile import PersistentTemporaryFile, SpooledTemporaryFile, base_dir
47 from calibre.utils.config import prefs, tweaks
48 from calibre.utils.date import UNDEFINED_DATE, is_date_undefined, timestampfromdt, utcnow
49 from calibre.utils.date import now as nowf
50 from calibre.utils.filenames import make_long_path_useable
51 from calibre.utils.icu import lower as icu_lower
52 from calibre.utils.icu import sort_key
53 from calibre.utils.iso8601 import parse_iso8601
54 from calibre.utils.localization import _, canonicalize_lang
55 from polyglot.builtins import cmp
56
57
58 class ExtraFile(NamedTuple):
59 relpath: str
60 file_path: str
61 stat_result: os.stat_result
62
63
64 cache_api: dict[str, bool | None] = {}
65
66
67 def api[T: types.FunctionType](f: T) -> T:
68 cache_api[f.__name__] = None
69 return f
70
71
72 def read_api[T: types.FunctionType](f: T) -> T:
73 cache_api[f.__name__] = False
74 return f
75
76
77 def write_api[T: types.FunctionType](f: T) -> T:
78 cache_api[f.__name__] = True
79 return f
80
81
82 def wrap_simple[**P, R](lock: RWLockWrapper, func: Callable[P, R]) -> Callable[P, R]:
83 @wraps(func)
84 def call_func_with_lock(*args: P.args, **kwargs: P.kwargs) -> R:
85 try:
86 with lock:
87 return func(*args, **kwargs)
88 except DowngradeLockError:
89 # We already have an exclusive lock, no need to acquire a shared
90 # lock. See the safe_read_lock properties' documentation for why
91 # this is necessary.
92 return func(*args, **kwargs)
93
94 return call_func_with_lock
95
96
97 def run_import_plugins(path_or_stream, fmt):
98 fmt = fmt.lower()
99 if hasattr(path_or_stream, 'seek'):
100 path_or_stream.seek(0)
101 pt = PersistentTemporaryFile('_import_plugin.' + fmt)
102 shutil.copyfileobj(path_or_stream, pt, 1024**2)
103 pt.close()
104 path = pt.name
105 else:
106 path = path_or_stream
107 return run_plugins_on_import(path, fmt)
108
109
110 def _add_newbook_tag(mi):
111 tags = prefs['new_book_tags']
112 if tags:
113 for tag in [t.strip() for t in tags]:
114 if tag:
115 if not mi.tags:
116 mi.tags = [tag]
117 elif tag not in mi.tags:
118 mi.tags.append(tag)
119
120
121 def _add_default_custom_column_values(mi, fm):
122 cols = fm.custom_field_metadata(include_composites=False)
123 for cc, col in cols.items():
124 dv = col['display'].get('default_value', None)
125 if dv is None:
126 continue
127 try:
128 if not (had_meta := mi.get_user_metadata(cc, make_copy=False)):
129 mi.set_user_metadata(cc, col)
130 dt = col['datatype']
131 if dt == 'datetime' and icu_lower(dv) == 'now':
132 dv = nowf()
133 if not had_meta or mi.get(cc) is None:
134 mi.set(cc, dv)
135 except Exception:
136 traceback.print_exc()
137
138
139 dynamic_category_preferences = frozenset({
140 'grouped_search_make_user_categories',
141 'grouped_search_terms',
142 'user_categories',
143 })
144
145
146 class Cache:
147 """
148 An in-memory cache of the metadata.db file from a calibre library.
149 This class also serves as a threadsafe API for accessing the database.
150 The in-memory cache is maintained in normal form for maximum performance.
151
152 SQLITE is simply used as a way to read and write from metadata.db robustly.
153 All table reading/sorting/searching/caching logic is re-implemented. This
154 was necessary for maximum performance and flexibility.
155 """
156
157 EventType = EventType
158 fts_indexing_sleep_time: float = 4 # seconds
159 server_library_id: str = ''
160
161 def __init__(self, backend, library_database_instance=None):
162 self.shutting_down = False
163 self.is_doing_rebuild_or_vacuum = False
164 self.backend = backend
165 self.maintain_page_counts = MaintainPageCounts(self)
166 self.maintain_page_counts.start()
167 # We want templates to have access to LibraryDatabase if we have it,
168 # otherwise this instance (Cache)
169 self.database_instance = weakref.ref(self) if library_database_instance is None else weakref.ref(library_database_instance)
170 self.event_dispatcher = EventDispatcher()
171 self.fields = {}
172 self.composites = {}
173 self.read_lock, self.write_lock = create_locks()
174 self.format_metadata_cache = defaultdict(dict)
175 self.formatter_template_cache = {}
176 self.dirtied_cache = {}
177 self.link_maps_cache = {}
178 self.extra_files_cache = {}
179 self.vls_for_books_cache = None
180 self.vls_for_books_lib_in_process = None
181 self.vls_cache_lock = Lock()
182 self.dirtied_sequence = 0
183 self.cover_caches = set()
184 self.clear_search_cache_count = 0
185
186 # Implement locking for all simple read/write API methods
187 # An unlocked version of the method is stored with the name starting
188 # with a leading underscore. Use the unlocked versions when the lock
189 # has already been acquired.
190 for name in dir(self):
191 if (is_write_api := cache_api.get(name)) is not None:
192 func = getattr(self, name)
193 # Wrap it in a lock
194 lock = self.write_lock if is_write_api else self.read_lock
195 setattr(self, name, wrap_simple(lock, func))
196
197 self._search_api = Search(self, 'saved_searches', self.field_metadata.get_search_terms())
198 self.initialize_dynamic()
199 self.initialize_fts()
200
201 @property
202 def new_api(self):
203 return self
204
205 @property
206 def library_id(self):
207 return self.backend.library_id
208
209 @property
210 def dbpath(self):
211 return self.backend.dbpath
212
213 @property
214 def is_fat_filesystem(self):
215 return self.backend.is_fat_filesystem
216
217 @property
218 def safe_read_lock(self):
219 """A safe read lock is a lock that does nothing if the thread already
220 has a write lock, otherwise it acquires a read lock. This is necessary
221 to prevent DowngradeLockErrors, which can happen when updating the
222 search cache in the presence of composite columns. Updating the search
223 cache holds an exclusive lock, but searching a composite column
224 involves reading field values via ProxyMetadata which tries to get a
225 shared lock. There may be other scenarios that trigger this as well.
226
227 This property returns a new lock object on every access. This lock
228 object is not recursive (for performance) and must only be used in a
229 with statement as ``with cache.safe_read_lock:`` otherwise bad things
230 will happen."""
231 return SafeReadLock(self.read_lock)
232
233 @write_api
234 def ensure_has_search_category(self, fail_on_existing=True):
235 if len(self._search_api.saved_searches.names()) > 0:
236 self.field_metadata.add_search_category(label='search', name=_('Saved searches'), fail_on_existing=fail_on_existing)
237
238 _ensure_has_search_category = ensure_has_search_category
239
240 def _initialize_dynamic_categories(self):
241 # Reconstruct the user categories, putting them into field_metadata
242 fm = self.field_metadata
243 fm.remove_dynamic_categories()
244 for user_cat in sorted(self._pref('user_categories', {}), key=sort_key):
245 cat_name = '@' + user_cat # add the '@' to avoid name collision
246 while cat_name:
247 try:
248 fm.add_user_category(label=cat_name, name=user_cat)
249 except ValueError:
250 break # Can happen since we are removing dots and adding parent categories ourselves
251 cat_name = cat_name.rpartition('.')[0]
252
253 # add grouped search term user categories
254 muc = frozenset(self._pref('grouped_search_make_user_categories', []))
255 for cat in sorted(self._pref('grouped_search_terms', {}), key=sort_key):
256 if cat in muc:
257 # There is a chance that these can be duplicates of an existing
258 # user category. Print the exception and continue.
259 try:
260 self.field_metadata.add_user_category(label='@' + cat, name=cat)
261 except ValueError:
262 traceback.print_exc()
263 self._ensure_has_search_category()
264
265 self.field_metadata.add_grouped_search_terms(self._pref('grouped_search_terms', {}))
266 self._refresh_search_locations()
267
268 @write_api
269 def initialize_dynamic(self):
270 self.backend.dirty_books_with_dirtied_annotations()
271 self.dirtied_cache = {x: i for i, x in enumerate(self.backend.dirtied_books())}
272 if self.dirtied_cache:
273 self.dirtied_sequence = max(self.dirtied_cache.values()) + 1
274 self._initialize_dynamic_categories()
275
276 _initialize_dynamic = initialize_dynamic
277
278 @write_api
279 def initialize_template_cache(self):
280 self.formatter_template_cache = {}
281
282 _initialize_template_cache = initialize_template_cache
283
284 @write_api
285 def set_user_template_functions(self, user_template_functions):
286 self.backend.set_user_template_functions(user_template_functions)
287
288 _set_user_template_functions = set_user_template_functions
289
290 @write_api
291 def clear_composite_caches(self, book_ids=None):
292 for field in self.composites.values():
293 field.clear_caches(book_ids=book_ids)
294
295 _clear_composite_caches = clear_composite_caches
296
297 @write_api
298 def clear_search_caches(self, book_ids=None):
299 self.clear_search_cache_count += 1
300 self._search_api.update_or_clear(self, book_ids)
301 self.vls_for_books_cache = None
302 self.vls_for_books_lib_in_process = None
303
304 _clear_search_caches = clear_search_caches
305
306 @write_api
307 def clear_extra_files_cache(self, book_id=None):
308 if book_id is None:
309 self.extra_files_cache = {}
310 else:
311 self.extra_files_cache.pop(book_id, None)
312
313 _clear_extra_files_cache = clear_extra_files_cache
314
315 @read_api
316 def last_modified(self):
317 return self.backend.last_modified()
318
319 _last_modified = last_modified
320
321 def __enter__(self):
322 self.backend.__enter__()
323
324 def __exit__(self, exc_type, exc_value, tb):
325 self.backend.__exit__(exc_type, exc_value, tb)
326
327 @write_api
328 def clear_caches(self, book_ids=None, template_cache=True, search_cache=True):
329 if template_cache:
330 self._initialize_template_cache() # Clear the formatter template cache
331 for field in self.fields.values():
332 if hasattr(field, 'clear_caches'):
333 field.clear_caches(book_ids=book_ids) # Clear the composite cache and ondevice caches
334 if book_ids:
335 for book_id in book_ids:
336 self.format_metadata_cache.pop(book_id, None)
337 else:
338 self.format_metadata_cache.clear()
339 if search_cache:
340 self._clear_search_caches(book_ids)
341 self._clear_link_map_cache(book_ids)
342
343 _clear_caches = clear_caches
344
345 @write_api
346 def clear_link_map_cache(self, book_ids=None):
347 if book_ids is None:
348 self.link_maps_cache = {}
349 else:
350 for book in book_ids:
351 self.link_maps_cache.pop(book, None)
352
353 _clear_link_map_cache = clear_link_map_cache
354
355 @write_api
356 def reload_from_db(self, clear_caches=True):
357 if clear_caches:
358 self._clear_caches()
359 with self.backend.conn: # Prevent other processes, such as calibredb from interrupting the reload by locking the db
360 self.backend.prefs.load_from_db()
361 self._search_api.saved_searches.load_from_db()
362 for field in self.fields.values():
363 if hasattr(field, 'table'):
364 field.table.read(self.backend) # Reread data from metadata.db
365
366 _reload_from_db = reload_from_db
367
368 @property
369 def field_metadata(self):
370 return self.backend.field_metadata
371
372 def _get_metadata(self, book_id, get_user_categories=True): # {{{
373 from calibre.ebooks.metadata.book.formatter import SafeFormat
374
375 mi = Metadata(None, template_cache=self.formatter_template_cache, formatter=SafeFormat())
376
377 mi._proxy_metadata = ProxyMetadata(self, book_id, formatter=mi.formatter)
378
379 author_ids = self._field_ids_for('authors', book_id)
380 adata = self._author_data(author_ids)
381 aut_list = [adata[i] for i in author_ids]
382 aum = []
383 aus = {}
384 for rec in aut_list:
385 aut = rec['name']
386 aum.append(aut)
387 aus[aut] = rec['sort']
388 mi.title = self._field_for('title', book_id, default_value=_('Unknown'))
389 mi.authors = aum
390 mi.author_sort = self._field_for('author_sort', book_id, default_value=_('Unknown'))
391 mi.author_sort_map = aus
392 mi.comments = self._field_for('comments', book_id)
393 mi.publisher = self._field_for('publisher', book_id)
394 n = utcnow()
395 mi.timestamp = self._field_for('timestamp', book_id, default_value=n)
396 mi.pubdate = self._field_for('pubdate', book_id, default_value=n)
397 mi.uuid = self._field_for('uuid', book_id, default_value='dummy')
398 mi.title_sort = self._field_for('sort', book_id, default_value=_('Unknown'))
399 mi.last_modified = self._field_for('last_modified', book_id, default_value=n)
400 formats = self._field_for('formats', book_id)
401 mi.format_metadata = {}
402 mi.languages = list(self._field_for('languages', book_id))
403 mi.pages = self._field_for('pages', book_id, default_value=0)
404 if not formats:
405 good_formats = None
406 else:
407 mi.format_metadata = FormatMetadata(self, book_id, formats)
408 good_formats = FormatsList(sorted(formats), mi.format_metadata)
409 # These three attributes are returned by the db2 get_metadata(),
410 # however, we don't actually use them anywhere other than templates, so
411 # they have been removed, to avoid unnecessary overhead. The templates
412 # all use _proxy_metadata.
413 # mi.book_size = self._field_for('size', book_id, default_value=0)
414 # mi.ondevice_col = self._field_for('ondevice', book_id, default_value='')
415 # mi.db_approx_formats = formats
416 mi.formats = good_formats
417 mi.has_cover = _('Yes') if self._field_for('cover', book_id, default_value=False) else ''
418 mi.tags = list(self._field_for('tags', book_id, default_value=()))
419 mi.series = self._field_for('series', book_id)
420 if mi.series:
421 mi.series_index = self._field_for('series_index', book_id, default_value=1.0)
422 mi.rating = self._field_for('rating', book_id)
423 mi.set_identifiers(self._field_for('identifiers', book_id, default_value={}))
424 mi.application_id = book_id
425 mi.id = book_id
426 for key, meta in self.field_metadata.custom_iteritems():
427 mi.set_user_metadata(key, meta)
428 if meta['datatype'] != 'composite':
429 # composites are evaluated on demand in metadata.book.base
430 # because their value is None
431 val = self._field_for(key, book_id)
432 if isinstance(val, tuple):
433 val = list(val)
434 extra = self._field_for(key + '_index', book_id)
435 mi.set(key, val=val, extra=extra)
436
437 mi.link_maps = self._get_all_link_maps_for_book(book_id)
438
439 user_cat_vals = {}
440 if get_user_categories:
441 user_cats = self._pref('user_categories', {})
442 for ucat in user_cats:
443 res = []
444 for name, cat, ign in user_cats[ucat]:
445 v = mi.get(cat, None)
446 if isinstance(v, list):
447 if name in v:
448 res.append([name, cat])
449 elif name == v:
450 res.append([name, cat])
451 user_cat_vals[ucat] = res
452 mi.user_categories = user_cat_vals
453
454 return mi
455
456 # }}}
457
458 @api
459 def init(self):
460 """
461 Initialize this cache with data from the backend.
462 """
463 with self.write_lock:
464 self.backend.read_tables()
465 bools_are_tristate = self.backend.prefs['bools_are_tristate']
466
467 for field, table in self.backend.tables.items():
468 self.fields[field] = create_field(field, table, bools_are_tristate, self.backend.get_template_functions, self.database_instance)
469 if table.metadata['datatype'] == 'composite':
470 self.composites[field] = self.fields[field]
471
472 self.fields['ondevice'] = create_field(
473 'ondevice',
474 VirtualTable('ondevice'),
475 bools_are_tristate,
476 self.backend.get_template_functions,
477 self.database_instance,
478 )
479
480 for name, field in self.fields.items():
481 if name[0] == '#' and name.endswith('_index'):
482 field.series_field = self.fields[name[: -len('_index')]]
483 self.fields[name[: -len('_index')]].index_field = field
484 elif name == 'series_index':
485 field.series_field = self.fields['series']
486 self.fields['series'].index_field = field
487 elif name == 'authors':
488 field.author_sort_field = self.fields['author_sort']
489 elif name == 'title':
490 field.title_sort_field = self.fields['sort']
491 if self.backend.prefs.get('full_page_scan_requested'):
492 self._queue_pages_scan(by_user=False)
493 if self.backend.prefs['update_all_last_mod_dates_on_start']:
494 self.update_last_modified(self.all_book_ids())
495 self.backend.prefs.set('update_all_last_mod_dates_on_start', False)
496
497 _init = init
498
499 # FTS API {{{
500 def initialize_fts(self):
501 self.fts_queue_thread = None
502 self.fts_measuring_rate = None
503 self.fts_num_done_since_start = 0
504 self.fts_job_queue = Queue()
505 self.fts_indexing_left = self.fts_indexing_total = 0
506 fts = self.backend.initialize_fts(weakref.ref(self))
507 if self.is_fts_enabled():
508 self.start_fts_pool()
509 return fts
510
511 def start_fts_pool(self):
512 from threading import Event, Thread
513
514 self.fts_dispatch_stop_event = Event()
515 self.fts_queue_thread = Thread(
516 name='FTSQueue',
517 target=Cache.dispatch_fts_jobs,
518 args=(self.fts_job_queue, self.fts_dispatch_stop_event, weakref.ref(self)),
519 daemon=True,
520 )
521 self.fts_queue_thread.start()
522 self.backend.fts.pool.initialize()
523 self.backend.fts.pool.initialized.wait()
524 self.queue_next_fts_job()
525
526 @read_api
527 def is_fts_enabled(self):
528 return self.backend.fts_enabled
529
530 _is_fts_enabled = is_fts_enabled
531
532 @write_api
533 def fts_start_measuring_rate(self, measure=True):
534 self.fts_measuring_rate = monotonic() if measure else None
535 self.fts_num_done_since_start = 0
536
537 _fts_start_measuring_rate = fts_start_measuring_rate
538
539 def _update_fts_indexing_numbers(self, job_time=None):
540 # this is called when new formats are added and when a format is
541 # indexed, but NOT when books or formats are deleted, so total may not
542 # be up to date.
543 nl = self.backend.fts.number_dirtied()
544 nt = self.backend.get('SELECT COUNT(*) FROM main.data')[0][0] or 0
545 if not nl:
546 self._fts_start_measuring_rate(measure=False)
547 if job_time is not None and self.fts_measuring_rate is not None:
548 self.fts_num_done_since_start += 1
549 if (self.fts_indexing_left, self.fts_indexing_total) != (nl, nt) or job_time is not None:
550 self.fts_indexing_left = nl
551 self.fts_indexing_total = nt
552 self.event_dispatcher(EventType.indexing_progress_changed, *self._fts_indexing_progress())
553
554 @read_api
555 def fts_indexing_progress(self):
556 rate = None
557 if self.fts_measuring_rate is not None and self.fts_num_done_since_start > 4:
558 rate = self.fts_num_done_since_start / (monotonic() - self.fts_measuring_rate)
559 return self.fts_indexing_left, self.fts_indexing_total, rate
560
561 _fts_indexing_progress = fts_indexing_progress
562
563 @write_api
564 def enable_fts(self, enabled=True, start_pool=True):
565 fts = self.backend.enable_fts(weakref.ref(self) if enabled else None)
566 if fts and start_pool: # used in the tests
567 self.start_fts_pool()
568 if not fts and self.fts_queue_thread:
569 self.fts_job_queue.put(None)
570 self.fts_queue_thread = None
571 self.fts_job_queue = Queue()
572 if fts:
573 self._update_fts_indexing_numbers()
574 return fts
575
576 _enable_fts = enable_fts
577
578 @write_api
579 def fts_unindex(self, book_id, fmt=None):
580 self.backend.fts_unindex(book_id, fmt=fmt)
581
582 _fts_unindex = fts_unindex
583
584 @staticmethod
585 def dispatch_fts_jobs(queue, stop_dispatch, dbref):
586 from .fts.text import is_fmt_extractable
587
588 def do_one():
589 self = dbref()
590 if self is None:
591 return False
592 start_time = monotonic()
593 with self.read_lock:
594 if not self.backend.fts_enabled:
595 return False
596 book_id, fmt = self.backend.get_next_fts_job()
597 if book_id is None:
598 return False
599 path = self._format_abspath(book_id, fmt)
600 if not path or not is_fmt_extractable(fmt):
601 with self.write_lock:
602 self.backend.remove_dirty_fts(book_id, fmt)
603 self._update_fts_indexing_numbers()
604 return True
605
606 with self.read_lock, open(path, 'rb') as src, PersistentTemporaryFile(suffix=f'.{fmt.lower()}') as pt:
607 sz = 0
608 h = hashlib.sha1()
609 while True:
610 chunk = src.read(DEFAULT_BUFFER_SIZE)
611 if not chunk:
612 break
613 sz += len(chunk)
614 h.update(chunk)
615 pt.write(chunk)
616 with self.write_lock:
617 queued = self.backend.queue_fts_job(book_id, fmt, pt.name, sz, h.hexdigest(), start_time)
618 if not queued: # means a dirtied book was removed from the dirty list because the text has not changed
619 self._update_fts_indexing_numbers(monotonic() - start_time)
620 return self.backend.fts_has_idle_workers
621
622 def loop_while_more_available():
623 self = dbref()
624 if not self or not self.backend.fts_enabled:
625 return
626 has_more = True
627 while has_more and not self.shutting_down and self.backend.fts_enabled and not stop_dispatch.is_set():
628 try:
629 has_more = do_one()
630 except Exception:
631 if self.backend.fts_enabled:
632 traceback.print_exc()
633 if stop_dispatch.wait(self.fts_indexing_sleep_time):
634 break
635
636 while True:
637 try:
638 x = queue.get()
639 except ShutDown:
640 break
641 if x is None:
642 break
643 loop_while_more_available()
644
645 @write_api
646 def queue_next_fts_job(self):
647 if not self.backend.fts_enabled:
648 return
649 self.fts_job_queue.put(True)
650 self._update_fts_indexing_numbers()
651
652 _queue_next_fts_job = queue_next_fts_job
653
654 @write_api
655 def commit_fts_result(self, book_id, fmt, fmt_size, fmt_hash, text, err_msg, start_time):
656 ans = self.backend.commit_fts_result(book_id, fmt, fmt_size, fmt_hash, text, err_msg)
657 self._update_fts_indexing_numbers(monotonic() - start_time)
658 return ans
659
660 _commit_fts_result = commit_fts_result
661
662 @write_api
663 def reindex_fts_book(self, book_id, *fmts):
664 if not self.is_fts_enabled():
665 return
666 if not fmts:
667 fmts = self._formats(book_id)
668 self.backend.reindex_fts_book(book_id, *fmts)
669 self._queue_next_fts_job()
670
671 _reindex_fts_book = reindex_fts_book
672
673 @api
674 def reindex_fts(self):
675 if not self.is_fts_enabled():
676 return
677 with self.write_lock:
678 self._shutdown_fts()
679 self._shutdown_fts(stage=2)
680 with self.write_lock:
681 self.backend.reindex_fts()
682 fts = self.initialize_fts()
683 fts.initialize(self.backend.conn) # ensure fts is pre-initialized needed for the tests
684 self._queue_next_fts_job()
685 return fts
686
687 _reindex_fts = reindex_fts
688
689 @write_api
690 def set_fts_num_of_workers(self, num):
691 existing = self.backend.fts_num_of_workers
692 if num != existing:
693 self.backend.fts_num_of_workers = num
694 if num > existing:
695 self._queue_next_fts_job()
696 return True
697 return False
698
699 _set_fts_num_of_workers = set_fts_num_of_workers
700
701 @write_api
702 def set_fts_speed(self, slow=True):
703 orig = self.fts_indexing_sleep_time
704 if slow:
705 self.fts_indexing_sleep_time = Cache.fts_indexing_sleep_time
706 changed = self._set_fts_num_of_workers(1)
707 else:
708 self.fts_indexing_sleep_time = 0.1
709 changed = self._set_fts_num_of_workers(max(1, detect_ncpus()))
710 changed = changed or orig != self.fts_indexing_sleep_time
711 if changed and self.fts_measuring_rate is not None:
712 self._fts_start_measuring_rate()
713 return changed
714
715 _set_fts_speed = set_fts_speed
716
717 @write_api # we need to use write locking as SQLITE gives a locked table error if multiple FTS queries are made at the same time
718 def fts_search(
719 self,
720 fts_engine_query,
721 use_stemming=True,
722 highlight_start=None,
723 highlight_end=None,
724 snippet_size=None,
725 restrict_to_book_ids=None,
726 return_text=True,
727 result_type=tuple,
728 process_each_result=None,
729 ):
730 return result_type(
731 self.backend.fts_search(
732 fts_engine_query,
733 use_stemming=use_stemming,
734 highlight_start=highlight_start,
735 highlight_end=highlight_end,
736 snippet_size=snippet_size,
737 return_text=return_text,
738 restrict_to_book_ids=restrict_to_book_ids,
739 process_each_result=process_each_result,
740 )
741 )
742
743 _fts_search = fts_search
744
745 # }}}
746
747 # Notes API {{{
748 @read_api
749 def notes_for(self, field, item_id) -> str:
750 "Return the notes document or an empty string if not found"
751 return self.backend.notes_for(field, item_id)
752
753 _notes_for = notes_for
754
755 @read_api
756 def notes_data_for(self, field, item_id) -> dict[str, Any] | None:
757 "Return all notes data as a dict or None if note does not exist"
758 return self.backend.notes_data_for(field, item_id)
759
760 _notes_data_for = notes_data_for
761
762 @read_api
763 def get_all_items_that_have_notes(self, field_name=None) -> set[int] | dict[str, set[int]]:
764 "Return all item_ids for items that have notes in the specified field or all fields if field_name is None"
765 return self.backend.get_all_items_that_have_notes(field_name)
766
767 _get_all_items_that_have_notes = get_all_items_that_have_notes
768
769 @read_api
770 def field_supports_notes(self, field=None) -> bool | frozenset:
771 "Return True iff the specified field supports notes. If field is None return frozenset of all fields that support notes."
772 if field is None:
773 return self.backend.notes.allowed_fields
774 return field in self.backend.notes.allowed_fields
775
776 _field_supports_notes = field_supports_notes
777
778 @read_api
779 def items_with_notes_in_book(self, book_id: int) -> dict[str, dict[int, str]]:
780 "Return a dict of field to items that have associated notes for that field for the specified book"
781 ans = {}
782 for k in self.backend.notes.allowed_fields:
783 try:
784 field = self.fields[k]
785 except KeyError:
786 continue
787 v = {}
788 for item_id in field.ids_for_book(book_id):
789 if self.backend.notes_for(k, item_id):
790 v[item_id] = field.table.id_map[item_id]
791 if v:
792 ans[k] = v
793 return ans
794
795 _items_with_notes_in_book = items_with_notes_in_book
796
797 @write_api
798 def set_notes_for(
799 self,
800 field,
801 item_id,
802 doc: str,
803 searchable_text: str = copy_marked_up_text,
804 resource_hashes=(),
805 remove_unused_resources=False,
806 ) -> int:
807 """
808 Set the notes document. If the searchable text is different from the document, specify it as searchable_text. If the document
809 references resources their hashes must be present in resource_hashes. Set remove_unused_resources to True to cleanup unused
810 resources, note that updating a note automatically cleans up resources pertaining to that note anyway.
811 """
812 ans = self.backend.set_notes_for(field, item_id, doc, searchable_text, resource_hashes, remove_unused_resources)
813 self.event_dispatcher(EventType.notes_changed, field, frozenset({item_id}))
814 return ans
815
816 _set_notes_for = set_notes_for
817
818 @write_api
819 def add_notes_resource(self, path_or_stream_or_data, name: str, mtime: float | None = None) -> int:
820 "Add the specified resource so it can be referenced by notes and return its content hash"
821 return self.backend.add_notes_resource(path_or_stream_or_data, name, mtime)
822
823 _add_notes_resource = add_notes_resource
824
825 @read_api
826 def get_notes_resource(self, resource_hash) -> dict | None:
827 "Return a dict containing the resource data and name or None if no resource with the specified hash is found"
828 return self.backend.get_notes_resource(resource_hash)
829
830 _get_notes_resource = get_notes_resource
831
832 @read_api
833 def notes_resources_used_by(self, field, item_id):
834 "Return the set of resource hashes of all resources used by the note for the specified item"
835 return frozenset(self.backend.notes_resources_used_by(field, item_id))
836
837 _notes_resources_used_by = notes_resources_used_by
838
839 @write_api
840 def unretire_note_for(self, field, item_id) -> int:
841 "Unretire a previously retired note for the specified item. Notes are retired when an item is removed from the database"
842 ans = self.backend.unretire_note_for(field, item_id)
843 self.event_dispatcher(EventType.notes_changed, field, frozenset({item_id}))
844 return ans
845
846 _unretire_note_for = unretire_note_for
847
848 @read_api
849 def export_note(self, field, item_id) -> str:
850 "Export the note as a single HTML document with embedded images as data: URLs"
851 return self.backend.export_note(field, item_id)
852
853 _export_note = export_note
854
855 @write_api
856 def import_note(self, field, item_id, path_to_html_file, path_is_data=False):
857 "Import a previously exported note or an arbitrary HTML file as the note for the specified item"
858 if path_is_data:
859 html = path_to_html_file
860 ctime = mtime = time()
861 basedir = base_dir()
862 else:
863 with open(path_to_html_file, 'rb') as f:
864 html = f.read()
865 st = os.stat(f.fileno())
866 try:
867 ctime, mtime = st.st_birthtime, st.st_mtime
868 except AttributeError:
869 ctime, mtime = st.st_ctime, st.st_mtime # type: ignore
870 basedir = os.path.dirname(os.path.abspath(path_to_html_file))
871 ans = self.backend.import_note(field, item_id, html, basedir, ctime, mtime)
872 self.event_dispatcher(EventType.notes_changed, field, frozenset({item_id}))
873 return ans
874
875 _import_note = import_note
876
877 @write_api # we need to use write locking as SQLITE gives a locked table error if multiple FTS queries are made at the same time
878 def search_notes(
879 self,
880 fts_engine_query='',
881 use_stemming=True,
882 highlight_start=None,
883 highlight_end=None,
884 snippet_size=None,
885 restrict_to_fields=(),
886 return_text=True,
887 result_type=tuple,
888 process_each_result=None,
889 limit=None,
890 ):
891 "Search the text of notes using an FTS index. If the query is empty return all notes."
892 return result_type(
893 self.backend.search_notes(
894 fts_engine_query,
895 use_stemming=use_stemming,
896 highlight_start=highlight_start,
897 highlight_end=highlight_end,
898 snippet_size=snippet_size,
899 return_text=return_text,
900 restrict_to_fields=restrict_to_fields,
901 process_each_result=process_each_result,
902 limit=limit,
903 )
904 )
905
906 _search_notes = search_notes
907
908 # }}}
909
910 # Cache Layer API {{{
911
912 @write_api
913 def add_listener(self, event_callback_function, check_already_added=False):
914 """
915 Register a callback function that will be called after certain actions are
916 taken on this database. The function must take three arguments:
917 (:class:`EventType`, library_id, event_type_specific_data)
918 """
919 self.event_dispatcher.library_id = getattr(self, 'server_library_id', self.library_id)
920 if check_already_added and event_callback_function in self.event_dispatcher:
921 return False
922 self.event_dispatcher.add_listener(event_callback_function)
923 return True
924
925 _add_listener = add_listener
926
927 @write_api
928 def remove_listener(self, event_callback_function):
929 self.event_dispatcher.remove_listener(event_callback_function)
930
931 _remove_listener = remove_listener
932
933 @read_api
934 def field_for(self, name, book_id, default_value=None):
935 """
936 Return the value of the field ``name`` for the book identified
937 by ``book_id``. If no such book exists or it has no defined
938 value for the field ``name`` or no such field exists, then
939 ``default_value`` is returned.
940
941 ``default_value`` is not used for title, title_sort, authors, author_sort
942 and series_index. This is because these always have values in the db.
943 ``default_value`` is used for all custom columns.
944
945 The returned value for is_multiple fields are always tuples, even when
946 no values are found (in other words, default_value is ignored). The
947 exception is identifiers for which the returned value is always a dictionary.
948 The returned tuples are always in link order, that is, the order in
949 which they were created.
950 """
951 if self.composites and name in self.composites:
952 return self.composite_for(name, book_id, default_value=default_value)
953 try:
954 field = self.fields[name]
955 except KeyError:
956 return default_value
957 if field.is_multiple:
958 default_value = field.default_value
959 try:
960 return field.for_book(book_id, default_value=default_value)
961 except KeyError, IndexError:
962 return default_value
963
964 _field_for = field_for
965
966 @read_api
967 def fast_field_for(self, field_obj, book_id, default_value=None):
968 "Same as field_for, except that it avoids the extra lookup to get the field object"
969 if field_obj.is_composite:
970 return field_obj.get_value_with_cache(book_id, self._get_proxy_metadata)
971 if field_obj.is_multiple:
972 default_value = field_obj.default_value
973 try:
974 return field_obj.for_book(book_id, default_value=default_value)
975 except KeyError, IndexError:
976 return default_value
977
978 _fast_field_for = fast_field_for
979
980 @read_api
981 def all_field_for(self, field, book_ids, default_value=None):
982 "Same as field_for, except that it operates on multiple books at once"
983 field_obj = self.fields[field]
984 return {book_id: self._fast_field_for(field_obj, book_id, default_value=default_value) for book_id in book_ids}
985
986 _all_field_for = all_field_for
987
988 @read_api
989 def composite_for(self, name, book_id, mi=None, default_value=''):
990 try:
991 f = self.fields[name]
992 except KeyError:
993 return default_value
994
995 if mi is None:
996 return f.get_value_with_cache(book_id, self._get_proxy_metadata)
997 else:
998 return f._render_composite_with_cache(book_id, mi, mi.formatter, mi.template_cache)
999
1000 _composite_for = composite_for
1001
1002 @read_api
1003 def field_ids_for(self, name, book_id):
1004 """
1005 Return the ids (as a tuple) for the values that the field ``name`` has on the book
1006 identified by ``book_id``. If there are no values, or no such book, or
1007 no such field, an empty tuple is returned.
1008 """
1009 try:
1010 return self.fields[name].ids_for_book(book_id)
1011 except KeyError, IndexError:
1012 return ()
1013
1014 _field_ids_for = field_ids_for
1015
1016 @read_api
1017 def books_for_field(self, name, item_id):
1018 """
1019 Return all the books associated with the item identified by
1020 ``item_id``, where the item belongs to the field ``name``.
1021
1022 Returned value is a set of book ids, or the empty set if the item
1023 or the field does not exist.
1024 """
1025 try:
1026 return self.fields[name].books_for(item_id)
1027 except KeyError, IndexError:
1028 return set()
1029
1030 _books_for_field = books_for_field
1031
1032 @read_api
1033 def books_by_year(self, field: str = 'pubdate', restrict_to_books: Iterable[int] | None = None) -> dict[int, set[int]]:
1034 books_by_year = defaultdict(set)
1035 m = self.field_metadata[field]
1036 value_column = m.get('column') or field
1037 table = m.get('table') or 'books'
1038 book_col = 'id' if table == 'books' else 'book'
1039 query = f'SELECT CAST(substr({value_column}, 1, 4) AS INTEGER), {book_col} FROM {table}'
1040 if restrict_to_books is None:
1041 for year, book_id in self.backend.execute(query):
1042 books_by_year[year].add(book_id)
1043 else:
1044 books = tuple(restrict_to_books)
1045 BATCH_SIZE = self.backend.max_number_of_variables
1046 for i in range(0, len(books), BATCH_SIZE):
1047 batch = books[i : i + BATCH_SIZE]
1048 placeholders = '?,' * len(batch)
1049 for year, book_id in self.backend.execute(query + f' WHERE {book_col} IN ({placeholders[:-1]})', batch):
1050 books_by_year[year].add(book_id)
1051 return dict(books_by_year)
1052
1053 _books_by_year = books_by_year
1054
1055 @read_api
1056 def books_by_month(self, field: str = 'pubdate', restrict_to_books: Iterable[int] | None = None) -> dict[tuple[int, int], set[int]]:
1057 m = self.field_metadata[field]
1058 value_column = m.get('column') or field
1059 table = m.get('table') or 'books'
1060 book_col = 'id' if table == 'books' else 'book'
1061
1062 query = f'''
1063 SELECT
1064 CAST(substr({value_column}, 1, 4) AS INTEGER),
1065 CAST(substr({value_column}, 6, 2) AS INTEGER),
1066 {book_col}
1067 FROM {table}
1068 '''
1069 ans = defaultdict(set)
1070 if restrict_to_books is None:
1071 for year, month, book_id in self.backend.execute(query):
1072 ans[(year, month)].add(book_id)
1073 else:
1074 books = tuple(restrict_to_books)
1075 BATCH_SIZE = self.backend.max_number_of_variables
1076 for i in range(0, len(books), BATCH_SIZE):
1077 batch = books[i : i + BATCH_SIZE]
1078 placeholders = '?,' * len(batch)
1079 for year, month, book_id in self.backend.execute(query + f' WHERE {book_col} IN ({placeholders[:-1]})', batch):
1080 ans[(year, month)].add(book_id)
1081 return dict(ans)
1082
1083 _books_by_month = books_by_month
1084
1085 @read_api
1086 def all_book_ids(self, type=frozenset):
1087 """
1088 Frozen set of all known book ids.
1089 """
1090 return type(self.fields['uuid'].table.book_col_map)
1091
1092 _all_book_ids = all_book_ids
1093
1094 @read_api
1095 def all_field_ids(self, name):
1096 """
1097 Frozen set of ids for all values in the field ``name``.
1098 """
1099 return frozenset(iter(self.fields[name]))
1100
1101 _all_field_ids = all_field_ids
1102
1103 @read_api
1104 def all_field_names(self, field):
1105 """Frozen set of all fields names (should only be used for many-one and many-many fields)"""
1106 if field == 'formats':
1107 return frozenset(self.fields[field].table.col_book_map)
1108
1109 try:
1110 return frozenset(self.fields[field].table.id_map.values())
1111 except AttributeError:
1112 raise ValueError(f'{field} is not a many-one or many-many field')
1113
1114 _all_field_names = all_field_names
1115
1116 @read_api
1117 def get_usage_count_by_id(self, field):
1118 """Return a mapping of id to usage count for all values of the specified
1119 field, which must be a many-one or many-many field."""
1120 try:
1121 return {k: len(v) for k, v in self.fields[field].table.col_book_map.items()}
1122 except AttributeError:
1123 raise ValueError(f'{field} is not a many-one or many-many field')
1124
1125 _get_usage_count_by_id = get_usage_count_by_id
1126
1127 @read_api
1128 def get_id_map(self, field):
1129 """Return a mapping of id numbers to values for the specified field.
1130 The field must be a many-one or many-many field, otherwise a ValueError
1131 is raised."""
1132 try:
1133 return self.fields[field].table.id_map.copy()
1134 except AttributeError:
1135 if field == 'title':
1136 return self.fields[field].table.book_col_map.copy()
1137 raise ValueError(f'{field} is not a many-one or many-many field')
1138
1139 _get_id_map = get_id_map
1140
1141 @read_api
1142 def get_item_name(self, field, item_id):
1143 """Return the item name for the item specified by item_id in the
1144 specified field. See also :meth:`get_id_map`."""
1145 return self.fields[field].table.id_map[item_id]
1146
1147 _get_item_name = get_item_name
1148
1149 @read_api
1150 def get_item_id(self, field, item_name, case_sensitive=False):
1151 """Return the item id for item_name or None if not found.
1152 This function is very slow if doing lookups for multiple names use either get_item_ids() or get_item_name_map().
1153 Similarly, case sensitive lookups are faster than case insensitive ones."""
1154 field = self.fields[field]
1155 if hasattr(field, 'item_ids_for_names'):
1156 d = field.item_ids_for_names(self.backend, (item_name,), case_sensitive)
1157 for v in d.values():
1158 return v
1159
1160 _get_item_id = get_item_id
1161
1162 @read_api
1163 def get_item_ids(self, field, item_names, case_sensitive=False):
1164 "Return a dict mapping item_name to the item id or None"
1165 field = self.fields[field]
1166 if hasattr(field, 'item_ids_for_names'):
1167 return field.item_ids_for_names(self.backend, item_names, case_sensitive)
1168 return dict.fromkeys(item_names)
1169
1170 _get_item_ids = get_item_ids
1171
1172 @read_api
1173 def get_item_name_map(self, field, normalize_func=None):
1174 "Return mapping of item values to ids"
1175 if normalize_func is None:
1176 return {v: k for k, v in self.fields[field].table.id_map.items()}
1177 return {normalize_func(v): k for k, v in self.fields[field].table.id_map.items()}
1178
1179 _get_item_name_map = get_item_name_map
1180
1181 @read_api
1182 def get_book_path(self, book_id, sep=os.sep, unsafe=False):
1183 """
1184 Return the relative book path for the given id.
1185 Prefer this because you can choose the directory separator, default use the OS one.
1186 If unsafe is True, allow to return None if the book_id is not in the library.
1187 """
1188 rslt = self._field_for('path', book_id)
1189 if rslt:
1190 if sep != '/':
1191 rslt = rslt.replace('/', sep)
1192 return rslt
1193 if unsafe:
1194 return None
1195 raise KeyError(f'No book with id {book_id!r} found in the library')
1196
1197 _get_book_path = get_book_path
1198
1199 @read_api
1200 def author_data(self, author_ids=None) -> dict[int, dict[str, str]]:
1201 """
1202 Return author data as a dictionary with keys: name, sort, link
1203
1204 If no authors with the specified ids are found an empty dictionary is
1205 returned. If author_ids is None, data for all authors is returned.
1206 """
1207 af = self.fields['authors']
1208 if author_ids is None:
1209 return {aid: af.author_data(aid) for aid in af.table.id_map}
1210 return {aid: af.author_data(aid) for aid in author_ids if aid in af.table.id_map}
1211
1212 _author_data = author_data
1213
1214 @read_api
1215 def author_sorts(self, author_ids=None) -> dict[int, str]:
1216 """
1217 Return author sorts for specified authors.
1218
1219 If no authors with the specified ids are found an empty dictionary is
1220 returned. If author_ids is None, data for all authors is returned.
1221 """
1222 af = self.fields['authors']
1223 m = af.table.asort_map
1224 if author_ids is None:
1225 return m.copy()
1226 return {aid: m[aid] for aid in author_ids if aid in m}
1227
1228 _author_sorts = author_sorts
1229
1230 @read_api
1231 def format_hash(self, book_id, fmt):
1232 """Return the hash of the specified format for the specified book. The
1233 kind of hash is backend dependent, but is usually SHA-256."""
1234 try:
1235 name = self.fields['formats'].format_fname(book_id, fmt)
1236 path = self._get_book_path(book_id)
1237 except Exception:
1238 raise NoSuchFormat(f'Record {book_id} has no fmt: {fmt}')
1239 return self.backend.format_hash(book_id, fmt, name, path)
1240
1241 _format_hash = format_hash
1242
1243 @api
1244 def format_metadata(self, book_id, fmt, allow_cache=True, update_db=False):
1245 """
1246 Return the path, size and mtime for the specified format for the specified book.
1247 You should not use path unless you absolutely have to,
1248 since accessing it directly breaks the threadsafe guarantees of this API. Instead use
1249 the :meth:`copy_format_to` method.
1250
1251 :param allow_cache: If ``True`` cached values are used, otherwise a
1252 slow filesystem access is done. The cache values could be out of date
1253 if access was performed to the filesystem outside of this API.
1254
1255 :param update_db: If ``True`` The max_size field of the database is updated for this book.
1256 """
1257 if not fmt:
1258 return {}
1259 fmt = fmt.upper()
1260 # allow_cache and update_db are mutually exclusive. Give priority to update_db
1261 if allow_cache and not update_db:
1262 x = self.format_metadata_cache[book_id].get(fmt, None)
1263 if x is not None:
1264 return x
1265 with self.safe_read_lock:
1266 try:
1267 name = self.fields['formats'].format_fname(book_id, fmt)
1268 path = self._get_book_path(book_id)
1269 except Exception:
1270 return {}
1271
1272 ans = {}
1273 if path and name:
1274 ans = self.backend.format_metadata(book_id, fmt, name, path)
1275 self.format_metadata_cache[book_id][fmt] = ans
1276 if update_db and 'size' in ans:
1277 with self.write_lock:
1278 max_size = self.fields['formats'].table.update_fmt(book_id, fmt, name, ans['size'], self.backend)
1279 self.fields['size'].table.update_sizes({book_id: max_size})
1280
1281 return ans
1282
1283 _format_metadata = format_metadata
1284
1285 @read_api
1286 def format_files(self, book_id):
1287 field = self.fields['formats']
1288 fmts = field.table.book_col_map.get(book_id, ())
1289 return {fmt: field.format_fname(book_id, fmt) for fmt in fmts}
1290
1291 _format_files = format_files
1292
1293 @read_api
1294 def format_db_size(self, book_id, fmt):
1295 field = self.fields['formats']
1296 return field.format_size(book_id, fmt)
1297
1298 _format_db_size = format_db_size
1299
1300 @write_api
1301 def refresh_format_sizes(self, size_map: dict[tuple[int, str], int]) -> set[int]:
1302 """
1303 Update the database with the on-disk sizes of the specified format files.
1304 ``size_map`` maps ``(book_id, fmt)`` to the size of the format file. Used
1305 to fix stored sizes that have gone stale because format files were
1306 changed outside of calibre. Returns the ids of the books whose stored
1307 sizes were changed.
1308 """
1309 formats_field = self.fields['formats']
1310 max_sizes = {}
1311 for (book_id, fmt), size in size_map.items():
1312 fmt = fmt.upper()
1313 stored_size = formats_field.format_size(book_id, fmt)
1314 if stored_size is None or stored_size == size:
1315 continue # the format was removed or its size is already correct
1316 name = formats_field.format_fname(book_id, fmt)
1317 max_sizes[book_id] = formats_field.table.update_fmt(book_id, fmt, name, size, self.backend)
1318 self.format_metadata_cache[book_id].pop(fmt, None)
1319 if max_sizes:
1320 self.fields['size'].table.update_sizes(max_sizes)
1321 self._clear_composite_caches(max_sizes)
1322 return set(max_sizes)
1323
1324 _refresh_format_sizes = refresh_format_sizes
1325
1326 @read_api
1327 def pref(self, name, default=None, namespace=None, get_default_from_defaults=False):
1328 "Return the value for the specified preference or the value specified as ``default`` if the preference is not set."
1329 p = self.backend.prefs
1330 if get_default_from_defaults:
1331 default = p.defaults.get(name, default)
1332 if namespace is not None:
1333 return p.get_namespaced(namespace, name, default)
1334 return p.get(name, default)
1335
1336 _pref = pref
1337
1338 @write_api
1339 def set_pref(self, name, val, namespace=None):
1340 "Set the specified preference to the specified value. See also :meth:`pref`."
1341 if namespace is not None:
1342 self.backend.prefs.set_namespaced(namespace, name, val)
1343 return
1344 self.backend.prefs.set(name, val)
1345 if name in ('grouped_search_terms', 'virtual_libraries'):
1346 self._clear_search_caches()
1347 if name in dynamic_category_preferences:
1348 self._initialize_dynamic_categories()
1349
1350 _set_pref = set_pref
1351
1352 @api
1353 def get_metadata(self, book_id, get_cover=False, get_user_categories=True, cover_as_data=False):
1354 """
1355 Return metadata for the book identified by book_id as a :class:`calibre.ebooks.metadata.book.base.Metadata` object.
1356 Note that the list of formats is not verified. If get_cover is True,
1357 the cover is returned, either a path to temp file as mi.cover or if
1358 cover_as_data is True then as mi.cover_data.
1359 """
1360
1361 # Check if virtual_libraries_for_books rebuilt its cache. If it did then
1362 # we must clear the composite caches so the new data can be taken into
1363 # account. Clearing the caches requires getting a write lock, so it must
1364 # be done outside of the closure of _get_metadata().
1365 composite_cache_needs_to_be_cleared = False
1366 with self.safe_read_lock:
1367 vl_cache_was_none = self.vls_for_books_cache is None
1368 mi = self._get_metadata(book_id, get_user_categories=get_user_categories)
1369 if vl_cache_was_none and self.vls_for_books_cache is not None:
1370 composite_cache_needs_to_be_cleared = True
1371 if composite_cache_needs_to_be_cleared:
1372 try:
1373 self.clear_composite_caches()
1374 except LockingError:
1375 # We can't clear the composite caches because a read lock is set.
1376 # As a consequence the value of a composite column that calls
1377 # virtual_libraries() might be wrong. Oh well. Log and keep running.
1378 print("Couldn't get write lock after vls_for_books_cache was loaded", file=sys.stderr)
1379 traceback.print_exc()
1380
1381 if get_cover:
1382 if cover_as_data:
1383 cdata = self.cover(book_id)
1384 if cdata:
1385 mi.cover_data = ('jpeg', cdata)
1386 else:
1387 mi.cover = self.cover(book_id, as_path=True)
1388
1389 return mi
1390
1391 @read_api
1392 def get_proxy_metadata(self, book_id):
1393 """Like :meth:`get_metadata` except that it returns a ProxyMetadata
1394 object that only reads values from the database on demand. This is much
1395 faster than get_metadata when only a small number of fields need to be
1396 accessed from the returned metadata object."""
1397 return ProxyMetadata(self, book_id)
1398
1399 _get_proxy_metadata = get_proxy_metadata
1400
1401 @api
1402 def cover(self, book_id, as_file=False, as_image=False, as_path=False, as_pixmap=False):
1403 """
1404 Return the cover image or None. By default, returns the cover as a
1405 bytestring.
1406
1407 WARNING: Using as_path will copy the cover to a temp file and return
1408 the path to the temp file. You should delete the temp file when you are
1409 done with it.
1410
1411 :param as_file: If True return the image as an open file object (a SpooledTemporaryFile)
1412 :param as_image: If True return the image as a QImage object
1413 :param as_pixmap: If True return the image as a QPixmap object
1414 :param as_path: If True return the image as a path pointing to a
1415 temporary file
1416 """
1417 if as_file:
1418 ret = SpooledTemporaryFile(SPOOL_SIZE)
1419 if not self.copy_cover_to(book_id, ret):
1420 ret.close()
1421 return
1422 ret.seek(0)
1423 elif as_path:
1424 pt = PersistentTemporaryFile('_dbcover.jpg')
1425 with pt:
1426 if not self.copy_cover_to(book_id, pt):
1427 return
1428 ret = pt.name
1429 elif as_pixmap or as_image:
1430 from qt.core import QImage, QPixmap
1431
1432 ret = QImage() if as_image else QPixmap()
1433 with self.safe_read_lock:
1434 path = self._format_abspath(book_id, '__COVER_INTERNAL__')
1435 if path:
1436 ret.load(path)
1437 else:
1438 buf = BytesIO()
1439 if not self.copy_cover_to(book_id, buf):
1440 return
1441 ret = buf.getvalue()
1442 return ret
1443
1444 _cover = cover
1445
1446 @read_api
1447 def cover_or_cache(self, book_id, timestamp, as_what='bytes'):
1448 try:
1449 path = self._get_book_path(book_id)
1450 except AttributeError, KeyError:
1451 return False, None, None
1452 return self.backend.cover_or_cache(path, timestamp, as_what)
1453
1454 _cover_or_cache = cover_or_cache
1455
1456 @read_api
1457 def cover_last_modified(self, book_id: int) -> datetime | None:
1458 try:
1459 path = self._get_book_path(book_id)
1460 except AttributeError, KeyError:
1461 return
1462 return self.backend.cover_last_modified(path)
1463
1464 _cover_last_modified = cover_last_modified
1465
1466 @read_api
1467 def cover_timestamp(self, book_id: int) -> float | None:
1468 try:
1469 path = self._get_book_path(book_id)
1470 except AttributeError, KeyError:
1471 return
1472 return self.backend.cover_timestamp(path)
1473
1474 _cover_timestamp = cover_timestamp
1475
1476 @read_api
1477 def copy_cover_to(self, book_id, dest, use_hardlink=False, report_file_size=None):
1478 """
1479 Copy the cover to the file like object ``dest``. Returns False
1480 if no cover exists or dest is the same file as the current cover.
1481 dest can also be a path in which case the cover is
1482 copied to it if and only if the path is different from the current path (taking
1483 case sensitivity into account).
1484 """
1485 try:
1486 path = self._get_book_path(book_id)
1487 except AttributeError, KeyError:
1488 return False
1489
1490 return self.backend.copy_cover_to(path, dest, use_hardlink=use_hardlink, report_file_size=report_file_size)
1491
1492 _copy_cover_to = copy_cover_to
1493
1494 @write_api
1495 def compress_covers(self, book_ids, jpeg_quality=100, progress_callback=None):
1496 """
1497 Compress the cover images for the specified books. A compression quality of 100
1498 will perform lossless compression, otherwise lossy compression.
1499
1500 The progress callback will be called with the book_id and the old and new sizes
1501 for each book that has been processed. If an error occurs, the new size will
1502 be a string with the error details.
1503 """
1504 jpeg_quality = max(10, min(jpeg_quality, 100))
1505 path_map = {}
1506 for book_id in book_ids:
1507 try:
1508 path_map[book_id] = self._get_book_path(book_id)
1509 except AttributeError, KeyError:
1510 continue
1511 self.backend.compress_covers(path_map, jpeg_quality, progress_callback)
1512
1513 _compress_covers = compress_covers
1514
1515 @read_api
1516 def copy_format_to(self, book_id, fmt, dest, use_hardlink=False, report_file_size=None):
1517 """
1518 Copy the format ``fmt`` to the file like object ``dest``. If the
1519 specified format does not exist, raises :class:`NoSuchFormat` error.
1520 dest can also be a path (to a file), in which case the format is copied to it, iff
1521 the path is different from the current path (taking case sensitivity
1522 into account).
1523 """
1524 fmt = (fmt or '').upper()
1525 try:
1526 name = self.fields['formats'].format_fname(book_id, fmt)
1527 path = self._get_book_path(book_id)
1528 except KeyError, AttributeError:
1529 raise NoSuchFormat(f'Record {book_id} has no {fmt} file')
1530
1531 return self.backend.copy_format_to(book_id, fmt, name, path, dest, use_hardlink=use_hardlink, report_file_size=report_file_size)
1532
1533 _copy_format_to = copy_format_to
1534
1535 @read_api
1536 def format_abspath(self, book_id, fmt):
1537 """
1538 Return absolute path to the e-book file of format `format`. You should
1539 almost never use this, as it breaks the threadsafe promise of this API.
1540 Instead use, :meth:`copy_format_to`.
1541
1542 Currently used only in calibredb list, the viewer, edit book,
1543 compare_format to original format, open with, bulk metadata edit and
1544 the catalogs (via get_data_as_dict()).
1545
1546 Apart from the viewer, open with and edit book, I don't believe any of
1547 the others do any file write I/O with the results of this call.
1548 """
1549 fmt = (fmt or '').upper()
1550 try:
1551 path = self._get_book_path(book_id)
1552 except Exception:
1553 return None
1554 if path:
1555 if fmt == '__COVER_INTERNAL__':
1556 return self.backend.cover_abspath(book_id, path)
1557 else:
1558 try:
1559 name = self.fields['formats'].format_fname(book_id, fmt)
1560 except Exception:
1561 return None
1562 if name:
1563 return self.backend.format_abspath(book_id, fmt, name, path)
1564
1565 _format_abspath = format_abspath
1566
1567 @read_api
1568 def has_format(self, book_id, fmt):
1569 "Return True iff the format exists on disk"
1570 fmt = (fmt or '').upper()
1571 try:
1572 name = self.fields['formats'].format_fname(book_id, fmt)
1573 path = self._get_book_path(book_id)
1574 except Exception:
1575 return False
1576 return self.backend.has_format(book_id, fmt, name, path)
1577
1578 _has_format = has_format
1579
1580 @api
1581 def save_original_format(self, book_id, fmt):
1582 "Save a copy of the specified format as ORIGINAL_FORMAT, overwriting any existing ORIGINAL_FORMAT."
1583 fmt = fmt.upper()
1584 if 'ORIGINAL' in fmt:
1585 raise ValueError('Cannot save original of an original fmt')
1586 fmtfile = self.format(book_id, fmt, as_file=True)
1587 if fmtfile is None:
1588 return False
1589 with fmtfile:
1590 nfmt = 'ORIGINAL_' + fmt
1591 return self.add_format(book_id, nfmt, fmtfile, run_hooks=False)
1592
1593 _save_original_format = save_original_format
1594
1595 @write_api
1596 def restore_original_format(self, book_id, original_fmt):
1597 """Restore the specified format from the previously saved
1598 ORIGINAL_FORMAT, if any. Return True on success. The ORIGINAL_FORMAT is
1599 deleted after a successful restore."""
1600 original_fmt = original_fmt.upper()
1601 fmt = original_fmt.partition('_')[2]
1602 try:
1603 ofmt_name = self.fields['formats'].format_fname(book_id, original_fmt)
1604 path = self._get_book_path(book_id)
1605 except Exception:
1606 return False
1607 if self.backend.is_format_accessible(book_id, original_fmt, ofmt_name, path):
1608 self.add_format(book_id, fmt, BytesIO(), run_hooks=False)
1609 fmt_name = self.fields['formats'].format_fname(book_id, fmt)
1610 file_size = self.backend.rename_format_file(book_id, ofmt_name, original_fmt, fmt_name, fmt, path)
1611 self.fields['formats'].table.update_fmt(book_id, fmt, fmt_name, file_size, self.backend)
1612 self._remove_formats({book_id: (original_fmt,)})
1613 return True
1614 return False
1615
1616 _restore_original_format = restore_original_format
1617
1618 @read_api
1619 def formats(self, book_id, verify_formats=True):
1620 """
1621 Return tuple of all formats for the specified book. If verify_formats
1622 is True, verifies that the files exist on disk.
1623 """
1624 ans = self.field_for('formats', book_id)
1625 if verify_formats and ans:
1626 try:
1627 path = self._get_book_path(book_id)
1628 except Exception:
1629 return ()
1630
1631 def verify(fmt):
1632 try:
1633 name = self.fields['formats'].format_fname(book_id, fmt)
1634 except Exception:
1635 return False
1636 return self.backend.has_format(book_id, fmt, name, path)
1637
1638 ans = tuple(x for x in ans if verify(x))
1639 return ans
1640
1641 _formats = formats
1642
1643 @api
1644 def format(self, book_id, fmt, as_file=False, as_path=False, preserve_filename=False):
1645 """
1646 Return the e-book format as a bytestring or `None` if the format doesn't exist,
1647 or we don't have permission to write to the e-book file.
1648
1649 :param as_file: If True the e-book format is returned as a file object. Note
1650 that the file object is a SpooledTemporaryFile, so if what you want to
1651 do is copy the format to another file, use :meth:`copy_format_to`
1652 instead for performance.
1653 :param as_path: Copies the format file to a temp file and returns the
1654 path to the temp file
1655 :param preserve_filename: If True and returning a path the filename is
1656 the same as that used in the library. Note that using
1657 this means that repeated calls yield the same
1658 temp file (which is re-created each time)
1659 """
1660 fmt = (fmt or '').upper()
1661 ext = ('.' + fmt.lower()) if fmt else ''
1662 if as_path:
1663 if preserve_filename:
1664 with self.safe_read_lock:
1665 try:
1666 fname = self.fields['formats'].format_fname(book_id, fmt)
1667 except Exception:
1668 return None
1669 fname += ext
1670
1671 bd = base_dir()
1672 d = os.path.join(bd, 'format_abspath')
1673 try:
1674 os.makedirs(d)
1675 except Exception:
1676 pass
1677 ret = os.path.join(d, fname)
1678 try:
1679 self.copy_format_to(book_id, fmt, ret)
1680 except NoSuchFormat:
1681 return None
1682 else:
1683 with PersistentTemporaryFile(ext) as pt:
1684 try:
1685 self.copy_format_to(book_id, fmt, pt)
1686 except NoSuchFormat:
1687 return None
1688 ret = pt.name
1689 elif as_file:
1690 with self.safe_read_lock:
1691 try:
1692 fname = self.fields['formats'].format_fname(book_id, fmt)
1693 except Exception:
1694 return None
1695 fname += ext
1696
1697 ret = SpooledTemporaryFile(SPOOL_SIZE)
1698 try:
1699 self.copy_format_to(book_id, fmt, ret)
1700 except NoSuchFormat:
1701 ret.close()
1702 return None
1703 ret.seek(0)
1704 # Various bits of code try to use the name as the default
1705 # title when reading metadata, so set it
1706 ret.name = fname
1707 else:
1708 buf = BytesIO()
1709 try:
1710 self.copy_format_to(book_id, fmt, buf)
1711 except NoSuchFormat:
1712 return None
1713
1714 ret = buf.getvalue()
1715
1716 return ret
1717
1718 _format = format
1719
1720 @read_api
1721 def newly_added_book_ids(self, count=5, book_ids=None) -> list[int]:
1722 ids_to_sort = self._all_book_ids(list) if book_ids is None else list(book_ids)
1723 ids_to_sort.sort(reverse=True)
1724 return ids_to_sort[:count]
1725
1726 _newly_added_book_ids = newly_added_book_ids
1727
1728 @read_api
1729 def size_stats(self) -> dict[str, int]:
1730 return self.backend.size_stats()
1731
1732 _size_stats = size_stats
1733
1734 @read_api
1735 def multisort(self, fields, ids_to_sort=None, virtual_fields=None):
1736 """
1737 Return a list of sorted book ids. If ids_to_sort is None, all book ids
1738 are returned.
1739
1740 fields must be a list of 2-tuples of the form (field_name,
1741 ascending=True or False). The most significant field is the first
1742 2-tuple.
1743 """
1744 ids_to_sort = self._all_book_ids() if ids_to_sort is None else ids_to_sort
1745 get_metadata = self._get_proxy_metadata
1746 lang_map = self.fields['languages'].book_value_map
1747 virtual_fields = virtual_fields or {}
1748
1749 fm = {'title': 'sort', 'authors': 'author_sort'}
1750
1751 def sort_key_func(field):
1752 "Handle series type fields, virtual fields and the id field"
1753 idx = field + '_index'
1754 is_series = idx in self.fields
1755 try:
1756 func = self.fields[fm.get(field, field)].sort_keys_for_books(get_metadata, lang_map)
1757 except KeyError:
1758 if field == 'id':
1759 return IDENTITY
1760 else:
1761 return virtual_fields[fm.get(field, field)].sort_keys_for_books(get_metadata, lang_map)
1762 if is_series:
1763 idx_func = self.fields[idx].sort_keys_for_books(get_metadata, lang_map)
1764
1765 def skf(book_id):
1766 return (func(book_id), idx_func(book_id))
1767
1768 return skf
1769 return func
1770
1771 # Sort only once on any given field
1772 fields = uniq(fields, operator.itemgetter(0))
1773
1774 if len(fields) == 1:
1775 keyfunc = sort_key_func(fields[0][0])
1776 reverse = not fields[0][1]
1777 try:
1778 return sorted(ids_to_sort, key=keyfunc, reverse=reverse)
1779 except Exception as err:
1780 print('Failed to sort database on field:', fields[0][0], 'with error:', err, file=sys.stderr)
1781 try:
1782 return sorted(ids_to_sort, key=type_safe_sort_key_function(keyfunc), reverse=reverse)
1783 except Exception as err:
1784 print('Failed to type-safe sort database on field:', fields[0][0], 'with error:', err, file=sys.stderr)
1785 return sorted(ids_to_sort, reverse=reverse)
1786 sort_key_funcs = tuple(sort_key_func(field) for field, order in fields)
1787 orders = tuple(1 if order else -1 for _, order in fields)
1788 Lazy = object() # Lazy load the sort keys for sub-sort fields
1789
1790 class SortKey:
1791 __slots__ = 'book_id', 'sort_key'
1792
1793 def __init__(self, book_id):
1794 self.book_id = book_id
1795 # Calculate only the first sub-sort key since that will always be used
1796 self.sort_key = [key(book_id) if i == 0 else Lazy for i, key in enumerate(sort_key_funcs)]
1797
1798 def compare_to_other(self, other):
1799 for i, (order, self_key, other_key) in enumerate(zip(orders, self.sort_key, other.sort_key)):
1800 if self_key is Lazy:
1801 self_key = self.sort_key[i] = sort_key_funcs[i](self.book_id)
1802 if other_key is Lazy:
1803 other_key = other.sort_key[i] = sort_key_funcs[i](other.book_id)
1804 ans = cmp(self_key, other_key)
1805 if ans != 0:
1806 return ans * order
1807 return 0
1808
1809 def __eq__(self, other):
1810 return self.compare_to_other(other) == 0
1811
1812 def __ne__(self, other):
1813 return self.compare_to_other(other) != 0
1814
1815 def __lt__(self, other):
1816 return self.compare_to_other(other) < 0
1817
1818 def __le__(self, other):
1819 return self.compare_to_other(other) <= 0
1820
1821 def __gt__(self, other):
1822 return self.compare_to_other(other) > 0
1823
1824 def __ge__(self, other):
1825 return self.compare_to_other(other) >= 0
1826
1827 return sorted(ids_to_sort, key=SortKey)
1828
1829 _multisort = multisort
1830
1831 @read_api
1832 def search(self, query, restriction='', virtual_fields=None, book_ids=None, allow_templates=True):
1833 """
1834 Search the database for the specified query, returning a set of matched book ids.
1835
1836 :param restriction: A restriction that is ANDed to the specified query. Note that
1837 restrictions are cached, therefore the search for a AND b will be slower than a with restriction b.
1838
1839 :param virtual_fields: Used internally (virtual fields such as on_device to search over).
1840
1841 :param book_ids: If not None, a set of book ids for which books will
1842 be searched instead of searching all books.
1843 """
1844 return self._search_api(self, query, restriction, virtual_fields=virtual_fields, book_ids=book_ids, allow_templates=allow_templates)
1845
1846 _search = search
1847
1848 @read_api
1849 def books_in_virtual_library(self, vl, search_restriction=None, virtual_fields=None):
1850 "Return the set of books in the specified virtual library"
1851 vl = self._pref('virtual_libraries', {}).get(vl) if vl else None
1852 if not vl and not search_restriction:
1853 return self.all_book_ids()
1854 # We utilize the search restriction cache to speed this up
1855 srch = partial(self._search, virtual_fields=virtual_fields)
1856 if vl:
1857 if search_restriction:
1858 return frozenset(srch('', vl) & srch('', search_restriction))
1859 return frozenset(srch('', vl))
1860 return frozenset(srch('', search_restriction))
1861
1862 _books_in_virtual_library = books_in_virtual_library
1863
1864 @read_api
1865 def number_of_books_in_virtual_library(self, vl=None, search_restriction=None):
1866 if not vl and not search_restriction:
1867 return len(self.fields['uuid'].table.book_col_map)
1868 return len(self.books_in_virtual_library(vl, search_restriction))
1869
1870 _number_of_books_in_virtual_library = number_of_books_in_virtual_library
1871
1872 @api
1873 def get_categories(self, sort='name', book_ids=None, already_fixed=None, first_letter_sort=False, uncollapsed_categories=None):
1874 "Used internally to implement the Tag Browser"
1875 try:
1876 with self.safe_read_lock:
1877 return get_categories(
1878 self,
1879 sort=sort,
1880 book_ids=book_ids,
1881 first_letter_sort=first_letter_sort,
1882 uncollapsed_categories=uncollapsed_categories,
1883 )
1884 except InvalidLinkTable as err:
1885 bad_field = err.field_name
1886 if bad_field == already_fixed:
1887 raise
1888 with self.write_lock:
1889 self.fields[bad_field].table.fix_link_table(self.backend)
1890 return self.get_categories(sort=sort, book_ids=book_ids, already_fixed=bad_field)
1891
1892 _get_categories = get_categories
1893
1894 @write_api
1895 def update_last_modified(self, book_ids, now=None):
1896 if book_ids:
1897 if now is None:
1898 now = nowf()
1899 f = self.fields['last_modified']
1900 f.writer.set_books({book_id: now for book_id in book_ids}, self.backend)
1901 if self.composites:
1902 self._clear_composite_caches(book_ids)
1903 self._clear_search_caches(book_ids)
1904
1905 _update_last_modified = update_last_modified
1906
1907 @write_api
1908 def mark_as_dirty(self, book_ids):
1909 self._update_last_modified(book_ids)
1910 already_dirtied = set(self.dirtied_cache).intersection(book_ids)
1911 new_dirtied = book_ids - already_dirtied
1912 already_dirtied = {book_id: self.dirtied_sequence + i for i, book_id in enumerate(already_dirtied)}
1913 if already_dirtied:
1914 self.dirtied_sequence = max(already_dirtied.values()) + 1
1915 self.dirtied_cache.update(already_dirtied)
1916 if new_dirtied:
1917 self.backend.dirty_books(new_dirtied)
1918 new_dirtied = {book_id: self.dirtied_sequence + i for i, book_id in enumerate(new_dirtied)}
1919 self.dirtied_sequence = max(new_dirtied.values()) + 1
1920 self.dirtied_cache.update(new_dirtied)
1921
1922 _mark_as_dirty = mark_as_dirty
1923
1924 @write_api
1925 def commit_dirty_cache(self):
1926 if self.dirtied_cache:
1927 self.backend.dirty_books(self.dirtied_cache)
1928
1929 _commit_dirty_cache = commit_dirty_cache
1930
1931 @write_api
1932 def check_dirtied_annotations(self):
1933 if not self.backend.dirty_books_with_dirtied_annotations():
1934 return
1935 book_ids = set(self.backend.dirtied_books())
1936 new_dirtied = book_ids - set(self.dirtied_cache)
1937 if new_dirtied:
1938 new_dirtied = {book_id: self.dirtied_sequence + i for i, book_id in enumerate(new_dirtied)}
1939 self.dirtied_sequence = max(new_dirtied.values()) + 1
1940 self.dirtied_cache.update(new_dirtied)
1941
1942 _check_dirtied_annotations = check_dirtied_annotations
1943
1944 @write_api
1945 def set_field(self, name, book_id_to_val_map, allow_case_change=True, do_path_update=True):
1946 """
1947 Set the values of the field specified by ``name``. Returns the set of all book ids that were affected by the change.
1948
1949 :param book_id_to_val_map: Mapping of book_ids to values that should be applied.
1950 :param allow_case_change: If True, the case of many-one or many-many fields will be changed.
1951 For example, if a book has the tag ``tag1`` and you set the tag for another book to ``Tag1``
1952 then the both books will have the tag ``Tag1`` if allow_case_change is True, otherwise they will
1953 both have the tag ``tag1``.
1954 :param do_path_update: Used internally, you should never change it.
1955 """
1956 f = self.fields[name]
1957 is_series = f.metadata['datatype'] == 'series'
1958 update_path = name in {'title', 'authors'}
1959 if update_path and iswindows:
1960 paths = (x for x in (self._get_book_path(book_id, sep='/', unsafe=True) for book_id in book_id_to_val_map) if x)
1961 self.backend.windows_check_if_files_in_use(paths)
1962
1963 if is_series:
1964 bimap, simap = {}, {}
1965 sfield = self.fields[name + '_index']
1966 for k, v in book_id_to_val_map.items():
1967 if isinstance(v, (str, bytes)):
1968 v, sid = get_series_values(v)
1969 else:
1970 v = sid = None
1971 if sid is None and name.startswith('#'):
1972 sid = self._fast_field_for(sfield, k)
1973 sid = 1.0 if sid is None else sid # The value to be set the db link table
1974 bimap[k] = v
1975 if sid is not None:
1976 simap[k] = sid
1977 book_id_to_val_map = bimap
1978
1979 dirtied = f.writer.set_books(book_id_to_val_map, self.backend, allow_case_change=allow_case_change)
1980
1981 if is_series and simap:
1982 sf = self.fields[f.name + '_index']
1983 dirtied |= sf.writer.set_books(simap, self.backend, allow_case_change=False)
1984
1985 if dirtied:
1986 if update_path and do_path_update:
1987 self._update_path(dirtied, mark_as_dirtied=False)
1988 self._mark_as_dirty(dirtied)
1989 self._clear_link_map_cache(dirtied)
1990 self.event_dispatcher(EventType.metadata_changed, name, dirtied)
1991 return dirtied
1992
1993 _set_field = set_field
1994
1995 # Page counts {{{
1996 @read_api
1997 def get_pages(self, book_id: int) -> Pages | None:
1998 "Return page count information for the specified book"
1999 for pages, algorithm, format, format_size, timestamp in self.backend.execute(
2000 f'SELECT pages,algorithm,format,format_size,timestamp FROM books_pages_link WHERE book={book_id:d} LIMIT 1'
2001 ):
2002 return Pages(int(pages), int(algorithm), str(format), int(format_size), parse_iso8601(timestamp, assume_utc=True))
2003
2004 _get_pages = get_pages
2005
2006 @read_api
2007 def pages_needs_scan(self, books: Iterable[int] = ()) -> set[int]:
2008 "Return the subset of books (or all books if empty) that are marked as needing a scan to update page count"
2009 books = tuple(books)
2010 if not books:
2011 return {r[0] for r in self.backend.execute('SELECT book FROM books_pages_link WHERE needs_scan=1')}
2012 ans = set()
2013 BATCH_SIZE = self.backend.max_number_of_variables
2014 for i in range(0, len(books), BATCH_SIZE):
2015 batch = books[i : i + BATCH_SIZE]
2016 placeholders = '?,' * len(batch)
2017 for (book_id,) in self.backend.execute(f'SELECT book FROM books_pages_link WHERE needs_scan=1 AND book IN ({placeholders[:-1]})', batch):
2018 ans.add(book_id)
2019 return ans
2020
2021 _pages_needs_scan = pages_needs_scan
2022
2023 @read_api
2024 def num_of_books_that_need_pages_counted(self) -> int:
2025 for (ans,) in self.backend.execute('SELECT COUNT(*) FROM books_pages_link WHERE needs_scan=1'):
2026 return ans
2027 return 0
2028
2029 _num_of_books_that_need_pages_counted = num_of_books_that_need_pages_counted
2030
2031 @write_api
2032 def mark_for_pages_recount(self, book_id: int = 0) -> None:
2033 "Mark all books for recount of pages"
2034 if book_id:
2035 self.backend.execute(f'UPDATE books_pages_link SET needs_scan=1 WHERE book={int(book_id)}')
2036 else:
2037 self.backend.execute('UPDATE books_pages_link SET needs_scan=1')
2038
2039 _mark_for_pages_recount = mark_for_pages_recount
2040
2041 @write_api
2042 def queue_pages_scan(self, book_id: int = 0, force: bool = False, by_user: bool = True) -> None:
2043 """
2044 Start a scan updating page counts for all books that need a scan.
2045 If book_id is specified, then only that book is scanned and it is always scanned.
2046 When `force` is True, the existing pages value, if any, is discarded so that
2047 the book is forcibly rescanned even if the existing value was up-to-date.
2048 """
2049 book_id = int(book_id)
2050 if book_id <= 0:
2051 if force:
2052 self.backend.execute('DELETE FROM books_pages_link')
2053 self.fields['pages'].table.book_col_map.clear()
2054 if len(self.fields['pages'].table.book_col_map) < len(self.fields['uuid'].table.book_col_map):
2055 self.backend.execute('INSERT OR IGNORE INTO books_pages_link(book,needs_scan) SELECT id,1 FROM books')
2056 if by_user:
2057 self._set_pref('full_page_scan_requested', True)
2058 elif force:
2059 self.backend.execute(f'DELETE FROM books_pages_link WHERE book={book_id}')
2060 self.fields['pages'].table.book_col_map.pop(book_id, None)
2061 self.backend.execute(f'INSERT INTO books_pages_link(book,needs_scan) VALUES ({book_id},1)')
2062 else:
2063 self.backend.execute(f'UPDATE books_pages_link SET needs_scan=1 WHERE book={book_id}')
2064 assert self.maintain_page_counts is not None
2065 self.maintain_page_counts.queue_scan(book_id)
2066
2067 _queue_pages_scan = queue_pages_scan
2068
2069 @property
2070 def page_count_failures_log_path(self) -> str:
2071 assert self.maintain_page_counts is not None
2072 return self.maintain_page_counts.failure_log_path
2073
2074 @write_api
2075 def set_pages(
2076 self,
2077 book_id: int,
2078 pages: int = 0,
2079 algorithm: int = 0,
2080 format: str = '',
2081 format_size: int = 0,
2082 ) -> None:
2083 "Set page count information for the specified book"
2084 now = sqlite_datetime(utcnow())
2085 self.backend.execute(
2086 '''
2087 INSERT INTO books_pages_link (book, pages, algorithm, format, format_size, timestamp, needs_scan) VALUES
2088 (?,?,?,?,?,?,0)
2089 ON CONFLICT(book) DO UPDATE SET
2090 pages = excluded.pages, algorithm = excluded.algorithm,
2091 format = excluded.format, format_size = excluded.format_size,
2092 timestamp = excluded.timestamp, needs_scan = excluded.needs_scan;
2093 ''',
2094 (book_id, int(pages), int(algorithm), format, int(format_size), now),
2095 )
2096 self.fields['pages'].table.book_col_map[book_id] = pages
2097 self._clear_composite_caches((book_id,))
2098
2099 _set_pages = set_pages
2100
2101 # }}}
2102
2103 @write_api
2104 def update_path(self, book_ids, mark_as_dirtied=True):
2105 for book_id in book_ids:
2106 title = self._field_for('title', book_id, default_value=_('Unknown'))
2107 try:
2108 author = self._field_for('authors', book_id, default_value=(_('Unknown'),))[0]
2109 except IndexError:
2110 author = _('Unknown')
2111 self.backend.update_path(book_id, title, author, self.fields['path'], self.fields['formats'])
2112 self.format_metadata_cache.pop(book_id, None)
2113 if mark_as_dirtied:
2114 self._mark_as_dirty(book_ids)
2115 self._clear_link_map_cache(book_ids)
2116
2117 _update_path = update_path
2118
2119 @read_api
2120 def get_a_dirtied_book(self):
2121 if self.dirtied_cache:
2122 return random.choice(tuple(self.dirtied_cache))
2123 return None
2124
2125 _get_a_dirtied_book = get_a_dirtied_book
2126
2127 def _metadata_as_object_for_dump(self, book_id):
2128 mi = self._get_metadata(book_id)
2129 # Always set cover to cover.jpg. Even if cover doesn't exist,
2130 # no harm done. This way no need to call dirtied when
2131 # cover is set/removed
2132 mi.cover = 'cover.jpg'
2133 mi.all_annotations = self._all_annotations_for_book(book_id)
2134 return mi
2135
2136 @read_api
2137 def get_metadata_for_dump(self, book_id):
2138 mi = None
2139 # get the current sequence number for this book to pass back to the
2140 # backup thread. This will avoid double calls in the case where the
2141 # thread has not done the work between the put and the get_metadata
2142 sequence = self.dirtied_cache.get(book_id, None)
2143 if sequence is not None:
2144 try:
2145 # While a book is being created, the path is empty. Don't bother to
2146 # try to write the opf, because it will go to the wrong folder.
2147 if self._get_book_path(book_id, unsafe=True):
2148 mi = self._metadata_as_object_for_dump(book_id)
2149 except Exception:
2150 # This almost certainly means that the book has been deleted while
2151 # the backup operation sat in the queue.
2152 traceback.print_exc()
2153 return mi, sequence
2154
2155 _get_metadata_for_dump = get_metadata_for_dump
2156
2157 @write_api
2158 def clear_dirtied(self, book_id, sequence):
2159 # Clear the dirtied indicator for the books. This is used when fetching
2160 # metadata, creating an OPF, and writing a file are separated into steps.
2161 # The last step is clearing the indicator
2162 dc_sequence = self.dirtied_cache.get(book_id, None)
2163 if dc_sequence is None or sequence is None or dc_sequence == sequence:
2164 self.backend.mark_book_as_clean(book_id)
2165 self.dirtied_cache.pop(book_id, None)
2166
2167 _clear_dirtied = clear_dirtied
2168
2169 @write_api
2170 def write_backup(self, book_id, raw):
2171 try:
2172 path = self._get_book_path(book_id)
2173 except Exception:
2174 return
2175
2176 self.backend.write_backup(path, raw)
2177
2178 _write_backup = write_backup
2179
2180 @read_api
2181 def dirty_queue_length(self):
2182 return len(self.dirtied_cache)
2183
2184 _dirty_queue_length = dirty_queue_length
2185
2186 @read_api
2187 def read_backup(self, book_id):
2188 """Return the OPF metadata backup for the book as a bytestring or None
2189 if no such backup exists."""
2190 try:
2191 path = self._get_book_path(book_id)
2192 except Exception:
2193 return
2194
2195 try:
2196 return self.backend.read_backup(path)
2197 except OSError:
2198 return None
2199
2200 _read_backup = read_backup
2201
2202 @write_api
2203 def dump_metadata(self, book_ids=None, remove_from_dirtied=True, callback=None):
2204 # Write metadata for each record to an individual OPF file. If callback
2205 # is not None, it is called once at the start with the number of book_ids
2206 # being processed. And once for every book_id, with arguments (book_id,
2207 # mi, ok).
2208 if book_ids is None:
2209 book_ids = set(self.dirtied_cache)
2210
2211 if callback is not None:
2212 callback(len(book_ids), True, False)
2213
2214 for book_id in book_ids:
2215 if self._get_book_path(book_id, unsafe=True) is None:
2216 if callback is not None:
2217 callback(book_id, None, False)
2218 continue
2219 mi, sequence = self._get_metadata_for_dump(book_id)
2220 if mi is None:
2221 if callback is not None:
2222 callback(book_id, mi, False)
2223 continue
2224 try:
2225 raw = metadata_to_opf(mi)
2226 self._write_backup(book_id, raw)
2227 if remove_from_dirtied:
2228 self._clear_dirtied(book_id, sequence)
2229 except Exception:
2230 pass
2231 if callback is not None:
2232 callback(book_id, mi, True)
2233
2234 _dump_metadata = dump_metadata
2235
2236 @write_api
2237 def set_cover(self, book_id_data_map):
2238 """Set the cover for this book. The data can be either a QImage,
2239 QPixmap, file object or bytestring. It can also be None, in which
2240 case any existing cover is removed."""
2241
2242 for book_id, data in book_id_data_map.items():
2243 try:
2244 path = self._get_book_path(book_id)
2245 except KeyError, AttributeError:
2246 self._update_path((book_id,))
2247 path = self._get_book_path(book_id)
2248
2249 self.backend.set_cover(book_id, path, data)
2250 for cc in self.cover_caches:
2251 cc.invalidate(book_id_data_map)
2252 return self._set_field('cover', {book_id: (0 if data is None else 1) for book_id, data in book_id_data_map.items()})
2253
2254 _set_cover = set_cover
2255
2256 @write_api
2257 def add_cover_cache(self, cover_cache):
2258 if not callable(cover_cache.invalidate):
2259 raise ValueError('Cover caches must have an invalidate method')
2260 self.cover_caches.add(cover_cache)
2261
2262 _add_cover_cache = add_cover_cache
2263
2264 @write_api
2265 def remove_cover_cache(self, cover_cache):
2266 self.cover_caches.discard(cover_cache)
2267
2268 _remove_cover_cache = remove_cover_cache
2269
2270 @write_api
2271 def set_metadata(
2272 self,
2273 book_id,
2274 mi,
2275 ignore_errors=False,
2276 force_changes=False,
2277 set_title=True,
2278 set_authors=True,
2279 allow_case_change=False,
2280 ):
2281 """
2282 Set metadata for the book `id` from the `Metadata` object `mi`
2283
2284 Setting force_changes=True will force set_metadata to update fields even
2285 if mi contains empty values. In this case, 'None' is distinguished from
2286 'empty'. If mi.XXX is None, the XXX is not replaced, otherwise it is.
2287 The tags, identifiers, and cover attributes are special cases. Tags and
2288 identifiers cannot be set to None so they will always be replaced if
2289 force_changes is true. You must ensure that mi contains the values you
2290 want the book to have. Covers are always changed if a new cover is
2291 provided, but are never deleted. Also note that force_changes has no
2292 effect on setting title or authors.
2293 """
2294 dirtied = set()
2295
2296 try:
2297 # Handle code passing in an OPF object instead of a Metadata object
2298 mi = mi.to_book_metadata()
2299 except AttributeError, TypeError:
2300 pass
2301
2302 def set_field(name, val):
2303 dirtied.update(self._set_field(name, {book_id: val}, do_path_update=False, allow_case_change=allow_case_change))
2304
2305 path_changed = False
2306 if set_title and mi.title:
2307 path_changed = True
2308 set_field('title', mi.title)
2309 authors_changed = False
2310 if set_authors:
2311 path_changed = True
2312 if not mi.authors:
2313 mi.authors = [_('Unknown')]
2314 authors = []
2315 for a in mi.authors:
2316 authors += string_to_authors(a)
2317 set_field('authors', authors)
2318 authors_changed = True
2319
2320 if path_changed:
2321 self._update_path({book_id})
2322
2323 def protected_set_field(name, val):
2324 try:
2325 set_field(name, val)
2326 except Exception:
2327 if ignore_errors:
2328 traceback.print_exc()
2329 else:
2330 raise
2331
2332 # force_changes has no effect on cover manipulation
2333 try:
2334 cdata = mi.cover_data[1]
2335 if cdata is None and isinstance(mi.cover, (str, bytes)) and mi.cover and os.access(mi.cover, os.R_OK):
2336 with open(mi.cover, 'rb') as f:
2337 cdata = f.read() or None
2338 if cdata is not None:
2339 self._set_cover({book_id: cdata})
2340 except Exception:
2341 if ignore_errors:
2342 traceback.print_exc()
2343 else:
2344 raise
2345
2346 try:
2347 with self.backend.conn: # Speed up set_metadata by not operating in autocommit mode
2348 for field in ('rating', 'series_index', 'timestamp'):
2349 val = getattr(mi, field)
2350 if val is not None:
2351 protected_set_field(field, val)
2352
2353 val = mi.get('author_sort', None)
2354 if authors_changed and (not val or mi.is_null('author_sort')):
2355 val = self._author_sort_from_authors(mi.authors)
2356 if authors_changed or (force_changes and val is not None) or not mi.is_null('author_sort'):
2357 protected_set_field('author_sort', val)
2358
2359 for field in ('publisher', 'series', 'tags', 'comments', 'languages', 'pubdate'):
2360 val = mi.get(field, None)
2361 if (force_changes and val is not None) or not mi.is_null(field):
2362 protected_set_field(field, val)
2363
2364 val = mi.get('title_sort', None)
2365 if (force_changes and val is not None) or not mi.is_null('title_sort'):
2366 protected_set_field('sort', val)
2367
2368 # identifiers will always be replaced if force_changes is True
2369 mi_idents = mi.get_identifiers()
2370 if force_changes:
2371 protected_set_field('identifiers', mi_idents)
2372 elif mi_idents:
2373 identifiers = self._field_for('identifiers', book_id, default_value={})
2374 for key, val in mi_idents.items():
2375 if val and val.strip(): # Don't delete an existing identifier
2376 identifiers[icu_lower(key)] = val
2377 protected_set_field('identifiers', identifiers)
2378
2379 user_mi = mi.get_all_user_metadata(make_copy=False)
2380 fm = self.field_metadata
2381 for key in user_mi:
2382 if (
2383 key in fm
2384 and user_mi[key]['datatype'] == fm[key]['datatype']
2385 and (user_mi[key]['datatype'] != 'text' or (user_mi[key]['is_multiple'] == fm[key]['is_multiple']))
2386 ):
2387 val = mi.get(key, None)
2388 if force_changes or val is not None:
2389 protected_set_field(key, val)
2390 idx = key + '_index'
2391 if idx in self.fields:
2392 extra = mi.get_extra(key)
2393 if extra is not None or force_changes:
2394 protected_set_field(idx, extra)
2395 except Exception:
2396 # sqlite will rollback the entire transaction, thanks to the with
2397 # statement, so we have to re-read everything form the db to ensure
2398 # the db and Cache are in sync
2399 self._reload_from_db()
2400 raise
2401 return dirtied
2402
2403 _set_metadata = set_metadata
2404
2405 def _do_add_format(self, book_id, fmt, stream, name=None, mtime=None):
2406 path = self._get_book_path(book_id, unsafe=True)
2407 if path is None:
2408 # Theoretically, this should never happen, but apparently it
2409 # does: https://www.mobileread.com/forums/showthread.php?t=233353
2410 self._update_path({book_id}, mark_as_dirtied=False)
2411 path = self._get_book_path(book_id)
2412
2413 title = self._field_for('title', book_id, default_value=_('Unknown'))
2414 try:
2415 author = self._field_for('authors', book_id, default_value=(_('Unknown'),))[0]
2416 except IndexError:
2417 author = _('Unknown')
2418
2419 size, fname = self.backend.add_format(book_id, fmt, stream, title, author, path, name, mtime=mtime)
2420 return size, fname
2421
2422 @api
2423 def add_format(self, book_id, fmt, stream_or_path, replace=True, run_hooks=True, dbapi=None):
2424 """
2425 Add a format to the specified book. Return True if the format was added successfully.
2426
2427 :param replace: If True replace existing format, otherwise if the format already exists, return False.
2428 :param run_hooks: If True, file type plugins are run on the format before and after being added.
2429 :param dbapi: Internal use only.
2430 """
2431 needs_close = False
2432 if run_hooks:
2433 # Run import plugins, the write lock is not held to cater for
2434 # broken plugins that might spin the event loop by popping up a
2435 # message in the GUI during the processing.
2436 npath = run_import_plugins(stream_or_path, fmt)
2437 fmt = os.path.splitext(npath)[-1].lower().replace('.', '').upper()
2438 stream_or_path = open(make_long_path_useable(npath), 'rb')
2439 needs_close = True
2440 fmt = check_ebook_format(stream_or_path, fmt)
2441
2442 with self.write_lock:
2443 if not self._has_id(book_id):
2444 raise NoSuchBook(book_id)
2445 fmt = (fmt or '').upper()
2446 self.format_metadata_cache[book_id].pop(fmt, None)
2447 try:
2448 name = self.fields['formats'].format_fname(book_id, fmt)
2449 except Exception:
2450 name = None
2451
2452 if name and not replace:
2453 if needs_close:
2454 stream_or_path.close()
2455 return False
2456
2457 if hasattr(stream_or_path, 'read'):
2458 stream = stream_or_path
2459 else:
2460 stream = open(make_long_path_useable(stream_or_path), 'rb')
2461 needs_close = True
2462 try:
2463 size, fname = self._do_add_format(book_id, fmt, stream, name)
2464 finally:
2465 if needs_close:
2466 stream.close()
2467 del stream
2468
2469 max_size = self.fields['formats'].table.update_fmt(book_id, fmt, fname, size, self.backend)
2470 self.fields['size'].table.update_sizes({book_id: max_size})
2471 self._update_last_modified((book_id,))
2472 self._queue_pages_scan(book_id)
2473 self.event_dispatcher(EventType.format_added, book_id, fmt)
2474
2475 if run_hooks:
2476 # Run post import plugins, the write lock is released so the plugin
2477 # can call api without a locking violation.
2478 run_plugins_on_postimport(dbapi or self, book_id, fmt)
2479 stream_or_path.close()
2480
2481 self.queue_next_fts_job()
2482 return True
2483
2484 _add_format = add_format
2485
2486 @write_api
2487 def remove_formats(self, formats_map, db_only=False):
2488 """
2489 Remove the specified formats from the specified books.
2490
2491 :param formats_map: A mapping of book_id to a list of formats to be removed from the book.
2492 :param db_only: If True, only remove the record for the format from the db, do not delete the actual format file from the filesystem.
2493 :return: A map of book id to set of formats actually deleted from the filesystem for that book
2494 """
2495 table = self.fields['formats'].table
2496 formats_map = {book_id: frozenset((f or '').upper() for f in fmts) for book_id, fmts in formats_map.items()}
2497 removed_map = {}
2498
2499 for book_id, fmts in formats_map.items():
2500 for fmt in fmts:
2501 self.format_metadata_cache[book_id].pop(fmt, None)
2502
2503 if not db_only:
2504 removes = defaultdict(set)
2505 metadata_map = {}
2506 for book_id, fmts in formats_map.items():
2507 try:
2508 path = self._get_book_path(book_id)
2509 except Exception:
2510 continue
2511 for fmt in fmts:
2512 try:
2513 name = self.fields['formats'].format_fname(book_id, fmt)
2514 except Exception:
2515 continue
2516 if name and path:
2517 removes[book_id].add((fmt, name, path))
2518 if removes[book_id]:
2519 metadata_map[book_id] = {
2520 'title': self._field_for('title', book_id),
2521 'authors': self._field_for('authors', book_id),
2522 }
2523 if removes:
2524 removed_map = self.backend.remove_formats(removes, metadata_map)
2525
2526 size_map = table.remove_formats(formats_map, self.backend)
2527 self.fields['size'].table.update_sizes(size_map)
2528
2529 for book_id, fmts in formats_map.items():
2530 for fmt in fmts:
2531 run_plugins_on_postdelete(self, book_id, fmt)
2532
2533 self._update_last_modified(tuple(formats_map))
2534 for book_id in formats_map:
2535 self._queue_pages_scan(book_id)
2536 self.event_dispatcher(EventType.formats_removed, formats_map)
2537 return removed_map
2538
2539 _remove_formats = remove_formats
2540
2541 @read_api
2542 def get_next_series_num_for(self, series, field='series', current_indices=False):
2543 """
2544 Return the next series index for the specified series, taking into account the various preferences that
2545 control next series number generation.
2546
2547 :param field: The series-like field (defaults to the builtin series column)
2548 :param current_indices: If True, returns a mapping of book_id to current series_index value instead.
2549 """
2550 books = ()
2551 sf = self.fields[field]
2552 if series:
2553 q = icu_lower(series)
2554 for val, book_ids in sf.iter_searchable_values(self._get_proxy_metadata, frozenset(self._all_book_ids())):
2555 if q == icu_lower(val):
2556 books = book_ids
2557 break
2558 idf = sf.index_field
2559 index_map = {book_id: self._fast_field_for(idf, book_id, default_value=1.0) for book_id in books}
2560 if current_indices:
2561 return index_map
2562 series_indices = sorted(index_map.values(), key=lambda s: s or 0)
2563 return _get_next_series_num_for_list(tuple(series_indices), unwrap=False)
2564
2565 _get_next_series_num_for = get_next_series_num_for
2566
2567 @read_api
2568 def author_sort_from_authors(self, authors, key_func=icu_lower):
2569 """Given a list of authors, return the author_sort string for the authors,
2570 preferring the author sort associated with the author over the computed
2571 string."""
2572 table = self.fields['authors'].table
2573 result = []
2574 rmap = {key_func(v): k for k, v in table.id_map.items()}
2575 for aut in authors:
2576 aid = rmap.get(key_func(aut), None)
2577 result.append(author_to_author_sort(aut) if aid is None else table.asort_map[aid])
2578 return ' & '.join(_f for _f in result if _f)
2579
2580 _author_sort_from_authors = author_sort_from_authors
2581
2582 @read_api
2583 def data_for_has_book(self):
2584 """Return data suitable for use in :meth:`has_book`. This can be used for an
2585 implementation of :meth:`has_book` in a worker process without access to the
2586 db."""
2587 try:
2588 return {icu_lower(title) for title in self.fields['title'].table.book_col_map.values()}
2589 except TypeError:
2590 # Some non-unicode titles in the db
2591 return {icu_lower(as_unicode(title)) for title in self.fields['title'].table.book_col_map.values()}
2592
2593 _data_for_has_book = data_for_has_book
2594
2595 @read_api
2596 def has_book(self, mi):
2597 """Return True iff the database contains an entry with the same title
2598 as the passed in Metadata object. The comparison is case-insensitive.
2599 See also :meth:`data_for_has_book`."""
2600 title = mi.title
2601 if title:
2602 if isinstance(title, bytes):
2603 title = title.decode(preferred_encoding, 'replace')
2604 q = icu_lower(title).strip()
2605 for title in self.fields['title'].table.book_col_map.values():
2606 if q == icu_lower(title):
2607 return True
2608 return False
2609
2610 _has_book = has_book
2611
2612 @read_api
2613 def has_id(self, book_id):
2614 "Return True iff the specified book_id exists in the db"
2615 return book_id in self.fields['title'].table.book_col_map
2616
2617 _has_id = has_id
2618
2619 @write_api
2620 def create_book_entry(self, mi, cover=None, add_duplicates=True, force_id=None, apply_import_tags=True, preserve_uuid=False):
2621 if mi.tags:
2622 mi.tags = list(mi.tags)
2623 if apply_import_tags:
2624 _add_newbook_tag(mi)
2625 _add_default_custom_column_values(mi, self.field_metadata)
2626 if not add_duplicates and self._has_book(mi):
2627 return
2628 series_index = self._get_next_series_num_for(mi.series) if mi.series_index is None else mi.series_index
2629 try:
2630 series_index = float(series_index)
2631 except Exception:
2632 try:
2633 series_index = float(self._get_next_series_num_for(mi.series))
2634 except Exception:
2635 series_index = 1.0
2636 if not mi.authors:
2637 mi.authors = (_('Unknown'),)
2638 aus = mi.author_sort if not mi.is_null('author_sort') else self._author_sort_from_authors(mi.authors)
2639 mi.title = mi.title or _('Unknown')
2640 if mi.is_null('title_sort'):
2641 mi.title_sort = title_sort(mi.title, lang=mi.languages[0] if mi.languages else None)
2642 if isinstance(aus, bytes):
2643 aus = aus.decode(preferred_encoding, 'replace')
2644 if isinstance(mi.title, bytes):
2645 mi.title = mi.title.decode(preferred_encoding, 'replace')
2646 if force_id is None:
2647 self.backend.execute('INSERT INTO books(title, series_index, author_sort) VALUES (?, ?, ?)', (mi.title, series_index, aus))
2648 else:
2649 self.backend.execute(
2650 'INSERT INTO books(id, title, series_index, author_sort) VALUES (?, ?, ?, ?)',
2651 (force_id, mi.title, series_index, aus),
2652 )
2653 book_id = self.backend.last_insert_rowid()
2654 self.event_dispatcher(EventType.book_created, book_id)
2655
2656 mi.timestamp = utcnow() if (mi.timestamp is None or is_date_undefined(mi.timestamp)) else mi.timestamp
2657 mi.pubdate = UNDEFINED_DATE if mi.pubdate is None else mi.pubdate
2658 if cover is not None:
2659 mi.cover, mi.cover_data = None, (None, cover)
2660 self._set_metadata(book_id, mi, ignore_errors=True)
2661 lm = getattr(mi, 'link_maps', None)
2662 if lm:
2663 for field, link_map in lm.items():
2664 if self._has_link_map(field):
2665 self._set_link_map(field, link_map, only_set_if_no_existing_link=True)
2666 if preserve_uuid and mi.uuid:
2667 self._set_field('uuid', {book_id: mi.uuid})
2668 # Update the caches for fields from the books table
2669 self.fields['size'].table.book_col_map[book_id] = 0
2670 row = next(self.backend.execute('SELECT sort, series_index, author_sort, uuid, has_cover FROM books WHERE id=?', (book_id,)))
2671 for field, val in zip(('sort', 'series_index', 'author_sort', 'uuid', 'cover'), row):
2672 if field == 'cover':
2673 val = bool(val)
2674 elif field == 'uuid':
2675 self.fields[field].table.uuid_to_id_map[val] = book_id
2676 self.fields[field].table.book_col_map[book_id] = val
2677
2678 return book_id
2679
2680 _create_book_entry = create_book_entry
2681
2682 @api
2683 def add_books(self, books, add_duplicates=True, apply_import_tags=True, preserve_uuid=False, run_hooks=True, dbapi=None):
2684 """
2685 Add the specified books to the library. Books should be an iterable of
2686 2-tuples, each 2-tuple of the form :code:`(mi, format_map)` where mi is a
2687 Metadata object and format_map is a dictionary of the form :code:`{fmt: path_or_stream}`,
2688 for example: :code:`{'EPUB': '/path/to/file.epub'}`.
2689
2690 Returns a pair of lists: :code:`ids, duplicates`. ``ids`` contains the book ids for all newly created books in the
2691 database. ``duplicates`` contains the :code:`(mi, format_map)` for all books that already exist in the database
2692 as per the simple duplicate detection heuristic used by :meth:`has_book`.
2693 """
2694 duplicates, ids = [], []
2695 for mi, format_map in books:
2696 book_id = self.create_book_entry(mi, add_duplicates=add_duplicates, apply_import_tags=apply_import_tags, preserve_uuid=preserve_uuid)
2697 if book_id is None:
2698 duplicates.append((mi, format_map))
2699 else:
2700 fmt_map = {}
2701 ids.append(book_id)
2702 for fmt, stream_or_path in format_map.items():
2703 if self.add_format(book_id, fmt, stream_or_path, dbapi=dbapi, run_hooks=run_hooks):
2704 fmt_map[fmt.lower()] = getattr(stream_or_path, 'name', stream_or_path) or '<stream>'
2705 run_plugins_on_postadd(dbapi or self, book_id, fmt_map)
2706 return ids, duplicates
2707
2708 _add_books = add_books
2709
2710 @write_api
2711 def remove_books(self, book_ids, permanent=False):
2712 """Remove the books specified by the book_ids from the database and delete
2713 their format files. If ``permanent`` is False, then the format files
2714 are placed in the per-library trash directory."""
2715 path_map = {}
2716 for book_id in book_ids:
2717 try:
2718 path = self._get_book_path(book_id)
2719 except Exception:
2720 path = None
2721 path_map[book_id] = path
2722 if not permanent and path:
2723 # ensure metadata.opf is written and up-to-date so we can restore the book
2724 try:
2725 mi = self._metadata_as_object_for_dump(book_id)
2726 raw = metadata_to_opf(mi)
2727 self.backend.write_backup(path, raw)
2728 except Exception:
2729 traceback.print_exc()
2730 self.backend.remove_books(path_map, permanent=permanent)
2731 for field in self.fields.values():
2732 try:
2733 table = field.table
2734 except AttributeError:
2735 continue # Some fields like ondevice do not have tables
2736 else:
2737 table.remove_books(book_ids, self.backend)
2738 self._search_api.discard_books(book_ids)
2739 self._clear_caches(book_ids=book_ids, template_cache=False, search_cache=False)
2740 for cc in self.cover_caches:
2741 cc.invalidate(book_ids)
2742 self.event_dispatcher(EventType.books_removed, book_ids)
2743
2744 _remove_books = remove_books
2745
2746 @read_api
2747 def author_sort_strings_for_books(self, book_ids):
2748 val_map = {}
2749 for book_id in book_ids:
2750 authors = self._field_ids_for('authors', book_id)
2751 adata = self._author_data(authors)
2752 val_map[book_id] = tuple(adata[aid]['sort'] for aid in authors)
2753 return val_map
2754
2755 _author_sort_strings_for_books = author_sort_strings_for_books
2756
2757 @write_api
2758 def rename_items(self, field, item_id_to_new_name_map, change_index=True, restrict_to_book_ids=None):
2759 """
2760 Rename items from a many-one or many-many field such as tags or series.
2761
2762 :param change_index: When renaming in a series-like field also change the series_index values.
2763 :param restrict_to_book_ids: An optional set of book ids for which the rename is to be performed, defaults to all books.
2764 """
2765 f = self.fields[field]
2766 affected_books = set()
2767 try:
2768 sv = f.metadata['is_multiple']['ui_to_list']
2769 except TypeError, KeyError, AttributeError:
2770 sv = None
2771
2772 if restrict_to_book_ids is not None:
2773 # We have a VL. Only change the item name for those books
2774 if not isinstance(restrict_to_book_ids, (Set, MutableSet)):
2775 restrict_to_book_ids = frozenset(restrict_to_book_ids)
2776 id_map = {}
2777 default_process_map = {}
2778 for old_id, new_name in item_id_to_new_name_map.items():
2779 new_names = tuple(x.strip() for x in new_name.split(sv)) if sv else (new_name,)
2780 # Get a list of books in the VL with the item
2781 books_with_id = f.books_for(old_id)
2782 books_to_process = books_with_id & restrict_to_book_ids
2783 if len(books_with_id) == len(books_to_process):
2784 # All the books with the ID are in the VL, so we can use
2785 # the normal processing
2786 default_process_map[old_id] = new_name
2787 elif books_to_process:
2788 affected_books.update(books_to_process)
2789 newvals = {}
2790 for book_id in books_to_process:
2791 # Get the current values, remove the one being renamed, then add
2792 # the new value(s) back.
2793 vals = self._field_for(field, book_id)
2794 # Check for is_multiple
2795 if isinstance(vals, tuple):
2796 # We must preserve order.
2797 vals = list(vals)
2798 # Don't need to worry about case here because we
2799 # are fetching its one-true spelling. But lets be
2800 # careful anyway
2801 try:
2802 dex = vals.index(self._get_item_name(field, old_id))
2803 # This can put the name back with a different case
2804 vals[dex] = new_names[0]
2805 # now add any other items if they aren't already there
2806 if len(new_names) > 1:
2807 set_vals = {icu_lower(x) for x in vals}
2808 for v in new_names[1:]:
2809 lv = icu_lower(v)
2810 if lv not in set_vals:
2811 vals.append(v)
2812 set_vals.add(lv)
2813 newvals[book_id] = vals
2814 except Exception:
2815 traceback.print_exc()
2816 else:
2817 newvals[book_id] = new_names[0]
2818 # Allow case changes
2819 self._set_field(field, newvals)
2820 id_map[old_id] = self._get_item_id(field, new_names[0])
2821 if default_process_map:
2822 ab, idm = self._rename_items(field, default_process_map, change_index=change_index)
2823 affected_books.update(ab)
2824 id_map.update(idm)
2825 self.event_dispatcher(EventType.items_renamed, field, affected_books, id_map)
2826 return affected_books, id_map
2827
2828 try:
2829 func = f.table.rename_item
2830 except AttributeError:
2831 raise ValueError(f'Cannot rename items for one-one fields: {field}')
2832 moved_books = set()
2833 id_map = {}
2834 for item_id, new_name in item_id_to_new_name_map.items():
2835 new_names = tuple(x.strip() for x in new_name.split(sv)) if sv else (new_name,)
2836 books, new_id = func(item_id, new_names[0], self.backend)
2837 affected_books.update(books)
2838 id_map[item_id] = new_id
2839 if new_id != item_id:
2840 moved_books.update(books)
2841 if len(new_names) > 1:
2842 # Add the extra items to the books
2843 extra = new_names[1:]
2844 self._set_field(field, {book_id: self._fast_field_for(f, book_id) + extra for book_id in books})
2845
2846 if affected_books:
2847 if field == 'authors':
2848 self._set_field(
2849 'author_sort',
2850 {k: ' & '.join(v) for k, v in self._author_sort_strings_for_books(affected_books).items()},
2851 )
2852 self._update_path(affected_books, mark_as_dirtied=False)
2853 elif change_index and hasattr(f, 'index_field') and tweaks['series_index_auto_increment'] != 'no_change':
2854 for book_id in moved_books:
2855 self._set_field(
2856 f.index_field.name,
2857 {book_id: self._get_next_series_num_for(self._fast_field_for(f, book_id), field=field)},
2858 )
2859 self._mark_as_dirty(affected_books)
2860 self._clear_link_map_cache(affected_books)
2861 self.event_dispatcher(EventType.items_renamed, field, affected_books, id_map)
2862 return affected_books, id_map
2863
2864 _rename_items = rename_items
2865
2866 @write_api
2867 def remove_items(self, field, item_ids, restrict_to_book_ids=None):
2868 """Delete all items in the specified field with the specified ids.
2869 Returns the set of affected book ids. ``restrict_to_book_ids`` is an
2870 optional set of books ids. If specified the items will only be removed
2871 from those books."""
2872 field = self.fields[field]
2873 if restrict_to_book_ids is not None and not isinstance(restrict_to_book_ids, (MutableSet, Set)):
2874 restrict_to_book_ids = frozenset(restrict_to_book_ids)
2875 affected_books = field.table.remove_items(item_ids, self.backend, restrict_to_book_ids=restrict_to_book_ids)
2876 if affected_books:
2877 if hasattr(field, 'index_field'):
2878 self._set_field(field.index_field.name, {bid: 1.0 for bid in affected_books})
2879 else:
2880 self._mark_as_dirty(affected_books)
2881 self._clear_link_map_cache(affected_books)
2882 self.event_dispatcher(EventType.items_removed, field, affected_books, item_ids)
2883 return affected_books
2884
2885 _remove_items = remove_items
2886
2887 @write_api
2888 def add_custom_book_data(self, name, val_map, delete_first=False):
2889 """Add data for name where val_map is a map of book_ids to values. If
2890 delete_first is True, all previously stored data for name will be
2891 removed."""
2892 missing = frozenset(val_map) - self._all_book_ids()
2893 if missing:
2894 raise ValueError(f'add_custom_book_data: no such book_ids: {missing}')
2895 self.backend.add_custom_data(name, val_map, delete_first)
2896
2897 _add_custom_book_data = add_custom_book_data
2898
2899 @read_api
2900 def get_custom_book_data(self, name, book_ids=(), default=None):
2901 """Get data for name. By default returns data for all book_ids, pass
2902 in a list of book ids if you only want some data. Returns a map of
2903 book_id to values. If a particular value could not be decoded, uses
2904 default for it."""
2905 return self.backend.get_custom_book_data(name, book_ids, default)
2906
2907 _get_custom_book_data = get_custom_book_data
2908
2909 @write_api
2910 def delete_custom_book_data(self, name, book_ids=()):
2911 """Delete data for name. By default deletes all data, if you only want
2912 to delete data for some book ids, pass in a list of book ids."""
2913 self.backend.delete_custom_book_data(name, book_ids)
2914
2915 _delete_custom_book_data = delete_custom_book_data
2916
2917 @read_api
2918 def get_ids_for_custom_book_data(self, name):
2919 """Return the set of book ids for which name has data."""
2920 return self.backend.get_ids_for_custom_book_data(name)
2921
2922 _get_ids_for_custom_book_data = get_ids_for_custom_book_data
2923
2924 @read_api
2925 def conversion_options(self, book_id, fmt='PIPE'):
2926 return self.backend.conversion_options(book_id, fmt)
2927
2928 _conversion_options = conversion_options
2929
2930 @read_api
2931 def has_conversion_options(self, ids, fmt='PIPE'):
2932 return self.backend.has_conversion_options(ids, fmt)
2933
2934 _has_conversion_options = has_conversion_options
2935
2936 @write_api
2937 def delete_conversion_options(self, book_ids, fmt='PIPE'):
2938 return self.backend.delete_conversion_options(book_ids, fmt)
2939
2940 _delete_conversion_options = delete_conversion_options
2941
2942 @write_api
2943 def set_conversion_options(self, options, fmt='PIPE'):
2944 """options must be a map of the form {book_id:conversion_options}"""
2945 return self.backend.set_conversion_options(options, fmt)
2946
2947 _set_conversion_options = set_conversion_options
2948
2949 @write_api
2950 def refresh_format_cache(self):
2951 self.fields['formats'].table.read(self.backend)
2952 self.format_metadata_cache.clear()
2953
2954 _refresh_format_cache = refresh_format_cache
2955
2956 @write_api
2957 def refresh_ondevice(self):
2958 self.fields['ondevice'].clear_caches()
2959 self.clear_search_caches()
2960 self.clear_composite_caches()
2961
2962 _refresh_ondevice = refresh_ondevice
2963
2964 @read_api
2965 def books_matching_device_book(self, lpath):
2966 ans = set()
2967 for book_id, (_x, _x, _x, _x, lpaths) in self.fields['ondevice'].cache.items():
2968 if lpath in lpaths:
2969 ans.add(book_id)
2970 return ans
2971
2972 _books_matching_device_book = books_matching_device_book
2973
2974 @read_api
2975 def tags_older_than(self, tag, delta=None, must_have_tag=None, must_have_authors=None):
2976 """
2977 Return the ids of all books having the tag ``tag`` that are older than
2978 the specified time. tag comparison is case insensitive.
2979
2980 :param delta: A timedelta object or None. If None, then all ids with
2981 the tag are returned.
2982
2983 :param must_have_tag: If not None the list of matches will be
2984 restricted to books that have this tag
2985
2986 :param must_have_authors: A list of authors. If not None the list of
2987 matches will be restricted to books that have these authors (case
2988 insensitive).
2989
2990 """
2991 tag_map = {icu_lower(v): k for k, v in self._get_id_map('tags').items()}
2992 tag = icu_lower(tag.strip())
2993 mht = icu_lower(must_have_tag.strip()) if must_have_tag else None
2994 tag_id, mht_id = tag_map.get(tag, None), tag_map.get(mht, None)
2995 ans = set()
2996 if mht_id is None and mht:
2997 return ans
2998 if tag_id is not None:
2999 tagged_books = self._books_for_field('tags', tag_id)
3000 if mht_id is not None and tagged_books:
3001 tagged_books = tagged_books.intersection(self._books_for_field('tags', mht_id))
3002 if tagged_books:
3003 if must_have_authors is not None:
3004 amap = {icu_lower(v): k for k, v in self._get_id_map('authors').items()}
3005 books = None
3006 for author in must_have_authors:
3007 abooks = self._books_for_field('authors', amap.get(icu_lower(author), None))
3008 books = abooks if books is None else books.intersection(abooks)
3009 if not books:
3010 break
3011 tagged_books = tagged_books.intersection(books or set())
3012 if delta is None:
3013 ans = tagged_books
3014 else:
3015 now = nowf()
3016 for book_id in tagged_books:
3017 ts = self._field_for('timestamp', book_id)
3018 if (now - ts) > delta:
3019 ans.add(book_id)
3020 return ans
3021
3022 _tags_older_than = tags_older_than
3023
3024 @write_api
3025 def set_sort_for_authors(self, author_id_to_sort_map, update_books=True):
3026 sort_map = self.fields['authors'].table.set_sort_names(author_id_to_sort_map, self.backend)
3027 changed_books = set()
3028 if update_books:
3029 val_map = {}
3030 for author_id in sort_map:
3031 books = self._books_for_field('authors', author_id)
3032 changed_books |= books
3033 for book_id in books:
3034 authors = self._field_ids_for('authors', book_id)
3035 adata = self._author_data(authors)
3036 sorts = [adata[x]['sort'] for x in authors]
3037 val_map[book_id] = ' & '.join(sorts)
3038 if val_map:
3039 self._set_field('author_sort', val_map)
3040 if changed_books:
3041 self._mark_as_dirty(changed_books)
3042 self._clear_link_map_cache(changed_books)
3043 return changed_books
3044
3045 _set_sort_for_authors = set_sort_for_authors
3046
3047 @write_api
3048 def set_link_for_authors(self, author_id_to_link_map):
3049 link_map = self.fields['authors'].table.set_links(author_id_to_link_map, self.backend)
3050 changed_books = set()
3051 for author_id in link_map:
3052 changed_books |= self._books_for_field('authors', author_id)
3053 if changed_books:
3054 self._mark_as_dirty(changed_books)
3055 self._clear_link_map_cache(changed_books)
3056 self.event_dispatcher(EventType.links_changed, 'authors', frozenset(author_id_to_link_map))
3057 return changed_books
3058
3059 _set_link_for_authors = set_link_for_authors
3060
3061 @read_api
3062 def has_link_map(self, field):
3063 return hasattr(getattr(self.fields.get(field), 'table', None), 'link_map')
3064
3065 _has_link_map = has_link_map
3066
3067 @read_api
3068 def get_link_map(self, for_field):
3069 """
3070 Return a dictionary of links for the supplied field.
3071
3072 :param for_field: the lookup name of the field for which the link map is desired
3073
3074 :return: {field_value:link_value, ...} for non-empty links
3075 """
3076 if for_field not in self.fields:
3077 raise ValueError(f'Lookup name {for_field} is not a valid name')
3078 table = self.fields[for_field].table
3079 lm = getattr(table, 'link_map', None)
3080 if lm is None:
3081 raise ValueError(f"Lookup name {for_field} doesn't have a link map")
3082 lm = table.link_map
3083 vm = table.id_map
3084 ans = {vm.get(fid): v for fid, v in lm.items() if v}
3085 ans.pop(None, None)
3086 return ans
3087
3088 _get_link_map = get_link_map
3089
3090 @read_api
3091 def link_for(self, field, item_id):
3092 """
3093 Return the link, if any, for the specified item or None if no link is found
3094 """
3095 f = self.fields.get(field)
3096 if f is not None:
3097 table = f.table
3098 lm = getattr(table, 'link_map', None)
3099 if lm is not None:
3100 return lm.get(item_id)
3101
3102 _link_for = link_for
3103
3104 @read_api
3105 def get_all_link_maps_for_book(self, book_id):
3106 """
3107 Returns all links for all fields referenced by book identified by book_id.
3108 If book_id doesn't exist then the method returns {}.
3109
3110 Example: Assume author A has link X, author B has link Y, tag S has link
3111 F, and tag T has link G. If book 1 has author A and tag T,
3112 this method returns {'authors':{'A':'X'}, 'tags':{'T', 'G'}}.
3113 If book 2's author is neither A nor B and has no tags, this method returns {}.
3114
3115 :param book_id: the book id in question.
3116
3117 :return: {field: {field_value, link_value}, ... for all fields with a field_value having a non-empty link value for that book
3118
3119 """
3120 if not self._has_id(book_id):
3121 # Works for book_id is None.
3122 return {}
3123 cached = self.link_maps_cache.get(book_id)
3124 if cached is not None:
3125 return cached
3126 links = {}
3127
3128 def add_links_for_field(f):
3129 field_ids = self._field_ids_for(f, book_id)
3130 if field_ids:
3131 table = self.fields[f].table
3132 lm = table.link_map
3133 id_link_map = {fid: lm.get(fid) for fid in field_ids}
3134 vm = table.id_map
3135 d = {vm.get(fid): v for fid, v in id_link_map.items() if v}
3136 d.pop(None, None)
3137 if d:
3138 links[f] = d
3139
3140 for field in ('authors', 'publisher', 'series', 'tags'):
3141 add_links_for_field(field)
3142 for field in self.field_metadata.custom_field_keys(include_composites=False):
3143 if self._has_link_map(field):
3144 add_links_for_field(field)
3145 self.link_maps_cache[book_id] = links
3146 return links
3147
3148 _get_all_link_maps_for_book = get_all_link_maps_for_book
3149
3150 @write_api
3151 def set_link_map(self, field, value_to_link_map, only_set_if_no_existing_link=False):
3152 """
3153 Sets links for item values in field.
3154 Note: this method doesn't change values not in the value_to_link_map
3155
3156 :param field: the lookup name
3157 :param value_to_link_map: dict(field_value:link, ...). Note that these are values, not field ids.
3158
3159 :return: books changed by setting the link
3160
3161 """
3162 if field not in self.fields:
3163 raise ValueError(f'Lookup name {field} is not a valid name')
3164 table = getattr(self.fields[field], 'table', None)
3165 if table is None:
3166 raise ValueError(f"Lookup name {field} doesn't have a link map")
3167 # Clear the links for book cache as we don't know what will be affected
3168 self.link_maps_cache = {}
3169
3170 fids = self._get_item_ids(field, value_to_link_map)
3171 if only_set_if_no_existing_link:
3172 lm = table.link_map
3173 id_to_link_map = {fid: value_to_link_map[k] for k, fid in fids.items() if fid is not None and not lm.get(fid)}
3174 else:
3175 id_to_link_map = {fid: value_to_link_map[k] for k, fid in fids.items() if fid is not None}
3176 result_map = table.set_links(id_to_link_map, self.backend)
3177 changed_books = set()
3178 for id_ in result_map:
3179 changed_books |= self._books_for_field(field, id_)
3180 if changed_books:
3181 self._mark_as_dirty(changed_books)
3182 self._clear_link_map_cache(changed_books)
3183 self.event_dispatcher(EventType.links_changed, field, frozenset(id_to_link_map))
3184 return changed_books
3185
3186 _set_link_map = set_link_map
3187
3188 @read_api
3189 def lookup_by_uuid(self, uuid):
3190 return self.fields['uuid'].table.lookup_by_uuid(uuid)
3191
3192 _lookup_by_uuid = lookup_by_uuid
3193
3194 @write_api
3195 def delete_custom_column(self, label=None, num=None):
3196 self.backend.delete_custom_column(label, num)
3197
3198 _delete_custom_column = delete_custom_column
3199
3200 @write_api
3201 def create_custom_column(self, label, name, datatype, is_multiple, editable=True, display={}):
3202 return self.backend.create_custom_column(label, name, datatype, is_multiple, editable=editable, display=display)
3203
3204 _create_custom_column = create_custom_column
3205
3206 @write_api
3207 def set_custom_column_metadata(self, num, name=None, label=None, is_editable=None, display=None, update_last_modified=False):
3208 changed = self.backend.set_custom_column_metadata(num, name=name, label=label, is_editable=is_editable, display=display)
3209 if changed:
3210 if update_last_modified:
3211 self._update_last_modified(self._all_book_ids())
3212 else:
3213 self.backend.prefs.set('update_all_last_mod_dates_on_start', True)
3214 return changed
3215
3216 _set_custom_column_metadata = set_custom_column_metadata
3217
3218 @read_api
3219 def get_books_for_category(self, category, item_id_or_composite_value):
3220 f = self.fields[category]
3221 if hasattr(f, 'get_books_for_val'):
3222 # Composite field
3223 return f.get_books_for_val(item_id_or_composite_value, self._get_proxy_metadata, self._all_book_ids())
3224 return self._books_for_field(f.name, int(item_id_or_composite_value))
3225
3226 _get_books_for_category = get_books_for_category
3227
3228 @read_api
3229 def split_if_is_multiple_composite(self, f, val):
3230 """
3231 If f is a composite column lookup key and the column is is_multiple then
3232 split v into unique non-empty values. The comparison is case sensitive.
3233 Order is not preserved. Return a list() for compatibility with proxy
3234 metadata field getters, for example tags.
3235 """
3236 fm = self.field_metadata.get(f, None)
3237 if fm and fm['datatype'] == 'composite' and fm['is_multiple']:
3238 sep = fm['is_multiple'].get('cache_to_list', ',')
3239 return list({v.strip() for v in val.split(sep) if v.strip()})
3240 return val
3241
3242 _split_if_is_multiple_composite = split_if_is_multiple_composite
3243
3244 @read_api
3245 def data_for_find_identical_books(self):
3246 """Return data that can be used to implement
3247 :meth:`find_identical_books` in a worker process without access to the
3248 db. See db.utils for an implementation."""
3249 at = self.fields['authors'].table
3250 author_map = defaultdict(set)
3251 for aid, author in at.id_map.items():
3252 author_map[icu_lower(author)].add(aid)
3253 return (
3254 author_map,
3255 at.col_book_map.copy(),
3256 self.fields['title'].table.book_col_map.copy(),
3257 self.fields['languages'].book_value_map.copy(),
3258 )
3259
3260 _data_for_find_identical_books = data_for_find_identical_books
3261
3262 @read_api
3263 def update_data_for_find_identical_books(self, book_id, data):
3264 author_map, author_book_map, title_map, lang_map = data
3265 title_map[book_id] = self._field_for('title', book_id)
3266 lang_map[book_id] = self._field_for('languages', book_id)
3267 at = self.fields['authors'].table
3268 for aid in at.book_col_map.get(book_id, ()):
3269 author_map[icu_lower(at.id_map[aid])].add(aid)
3270 try:
3271 author_book_map[aid].add(book_id)
3272 except KeyError:
3273 author_book_map[aid] = {book_id}
3274
3275 _update_data_for_find_identical_books = update_data_for_find_identical_books
3276
3277 @read_api
3278 def find_identical_books(self, mi, search_restriction='', book_ids=None):
3279 """Finds books that have a superset of the authors in mi and the same
3280 title (title is fuzzy matched). See also :meth:`data_for_find_identical_books`."""
3281 from calibre.db.utils import fuzzy_title
3282
3283 identical_book_ids = set()
3284 langq = tuple(x for x in map(canonicalize_lang, mi.languages or ()) if x and x != 'und')
3285 if mi.authors:
3286 try:
3287 quathors = mi.authors[:20] # Too many authors causes parsing of the search expression to fail
3288 query = ' and '.join('authors:"={}"'.format(a.replace('"', '')) for a in quathors)
3289 qauthors = mi.authors[20:]
3290 except ValueError:
3291 return identical_book_ids
3292 try:
3293 book_ids = self._search(query, restriction=search_restriction, book_ids=book_ids)
3294 except Exception:
3295 traceback.print_exc()
3296 return identical_book_ids
3297 if qauthors and book_ids:
3298 matches = set()
3299 qauthors = {icu_lower(x) for x in qauthors}
3300 for book_id in book_ids:
3301 aut = self._field_for('authors', book_id)
3302 if aut:
3303 aut = {icu_lower(x) for x in aut}
3304 if aut.issuperset(qauthors):
3305 matches.add(book_id)
3306 book_ids = matches
3307
3308 for book_id in book_ids:
3309 fbook_title = self._field_for('title', book_id)
3310 fbook_title = fuzzy_title(fbook_title)
3311 mbook_title = fuzzy_title(mi.title)
3312 if fbook_title == mbook_title:
3313 bl = self._field_for('languages', book_id)
3314 if not langq or not bl or bl == langq:
3315 identical_book_ids.add(book_id)
3316 return identical_book_ids
3317
3318 _find_identical_books = find_identical_books
3319
3320 @read_api
3321 def get_top_level_move_items(self):
3322 all_paths = {self._get_book_path(book_id, sep='/').partition('/')[0] for book_id in self._all_book_ids()}
3323 return self.backend.get_top_level_move_items(all_paths)
3324
3325 _get_top_level_move_items = get_top_level_move_items
3326
3327 @write_api
3328 def move_library_to(self, newloc, progress=None, abort=None):
3329 def progress_callback(item_name, item_count, total):
3330 try:
3331 if progress is not None:
3332 progress(item_name, item_count, total)
3333 except Exception:
3334 traceback.print_exc()
3335
3336 all_paths = {self._get_book_path(book_id, sep='/').partition('/')[0] for book_id in self._all_book_ids()}
3337 self.backend.move_library_to(all_paths, newloc, progress=progress_callback, abort=abort)
3338
3339 _move_library_to = move_library_to
3340
3341 @read_api
3342 def saved_search_names(self):
3343 return self._search_api.saved_searches.names()
3344
3345 _saved_search_names = saved_search_names
3346
3347 @read_api
3348 def saved_search_lookup(self, name):
3349 return self._search_api.saved_searches.lookup(name)
3350
3351 _saved_search_lookup = saved_search_lookup
3352
3353 @write_api
3354 def saved_search_set_all(self, smap):
3355 self._search_api.saved_searches.set_all(smap)
3356 self._clear_search_caches()
3357
3358 _saved_search_set_all = saved_search_set_all
3359
3360 @write_api
3361 def saved_search_delete(self, name):
3362 self._search_api.saved_searches.delete(name)
3363 self._clear_search_caches()
3364
3365 _saved_search_delete = saved_search_delete
3366
3367 @write_api
3368 def saved_search_add(self, name, val):
3369 self._search_api.saved_searches.add(name, val)
3370
3371 _saved_search_add = saved_search_add
3372
3373 @write_api
3374 def saved_search_rename(self, old_name, new_name):
3375 self._search_api.saved_searches.rename(old_name, new_name)
3376 self._clear_search_caches()
3377
3378 _saved_search_rename = saved_search_rename
3379
3380 @write_api
3381 def change_search_locations(self, newlocs):
3382 self._search_api.change_locations(newlocs)
3383
3384 _change_search_locations = change_search_locations
3385
3386 @write_api
3387 def refresh_search_locations(self):
3388 self._search_api.change_locations(self.field_metadata.get_search_terms())
3389
3390 _refresh_search_locations = refresh_search_locations
3391
3392 @write_api
3393 def dump_and_restore(self, callback=None, sql=None):
3394 return self.backend.dump_and_restore(callback=callback, sql=sql)
3395
3396 _dump_and_restore = dump_and_restore
3397
3398 @write_api
3399 def vacuum(self, include_fts_db=False, include_notes_db=True, rebuild_annotations_fts=False):
3400 self.is_doing_rebuild_or_vacuum = True
3401 try:
3402 self.backend.vacuum(include_fts_db, include_notes_db, rebuild_annotations_fts)
3403 finally:
3404 self.is_doing_rebuild_or_vacuum = False
3405
3406 _vacuum = vacuum
3407
3408 def __del__(self):
3409 self.close()
3410
3411 def _shutdown_fts(self, stage=1):
3412 if stage == 1:
3413 self.backend.shutdown_fts()
3414 if self.fts_queue_thread is not None:
3415 self.fts_job_queue.shutdown(True)
3416 if hasattr(self, 'fts_dispatch_stop_event'):
3417 self.fts_dispatch_stop_event.set()
3418 return
3419 # the fts supervisor thread could be in the middle of committing a
3420 # result to the db, so holding a lock here will cause a deadlock
3421 if self.fts_queue_thread is not None:
3422 t, self.fts_queue_thread = self.fts_queue_thread, None
3423 if not sys.is_finalizing():
3424 t.join()
3425 self.backend.join_fts()
3426
3427 @api
3428 def close(self):
3429 with self.write_lock:
3430 if hasattr(self, 'close_called'):
3431 return
3432 self.close_called = True
3433 self.shutting_down = True
3434 m, self.maintain_page_counts = self.maintain_page_counts, None
3435 m.shutdown()
3436 self.event_dispatcher.close()
3437 self._shutdown_fts()
3438 try:
3439 from calibre.customize.ui import available_library_closed_plugins
3440 except ImportError:
3441 pass # happens during interpreter shutdown
3442 else:
3443 for plugin in available_library_closed_plugins():
3444 try:
3445 plugin.run(self)
3446 except Exception:
3447 traceback.print_exc()
3448 self._shutdown_fts(stage=2)
3449 if m is not None:
3450 m.wait_for_worker_shutdown()
3451 with self.write_lock:
3452 self.backend.close()
3453
3454 _close = close
3455
3456 @property
3457 def is_closed(self):
3458 return self.backend.is_closed
3459
3460 @write_api
3461 def clear_trash_bin(self):
3462 self.backend.clear_trash_dir()
3463
3464 _clear_trash_bin = clear_trash_bin
3465
3466 @read_api
3467 def list_trash_entries(self):
3468 books, formats = self.backend.list_trash_entries()
3469 ff = []
3470 for e in formats:
3471 if self._has_id(e.book_id):
3472 ff.append(e)
3473 e.cover_path = self.format_abspath(e.book_id, '__COVER_INTERNAL__')
3474 return books, formats
3475
3476 _list_trash_entries = list_trash_entries
3477
3478 @read_api
3479 def copy_format_from_trash(self, book_id, fmt, dest):
3480 fmt = fmt.upper()
3481 fpath = self.backend.path_for_trash_format(book_id, fmt)
3482 if not fpath:
3483 raise ValueError(f'No format {fmt} found in book {book_id}')
3484 shutil.copyfile(fpath, dest)
3485
3486 _copy_format_from_trash = copy_format_from_trash
3487
3488 @write_api
3489 def move_format_from_trash(self, book_id, fmt):
3490 """Undelete a format from the trash directory"""
3491 if not self._has_id(book_id):
3492 raise ValueError(f'A book with the id {book_id} does not exist')
3493 fmt = fmt.upper()
3494 try:
3495 name = self.fields['formats'].format_fname(book_id, fmt)
3496 except Exception:
3497 name = None
3498 fpath = self.backend.path_for_trash_format(book_id, fmt)
3499 if not fpath:
3500 raise ValueError(f'No format {fmt} found in book {book_id}')
3501 size, fname = self._do_add_format(book_id, fmt, fpath, name)
3502 self.format_metadata_cache.pop(book_id, None)
3503 max_size = self.fields['formats'].table.update_fmt(book_id, fmt, fname, size, self.backend)
3504 self.fields['size'].table.update_sizes({book_id: max_size})
3505 self._queue_pages_scan(book_id)
3506 self.event_dispatcher(EventType.format_added, book_id, fmt)
3507 self.backend.remove_trash_formats_dir_if_empty(book_id)
3508
3509 _move_format_from_trash = move_format_from_trash
3510
3511 @read_api
3512 def copy_book_from_trash(self, book_id, dest: str):
3513 self.backend.copy_book_from_trash(book_id, dest)
3514
3515 _copy_book_from_trash = copy_book_from_trash
3516
3517 @write_api
3518 def move_book_from_trash(self, book_id):
3519 """Undelete a book from the trash directory"""
3520 if self._has_id(book_id):
3521 raise ValueError(f'A book with the id {book_id} already exists')
3522 mi, annotations, formats = self.backend.get_metadata_for_trash_book(book_id)
3523 mi.cover = None
3524 self._create_book_entry(mi, add_duplicates=True, force_id=book_id, apply_import_tags=False, preserve_uuid=True)
3525 path = self._get_book_path(book_id)
3526 self.backend.move_book_from_trash(book_id, path)
3527 self.format_metadata_cache.pop(book_id, None)
3528 f = self.fields['formats'].table
3529 max_size = 0
3530 for fmt, size, fname in formats:
3531 max_size = max(max_size, f.update_fmt(book_id, fmt, fname, size, self.backend))
3532 self.fields['size'].table.update_sizes({book_id: max_size})
3533 cover = self.backend.cover_abspath(book_id, path)
3534 if cover and os.path.exists(cover):
3535 self._set_field('cover', {book_id: 1})
3536 if annotations:
3537 self._restore_annotations(book_id, annotations)
3538 self._queue_pages_scan(book_id)
3539
3540 _move_book_from_trash = move_book_from_trash
3541
3542 @write_api
3543 def delete_trash_entry(self, book_id, category):
3544 "Delete an entry from the trash. Here category is 'b' for books and 'f' for formats."
3545 self.backend.delete_trash_entry(book_id, category)
3546
3547 _delete_trash_entry = delete_trash_entry
3548
3549 @write_api
3550 def expire_old_trash(self):
3551 "Expire entries from the trash that are too old"
3552 self.backend.expire_old_trash()
3553
3554 _expire_old_trash = expire_old_trash
3555
3556 @write_api
3557 def restore_book(self, book_id, mi, last_modified, path, formats, annotations=()):
3558 """Restore the book entry in the database for a book that already exists on the filesystem"""
3559 cover, mi.cover = mi.cover, None
3560 self._create_book_entry(mi, add_duplicates=True, force_id=book_id, apply_import_tags=False, preserve_uuid=True)
3561 self._update_last_modified((book_id,), last_modified)
3562 if cover and os.path.exists(cover):
3563 self._set_field('cover', {book_id: 1})
3564 f = self.fields['formats'].table
3565 for fmt, size, fname in formats:
3566 f.update_fmt(book_id, fmt, fname, size, self.backend)
3567 self.fields['path'].table.set_path(book_id, path, self.backend)
3568 if annotations:
3569 self._restore_annotations(book_id, annotations)
3570 self._queue_pages_scan(book_id)
3571
3572 _restore_book = restore_book
3573
3574 @read_api
3575 def virtual_libraries_for_books(self, book_ids, virtual_fields=None):
3576 # use a primitive lock to ensure that only one thread is updating
3577 # the cache and that recursive calls don't do the update. This
3578 # method can recurse via self._search()
3579 with try_lock(self.vls_cache_lock) as got_lock:
3580 # Using a list is slightly faster than a set.
3581 c = defaultdict(list)
3582 if not got_lock:
3583 # We get here if resolving the books in a VL triggers another VL
3584 # cache calculation. This can be 'real' recursion, for example a
3585 # VL expression using a template that calls virtual_libraries(),
3586 # or a search using a location of 'all' that causes evaluation
3587 # of a composite that uses virtual_libraries(). The first case
3588 # is an error and the exception message should appear somewhere.
3589 # However, the error can seem nondeterministic. It might not be
3590 # raised if the use is via a composite and that composite is
3591 # evaluated before it is used in the search. The second case is
3592 # also an error but if the composite isn't used in a VL then the
3593 # eventual answer will be correct because get_metadata() will
3594 # clear the caches.
3595 raise ValueError(_('Recursion detected while processing Virtual library "%s"') % self.vls_for_books_lib_in_process)
3596 if self.vls_for_books_cache is None:
3597 libraries = self._pref('virtual_libraries', {})
3598 for lib, expr in libraries.items():
3599 book = None
3600 self.vls_for_books_lib_in_process = lib
3601 try:
3602 for book in self._search(expr, virtual_fields=virtual_fields):
3603 c[book].append(lib)
3604 except Exception as e:
3605 if book:
3606 c[book].append(_('[Error in Virtual library {0}: {1}]').format(lib, str(e)))
3607 self.vls_for_books_cache = {b: tuple(sorted(libs, key=sort_key)) for b, libs in c.items()}
3608 if not book_ids:
3609 book_ids = self._all_book_ids()
3610 # book_ids is usually 1 long. The loop will be faster than a comprehension
3611 r = {}
3612 default = ()
3613 for b in book_ids:
3614 r[b] = self.vls_for_books_cache.get(b, default)
3615 return r
3616
3617 _virtual_libraries_for_books = virtual_libraries_for_books
3618
3619 @read_api
3620 def user_categories_for_books(self, book_ids, proxy_metadata_map=None):
3621 """Return the user categories for the specified books.
3622 proxy_metadata_map is optional and is useful for a performance boost,
3623 in contexts where a ProxyMetadata object for the books already exists.
3624 It should be a mapping of book_ids to their corresponding ProxyMetadata
3625 objects.
3626 """
3627 user_cats = self._pref('user_categories', {})
3628 pmm = proxy_metadata_map or {}
3629 ans = {}
3630
3631 for book_id in book_ids:
3632 proxy_metadata = pmm.get(book_id) or self._get_proxy_metadata(book_id)
3633 user_cat_vals = ans[book_id] = {}
3634 for ucat, categories in user_cats.items():
3635 user_cat_vals[ucat] = res = []
3636 for name, cat, ign in categories:
3637 try:
3638 field_obj = self.fields[cat]
3639 except KeyError:
3640 continue
3641
3642 if field_obj.is_composite:
3643 v = field_obj.get_value_with_cache(book_id, lambda x: proxy_metadata)
3644 else:
3645 v = self._fast_field_for(field_obj, book_id)
3646
3647 if isinstance(v, (list, tuple)):
3648 if name in v:
3649 res.append([name, cat])
3650 elif name == v:
3651 res.append([name, cat])
3652 return ans
3653
3654 _user_categories_for_books = user_categories_for_books
3655
3656 @write_api
3657 def embed_metadata(self, book_ids, only_fmts=None, report_error=None, report_progress=None):
3658 """Update metadata in all formats of the specified book_ids to current metadata in the database."""
3659 field = self.fields['formats']
3660 from calibre.customize.ui import apply_null_metadata
3661 from calibre.ebooks.metadata.meta import set_metadata
3662 from calibre.ebooks.metadata.opf2 import pretty_print
3663
3664 if only_fmts:
3665 only_fmts = {f.lower() for f in only_fmts}
3666
3667 def doit(fmt, mi, stream):
3668 with apply_null_metadata, pretty_print:
3669 set_metadata(stream, mi, stream_type=fmt, report_error=report_error)
3670 stream.seek(0, os.SEEK_END)
3671 return stream.tell()
3672
3673 for i, book_id in enumerate(book_ids):
3674 fmts = field.table.book_col_map.get(book_id, ())
3675 if not fmts:
3676 continue
3677 mi = self._get_metadata(book_id)
3678 buf = BytesIO()
3679 self._copy_cover_to(book_id, buf)
3680 cdata = buf.getvalue()
3681 if cdata:
3682 mi.cover_data = ('jpeg', cdata)
3683 try:
3684 path = self._get_book_path(book_id)
3685 except Exception:
3686 continue
3687 for fmt in fmts:
3688 if only_fmts is not None and fmt.lower() not in only_fmts:
3689 continue
3690 try:
3691 name = self.fields['formats'].format_fname(book_id, fmt)
3692 except Exception:
3693 continue
3694 if name and path:
3695 try:
3696 new_size = self.backend.apply_to_format(book_id, path, name, fmt, partial(doit, fmt, mi))
3697 except Exception as e:
3698 if report_error is not None:
3699 tb = traceback.format_exc()
3700 if iswindows and isinstance(e, PermissionError) and e.filename and isinstance(e.filename, str):
3701 from calibre_extensions import winutil
3702
3703 try:
3704 p = winutil.get_processes_using_files(e.filename)
3705 except OSError:
3706 pass
3707 else:
3708 path_map = {x['path']: x for x in p}
3709 tb = _('Could not open the file: "{}". It is already opened in the following programs:').format(e.filename)
3710 for path, x in path_map.items():
3711 tb += '\n' + f'{x["app_name"]}: {path}'
3712 report_error(mi, fmt, tb)
3713 new_size = None
3714 else:
3715 raise
3716 if new_size is not None:
3717 self.format_metadata_cache[book_id].get(fmt, {})['size'] = new_size
3718 max_size = self.fields['formats'].table.update_fmt(book_id, fmt, name, new_size, self.backend)
3719 self.fields['size'].table.update_sizes({book_id: max_size})
3720 if report_progress is not None:
3721 report_progress(i + 1, len(book_ids), mi)
3722
3723 _embed_metadata = embed_metadata
3724
3725 @read_api
3726 def get_last_read_positions(self, book_id, fmt='', user='', order_by='', limit=0):
3727 q = 'SELECT device,cfi,epoch,pos_frac,format,user FROM last_read_positions WHERE book=?'
3728 bindings = [book_id]
3729 if fmt:
3730 q += ' AND format=?'
3731 bindings.append(fmt.upper())
3732 if user:
3733 q += ' AND user=?'
3734 bindings.append(user)
3735 if order_by in ('pos_frac', 'epoch'):
3736 q += f' ORDER BY {order_by} DESC'
3737 if limit:
3738 q += f' LIMIT {int(limit)}'
3739 return tuple(
3740 {'device': device, 'cfi': cfi, 'epoch': epoch, 'pos_frac': pos_frac, 'format': format, 'user': user}
3741 for device, cfi, epoch, pos_frac, format, user in self.backend.execute(q, tuple(bindings))
3742 )
3743
3744 _get_last_read_positions = get_last_read_positions
3745
3746 @write_api
3747 def set_last_read_position(self, book_id, fmt, user='_', device='_', cfi=None, epoch=None, pos_frac=0):
3748 self.backend.set_last_read_position(book_id, fmt, user, device, cfi, epoch, pos_frac)
3749
3750 _set_last_read_position = set_last_read_position
3751
3752 @write_api # doesn't need write access but sqlite does require only a single thread to access the db during backup
3753 def export_library(self, library_key, exporter, progress=None, abort=None):
3754 from polyglot.binary import as_hex_unicode
3755
3756 key_prefix = as_hex_unicode(library_key)
3757 book_ids = self._all_book_ids()
3758 total = len(book_ids) + 2
3759 has_fts = self.is_fts_enabled()
3760 if has_fts:
3761 total += 1
3762 poff = 0
3763
3764 def report_progress(fname):
3765 nonlocal poff
3766 if progress is not None:
3767 progress(fname, poff, total)
3768 poff += 1
3769
3770 @contextmanager
3771 def tempfile_for_export(which: str) -> Iterator[str]:
3772 import tempfile
3773
3774 fd, ans = tempfile.mkstemp(suffix=f'-{which}.db', dir=exporter.base)
3775 os.close(fd)
3776 try:
3777 yield ans
3778 finally:
3779 os.remove(ans)
3780
3781 report_progress('metadata.db')
3782 with tempfile_for_export('metadata') as tf:
3783 self.backend.backup_database(tf)
3784 dbkey = key_prefix + ':::' + 'metadata.db'
3785 with open(tf, 'rb') as f:
3786 exporter.add_file(f, dbkey)
3787 if has_fts:
3788 report_progress('full-text-search.db')
3789 with tempfile_for_export('fts') as tf:
3790 self.backend.backup_fts_database(tf)
3791 ftsdbkey = key_prefix + ':::full-text-search.db'
3792 with open(tf, 'rb') as f:
3793 exporter.add_file(f, ftsdbkey)
3794 with tempfile_for_export('notes') as tf, open(tf, 'r+b') as pt:
3795 self.backend.export_notes_data(pt)
3796 pt.flush()
3797 pt.seek(0)
3798 report_progress('notes.db')
3799 notesdbkey = key_prefix + ':::notes.db'
3800 exporter.add_file(pt, notesdbkey)
3801
3802 format_metadata = {}
3803 extra_files = {}
3804 metadata = {
3805 'format_data': format_metadata,
3806 'metadata.db': dbkey,
3807 'notes.db': notesdbkey,
3808 'total': total,
3809 'extra_files': extra_files,
3810 }
3811 if has_fts:
3812 metadata['full-text-search.db'] = ftsdbkey
3813 for i, book_id in enumerate(book_ids):
3814 if abort is not None and abort.is_set():
3815 return
3816 if progress is not None:
3817 report_progress(self._field_for('title', book_id))
3818 format_metadata[book_id] = fm = {}
3819 for fmt in self._formats(book_id):
3820 mdata = self.format_metadata(book_id, fmt)
3821 key = f'{key_prefix}:{book_id}:{fmt}'
3822 fm[fmt] = key
3823 mtime = mdata.get('mtime')
3824 if mtime is not None:
3825 mtime = timestampfromdt(mtime)
3826 with exporter.start_file(key, mtime=mtime) as dest:
3827 self._copy_format_to(book_id, fmt, dest)
3828 cover_key = '{}:{}:{}'.format(key_prefix, book_id, '.cover')
3829 with exporter.start_file(cover_key) as dest:
3830 if not self.copy_cover_to(book_id, dest):
3831 dest.discard()
3832 else:
3833 fm['.cover'] = cover_key
3834 bp = self._get_book_path(book_id, sep='/', unsafe=True)
3835 extra_files[book_id] = ef = {}
3836 if bp:
3837 for relpath, fobj, stat_result in self.backend.iter_extra_files(book_id, bp, self.fields['formats']):
3838 key = f'{key_prefix}:{book_id}:.|{relpath}'
3839 with exporter.start_file(key, mtime=stat_result.st_mtime) as dest:
3840 shutil.copyfileobj(fobj, dest)
3841 ef[relpath] = key
3842 exporter.set_metadata(library_key, metadata)
3843 if progress is not None:
3844 progress(_('Completed'), total, total)
3845
3846 _export_library = export_library
3847
3848 @read_api
3849 def annotations_map_for_book(self, book_id, fmt, user_type='local', user='viewer'):
3850 """
3851 Return a map of annotation type -> annotation data for the specified book_id, format, user and user_type.
3852 """
3853 ans = {}
3854 for annot in self.backend.annotations_for_book(book_id, fmt, user_type, user):
3855 ans.setdefault(annot['type'], []).append(annot)
3856 return ans
3857
3858 _annotations_map_for_book = annotations_map_for_book
3859
3860 @read_api
3861 def all_annotations_for_book(self, book_id):
3862 """
3863 Return a tuple containing all annotations for the specified book_id as a dict with keys:
3864 `format`, `user_type`, `user`, `annotation`. Here, annotation is the annotation data.
3865 """
3866 return tuple(self.backend.all_annotations_for_book(book_id))
3867
3868 _all_annotations_for_book = all_annotations_for_book
3869
3870 @read_api
3871 def annotation_count_for_book(self, book_id):
3872 """
3873 Return the number of annotations for the specified book available in the database.
3874 """
3875 return self.backend.annotation_count_for_book(book_id)
3876
3877 _annotation_count_for_book = annotation_count_for_book
3878
3879 @read_api
3880 def all_annotation_users(self):
3881 """
3882 Return a tuple of all (user_type, user name) that have annotations.
3883 """
3884 return tuple(self.backend.all_annotation_users())
3885
3886 _all_annotation_users = all_annotation_users
3887
3888 @read_api
3889 def all_annotation_types(self):
3890 """
3891 Return a tuple of all annotation types in the database.
3892 """
3893 return tuple(self.backend.all_annotation_types())
3894
3895 _all_annotation_types = all_annotation_types
3896
3897 @read_api
3898 def all_annotations(
3899 self,
3900 restrict_to_user=None,
3901 limit=None,
3902 annotation_type=None,
3903 annotation_style=None,
3904 ignore_removed=False,
3905 restrict_to_book_ids=None,
3906 ):
3907 """
3908 Return a tuple of all annotations matching the specified criteria.
3909 `ignore_removed` controls whether removed (deleted) annotations are also returned.
3910 Removed annotations are just a skeleton used for merging of annotations.
3911 """
3912 return tuple(self.backend.all_annotations(restrict_to_user, limit, annotation_type, annotation_style, ignore_removed, restrict_to_book_ids))
3913
3914 _all_annotations = all_annotations
3915
3916 @read_api
3917 def all_annotation_styles(self):
3918 """
3919 Return a tuple of all built-in annotation styles.
3920 """
3921 return self.backend.all_annotation_styles()
3922
3923 _all_annotation_styles = all_annotation_styles
3924
3925 @read_api
3926 def search_annotations(
3927 self,
3928 fts_engine_query,
3929 use_stemming=True,
3930 highlight_start=None,
3931 highlight_end=None,
3932 snippet_size=None,
3933 annotation_type=None,
3934 restrict_to_book_ids=None,
3935 restrict_to_user=None,
3936 ignore_removed=False,
3937 annotation_style=None,
3938 ):
3939 """
3940 Return of a tuple of annotations matching the specified Full-text query.
3941 """
3942 return tuple(
3943 self.backend.search_annotations(
3944 fts_engine_query,
3945 use_stemming,
3946 highlight_start,
3947 highlight_end,
3948 snippet_size,
3949 annotation_type,
3950 restrict_to_book_ids,
3951 restrict_to_user,
3952 ignore_removed,
3953 annotation_style,
3954 )
3955 )
3956
3957 _search_annotations = search_annotations
3958
3959 @write_api
3960 def delete_annotations(self, annot_ids):
3961 """
3962 Delete annotations with the specified ids.
3963 """
3964 self.backend.delete_annotations(annot_ids)
3965
3966 _delete_annotations = delete_annotations
3967
3968 @write_api
3969 def update_annotations(self, annot_id_map):
3970 """
3971 Update annotations.
3972 """
3973 self.backend.update_annotations(annot_id_map)
3974
3975 _update_annotations = update_annotations
3976
3977 @write_api
3978 def restore_annotations(self, book_id, annotations):
3979 from calibre.utils.date import EPOCH
3980 from calibre.utils.iso8601 import parse_iso8601
3981
3982 umap = defaultdict(list)
3983 for adata in annotations:
3984 key = adata['user_type'], adata['user'], adata['format']
3985 a = adata['annotation']
3986 ts = (parse_iso8601(a['timestamp']) - EPOCH).total_seconds()
3987 umap[key].append((a, ts))
3988 for (user_type, user, fmt), annots_list in umap.items():
3989 self._set_annotations_for_book(book_id, fmt, annots_list, user_type=user_type, user=user)
3990
3991 _restore_annotations = restore_annotations
3992
3993 @write_api
3994 def set_annotations_for_book(self, book_id, fmt, annots_list, user_type='local', user='viewer'):
3995 """
3996 Set all annotations for the specified book_id, fmt, user_type and user.
3997 """
3998 self.backend.set_annotations_for_book(book_id, fmt, annots_list, user_type, user)
3999
4000 _set_annotations_for_book = set_annotations_for_book
4001
4002 @write_api
4003 def merge_annotations_for_book(self, book_id, fmt, annots_list, user_type='local', user='viewer'):
4004 """
4005 Merge the specified annotations into the existing annotations for book_id, fm, user_type, and user.
4006 """
4007 from calibre.utils.date import EPOCH
4008 from calibre.utils.iso8601 import parse_iso8601
4009
4010 amap = self._annotations_map_for_book(book_id, fmt, user_type=user_type, user=user)
4011 merge_annotations(annots_list, amap)
4012 alist = []
4013 for val in amap.values():
4014 for annot in val:
4015 ts = (parse_iso8601(annot['timestamp']) - EPOCH).total_seconds()
4016 alist.append((annot, ts))
4017 self._set_annotations_for_book(book_id, fmt, alist, user_type=user_type, user=user)
4018
4019 _merge_annotations_for_book = merge_annotations_for_book
4020
4021 @write_api
4022 def save_annotations_list(self, book_id: int, book_fmt: str, sync_annots_user: str, alist: list[dict]) -> None:
4023 self.backend.save_annotations_list(book_id, book_fmt, sync_annots_user, alist)
4024
4025 _save_annotations_list = save_annotations_list
4026
4027 @write_api
4028 def reindex_annotations(self):
4029 self.backend.reindex_annotations()
4030
4031 _reindex_annotations = reindex_annotations
4032
4033 @read_api
4034 def are_paths_inside_book_dir(self, book_id, paths, sub_path=''):
4035 try:
4036 path = self._get_book_path(book_id)
4037 except Exception:
4038 return set()
4039 return {x for x in paths if self.backend.is_path_inside_book_dir(x, path, sub_path)}
4040
4041 _are_paths_inside_book_dir = are_paths_inside_book_dir
4042
4043 @write_api
4044 def add_extra_files(self, book_id, map_of_relpath_to_stream_or_path, replace=True, auto_rename=False):
4045 "Add extra data files"
4046 path = self._get_book_path(book_id)
4047 added = {}
4048 for relpath, stream_or_path in map_of_relpath_to_stream_or_path.items():
4049 added[relpath] = bool(self.backend.add_extra_file(relpath, stream_or_path, path, replace, auto_rename))
4050 self._clear_extra_files_cache(book_id)
4051 return added
4052
4053 _add_extra_files = add_extra_files
4054
4055 @write_api
4056 def rename_extra_files(self, book_id, map_of_relpath_to_new_relpath, replace=False):
4057 "Rename extra data files"
4058 path = self._get_book_path(book_id)
4059 renamed = set()
4060 for relpath, newrelpath in map_of_relpath_to_new_relpath.items():
4061 if self.backend.rename_extra_file(relpath, newrelpath, path, replace):
4062 renamed.add(relpath)
4063 self._clear_extra_files_cache(book_id)
4064 return renamed
4065
4066 _rename_extra_files = rename_extra_files
4067
4068 @write_api
4069 def merge_extra_files(self, dest_id, src_ids, replace=False):
4070 "Merge the extra files from src_ids into dest_id. Conflicting files are auto-renamed unless replace=True in which case they are replaced."
4071 added = set()
4072 path = self._field_for('path', dest_id)
4073 if path:
4074 path = path.replace('/', os.sep)
4075 for src_id in src_ids:
4076 book_path = self._field_for('path', src_id)
4077 if book_path:
4078 book_path = book_path.replace('/', os.sep)
4079 for relpath, file_path, stat_result in self.backend.iter_extra_files(src_id, book_path, self.fields['formats'], yield_paths=True):
4080 added.add(self.backend.add_extra_file(relpath, file_path, path, replace=replace, auto_rename=True))
4081 self._clear_extra_files_cache(dest_id)
4082 return added
4083
4084 _merge_extra_files = merge_extra_files
4085
4086 @write_api
4087 def remove_extra_files(self, book_id: int, relpaths: Iterable[str], permanent=False) -> dict[str, Exception | None]:
4088 """
4089 Delete the specified extra files, either to Recycle Bin or permanently.
4090 """
4091 path = self._get_book_path(book_id, sep='/', unsafe=True)
4092 if path:
4093 self._clear_extra_files_cache(book_id)
4094 return self.backend.remove_extra_files(path, relpaths, permanent)
4095 return dict.fromkeys(relpaths)
4096
4097 _remove_extra_files = remove_extra_files
4098
4099 @read_api
4100 def list_extra_files(self, book_id, use_cache=False, pattern='') -> tuple[ExtraFile, ...]:
4101 """
4102 Get information about extra files in the book's directory.
4103
4104 :param book_id: the database book id for the book
4105 :param pattern: the pattern of filenames to search for. Empty pattern matches all extra files. Patterns must use / as separator.
4106 Use the DATA_FILE_PATTERN constant to match files inside the data directory.
4107
4108 :return: A tuple of all extra files matching the specified pattern. Each element of the tuple is
4109 ExtraFile(relpath, file_path, stat_result). Where relpath is the relative path of the file
4110 to the book directory using / as a separator.
4111 stat_result is the result of calling os.stat() on the file.
4112 """
4113 ans = self.extra_files_cache.setdefault(book_id, {}).get(pattern)
4114 if ans is None or not use_cache:
4115 ans = []
4116 path = self._get_book_path(book_id, sep='/', unsafe=True)
4117 if path:
4118 for relpath, file_path, stat_result in self.backend.iter_extra_files(book_id, path, self.fields['formats'], yield_paths=True, pattern=pattern):
4119 ans.append(ExtraFile(relpath, file_path, stat_result))
4120 self.extra_files_cache[book_id][pattern] = ans = tuple(ans)
4121 return ans
4122
4123 _list_extra_files = list_extra_files
4124
4125 @read_api
4126 def copy_extra_file_to(self, book_id, relpath, stream_or_path):
4127 path = self._get_book_path(book_id)
4128 self.backend.copy_extra_file_to(book_id, path, relpath, stream_or_path)
4129
4130 _copy_extra_file_to = copy_extra_file_to
4131
4132 @write_api
4133 def merge_book_metadata(self, dest_id, src_ids, replace_cover=False, save_alternate_cover=False):
4134 dest_mi = self.get_metadata(dest_id)
4135 merged_identifiers = self._field_for('identifiers', dest_id) or {}
4136 orig_dest_comments = dest_mi.comments
4137 dest_cover = orig_dest_cover = self.cover(dest_id)
4138 had_orig_cover = bool(dest_cover)
4139 alternate_covers = []
4140 from calibre.utils.date import is_date_undefined
4141
4142 def is_null_date(x):
4143 return x is None or is_date_undefined(x)
4144
4145 for src_id in src_ids:
4146 src_mi = self.get_metadata(src_id)
4147
4148 if src_mi.comments and orig_dest_comments != src_mi.comments:
4149 if not dest_mi.comments:
4150 dest_mi.comments = src_mi.comments
4151 else:
4152 dest_mi.comments = str(dest_mi.comments) + '\n\n' + str(src_mi.comments)
4153 if src_mi.title and dest_mi.is_null('title'):
4154 dest_mi.title = src_mi.title
4155 dest_mi.title_sort = src_mi.title_sort
4156 if (src_mi.authors and src_mi.authors[0] != _('Unknown')) and (not dest_mi.authors or dest_mi.authors[0] == _('Unknown')):
4157 dest_mi.authors = src_mi.authors
4158 dest_mi.author_sort = src_mi.author_sort
4159 if src_mi.tags:
4160 if not dest_mi.tags:
4161 dest_mi.tags = src_mi.tags
4162 else:
4163 dest_mi.tags.extend(src_mi.tags)
4164 if not dest_cover or replace_cover:
4165 src_cover = self.cover(src_id)
4166 if src_cover:
4167 if save_alternate_cover and dest_cover:
4168 alternate_covers.append(dest_cover)
4169 dest_cover = src_cover
4170 replace_cover = False
4171 elif save_alternate_cover:
4172 src_cover = self.cover(src_id)
4173 if src_cover:
4174 alternate_covers.append(src_cover)
4175 if not dest_mi.publisher:
4176 dest_mi.publisher = src_mi.publisher
4177 if not dest_mi.rating:
4178 dest_mi.rating = src_mi.rating
4179 if not dest_mi.series:
4180 dest_mi.series = src_mi.series
4181 dest_mi.series_index = src_mi.series_index
4182 if is_null_date(dest_mi.pubdate) and not is_null_date(src_mi.pubdate):
4183 dest_mi.pubdate = src_mi.pubdate
4184
4185 src_identifiers = (src_mi.get_identifiers() or {}).copy()
4186 src_identifiers.update(merged_identifiers)
4187 merged_identifiers = src_identifiers.copy()
4188
4189 if merged_identifiers:
4190 dest_mi.set_identifiers(merged_identifiers)
4191 self._set_metadata(dest_id, dest_mi, ignore_errors=False)
4192
4193 if dest_cover and (not had_orig_cover or dest_cover is not orig_dest_cover):
4194 self._set_cover({dest_id: dest_cover})
4195 if alternate_covers:
4196 existing = {x[0] for x in self._list_extra_files(dest_id)}
4197 h, ext = os.path.splitext(COVER_FILE_NAME)
4198 template = f'{DATA_DIR_NAME}/{h}-{{:03d}}{ext}'
4199 for cdata in alternate_covers:
4200 for i in range(1, 1000):
4201 q = template.format(i)
4202 if q not in existing:
4203 existing.add(q)
4204 self._add_extra_files(dest_id, {q: BytesIO(cdata)}, replace=False, auto_rename=True)
4205 break
4206
4207 for key in self.field_metadata: # loop through all defined fields
4208 fm = self.field_metadata[key]
4209 if not fm['is_custom']:
4210 continue
4211 dt = fm['datatype']
4212 label = fm['label']
4213 try:
4214 field = self.field_metadata.label_to_key(label)
4215 except ValueError:
4216 continue
4217 # Get orig_dest_comments before it gets changed
4218 if dt == 'comments':
4219 orig_dest_value = self._field_for(field, dest_id)
4220
4221 for src_id in src_ids:
4222 dest_value = self._field_for(field, dest_id)
4223 src_value = self._field_for(field, src_id)
4224 if dt == 'comments' and src_value and src_value != orig_dest_value:
4225 if not dest_value:
4226 self._set_field(field, {dest_id: src_value})
4227 else:
4228 dest_value = str(dest_value) + '\n\n' + str(src_value)
4229 self._set_field(field, {dest_id: dest_value})
4230 if dt in {'bool', 'int', 'float', 'rating', 'datetime'} and dest_value is None:
4231 self._set_field(field, {dest_id: src_value})
4232 if dt == 'series' and not dest_value and src_value:
4233 src_index = self._field_for(field + '_index', src_id)
4234 self._set_field(field, {dest_id: src_value})
4235 self._set_field(field + '_index', {dest_id: src_index})
4236 if (dt == 'enumeration' or (dt == 'text' and not fm['is_multiple'])) and not dest_value:
4237 self._set_field(field, {dest_id: src_value})
4238 if dt == 'text' and fm['is_multiple'] and src_value:
4239 if not dest_value:
4240 dest_value = src_value
4241 else:
4242 dest_value = list(dest_value)
4243 dest_value.extend(src_value)
4244 self._set_field(field, {dest_id: dest_value})
4245
4246 _merge_book_metadata = merge_book_metadata
4247
4248 @read_api
4249 def clone_for_readonly_access(self, dest_dir: str) -> str:
4250 return self.backend.clone_for_readonly_access(dest_dir)
4251
4252 _clone_for_readonly_access = clone_for_readonly_access
4253
4254
4255 def import_library(library_key, importer, library_path, progress=None, abort=None):
4256 from calibre.db.backend import DB
4257
4258 metadata = importer.metadata[library_key]
4259 total = metadata['total']
4260 poff = 0
4261
4262 def report_progress(fname):
4263 nonlocal poff
4264 if progress is not None:
4265 progress(fname, poff, total)
4266 poff += 1
4267
4268 report_progress('metadata.db')
4269 if abort is not None and abort.is_set():
4270 return
4271 importer.save_file(metadata['metadata.db'], 'metadata.db for ' + library_path, os.path.join(library_path, 'metadata.db'))
4272 if 'full-text-search.db' in metadata:
4273 if progress is not None:
4274 progress('full-text-search.db', 1, total)
4275 if abort is not None and abort.is_set():
4276 return
4277 poff += 1
4278 importer.save_file(
4279 metadata['full-text-search.db'],
4280 'full-text-search.db for ' + library_path,
4281 os.path.join(library_path, 'full-text-search.db'),
4282 )
4283 if abort is not None and abort.is_set():
4284 return
4285 if 'notes.db' in metadata:
4286 import zipfile
4287
4288 notes_dir = os.path.join(library_path, NOTES_DIR_NAME)
4289 os.makedirs(notes_dir, exist_ok=True)
4290 with importer.start_file(metadata['notes.db'], 'notes.db for ' + library_path) as stream:
4291 stream.check_hash = False
4292 with zipfile.ZipFile(stream) as zf:
4293 for zi in zf.infolist():
4294 tpath = zf.extract(zi, notes_dir, None)
4295 date_time = mktime(datetime(*zi.date_time).timetuple())
4296 os.utime(tpath, (date_time, date_time))
4297 if abort is not None and abort.is_set():
4298 return
4299 if importer.corrupted_files:
4300 raise ValueError('Corrupted files:\n' + '\n'.join(importer.corrupted_files))
4301 cache = Cache(DB(library_path, load_user_formatter_functions=False))
4302 cache.init()
4303 with cache.write_lock:
4304 format_data = {int(book_id): data for book_id, data in metadata['format_data'].items()}
4305 extra_files = {int(book_id): data for book_id, data in metadata.get('extra_files', {}).items()}
4306 for i, (book_id, fmt_key_map) in enumerate(format_data.items()):
4307 if abort is not None and abort.is_set():
4308 return
4309 title = cache._field_for('title', book_id)
4310 if progress is not None:
4311 progress(title, i + poff, total)
4312 cache._update_path((book_id,), mark_as_dirtied=False)
4313 for fmt, fmtkey in fmt_key_map.items():
4314 if fmt == '.cover':
4315 with importer.start_file(fmtkey, _('Cover for %s') % title) as stream:
4316 path = cache._get_book_path(book_id)
4317 cache.backend.set_cover(book_id, path, stream, no_processing=True)
4318 else:
4319 with importer.start_file(fmtkey, _('{0} format for {1}').format(fmt.upper(), title)) as stream:
4320 size, fname = cache._do_add_format(book_id, fmt, stream, mtime=stream.mtime)
4321 cache.fields['formats'].table.update_fmt(book_id, fmt, fname, size, cache.backend)
4322 for relpath, efkey in extra_files.get(book_id, {}).items():
4323 with importer.start_file(efkey, _('Extra file {0} for book {1}').format(relpath, title)) as stream:
4324 path = cache._get_book_path(book_id)
4325 cache.backend.add_extra_file(relpath, stream, path)
4326 cache.dump_metadata({book_id})
4327 if importer.corrupted_files:
4328 raise ValueError('Corrupted files:\n' + '\n'.join(importer.corrupted_files))
4329 if progress is not None:
4330 progress(_('Completed'), total, total)
4331 return cache
4332
4333
4334 # }}}