Debian Codegraph @

src/pyj/read_book/db.pyj

raw
1 # vim:fileencoding=utf-8
2 # License: GPL v3 Copyright: 2016, Kovid Goyal <kovid at kovidgoyal.net>
3 from __python__ import bound_methods, hash_literals
4
5 from encodings import base64decode, base64encode
6 from gettext import gettext as _
7
8 from book_list.router import is_reading_book
9 from modals import error_dialog
10 from read_book.annotations import merge_annotation_maps
11 from session import get_interface_data
12 from utils import username_key
13
14
15 def upgrade_schema(idb, old_version, new_version, transaction):
16 print('upgrade_schema:', old_version, new_version)
17 if not idb.objectStoreNames.contains('books'):
18 idb.createObjectStore('books', {'keyPath': 'key'})
19 if not idb.objectStoreNames.contains('files'):
20 idb.createObjectStore('files')
21 if not idb.objectStoreNames.contains('mathjax'):
22 idb.createObjectStore('mathjax')
23 if not idb.objectStoreNames.contains('objects'):
24 idb.createObjectStore('objects', {'keyPath': 'key'})
25
26 # Create indices
27 books_store = transaction.objectStore('books')
28 if not books_store.indexNames.contains('recent_index'):
29 books_store.createIndex('recent_index', 'recent_date')
30
31
32 def file_store_name(book, name):
33 return book.book_hash + ' ' + name
34
35
36 def get_error_details(event):
37 desc = event.target
38 if desc.error and desc.error.toString:
39 desc = desc.error.toString()
40 elif desc.errorCode:
41 desc = desc.errorCode
42 elif desc.error:
43 desc = desc.error
44 if desc.name is 'QuotaExceededError':
45 desc = _('Offline storage quota exceeded! Try deleting some stored books first.')
46 elif desc.name:
47 desc = desc.name
48 return desc or 'Unknown Error'
49
50
51 def new_book(key, metadata):
52 return {
53 'key': key,
54 'is_complete': False,
55 'stored_files': {},
56 'book_hash': None,
57 'metadata': metadata,
58 'manifest': None,
59 'cover_width': None,
60 'cover_height': None,
61 'cover_name': None,
62 'recent_date': new Date(),
63 'last_read': {},
64 'last_read_position': {},
65 'saved_reading_rates': {},
66 'annotations_map': {},
67 }
68
69
70 DB_NAME = 'calibre'
71 DB_VERSION = 1
72
73
74 def indexed_db_api():
75 if window?:
76 return window.indexedDB
77 if self?:
78 return self.indexedDB
79
80
81 class DB:
82 def __init__(self, callback, show_read_book_error):
83 self.initialized = False
84 self.is_ok = False
85 self.initialize_error_msg = None
86 self.callback = callback
87 self.show_read_book_error = show_read_book_error
88 self.initialize_stage1()
89
90 def show_error(self, title, msg, det_msg):
91 if not window? or is_reading_book():
92 self.show_read_book_error(title, msg, det_msg)
93 else:
94 error_dialog(title, msg, det_msg)
95
96 def initialize_stage1(self):
97 idb = indexed_db_api()
98 if not idb:
99 self.initialize_error_msg = _(
100 'Your browser does not support IndexedDB. Cannot read books.'
101 ' Consider using a modern browser, such as Chrome or Firefox.'
102 )
103 self.initialized = True
104 # Callers assume __init__ has finished before the callback is
105 # called, since we are called in __init__, only callback after
106 # event loop ticks over
107 setTimeout(self.callback, 0)
108 return
109
110 request = idb.open(DB_NAME, DB_VERSION)
111
112 request.onupgradeneeded = def (event):
113 upgrade_schema(event.target.result, event.oldVersion, event.newVersion, event.target.transaction)
114
115 request.onblocked = def (event):
116 self.initialize_error_msg = _('Please close all other browser tabs with calibre open')
117 self.initialized = True
118 console.log(event)
119 self.callback()
120
121 request.onerror = def (event):
122 self.initialize_error_msg = _('You must allow calibre to use IndexedDB storage in your browser to read books')
123 self.initialized = True
124 console.log(event)
125 self.callback()
126
127 request.onsuccess = def (event):
128 blob = Blob(['test'], {'type': 'text/plain'})
129 idb = event.target.result
130 store = idb.transaction(['files'], 'readwrite').objectStore('files')
131 try:
132 req = store.put(blob, ':-test-blob-:')
133 except:
134 self.initialize_stage2(idb, False)
135 return
136 req.onsuccess = def (event):
137 self.initialize_stage2(idb, True)
138 req.onerror = def (event):
139 # We use setTimeout as otherwise the idb.onerror handler is
140 # called with this error on Safari
141 setTimeout(self.initialize_stage2.bind(None, idb, False), 0)
142
143 def initialize_stage2(self, idb, supports_blobs):
144 self.idb = idb
145 self.supports_blobs = supports_blobs
146 self.initialized = True
147 self.is_ok = True
148 if not supports_blobs:
149 print('WARNING: browser does not support blob storage, calibre falling back to base64 encoding')
150
151 idb.onerror = def (event):
152 self.display_error(None, event)
153 if console.dir:
154 console.dir(event)
155 else:
156 console.log(event)
157
158 idb.onversionchange = def (event):
159 idb.close()
160 self.show_error(_('Database upgraded!'), _(
161 'A newer version of calibre is available, please click the Reload button in your browser.'))
162
163 def try_persist(lock):
164 if not lock:
165 return # another tab is already requesting persistence
166 return window.navigator.storage.persisted().then(def (already_persistent):
167 if not already_persistent:
168 return window.navigator.storage.persist()
169 )
170 if window?.navigator?.storage?.persist:
171 if window.navigator.locks:
172 window.navigator.locks.request('calibre-storage-persist', {'ifAvailable': True}, try_persist)
173 else:
174 window.navigator.storage.persisted().then(def (already_persistent):
175 if not already_persistent:
176 window.navigator.storage.persist()
177 )
178 self.callback()
179
180 def display_error(self, msg, event):
181 if event.already_displayed_by_calibre:
182 return
183 event.already_displayed_by_calibre = True
184 msg = msg or _(
185 'There was an error while interacting with the'
186 ' database used to store books for offline reading. Click "Show details" for more information.')
187 self.show_error(_('Cannot read book'), msg, get_error_details(event))
188
189 def do_op(self, stores, data, error_msg, proceed, op='get', store=None):
190 store = store or stores[0]
191 if op is 'get':
192 transaction = self.idb.transaction(stores)
193 req = transaction.objectStore(store).get(data)
194 req.onsuccess = def (event):
195 if proceed:
196 proceed(req.result)
197 elif op is 'put':
198 transaction = self.idb.transaction(stores, 'readwrite')
199 req = transaction.objectStore(store).put(data)
200 if proceed:
201 req.onsuccess = proceed
202 elif op is 'delete':
203 transaction = self.idb.transaction(stores, 'readwrite')
204 req = transaction.objectStore(store).delete(data)
205 if proceed:
206 req.onsuccess = proceed
207 req.onerror = def (event):
208 self.display_error(error_msg, event)
209
210 def set_pending_annot_upload(self, library_id, book_id, fmt, amap):
211 key = f'pending-annot-upload:{library_id}/{book_id}/{fmt}'
212 entry = {'key': key, 'library_id': library_id, 'book_id': book_id, 'fmt': fmt, 'amap': amap}
213 self.do_op(['objects'], entry, _('Failed to save pending annotation upload'),
214 def (): None;, op='put')
215
216 def get_all_pending_annot_uploads(self, proceed):
217 transaction = self.idb.transaction(['objects'])
218 store = transaction.objectStore('objects')
219 key_range = IDBKeyRange.bound('pending-annot-upload:', 'pending-annot-upload:\uffff') # noqa
220 entries = []
221 req = store.openCursor(key_range)
222 req.onsuccess = def (event):
223 cursor = event.target.result
224 if cursor:
225 entries.push(cursor.value)
226 cursor.continue ()
227 else:
228 proceed(entries)
229 req.onerror = def (event):
230 self.display_error(_('Failed to read pending annotation uploads'), event)
231 proceed([])
232
233 def clear_pending_annot_upload(self, library_id, book_id, fmt, proceed):
234 key = f'pending-annot-upload:{library_id}/{book_id}/{fmt}'
235 self.do_op(['objects'], key,
236 _('Failed to clear pending annotation upload'), proceed or def (): None;, op='delete')
237
238 def get_book(self, library_id, book_id, fmt, metadata, proceed):
239 fmt = fmt.toUpperCase()
240 # The key has to be a JavaScript array as otherwise it cannot be stored
241 # into indexed db, because the RapydScript list has properties that
242 # refer to non-serializable objects like functions.
243 book_id = int(book_id)
244 key = [library_id, book_id, fmt]
245 self.do_op(['books'], key, _('Failed to read from the books database'), def (result):
246 if result and not result.annotations_map:
247 result.annotations_map = {}
248 proceed(result or new_book(key, metadata))
249 )
250
251 def get_mathjax_info(self, proceed):
252 self.do_op(['objects'], 'mathjax-info', _('Failed to read from the objects database'), def (result):
253 proceed(result or {'key': 'mathjax-info'})
254 )
255
256 def save_manifest(self, book, manifest, proceed):
257 book.manifest = manifest
258 book.metadata = manifest.metadata
259 book.book_hash = manifest.book_hash.hash
260 book.stored_files = {}
261 book.is_complete = False
262 newest_epoch = newest_pos = None
263 for pos in manifest.last_read_positions:
264 if newest_epoch is None or pos.epoch > newest_epoch:
265 newest_epoch = pos.epoch
266 newest_pos = pos.cfi
267 username = get_interface_data().username
268 unkey = username_key(username)
269 if newest_pos and username:
270 book.last_read[unkey] = new Date(newest_epoch * 1000)
271 book.last_read_position[unkey] = newest_pos
272 if manifest.annotations_map:
273 book.annotations_map[unkey] = manifest.annotations_map
274
275 v'delete manifest.metadata'
276 v'delete manifest.last_read_positions'
277 v'delete manifest.annotations_map'
278 self.do_op(['books'], book, _('Failed to write to the books database'), proceed, op='put')
279
280 def store_file(self, book, name, xhr, proceed, is_cover):
281 store_as_text = xhr.responseType is 'text' or not xhr.responseType
282 fname = file_store_name(book, name)
283 needs_encoding = not store_as_text and not self.supports_blobs
284 mt = book.manifest.files[name]?.mimetype
285 if not mt and is_cover:
286 mt = 'image/jpeg'
287 book.stored_files[fname] = {'encoded': needs_encoding, 'mimetype': mt, 'store_as_text': store_as_text}
288 if is_cover:
289 self.store_cover(book, needs_encoding, xhr.response, name, fname, proceed)
290 else:
291 self.store_file_stage2(needs_encoding, xhr.response, name, fname, proceed)
292
293 def store_cover(self, book, needs_encoding, data, name, fname, proceed):
294 blob = data
295 if needs_encoding:
296 blob = Blob([data], {'type': 'image/jpeg'})
297 url = window.URL.createObjectURL(blob)
298 img = new Image()
299 book.cover_name = name
300 proceeded = False
301
302 def done():
303 nonlocal proceeded
304 if not proceeded:
305 proceeded = True
306 window.URL.revokeObjectURL(url)
307 self.store_file_stage2(needs_encoding, data, name, fname, proceed)
308
309 img.onload = def ():
310 book.cover_width = this.width
311 book.cover_height = this.height
312 done()
313
314 img.onerror = def ():
315 print('WARNING: Failed to read dimensions of cover')
316 done()
317 img.src = url
318
319 def store_file_stage2(self, needs_encoding, data, name, fname, proceed):
320 if needs_encoding:
321 data = base64encode(Uint8Array(data))
322 req = self.idb.transaction(['files'], 'readwrite').objectStore('files').put(data, fname)
323 req.onsuccess = def (event):
324 proceed()
325 req.onerror = def (event):
326 proceed(_('Failed to store book data ({0}) with error: {1}').format(name, get_error_details(event)))
327
328 def clear_mathjax(self, proceed):
329 self.idb.transaction(['mathjax'], 'readwrite').objectStore('mathjax').clear().onsuccess = proceed
330
331 def store_mathjax_file(self, name, xhr, proceed):
332 data = xhr.response
333 if not self.supports_blobs:
334 data = base64encode(Uint8Array(data))
335 req = self.idb.transaction(['mathjax'], 'readwrite').objectStore('mathjax').put(data, name)
336 req.onsuccess = def (event): proceed()
337 req.onerror = def (event):
338 proceed(_('Failed to store mathjax file ({0}) with error: {1}').format(name, get_error_details(event)))
339
340 def finish_book(self, book, proceed):
341 book.is_complete = True
342 self.do_op(['books'], book, _('Failed to write to the books database'), proceed, op='put')
343
344 def finish_mathjax(self, mathjax_info, proceed):
345 self.do_op(['objects'], mathjax_info, _('Failed to write to the objects database'), proceed, op='put')
346
347 def update_last_read_time(self, book):
348 unkey = username_key(get_interface_data().username)
349 now = new Date()
350 book.last_read[unkey] = book.recent_date = now
351 self.do_op(['books'], book, _('Failed to write to the books database'), op='put')
352
353 def update_metadata(self, book, new_metadata):
354 if book.metadata:
355 for key in Object.keys(new_metadata):
356 book.metadata[key] = new_metadata[key]
357 self.do_op(['books'], book, _('Failed to write to the books database'), def (): None;, op='put')
358
359 def save_reading_rates(self, book, rates):
360 book.saved_reading_rates = rates
361 self.do_op(['books'], book, _('Failed to write to the books database'), def (): None;, op='put')
362
363 def update_annotations_data_from_key(self, library_id, book_id, fmt, new_data):
364 unkey = username_key(get_interface_data().username)
365 self.get_book(library_id, book_id, fmt, None, def (book):
366 if book.metadata: # book exists
367 changed = False
368 if new_data.last_read_position:
369 book.last_read[unkey] = book.recent_date = new_data.last_read
370 book.last_read_position[unkey] = new_data.last_read_position
371 changed = True
372 if not book.annotations_map:
373 book.annotations_map = v'{}'
374 if new_data.annotations_map:
375 existing = book.annotations_map[unkey]
376 if not existing:
377 changed = True
378 book.annotations_map[unkey] = new_data.annotations_map
379 else:
380 updated, merged = merge_annotation_maps(existing, new_data.annotations_map)
381 if updated:
382 changed = True
383 book.annotations_map[unkey] = merged
384 if changed:
385 self.do_op(['books'], book, _('Failed to write to the books database'), def (): None;, op='put')
386 )
387
388 def get_file(self, book, name, proceed):
389 key = file_store_name(book, name)
390 err = _(
391 'Failed to read the file {0} for the book {1} from the browser offline cache.'
392 ).format(name, book.metadata.title)
393 self.do_op(['files'], key, err, def (result):
394 if not result:
395 # File was not found in the cache, this happens in Chrome if
396 # the Application Cache is full. IndexedDB put succeeds,
397 # but get fails. Browsers are a horrible travesty.
398 bad = err + _(
399 ' This usually means the cache is close to full. Clear the browser'
400 ' cache from the browser settings.')
401 self.show_error(_('Cannot read file from book'), bad)
402 return
403 fdata = book.stored_files[key]
404 mt = fdata.mimetype or 'application/octet-stream'
405 if fdata.encoded:
406 result = Blob([base64decode(result)], {'type': mt})
407 proceed(result, name, mt, book)
408 )
409
410 def get_book_file(self, book_hash, stored_files, name, proceed):
411 stores = ['files']
412 key = file_store_name({'book_hash': book_hash}, name)
413 transaction = self.idb.transaction(stores)
414 req = transaction.objectStore(stores[0]).get(key)
415 req.onsuccess = def (event):
416 if not req.result:
417 proceed({'ok': False, 'name': name, 'details': 'ENOENT'})
418 return
419 fdata = stored_files[key]
420 mt = fdata.mimetype or 'application/octet-stream'
421 if fdata.encoded:
422 result = Blob([base64decode(result)], {'type': mt})
423 proceed({'ok': True, 'result': req.result, 'name': name, 'mt': mt})
424 req.onerror = def (event):
425 details = get_error_details(event)
426 proceed({'ok': False, 'name': name, 'details': details})
427
428 def get_mathjax_files(self, proceed):
429 c = self.idb.transaction('mathjax').objectStore('mathjax').openCursor()
430 c.onerror = def (event):
431 err = _('Failed to read the MathJax files from the browser offline cache.')
432 self.display_error(err, event)
433 data = {}
434 c.onsuccess = def (event):
435 cursor = event.target.result
436 if cursor:
437 name, result = cursor.key, cursor.value
438 if not isinstance(result, Blob):
439 mt = 'application/x-font-woff' if name.endswith('.woff') else 'text/javascript'
440 result = Blob([base64decode(result)], {'type': mt})
441 data[name] = result
442 cursor.continue ()
443 else:
444 proceed(data)
445
446 def get_books_by_recent_date(self, proceed, limit=None):
447 c = self.idb.transaction(['books'], 'readonly').objectStore('books').index('recent_index').openCursor(None, 'prev')
448 books = []
449 c.onerror = def (event):
450 err = _('Failed to read recent books from local storage')
451 self.display_error(err, event)
452 c.onsuccess = def (ev):
453 cursor = ev.target.result
454 if cursor:
455 books.push(cursor.value)
456 if (limit is not None and books.length >= limit) or not cursor:
457 proceed(books)
458 return
459 if cursor:
460 cursor.continue ()
461
462 def get_recently_read_books(self, proceed, limit):
463 self.get_books_by_recent_date(proceed, limit or 3)
464
465 def get_downloaded_books(self, proceed):
466 self.get_books_by_recent_date(proceed)
467
468 def has_book_matching(self, library_id, book_id, proceed):
469 # Should really be using a multiEntry index to avoid iterating over all
470 # books in JS, but since the number of stored books is not that large,
471 # I can't be bothered.
472 c = self.idb.transaction(['books'], 'readonly').objectStore('books').index('recent_index').openCursor(None, 'prev')
473 c.onerror = def (event):
474 proceed(False)
475 book_id = int(book_id)
476 c.onsuccess = def (ev):
477 cursor = ev.target.result
478 if cursor:
479 book = cursor.value
480 if book.key[0] is library_id and book.key[1] is book_id:
481 proceed(True)
482 return
483 cursor.continue ()
484 else:
485 proceed(False)
486
487 def delete_book(self, book, proceed):
488 c = self.idb.transaction(['books', 'files'], 'readwrite')
489 files = c.objectStore('files')
490 books = c.objectStore('books')
491 filenames = Object.keys(book.stored_files)
492 c.oncomplete = def (event):
493 proceed(book)
494 c.onerror = def (event):
495 proceed(book, c.error.toString())
496
497 def next_step():
498 if filenames.length:
499 r = files.delete(filenames.pop())
500 r.onsuccess = next_step
501 else:
502 books.delete(book.key)
503 next_step()
504
505 def delete_books_matching(self, library_id, book_id, proceed):
506 c = self.idb.transaction(['books'], 'readonly').objectStore('books').index('recent_index').openCursor(None, 'prev')
507 c.onerror = def (event):
508 pass
509 book_id = int(book_id)
510 matches = []
511
512 def delete_all():
513 if matches.length:
514 book = matches.pop()
515 self.delete_book(book, delete_all)
516 else:
517 if proceed:
518 proceed()
519
520 c.onsuccess = def (ev):
521 cursor = ev.target.result
522 if cursor:
523 book = cursor.value
524 if book.key[0] is library_id and book.key[1] is book_id:
525 matches.push(book)
526 cursor.continue ()
527 else:
528 delete_all()
529
530
531 def get_db(callback, show_read_book_error):
532 if not get_db.ans:
533 get_db.ans = DB(callback, show_read_book_error)
534 return get_db.ans