Debian Codegraph @

src/calibre/db/write.py

raw
1 #!/usr/bin/env python
2 # License: GPLv3 Copyright: 2013, Kovid Goyal <kovid at kovidgoyal.net>
3
4 import re
5 from datetime import datetime
6 from functools import partial
7
8 from calibre.constants import preferred_encoding
9 from calibre.ebooks.metadata import author_to_author_sort, title_sort
10 from calibre.utils.date import UNDEFINED_DATE, is_date_undefined, isoformat, parse_date, parse_only_date
11 from calibre.utils.icu import lower as icu_lower
12 from calibre.utils.icu import strcmp
13 from calibre.utils.localization import _, canonicalize_lang
14
15 missing = object()
16
17 # Convert data into values suitable for the db {{{
18
19
20 def sqlite_datetime(x):
21 return isoformat(x, sep=' ') if isinstance(x, datetime) else x
22
23
24 def single_text(x):
25 if x is None:
26 return x
27 if not isinstance(x, str):
28 x = x.decode(preferred_encoding, 'replace')
29 x = x.strip()
30 return x or None
31
32
33 series_index_pat = re.compile(r'(.*)\s+\[([.0-9]+)\]$')
34
35
36 def get_series_values(val):
37 if not val:
38 return val, None
39 match = series_index_pat.match(val.strip())
40 if match is not None:
41 idx = match.group(2)
42 try:
43 idx = float(idx)
44 return match.group(1).strip(), idx
45 except Exception:
46 pass
47 return val, None
48
49
50 def multiple_text(sep, ui_sep, x):
51 if not x:
52 return ()
53 if isinstance(x, bytes):
54 x = x.decode(preferred_encoding, 'replace')
55 if isinstance(x, str):
56 x = x.split(sep)
57 else:
58 x = (y.decode(preferred_encoding, 'replace') if isinstance(y, bytes) else y for y in x)
59 ui_sep = ui_sep.strip()
60 repsep = ',' if ui_sep == ';' else ';'
61 x = (y.strip().replace(ui_sep, repsep) for y in x if y.strip())
62 return tuple(' '.join(y.split()) for y in x if y)
63
64
65 def adapt_datetime(x):
66 if isinstance(x, (str, bytes)):
67 x = parse_date(x, assume_utc=False, as_utc=False)
68 if x and is_date_undefined(x):
69 x = UNDEFINED_DATE
70 return x
71
72
73 def adapt_date(x):
74 if isinstance(x, (str, bytes)):
75 x = parse_only_date(x)
76 if x is None or is_date_undefined(x):
77 x = UNDEFINED_DATE
78 return x
79
80
81 def adapt_number(typ, x):
82 if x is None:
83 return None
84 if isinstance(x, (str, bytes)):
85 if isinstance(x, bytes):
86 x = x.decode(preferred_encoding, 'replace')
87 if not x or x.lower() == 'none':
88 return None
89 return typ(x)
90
91
92 def adapt_bool(x):
93 if isinstance(x, (str, bytes)):
94 if isinstance(x, bytes):
95 x = x.decode(preferred_encoding, 'replace')
96 x = x.lower()
97 if x == 'true':
98 x = True
99 elif x == 'false':
100 x = False
101 elif x in {'none', ''}:
102 x = None
103 else:
104 x = bool(int(x))
105 return x if x is None else bool(x)
106
107
108 def adapt_languages(to_tuple, x):
109 ans = []
110 for lang in to_tuple(x):
111 lc = canonicalize_lang(lang)
112 if not lc or lc in ans or lc in ('und', 'zxx', 'mis', 'mul'):
113 continue
114 ans.append(lc)
115 return tuple(ans)
116
117
118 def clean_identifier(typ, val):
119 typ = icu_lower(typ or '').strip().replace(':', '').replace(',', '')
120 val = (val or '').strip().replace(',', '|')
121 return typ, val
122
123
124 def adapt_identifiers(to_tuple, x):
125 if not isinstance(x, dict):
126 x = dict(y.partition(':')[0::2] for y in to_tuple(x))
127 ans = {}
128 for k, v in x.items():
129 k, v = clean_identifier(k, v)
130 if k and v:
131 ans[k] = v
132 return ans
133
134
135 def adapt_series_index(x):
136 ret = adapt_number(float, x)
137 return 1.0 if ret is None else ret
138
139
140 def get_adapter(name, metadata):
141 dt = metadata['datatype']
142 if dt == 'text':
143 if metadata['is_multiple']:
144 m = metadata['is_multiple']
145 ans = partial(multiple_text, m['ui_to_list'], m['list_to_ui'])
146 else:
147 ans = single_text
148 elif dt == 'series':
149 ans = single_text
150 elif dt == 'datetime':
151 ans = adapt_date if name == 'pubdate' else adapt_datetime
152 elif dt == 'int':
153 ans = partial(adapt_number, int)
154 elif dt == 'float':
155 ans = partial(adapt_number, float)
156 elif dt == 'bool':
157 ans = adapt_bool
158 elif dt == 'comments':
159 ans = single_text
160 elif dt == 'rating':
161
162 def ans(x):
163 return None if x in {None, 0} else min(10, max(0, adapt_number(int, x)))
164
165 elif dt == 'enumeration':
166 ans = single_text
167 elif dt == 'composite':
168
169 def ans(x):
170 return x
171
172 if name == 'title':
173 return lambda x: ans(x) or _('Unknown')
174 if name == 'author_sort':
175 return lambda x: ans(x) or ''
176 if name == 'authors':
177 return lambda x: tuple(y.replace('|', ',') for y in ans(x)) or (_('Unknown'),)
178 if name in {'timestamp', 'last_modified'}:
179 return lambda x: ans(x) or UNDEFINED_DATE
180 if name == 'series_index':
181 return adapt_series_index
182 if name == 'languages':
183 return partial(adapt_languages, ans)
184 if name == 'identifiers':
185 return partial(adapt_identifiers, ans)
186
187 return ans
188
189
190 # }}}
191
192 # One-One fields {{{
193
194
195 def one_one_in_books(book_id_val_map, db, field, *args):
196 "Set a one-one field in the books table"
197 # Ignore those items whose value is the same as the current value
198 # We can't do this for the cover because the file might change without
199 # the presence-of-cover flag changing
200 if field.name != 'cover':
201 g = field.table.book_col_map.get
202 book_id_val_map = {k: v for k, v in book_id_val_map.items() if v != g(k, missing)}
203 if book_id_val_map:
204 sequence = ((sqlite_datetime(v), k) for k, v in book_id_val_map.items())
205 db.executemany('UPDATE books SET {}=? WHERE id=?'.format(field.metadata['column']), sequence)
206 field.table.book_col_map.update(book_id_val_map)
207 return set(book_id_val_map)
208
209
210 def set_uuid(book_id_val_map, db, field, *args):
211 field.table.update_uuid_cache(book_id_val_map)
212 return one_one_in_books(book_id_val_map, db, field, *args)
213
214
215 def set_title(book_id_val_map, db, field, *args):
216 ans = one_one_in_books(book_id_val_map, db, field, *args)
217 # Set the title sort field if the title changed
218 field.title_sort_field.writer.set_books({k: title_sort(v) for k, v in book_id_val_map.items() if k in ans}, db)
219 return ans
220
221
222 def one_one_in_other(book_id_val_map, db, field, *args):
223 "Set a one-one field in the non-books table, like comments"
224 # Ignore those items whose value is the same as the current value
225 g = field.table.book_col_map.get
226 book_id_val_map = {k: v for k, v in book_id_val_map.items() if v != g(k, missing)}
227 deleted = tuple((k,) for k, v in book_id_val_map.items() if v is None)
228 if deleted:
229 db.executemany('DELETE FROM {} WHERE book=?'.format(field.metadata['table']), deleted)
230 for book_id in deleted:
231 field.table.book_col_map.pop(book_id[0], None)
232 updated = {k: v for k, v in book_id_val_map.items() if v is not None}
233 if updated:
234 db.executemany(
235 'INSERT OR REPLACE INTO {}(book,{}) VALUES (?,?)'.format(field.metadata['table'], field.metadata['column']),
236 ((k, sqlite_datetime(v)) for k, v in updated.items()),
237 )
238 field.table.book_col_map.update(updated)
239 return set(book_id_val_map)
240
241
242 def custom_series_index(book_id_val_map, db, field, *args):
243 series_field = field.series_field
244 sequence = []
245 for book_id, sidx in book_id_val_map.items():
246 ids = series_field.ids_for_book(book_id)
247 if sidx is None:
248 sidx = 1.0
249 if ids:
250 if field.table.book_col_map.get(book_id, missing) != sidx:
251 sequence.append((sidx, book_id, ids[0]))
252 field.table.book_col_map[book_id] = sidx
253 else:
254 # the series has been deleted from the book, which means no row for
255 # it exists in the series table. The series_index value should be
256 # removed from the in-memory table as well, to ensure this book
257 # sorts the same as other books with no series.
258 field.table.remove_books((book_id,), db)
259 if sequence:
260 db.executemany(
261 'UPDATE {} SET {}=? WHERE book=? AND value=?'.format(field.metadata['table'], field.metadata['column']),
262 sequence,
263 )
264 return {s[1] for s in sequence}
265
266
267 # }}}
268
269 # Many-One fields {{{
270
271
272 def safe_lower(x):
273 try:
274 return icu_lower(x)
275 except TypeError, ValueError, KeyError, AttributeError:
276 return x
277
278
279 def get_db_id(val, db, m, table, kmap, rid_map, allow_case_change, case_changes, val_map, is_authors=False):
280 """Get the db id for the value val. If val does not exist in the db it is
281 inserted into the db."""
282 kval = kmap(val)
283 item_id = rid_map.get(kval, None)
284 if item_id is None:
285 if is_authors:
286 aus = author_to_author_sort(val)
287 db.execute('INSERT INTO authors(name,sort) VALUES (?,?)', (val.replace(',', '|'), aus))
288 else:
289 db.execute('INSERT INTO {}({}) VALUES (?)'.format(m['table'], m['column']), (val,))
290 item_id = rid_map[kval] = db.last_insert_rowid()
291 table.id_map[item_id] = val
292 table.col_book_map[item_id] = set()
293 if is_authors:
294 table.asort_map[item_id] = aus
295 if hasattr(table, 'link_map'):
296 table.link_map[item_id] = ''
297 if table.supports_notes:
298 db.unretire_note(table.name, item_id, val)
299 elif allow_case_change and val != table.id_map[item_id]:
300 case_changes[item_id] = val
301 val_map[val] = item_id
302
303
304 def change_case(case_changes, dirtied, db, table, m, is_authors=False):
305 if is_authors:
306 vals = ((val.replace(',', '|'), item_id) for item_id, val in case_changes.items())
307 else:
308 vals = ((val, item_id) for item_id, val in case_changes.items())
309 db.executemany('UPDATE {} SET {}=? WHERE id=?'.format(m['table'], m['column']), vals)
310 for item_id, val in case_changes.items():
311 table.id_map[item_id] = val
312 dirtied.update(table.col_book_map[item_id])
313 if is_authors:
314 table.asort_map[item_id] = author_to_author_sort(val)
315
316
317 def many_one(book_id_val_map, db, field, allow_case_change, *args):
318 dirtied = set()
319 m = field.metadata
320 table = field.table
321 dt = m['datatype']
322 is_custom_series = dt == 'series' and table.name.startswith('#')
323
324 # Map values to db ids, including any new values
325 kmap = safe_lower if dt in {'text', 'series'} else lambda x: x
326 rid_map = {kmap(item): item_id for item_id, item in table.id_map.items()}
327 if len(rid_map) != len(table.id_map):
328 # table has some entries that differ only in case, fix it
329 table.fix_case_duplicates(db)
330 rid_map = {kmap(item): item_id for item_id, item in table.id_map.items()}
331 val_map = {None: None}
332 case_changes = {}
333 for val in book_id_val_map.values():
334 if val is not None:
335 get_db_id(val, db, m, table, kmap, rid_map, allow_case_change, case_changes, val_map)
336
337 if case_changes:
338 change_case(case_changes, dirtied, db, table, m)
339
340 book_id_item_id_map = {k: val_map[v] for k, v in book_id_val_map.items()}
341
342 # Ignore those items whose value is the same as the current value
343 book_id_item_id_map = {k: v for k, v in book_id_item_id_map.items() if v != table.book_col_map.get(k, None)}
344 dirtied |= set(book_id_item_id_map)
345
346 # Update the book->col and col->book maps
347 deleted = set()
348 updated = {}
349 for book_id, item_id in book_id_item_id_map.items():
350 old_item_id = table.book_col_map.get(book_id, None)
351 if old_item_id is not None:
352 table.col_book_map[old_item_id].discard(book_id)
353 if item_id is None:
354 table.book_col_map.pop(book_id, None)
355 deleted.add(book_id)
356 else:
357 table.book_col_map[book_id] = item_id
358 table.col_book_map[item_id].add(book_id)
359 updated[book_id] = item_id
360
361 # Update the db link table
362 if deleted:
363 db.executemany(f'DELETE FROM {table.link_table} WHERE book=?', ((k,) for k in deleted))
364 if updated:
365 sql = (
366 'DELETE FROM {0} WHERE book=?; INSERT INTO {0}(book,{1},extra) VALUES(?, ?, 1.0)'
367 if is_custom_series
368 else 'DELETE FROM {0} WHERE book=?; INSERT INTO {0}(book,{1}) VALUES(?, ?)'
369 )
370 db.executemany(
371 sql.format(table.link_table, m['link_column']),
372 ((book_id, book_id, item_id) for book_id, item_id in updated.items()),
373 )
374
375 # Remove no longer used items
376 remove = {item_id: item_val for item_id, item_val in table.id_map.items() if not table.col_book_map.get(item_id, False)}
377 if remove:
378 if table.supports_notes:
379 db.clear_notes_for_category_items(table.name, remove)
380 db.executemany('DELETE FROM {} WHERE id=?'.format(m['table']), ((item_id,) for item_id in remove))
381 for item_id in remove:
382 del table.id_map[item_id]
383 table.col_book_map.pop(item_id, None)
384
385 return dirtied
386
387
388 # }}}
389
390 # Many-Many fields {{{
391
392
393 def uniq(vals, kmap=lambda x: x):
394 """Remove all duplicates from vals, while preserving order. kmap must be a
395 callable that returns a hashable value for every item in vals"""
396 vals = vals or ()
397 lvals = (kmap(x) for x in vals)
398 seen = set()
399 seen_add = seen.add
400 return tuple(x for x, k in zip(vals, lvals) if k not in seen and not seen_add(k)) # ty: ignore[redundant-condition]
401
402
403 def many_many(book_id_val_map, db, field, allow_case_change, *args):
404 dirtied = set()
405 m = field.metadata
406 table = field.table
407 dt = m['datatype']
408 is_authors = field.name == 'authors'
409
410 # Map values to db ids, including any new values
411 kmap = safe_lower if dt == 'text' else lambda x: x
412 rid_map = {kmap(item): item_id for item_id, item in table.id_map.items()}
413 if len(rid_map) != len(table.id_map):
414 # table has some entries that differ only in case, fix it
415 table.fix_case_duplicates(db)
416 rid_map = {kmap(item): item_id for item_id, item in table.id_map.items()}
417 val_map = {}
418 case_changes = {}
419 book_id_val_map = {k: uniq(vals, kmap) for k, vals in book_id_val_map.items()}
420 for vals in book_id_val_map.values():
421 for val in vals:
422 get_db_id(val, db, m, table, kmap, rid_map, allow_case_change, case_changes, val_map, is_authors=is_authors)
423
424 if case_changes:
425 change_case(case_changes, dirtied, db, table, m, is_authors=is_authors)
426 if is_authors:
427 for item_id, val in case_changes.items():
428 for book_id in table.col_book_map[item_id]:
429 current_sort = field.db_author_sort_for_book(book_id)
430 new_sort = field.author_sort_for_book(book_id)
431 if strcmp(current_sort, new_sort) == 0:
432 # The sort strings differ only by case, update the db
433 # sort
434 field.author_sort_field.writer.set_books({book_id: new_sort}, db)
435
436 book_id_item_id_map = {k: tuple(val_map[v] for v in vals) for k, vals in book_id_val_map.items()}
437
438 # Ignore those items whose value is the same as the current value
439 g = table.book_col_map.get
440 not_set = ()
441 book_id_item_id_map = {k: v for k, v in book_id_item_id_map.items() if v != g(k, not_set)}
442 dirtied |= set(book_id_item_id_map)
443
444 # Update the book->col and col->book maps
445 deleted = set()
446 updated = {}
447 for book_id, item_ids in book_id_item_id_map.items():
448 old_item_ids = table.book_col_map.get(book_id, None)
449 if old_item_ids:
450 for old_item_id in old_item_ids:
451 table.col_book_map[old_item_id].discard(book_id)
452 if item_ids:
453 table.book_col_map[book_id] = item_ids
454 for item_id in item_ids:
455 table.col_book_map[item_id].add(book_id)
456 updated[book_id] = item_ids
457 else:
458 table.book_col_map.pop(book_id, None)
459 deleted.add(book_id)
460
461 # Update the db link table
462 if deleted:
463 db.executemany(f'DELETE FROM {table.link_table} WHERE book=?', ((k,) for k in deleted))
464 if updated:
465 vals = ((book_id, val) for book_id, vals in updated.items() for val in vals)
466 db.executemany(f'DELETE FROM {table.link_table} WHERE book=?', ((k,) for k in updated))
467 db.executemany('INSERT INTO {}(book,{}) VALUES(?, ?)'.format(table.link_table, m['link_column']), vals)
468 if is_authors:
469 aus_map = {book_id: field.author_sort_for_book(book_id) for book_id in updated}
470 field.author_sort_field.writer.set_books(aus_map, db)
471
472 # Remove no longer used items
473 remove = {item_id: item_val for item_id, item_val in table.id_map.items() if not table.col_book_map.get(item_id, False)}
474 if remove:
475 if table.supports_notes:
476 db.clear_notes_for_category_items(table.name, remove)
477 db.executemany('DELETE FROM {} WHERE id=?'.format(m['table']), ((item_id,) for item_id in remove))
478 for item_id in remove:
479 del table.id_map[item_id]
480 table.col_book_map.pop(item_id, None)
481 if is_authors:
482 table.asort_map.pop(item_id, None)
483 if hasattr(table, 'link_map'):
484 table.link_map.pop(item_id, None)
485 return dirtied
486
487
488 # }}}
489
490
491 def identifiers(book_id_val_map, db, field, *args): # {{{
492 # Ignore those items whose value is the same as the current value
493 g = field.table.book_col_map.get
494 book_id_val_map = {k: v for k, v in book_id_val_map.items() if v != g(k, missing)}
495
496 table = field.table
497 updates = set()
498 for book_id, identifiers in book_id_val_map.items():
499 if book_id not in table.book_col_map:
500 table.book_col_map[book_id] = {}
501 current_ids = table.book_col_map[book_id]
502 remove_keys = set(current_ids) - set(identifiers)
503 for key in remove_keys:
504 table.col_book_map.get(key, set()).discard(book_id)
505 current_ids.pop(key, None)
506 current_ids.update(identifiers)
507 for key, val in identifiers.items():
508 if key not in table.col_book_map:
509 table.col_book_map[key] = set()
510 table.col_book_map[key].add(book_id)
511 updates.add((book_id, key, val))
512 db.executemany('DELETE FROM identifiers WHERE book=?', ((x,) for x in book_id_val_map))
513 if updates:
514 db.executemany('INSERT OR REPLACE INTO identifiers (book, type, val) VALUES (?, ?, ?)', tuple(updates))
515 return set(book_id_val_map)
516
517
518 # }}}
519
520
521 def dummy(book_id_val_map, *args):
522 return set()
523
524
525 class Writer:
526 def __init__(self, field):
527 self.adapter = get_adapter(field.name, field.metadata)
528 self.name = field.name
529 self.field = field
530 dt = field.metadata['datatype']
531 self.accept_vals = lambda x: True
532 if dt == 'composite' or field.name in {'id', 'size', 'path', 'formats', 'news'}:
533 self.set_books_func = dummy
534 elif self.name[0] == '#' and self.name.endswith('_index'):
535 self.set_books_func = custom_series_index
536 elif self.name == 'identifiers':
537 self.set_books_func = identifiers
538 elif self.name == 'uuid':
539 self.set_books_func = set_uuid
540 elif self.name == 'title':
541 self.set_books_func = set_title
542 elif field.is_many_many:
543 self.set_books_func = many_many
544 elif field.is_many:
545 self.set_books_func = self.set_books_for_enum if dt == 'enumeration' else many_one
546 else:
547 self.set_books_func = one_one_in_books if field.metadata['table'] == 'books' else one_one_in_other
548 if self.name in {'timestamp', 'uuid', 'sort'}:
549 self.accept_vals = bool
550
551 def set_books(self, book_id_val_map, db, allow_case_change=True):
552 book_id_val_map = {k: self.adapter(v) for k, v in book_id_val_map.items() if self.accept_vals(v)}
553 if not book_id_val_map:
554 return set()
555 dirtied = self.set_books_func(book_id_val_map, db, self.field, allow_case_change)
556 return dirtied
557
558 def set_books_for_enum(self, book_id_val_map, db, field, allow_case_change):
559 allowed = set(field.metadata['display']['enum_values'])
560 book_id_val_map = {k: v for k, v in book_id_val_map.items() if v is None or v in allowed}
561 if not book_id_val_map:
562 return set()
563 return many_one(book_id_val_map, db, field, False)