Debian Codegraph @

src/calibre/db/search.py

raw
1 #!/usr/bin/env python
2 # License: GPLv3 Copyright: 2013, Kovid Goyal <kovid at kovidgoyal.net>
3
4 import operator
5 import unicodedata
6 import weakref
7 from collections import OrderedDict, deque
8 from collections.abc import Callable
9 from datetime import timedelta
10 from functools import partial
11
12 import regex
13
14 from calibre.constants import DEBUG, preferred_encoding
15 from calibre.db.utils import force_to_bool
16 from calibre.utils.config_base import prefs
17 from calibre.utils.date import UNDEFINED_DATE, dt_as_local, now, parse_date
18 from calibre.utils.icu import lower as icu_lower
19 from calibre.utils.icu import primary_contains, primary_no_punc_contains, sort_key
20 from calibre.utils.localization import _, canonicalize_lang, lang_map
21 from calibre.utils.search_query_parser import ParseException, SearchQueryParser
22
23 CONTAINS_MATCH = 0
24 EQUALS_MATCH = 1
25 REGEXP_MATCH = 2
26 ACCENT_MATCH = 3
27
28
29 class TemplatesNotAllowed(ParseException):
30 pass
31
32
33 # Utils {{{
34
35
36 def _matchkind(query, case_sensitive=False):
37 matchkind = CONTAINS_MATCH
38 if len(query) > 1:
39 if query.startswith('\\'):
40 query = query[1:]
41 elif query.startswith('='):
42 matchkind = EQUALS_MATCH
43 query = query[1:]
44 elif query.startswith('~'):
45 matchkind = REGEXP_MATCH
46 query = query[1:]
47 elif query.startswith('^'):
48 matchkind = ACCENT_MATCH
49 query = query[1:]
50
51 if not case_sensitive and matchkind != REGEXP_MATCH:
52 # leave case in regexps because it can be significant e.g. \S \W \D
53 query = icu_lower(query)
54 return matchkind, query
55
56
57 def _match(query, value, matchkind, use_primary_find_in_search=True, case_sensitive=False):
58 if query.startswith('..'):
59 query = query[1:]
60 sq = query[1:]
61 internal_match_ok = True
62 else:
63 internal_match_ok = False
64 for t in value:
65 if not case_sensitive:
66 t = icu_lower(t)
67 if matchkind == EQUALS_MATCH:
68 if internal_match_ok:
69 if query == t:
70 return True
71 return sq in (c.strip() for c in t.split('.') if c.strip())
72 elif query[0] == '.':
73 if t.startswith(query[1:]):
74 ql = len(query) - 1
75 if len(t) == ql or t[ql : ql + 1] == '.':
76 return True
77 elif query == t:
78 return True
79 elif matchkind == REGEXP_MATCH:
80 flags = regex.UNICODE | regex.VERSION1 | regex.FULLCASE | (0 if case_sensitive else regex.IGNORECASE)
81 try:
82 if regex.search(query, t, flags) is not None:
83 return True
84 except regex.error as e:
85 raise ParseException(_('Invalid regular expression: {!r} with error: {}').format(query, str(e)))
86 elif matchkind == ACCENT_MATCH:
87 if primary_contains(query, t):
88 return True
89 elif matchkind == CONTAINS_MATCH:
90 if not case_sensitive and use_primary_find_in_search:
91 if primary_no_punc_contains(query, t):
92 return True
93 elif query in t:
94 return True
95 return False
96
97
98 # }}}
99
100
101 class DateSearch: # {{{
102 def __init__(self):
103 self.operators = OrderedDict((
104 ('!=', self.ne),
105 ('>=', self.ge),
106 ('<=', self.le),
107 ('=', self.eq),
108 ('>', self.gt),
109 ('<', self.lt),
110 ))
111 self.local_today = {'_today', 'today', icu_lower(_('today'))}
112 self.local_yesterday = {'_yesterday', 'yesterday', icu_lower(_('yesterday'))}
113 self.local_thismonth = {'_thismonth', 'thismonth', icu_lower(_('thismonth'))}
114 self.daysago_pat = regex.compile(r'({}|daysago|_daysago)$'.format(_('daysago')), flags=regex.UNICODE | regex.VERSION1)
115
116 def eq(self, dbdate, query, field_count):
117 if dbdate.year == query.year:
118 if field_count == 1:
119 return True
120 if dbdate.month == query.month:
121 if field_count == 2:
122 return True
123 return dbdate.day == query.day
124 return False
125
126 def ne(self, *args):
127 return not self.eq(*args)
128
129 def gt(self, dbdate, query, field_count):
130 if dbdate.year > query.year:
131 return True
132 if field_count > 1 and dbdate.year == query.year:
133 if dbdate.month > query.month:
134 return True
135 return field_count == 3 and dbdate.month == query.month and dbdate.day > query.day
136 return False
137
138 def le(self, *args):
139 return not self.gt(*args)
140
141 def lt(self, dbdate, query, field_count):
142 if dbdate.year < query.year:
143 return True
144 if field_count > 1 and dbdate.year == query.year:
145 if dbdate.month < query.month:
146 return True
147 return field_count == 3 and dbdate.month == query.month and dbdate.day < query.day
148 return False
149
150 def ge(self, *args):
151 return not self.lt(*args)
152
153 def __call__(self, query, field_iter):
154 matches = set()
155 if len(query) < 2:
156 return matches
157
158 if query == 'false':
159 for v, book_ids in field_iter():
160 if isinstance(v, (bytes, str)):
161 if isinstance(v, bytes):
162 v = v.decode(preferred_encoding, 'replace')
163 v = parse_date(v)
164 if v is None or v <= UNDEFINED_DATE:
165 matches |= book_ids
166 return matches
167
168 if query == 'true':
169 for v, book_ids in field_iter():
170 if isinstance(v, (bytes, str)):
171 if isinstance(v, bytes):
172 v = v.decode(preferred_encoding, 'replace')
173 v = parse_date(v)
174 if v is not None and v > UNDEFINED_DATE:
175 matches |= book_ids
176 return matches
177
178 for k, relop in self.operators.items():
179 if query.startswith(k):
180 query = query[len(k) :]
181 break
182 else:
183 relop = self.operators['=']
184
185 if query in self.local_today:
186 qd = now()
187 field_count = 3
188 elif query in self.local_yesterday:
189 qd = now() - timedelta(1)
190 field_count = 3
191 elif query in self.local_thismonth:
192 qd = now()
193 field_count = 2
194 else:
195 m = self.daysago_pat.search(query)
196 if m is not None:
197 num = query[: -len(m.group(1))]
198 try:
199 qd = now() - timedelta(int(num))
200 except Exception:
201 raise ParseException(_('Number conversion error: {0}').format(num))
202 field_count = 3
203 else:
204 try:
205 qd = parse_date(query, as_utc=False)
206 except Exception:
207 raise ParseException(_('Date conversion error: {0}').format(query))
208 if '-' in query:
209 field_count = query.count('-') + 1
210 else:
211 field_count = query.count('/') + 1
212
213 for v, book_ids in field_iter():
214 if isinstance(v, (str, bytes)):
215 v = parse_date(v)
216 if v is not None and relop(dt_as_local(v), qd, field_count):
217 matches |= book_ids
218
219 return matches
220
221
222 # }}}
223
224
225 class NumericSearch: # {{{
226 def __init__(self):
227 self.operators = OrderedDict((
228 ('!=', operator.ne),
229 ('>=', operator.ge),
230 ('<=', operator.le),
231 ('=', operator.eq),
232 ('>', operator.gt),
233 ('<', operator.lt),
234 ))
235
236 def __call__(self, query, field_iter, location, datatype, candidates, is_many=False):
237 matches = set()
238 if not query:
239 return matches
240
241 q = ''
242 cast = adjust = lambda x: x
243 dt = datatype
244
245 if is_many and query in {'true', 'false'}:
246 if datatype == 'rating':
247
248 def valcheck(x):
249 return x is not None and x > 0
250
251 else:
252
253 def valcheck(x):
254 return True
255
256 found = set()
257 for val, book_ids in field_iter():
258 if valcheck(val):
259 found |= book_ids
260 return found if query == 'true' else candidates - found
261
262 if query == 'false':
263 if location == 'cover':
264
265 def relop(x, y):
266 return not bool(x)
267
268 else:
269
270 def relop(x, y):
271 return x is None
272
273 elif query == 'true':
274 if location == 'cover':
275
276 def relop(x, y):
277 return bool(x)
278
279 else:
280
281 def relop(x, y):
282 return x is not None
283
284 else:
285 for k, relop in self.operators.items():
286 if query.startswith(k):
287 query = query[len(k) :]
288 break
289 else:
290 relop = self.operators['=']
291
292 cast: Callable[..., int | float]
293 if dt == 'rating':
294
295 def _cast_for_rating(x):
296 return 0 if x is None else int(x)
297
298 cast = _cast_for_rating
299
300 def adjust(x):
301 return x // 2
302
303 else:
304 # Datatype is empty if the source is a template. Assume float
305 cast = float if dt in ('float', 'composite', 'half-rating', '') else int
306
307 mult = 1.0
308 if len(query) > 1:
309 mult = query[-1].lower()
310 mult = {'k': 1024.0, 'm': 1024.0**2, 'g': 1024.0**3}.get(mult, 1.0)
311 if mult != 1.0:
312 query = query[:-1]
313 else:
314 mult = 1.0
315
316 try:
317 q = cast(query) * mult
318 except Exception:
319 raise ParseException(_('Non-numeric value in query: {0}').format(query))
320 if dt == 'half-rating':
321 q = round(q * 2)
322 cast = int
323
324 qfalse = query == 'false'
325 for val, book_ids in field_iter():
326 if val is None:
327 if qfalse:
328 matches |= book_ids
329 continue
330 try:
331 v = cast(val)
332 except Exception:
333 raise ParseException(_('Non-numeric value in column {0}: {1}').format(location, val))
334 if v:
335 v = adjust(v)
336 if relop(v, q):
337 matches |= book_ids
338 return matches
339
340
341 # }}}
342
343 nfc = partial(unicodedata.normalize, 'NFC')
344 nfkc = partial(unicodedata.normalize, 'NFKC')
345 nfkd = partial(unicodedata.normalize, 'NFKD')
346 nfd = partial(unicodedata.normalize, 'NFD')
347
348
349 def local_forms(x: str) -> set[str]:
350 x = icu_lower(x)
351 return {nfc(x), nfd(x), nfkc(x), nfkd(x)}
352
353
354 class BooleanSearch: # {{{
355 def __init__(self):
356 local_no = local_forms(_('No')) | local_forms(_('no'))
357 local_yes = local_forms(_('Yes')) | local_forms(_('yes'))
358 local_unchecked = local_forms(_('unchecked'))
359 local_checked = local_forms(_('checked'))
360 local_empty = local_forms(_('empty'))
361 local_blank = local_forms(_('blank'))
362
363 self.valid_bool_values = frozenset(
364 {
365 '_no',
366 'false',
367 'no',
368 'unchecked',
369 '_unchecked',
370 'checked',
371 '_checked',
372 '_yes',
373 'true',
374 'yes',
375 'blank',
376 '_blank',
377 '_empty',
378 'empty',
379 }
380 | local_no
381 | local_yes
382 | local_unchecked
383 | local_checked
384 | local_empty
385 | local_blank
386 )
387 self.no_and_unchecked = frozenset({'unchecked', '_unchecked', 'no', '_no', 'false'} | local_no | local_unchecked)
388 self.no_and_unchecked_with_true = frozenset({'unchecked', '_unchecked', 'no', '_no', 'true'} | local_no | local_unchecked)
389 self.yes_and_checked = frozenset({'checked', '_checked', 'yes', '_yes', 'true'} | local_yes | local_checked)
390 self.empty_and_blank = frozenset({'blank', '_blank', 'empty', '_empty', 'false'} | local_empty | local_blank)
391
392 def __call__(self, query, field_iter, bools_are_tristate):
393 matches = set()
394 if query not in self.valid_bool_values:
395 raise ParseException(_('Invalid boolean query "{0}"').format(query))
396 for val, book_ids in field_iter():
397 val = force_to_bool(val)
398 if not bools_are_tristate:
399 if val is None or not val: # item is None or set to false
400 if query in self.no_and_unchecked:
401 matches |= book_ids
402 elif query in self.yes_and_checked:
403 matches |= book_ids
404 elif val is None:
405 if query in self.empty_and_blank:
406 matches |= book_ids
407 elif not val: # is not None and false
408 if query in self.no_and_unchecked_with_true:
409 matches |= book_ids
410 elif query in self.yes_and_checked:
411 matches |= book_ids
412 return matches
413
414
415 # }}}
416
417
418 class KeyPairSearch: # {{{
419 def __call__(self, query, field_iter, candidates, use_primary_find):
420 matches = set()
421 if ':' in query:
422 q = [q.strip() for q in query.partition(':')[0::2]]
423 keyq, valq = q
424 keyq_mkind, keyq = _matchkind(keyq)
425 valq_mkind, valq = _matchkind(valq)
426 else:
427 keyq = keyq_mkind = ''
428 valq_mkind, valq = _matchkind(query)
429
430 if valq in {'true', 'false'}:
431 found = set()
432 if keyq:
433 for val, book_ids in field_iter():
434 if val and val.get(keyq, False):
435 found |= book_ids
436 else:
437 for val, book_ids in field_iter():
438 if val:
439 found |= book_ids
440 return found if valq == 'true' else candidates - found
441
442 for m, book_ids in field_iter():
443 for key, val in m.items():
444 if keyq and not _match(keyq, (key,), keyq_mkind, use_primary_find_in_search=use_primary_find):
445 continue
446 if valq and not _match(valq, (val,), valq_mkind, use_primary_find_in_search=use_primary_find):
447 continue
448 matches |= book_ids
449 break
450
451 return matches
452
453
454 # }}}
455
456
457 class SavedSearchQueries: # {{{
458 queries = {}
459 opt_name = ''
460
461 def __init__(self, db, _opt_name):
462 self.opt_name = _opt_name
463 try:
464 self._db = weakref.ref(db)
465 except TypeError:
466 # db could be None
467 self._db = lambda: None
468 self.load_from_db()
469
470 def load_from_db(self):
471 db = self.db
472 if db is not None:
473 self.queries = db._pref(self.opt_name, default={})
474 else:
475 self.queries = {}
476
477 @property
478 def db(self):
479 return self._db()
480
481 def force_unicode(self, x):
482 if not isinstance(x, str):
483 x = x.decode(preferred_encoding, 'replace')
484 return x
485
486 def add(self, name, value):
487 db = self.db
488 if db is not None:
489 self.queries[self.force_unicode(name)] = self.force_unicode(value).strip()
490 db._set_pref(self.opt_name, self.queries)
491
492 def lookup(self, name):
493 sn = self.force_unicode(name).lower()
494 for n, q in self.queries.items():
495 if sn == n.lower():
496 return q
497 return None
498
499 def delete(self, name):
500 db = self.db
501 if db is not None:
502 self.queries.pop(self.force_unicode(name), False)
503 db._set_pref(self.opt_name, self.queries)
504
505 def rename(self, old_name, new_name):
506 db = self.db
507 if db is not None:
508 self.queries[self.force_unicode(new_name)] = self.queries.get(self.force_unicode(old_name), None)
509 self.queries.pop(self.force_unicode(old_name), False)
510 db._set_pref(self.opt_name, self.queries)
511
512 def set_all(self, smap):
513 db = self.db
514 if db is not None:
515 self.queries = smap
516 db._set_pref(self.opt_name, smap)
517
518 def names(self):
519 return sorted(self.queries, key=sort_key)
520
521
522 # }}}
523
524
525 class Parser(SearchQueryParser): # {{{
526 def __init__(
527 self,
528 dbcache,
529 all_book_ids,
530 gst,
531 date_search,
532 num_search,
533 bool_search,
534 keypair_search,
535 limit_search_columns,
536 limit_search_columns_to,
537 locations,
538 virtual_fields,
539 lookup_saved_search,
540 parse_cache,
541 allow_templates=True,
542 ):
543 self.allow_templates = allow_templates
544 self.dbcache, self.all_book_ids = dbcache, all_book_ids
545 self.all_search_locations = frozenset(locations)
546 self.grouped_search_terms = gst
547 self.date_search, self.num_search = date_search, num_search
548 self.bool_search, self.keypair_search = bool_search, keypair_search
549 self.limit_search_columns, self.limit_search_columns_to = (limit_search_columns, limit_search_columns_to)
550 self.virtual_fields = virtual_fields or {}
551 if 'marked' not in self.virtual_fields:
552 self.virtual_fields['marked'] = self
553 if 'in_tag_browser' not in self.virtual_fields:
554 self.virtual_fields['in_tag_browser'] = self
555 SearchQueryParser.__init__(self, locations, optimize=True, lookup_saved_search=lookup_saved_search, parse_cache=parse_cache)
556
557 @property
558 def field_metadata(self):
559 return self.dbcache.field_metadata
560
561 def universal_set(self):
562 return self.all_book_ids
563
564 def field_iter(self, name, candidates):
565 get_metadata = self.dbcache._get_proxy_metadata
566 try:
567 field = self.dbcache.fields[name]
568 except KeyError:
569 field = self.virtual_fields[name]
570 self.virtual_field_used = True
571 return field.iter_searchable_values(get_metadata, candidates)
572
573 def iter_searchable_values(self, *args, **kwargs):
574 for x in ():
575 yield x, set()
576
577 def parse(self, *args, **kwargs):
578 self.virtual_field_used = False
579 return SearchQueryParser.parse(self, *args, **kwargs)
580
581 def get_matches(self, location, query, candidates=None, allow_recursion=True):
582 # If candidates is not None, it must not be modified. Changing its
583 # value will break query optimization in the search parser
584 matches = set()
585
586 if candidates is None:
587 candidates = self.all_book_ids
588 if not candidates or not query or not query.strip():
589 return matches
590 if location not in self.all_search_locations:
591 return matches
592
593 if location == 'vl':
594 vl = self.dbcache._pref('virtual_libraries', {}).get(query) if query else None
595 if not vl:
596 raise ParseException(_('No such Virtual library: {}').format(query))
597 try:
598 return candidates & self.dbcache.books_in_virtual_library(query, virtual_fields=self.virtual_fields)
599 except RuntimeError:
600 raise ParseException(_('Virtual library search is recursive: {}').format(query))
601
602 if len(location) > 2 and location.startswith('@') and location[1:] in self.grouped_search_terms:
603 location = location[1:]
604
605 # get metadata key associated with the search term. Eliminates
606 # dealing with plurals and other aliases
607 original_location = location
608 location = self.field_metadata.search_term_to_field_key(icu_lower(location.strip()))
609 # grouped search terms
610 if isinstance(location, list):
611 if allow_recursion:
612 if query.lower() == 'false':
613 invert = True
614 query = 'true'
615 else:
616 invert = False
617 for loc in location:
618 c = candidates.copy()
619 m = self.get_matches(loc, query, candidates=c, allow_recursion=False)
620 matches |= m
621 c -= m
622 if len(c) == 0:
623 break
624 if invert:
625 matches = self.all_book_ids - matches
626 return matches
627 raise ParseException(_('Recursive query group detected: {0}').format(query))
628
629 # If the user has asked to restrict searching over all field, apply
630 # that restriction
631 if location == 'all' and self.limit_search_columns and self.limit_search_columns_to:
632 terms = set()
633 for l in self.limit_search_columns_to:
634 l = icu_lower(l.strip())
635 if l and l != 'all' and l in self.all_search_locations:
636 terms.add(l)
637 if terms:
638 c = candidates.copy()
639 for l in terms:
640 try:
641 m = self.get_matches(l, query, candidates=c, allow_recursion=allow_recursion)
642 matches |= m
643 c -= m
644 if len(c) == 0:
645 break
646 except Exception:
647 pass
648 return matches
649
650 upf = prefs['use_primary_find_in_search']
651
652 if location in self.field_metadata:
653 fm = self.field_metadata[location]
654 dt = fm['datatype']
655
656 # take care of dates special case
657 if dt == 'datetime' or (dt == 'composite' and fm['display'].get('composite_sort', '') == 'date'):
658 if location == 'date':
659 location = 'timestamp'
660 return self.date_search(icu_lower(query), partial(self.field_iter, location, candidates))
661
662 # take care of numbers special case
663 if dt in ('rating', 'int', 'float') or (dt == 'composite' and fm['display'].get('composite_sort', '') == 'number'):
664 if location == 'id':
665 is_many = False
666
667 def fi(default_value=None):
668 for qid in candidates:
669 yield qid, {qid}
670
671 else:
672 field = self.dbcache.fields[location]
673 fi, is_many = partial(self.field_iter, location, candidates), field.is_many
674 if dt == 'rating' and fm['display'].get('allow_half_stars'):
675 dt = 'half-rating'
676 return self.num_search(icu_lower(query), fi, location, dt, candidates, is_many=is_many)
677
678 # take care of the 'count' operator for is_multiples
679 if fm['is_multiple'] and len(query) > 1 and query[0] == '#' and query[1] in '=<>!':
680 return self.num_search(
681 icu_lower(query[1:]),
682 partial(
683 self.dbcache.fields[location].iter_counts,
684 candidates,
685 get_metadata=self.dbcache._get_proxy_metadata,
686 ),
687 location,
688 dt,
689 candidates,
690 )
691
692 # take care of boolean special case
693 if dt == 'bool':
694 return self.bool_search(
695 icu_lower(query),
696 partial(self.field_iter, location, candidates),
697 self.dbcache._pref('bools_are_tristate'),
698 )
699
700 # special case: colon-separated fields such as identifiers. isbn
701 # is a special case within the case
702 if fm.get('is_csp', False):
703 field_iter = partial(self.field_iter, location, candidates)
704 if location == 'identifiers' and original_location == 'isbn':
705 return self.keypair_search('=isbn:' + query, field_iter, candidates, upf)
706 return self.keypair_search(query, field_iter, candidates, upf)
707
708 # check for user categories
709 if len(location) >= 2 and location.startswith('@'):
710 return self.get_user_category_matches(location[1:], icu_lower(query), candidates)
711
712 # Everything else (and 'all' matches)
713 case_sensitive = prefs['case_sensitive']
714
715 if location == 'template':
716 if not self.allow_templates:
717 raise TemplatesNotAllowed(_('Templates are not allowed in search expressions'))
718 try:
719 template, sep, query = regex.split(r'#@#:([tdnb]):', query, flags=regex.IGNORECASE)
720 if sep:
721 sep = sep.lower()
722 else:
723 sep = 't'
724 except Exception:
725 if DEBUG:
726 import traceback
727
728 traceback.print_exc()
729 raise ParseException(_('search template: missing or invalid separator. Valid separators are: {}').format('#@#:[tdnb]:'))
730 matchkind, query = _matchkind(query, case_sensitive=case_sensitive)
731 matches = set()
732 error_string = '*@*TEMPLATE_ERROR*@*'
733 template_cache = {}
734 global_vars = {'_candidates': candidates}
735 for book_id in candidates:
736 mi = self.dbcache.get_proxy_metadata(book_id)
737 val = mi.formatter.safe_format(
738 template,
739 {},
740 error_string,
741 mi,
742 column_name='search template',
743 template_cache=template_cache,
744 global_vars=global_vars,
745 database=self.dbcache,
746 )
747 if val.startswith(error_string):
748 raise ParseException(val[len(error_string) :])
749 if sep == 't':
750 if _match(
751 query,
752 [
753 val,
754 ],
755 matchkind,
756 use_primary_find_in_search=upf,
757 case_sensitive=case_sensitive,
758 ):
759 matches.add(book_id)
760 elif sep == 'n' and val:
761 matches.update(
762 self.num_search(
763 icu_lower(query),
764 {
765 val: {
766 book_id,
767 }
768 }.items,
769 '',
770 '',
771 {
772 book_id,
773 },
774 is_many=False,
775 )
776 )
777 elif sep == 'd' and val:
778 matches.update(
779 self.date_search(
780 icu_lower(query),
781 {
782 val: {
783 book_id,
784 }
785 }.items,
786 )
787 )
788 elif sep == 'b':
789 matches.update(
790 self.bool_search(
791 icu_lower(query),
792 {
793 'True' if val else 'False': {
794 book_id,
795 }
796 }.items,
797 False,
798 )
799 )
800
801 return matches
802
803 matchkind, query = _matchkind(query, case_sensitive=case_sensitive)
804 all_locs = set()
805 text_fields = set()
806 field_metadata = {}
807
808 for x, fm in self.field_metadata.iter_items():
809 if x.startswith('@'):
810 continue
811 if fm['search_terms'] and x not in {'series_sort', 'id'}:
812 if x not in self.virtual_fields and x != 'uuid':
813 # We don't search virtual fields because if we do, search
814 # caching will not be used
815 all_locs.add(x)
816 field_metadata[x] = fm
817 if fm['datatype'] in {'composite', 'text', 'comments', 'series', 'enumeration'}:
818 text_fields.add(x)
819
820 locations = all_locs if location == 'all' else {location}
821
822 current_candidates = set(candidates)
823
824 try:
825 rating_query = int(float(query)) * 2
826 except Exception:
827 rating_query = None
828
829 try:
830 int_query = int(float(query))
831 except Exception:
832 int_query = None
833
834 try:
835 float_query = float(query)
836 except Exception:
837 float_query = None
838
839 for location in locations:
840 current_candidates -= matches
841 q = query
842 if location == 'languages':
843 q = canonicalize_lang(query)
844 if q is None:
845 lm = lang_map()
846 rm = {v.lower(): k for k, v in lm.items()}
847 q = rm.get(query, query)
848
849 if matchkind == CONTAINS_MATCH and q.lower() in {'true', 'false'}:
850 found = set()
851 for val, book_ids in self.field_iter(location, current_candidates):
852 if val and (not hasattr(val, 'strip') or val.strip()):
853 found |= book_ids
854 matches |= found if q.lower() == 'true' else (current_candidates - found)
855 continue
856
857 dt = field_metadata.get(location, {}).get('datatype', None)
858 if dt == 'rating':
859 if rating_query is not None:
860 for val, book_ids in self.field_iter(location, current_candidates):
861 if val == rating_query:
862 matches |= book_ids
863 continue
864
865 if dt == 'float':
866 if float_query is not None:
867 for val, book_ids in self.field_iter(location, current_candidates):
868 if val == float_query:
869 matches |= book_ids
870 continue
871
872 if dt == 'int':
873 if int_query is not None:
874 for val, book_ids in self.field_iter(location, current_candidates):
875 if val == int_query:
876 matches |= book_ids
877 continue
878
879 if location in text_fields:
880 for val, book_ids in self.field_iter(location, current_candidates):
881 if val is not None:
882 if isinstance(val, (str, bytes)):
883 val = (val,)
884 if _match(q, val, matchkind, use_primary_find_in_search=upf, case_sensitive=case_sensitive):
885 matches |= book_ids
886
887 if location == 'series_sort':
888 book_lang_map = self.dbcache.fields['languages'].book_value_map
889 for val, book_ids in self.dbcache.fields['series'].iter_searchable_values_for_sort(current_candidates, book_lang_map):
890 if val is not None:
891 if _match(q, (val,), matchkind, use_primary_find_in_search=upf, case_sensitive=case_sensitive):
892 matches |= book_ids
893
894 return matches
895
896 def get_user_category_matches(self, location, query, candidates):
897 matches = set()
898 if len(query) < 2:
899 return matches
900
901 user_cats = self.dbcache._pref('user_categories')
902 c = set(candidates)
903
904 if query.startswith('.'):
905 check_subcats = True
906 query = query[1:]
907 else:
908 check_subcats = False
909
910 for key in user_cats:
911 if key == location or (check_subcats and key.startswith(location + '.')):
912 for item, category, ign in user_cats[key]:
913 s = self.get_matches(category, '=' + item, candidates=c)
914 c -= s
915 matches |= s
916 if query == 'false':
917 return candidates - matches
918 return matches
919
920
921 # }}}
922
923
924 class LRUCache: # {{{
925 "A simple Least-Recently-Used cache"
926
927 def __init__(self, limit=50):
928 self.item_map = {}
929 self.age_map = deque()
930 self.limit = limit
931
932 def _move_up(self, key):
933 if key != self.age_map[-1]:
934 self.age_map.remove(key)
935 self.age_map.append(key)
936
937 def add(self, key, val):
938 if key in self.item_map:
939 self._move_up(key)
940 return
941
942 if len(self.age_map) >= self.limit:
943 self.item_map.pop(self.age_map.popleft())
944
945 self.item_map[key] = val
946 self.age_map.append(key)
947
948 __setitem__ = add
949
950 def get(self, key, default=None):
951 ans = self.item_map.get(key, default)
952 if ans is not default:
953 self._move_up(key)
954 return ans
955
956 def clear(self):
957 self.item_map.clear()
958 self.age_map.clear()
959
960 def pop(self, key, default=None):
961 self.item_map.pop(key, default)
962 try:
963 self.age_map.remove(key)
964 except ValueError:
965 pass
966
967 def __contains__(self, key):
968 return key in self.item_map
969
970 def __len__(self):
971 return len(self.age_map)
972
973 def __getitem__(self, key):
974 return self.get(key)
975
976 def __iter__(self):
977 yield from self.item_map.items()
978
979
980 # }}}
981
982
983 class Search:
984 MAX_CACHE_UPDATE = 50
985
986 def __init__(self, db, opt_name, all_search_locations=()):
987 self.all_search_locations = all_search_locations
988 self.date_search = DateSearch()
989 self.num_search = NumericSearch()
990 self.bool_search = BooleanSearch()
991 self.keypair_search = KeyPairSearch()
992 self.saved_searches = SavedSearchQueries(db, opt_name)
993 self.cache = LRUCache()
994 self.parse_cache = LRUCache(limit=100)
995
996 def get_saved_searches(self):
997 return self.saved_searches
998
999 def change_locations(self, newlocs):
1000 if frozenset(newlocs) != frozenset(self.all_search_locations):
1001 self.clear_caches()
1002 self.parse_cache.clear()
1003 self.all_search_locations = newlocs
1004
1005 def update_or_clear(self, dbcache, book_ids=None):
1006 if book_ids and (len(book_ids) * len(self.cache)) <= self.MAX_CACHE_UPDATE:
1007 self.update_caches(dbcache, book_ids)
1008 else:
1009 self.clear_caches()
1010
1011 def clear_caches(self):
1012 self.cache.clear()
1013
1014 def update_caches(self, dbcache, book_ids):
1015 sqp = self.create_parser(dbcache)
1016 try:
1017 return self._update_caches(sqp, book_ids)
1018 finally:
1019 sqp.dbcache = sqp.lookup_saved_search = None
1020
1021 def discard_books(self, book_ids):
1022 book_ids = set(book_ids)
1023 for query, result in self.cache:
1024 result.difference_update(book_ids)
1025
1026 def _update_caches(self, sqp, book_ids):
1027 book_ids = sqp.all_book_ids = set(book_ids)
1028 remove = set()
1029 for query, result in tuple(self.cache):
1030 try:
1031 matches = sqp.parse(query)
1032 except ParseException:
1033 remove.add(query)
1034 else:
1035 # remove books that no longer match
1036 result.difference_update(book_ids - matches)
1037 # add books that now match but did not before
1038 result.update(matches)
1039 for query in remove:
1040 self.cache.pop(query)
1041
1042 def create_parser(self, dbcache, virtual_fields=None, allow_templates=True):
1043 return Parser(
1044 dbcache,
1045 set(),
1046 dbcache._pref('grouped_search_terms'),
1047 self.date_search,
1048 self.num_search,
1049 self.bool_search,
1050 self.keypair_search,
1051 prefs['limit_search_columns'],
1052 prefs['limit_search_columns_to'],
1053 self.all_search_locations,
1054 virtual_fields,
1055 self.saved_searches.lookup,
1056 self.parse_cache,
1057 allow_templates=allow_templates,
1058 )
1059
1060 def __call__(self, dbcache, query, search_restriction, virtual_fields=None, book_ids=None, allow_templates=True):
1061 """
1062 Return the set of ids of all records that match the specified
1063 query and restriction
1064 """
1065 # We construct a new parser instance per search as the parse is not
1066 # thread safe.
1067 sqp = self.create_parser(dbcache, virtual_fields, allow_templates)
1068 try:
1069 return self._do_search(sqp, query, search_restriction, dbcache, book_ids=book_ids)
1070 finally:
1071 sqp.dbcache = sqp.lookup_saved_search = None
1072
1073 def query_is_cacheable(self, sqp, dbcache, query):
1074 if query:
1075 for name, value in sqp.get_queried_fields(query):
1076 if name == 'template':
1077 return False
1078 elif name in dbcache.field_metadata.all_field_keys():
1079 fm = dbcache.field_metadata[name]
1080 if fm['datatype'] == 'datetime':
1081 return False
1082 if fm['datatype'] == 'composite':
1083 if fm.get('display', {}).get('composite_sort', '') == 'date':
1084 return False
1085 return True
1086
1087 def _do_search(self, sqp, query, search_restriction, dbcache, book_ids=None):
1088 """Do the search, caching the results. Results are cached only if the
1089 search is on the full library and no virtual field is searched on"""
1090 if isinstance(search_restriction, bytes):
1091 search_restriction = search_restriction.decode('utf-8')
1092 if isinstance(query, bytes):
1093 query = query.decode('utf-8')
1094
1095 query = query.strip()
1096 use_cache = self.query_is_cacheable(sqp, dbcache, query)
1097
1098 if use_cache and book_ids is None and query and not search_restriction:
1099 cached = self.cache.get(query)
1100 if cached is not None:
1101 return cached
1102
1103 restricted_ids = all_book_ids = dbcache._all_book_ids(type=set)
1104 if search_restriction and search_restriction.strip():
1105 sr = search_restriction.strip()
1106 sqp.all_book_ids = all_book_ids if book_ids is None else book_ids
1107 if self.query_is_cacheable(sqp, dbcache, sr):
1108 cached = self.cache.get(sr)
1109 if cached is None:
1110 restricted_ids = sqp.parse(sr)
1111 if not sqp.virtual_field_used and sqp.all_book_ids is all_book_ids:
1112 self.cache.add(sr, restricted_ids)
1113 else:
1114 restricted_ids = cached
1115 if book_ids is not None:
1116 restricted_ids = book_ids.intersection(restricted_ids)
1117 else:
1118 restricted_ids = sqp.parse(sr)
1119 elif book_ids is not None:
1120 restricted_ids = book_ids
1121
1122 if not query:
1123 return restricted_ids
1124
1125 if use_cache and restricted_ids is all_book_ids:
1126 cached = self.cache.get(query)
1127 if cached is not None:
1128 return cached
1129
1130 sqp.all_book_ids = restricted_ids
1131 result = sqp.parse(query)
1132
1133 if not sqp.virtual_field_used and sqp.all_book_ids is all_book_ids:
1134 self.cache.add(query, result)
1135
1136 return result