Debian Codegraph @

src/pyj/book_list/fts.pyj

raw
1 # vim:fileencoding=utf-8
2 # License: GPL v3 Copyright: 2022, Kovid Goyal <kovid at kovidgoyal.net>
3 from __python__ import bound_methods, hash_literals
4
5 from elementmaker import E
6 from gettext import gettext as _
7
8 from ajax import ajax, ajax_send
9 from book_list.cover_grid import THUMBNAIL_MAX_HEIGHT, THUMBNAIL_MAX_WIDTH
10 from book_list.globals import get_current_query, get_session_data
11 from book_list.library_data import current_library_id, download_url, library_data
12 from book_list.router import home, open_book_url, push_state
13 from book_list.top_bar import create_top_bar
14 from book_list.ui import query_as_href, set_panel_handler
15 from book_list.views import create_image
16 from complete import create_search_bar
17 from dom import add_extra_css, clear, set_css, svgicon
18 from modals import create_custom_dialog, error_dialog, info_dialog, question_dialog
19 from widgets import create_spinner
20
21 overall_container_id = ''
22 current_fts_query = {}
23 query_id_counter = 0
24
25 add_extra_css(def ():
26 sel = '.fts-help-display '
27 style = f'{sel} ' + '{ padding-top: 0.5ex }\n'
28 style += f'{sel} div' + ' { margin-top: 0.5ex }\n'
29 style += f'{sel} .h' + ' { font-weight: bold; padding-bottom: 0.25ex }\n'
30 style += f'{sel} .bq' + ' { margin-left: 1em; margin-top: 0.5ex; margin-bottom: 0.5ex; font-style: normal; font-weight: 700 }\n'
31 style += f'{sel} p' + ' { margin: 0}\n'
32 style += '.fts-highlight-match { font-style: normal; font-weight: bold }\n'
33 return style
34 )
35
36
37 def component(name):
38 c = document.getElementById(overall_container_id)
39 if c:
40 return c.querySelector(f'[data-component="{name}"]')
41
42
43 def showing_search_panel():
44 c = component('search')
45 return bool(c and c.style.display is 'block')
46
47
48 def make_new_fts_query(q):
49 nonlocal current_fts_query, query_id_counter
50 query_id_counter += 1
51 current_fts_query = {'query_id': query_id_counter + ''}
52 if current_library_id():
53 current_fts_query.library_id = current_library_id()
54 Object.assign(current_fts_query, q)
55 xhr = ajax('fts/search', on_initial_fts_fetched, query=current_fts_query, bypass_cache=True)
56 xhr.send()
57
58
59 def enable_indexing():
60 def on_response(end_type, xhr, ev):
61 if end_type is 'abort' or not showing_search_panel():
62 return
63 if end_type is not 'load':
64 if xhr.status is 403:
65 return error_dialog(_('Permission denied'), _(
66 'You do not have permission to enable indexing. Only logged in users with write permission are allowed.'), xhr.error_html)
67 return error_dialog(_('Failed to enable indexing'), _('Enabling indexing failed. Click "Show details" for more information.'), xhr.error_html)
68 library_data.fts_enabled = True
69 info_dialog(
70 _('Indexing enabled'),
71 _('Indexing of this library has been enabled. Depending on library size it can take a long time to index all books.')
72 )
73 ajax_send(f'fts/indexing', True, on_response)
74
75
76 def report_fts_not_enabled():
77 question_dialog(_('Full text searching disabled'), _(
78 'Full text search indexing has not been enabled for this library. Once enabled, indexing'
79 ' will take some time to complete after which searching will work. Enable indexing?'),
80 def (yes):
81 if yes:
82 enable_indexing()
83 )
84
85
86 def on_initial_fts_fetched(end_type, xhr, ev):
87 if end_type is 'abort' or not showing_search_panel():
88 return
89 if end_type is not 'load':
90 clear_to_waiting_for_results(_('No matches found'))
91 if xhr.status is 428:
92 return report_fts_not_enabled()
93 return error_dialog(_('Failed to search'), _('The search failed. Click "Show details" for more information.'), xhr.error_html)
94 try:
95 results = JSON.parse(xhr.responseText)
96 except Exception as err:
97 return error_dialog(_('Server error'), _('Failed to parse search response from server.'), err + '')
98 if results.query_id + '' is not current_fts_query.query_id:
99 return
100 current_fts_query.results = results
101 show_initial_results()
102
103
104 def execute_search_interactive():
105 nonlocal current_fts_query
106 query = component('query').querySelector('input').value
107 if not query or query.length < 1:
108 error_dialog(_('No search query specified'), _('A search word/phrase must be specified before attempting to search'))
109 clear_to_help()
110 return
111 q = get_current_query()
112 q.fts_panel = 'search'
113 q.fts_query = query
114 q.fts_use_stemming = 'y' if component('related_words').checked else 'n'
115 if q.restricted is 'y':
116 q.restriction = component('restrict').querySelector('input').value
117 current_fts_query = {}
118 push_state(q, replace=True)
119
120
121 def build_search_page():
122 # search input container
123 container = component('search')
124 clear(container)
125 search_bar = create_search_bar(
126 execute_search_interactive,
127 'search-books-fts',
128 tooltip=_('Search for books'),
129 placeholder=_('Enter words to search for'),
130 button='icon-button'
131 )
132 set_css(search_bar, flex_grow='10', margin_right='0.5em')
133 search_bar.dataset.component = 'query'
134 container.appendChild(search_bar)
135 sd = get_session_data()
136 related_words = E.label(E.input(
137 type='checkbox', data_component='related_words', checked=bool(sd.get('fts_related_words'))),
138 onchange=def ():
139 sd = get_session_data()
140 rw = bool(component('related_words').checked)
141 if rw is not sd.get('fts_related_words'):
142 get_session_data().set('fts_related_words', rw)
143 , ' ' + _('Match on related words'),
144 title = _(
145 'With this option searching for words will also match on any related words (supported in several languages). For'
146 ' example, in the English language: {0} matches {1} and {2} as well').format(
147 'correction', 'correcting', 'corrected')
148 )
149 container.appendChild(E.div(style='padding-top: 1ex;', related_words))
150
151 restrict_bar = create_search_bar(
152 execute_search_interactive,
153 'search-books-fts-restrict',
154 tooltip=_('Restrict the results to only books matching this query'),
155 placeholder=_('Restrict to books matching...')
156 )
157 set_css(restrict_bar, flex_grow='10', margin_right='0.5em')
158 restrict_bar.dataset.component = 'restrict'
159 container.appendChild(E.div(style='padding-top: 1ex; display: flex; width: 100%; align-items: center', svgicon('search'), E.span('\xa0'), restrict_bar))
160
161 # Search help
162 container.appendChild(E.div(style='border-top: 1px currentColor solid; margin-top: 1ex', data_component='results'))
163
164
165 def clear_to_waiting_for_results(msg):
166 container = component('results')
167 if not container:
168 return
169 clear(container)
170 msg = msg or (_('Searching, please wait') + '…')
171 container.appendChild(E.div(
172 style='margin: auto; width: 100%; text-align: center; margin-top: 4ex',
173 create_spinner(), '\xa0' + msg
174 ))
175
176
177 def toggle_restrict():
178 container = component('restrict')
179 if not container:
180 return
181 q = get_current_query()
182 q.restricted = 'n' if q.restricted is 'y' else 'y'
183 push_state(q, replace=True)
184
185
186 def clear_to_help():
187 container = component('results')
188 if not container:
189 return
190 clear(container)
191 container.appendChild(E.div(class_='fts-help-display'))
192 restrict = get_current_query().restricted is 'y'
193 container.appendChild(E.div(
194 style='margin-top: 1ex',
195 E.a(_('Re-index all books in this library'), class_='blue-link', href='javascript:void(0)', onclick=reindex_all),
196 E.span('\xa0\xa0'),
197 E.a(_('Disable full text search'), class_='blue-link', href='javascript:void(0)', onclick=disable_fts),
198 E.span('\xa0\xa0'),
199 E.a(
200 _('Search all books') if restrict else _('Search a subset of books'),
201 class_='blue-link', href='javascript:void(0)', onclick=toggle_restrict
202 ),
203 ))
204 container = container.firstChild
205 fts_url = 'https://www.sqlite.org/fts5.html#full_text_query_syntax'
206 html = _('''
207 <div class="h">Search for single words</div>
208 <p>Simply type the word:</p>
209 <div class="bq">awesome<br>calibre</div>
210
211 <div class="h">Search for phrases</div>
212 <p>Enclose the phrase in quotes:</p>
213 <div class="bq">"early run"<br>"song of love"</div>
214
215 <div class="h">Boolean searches</div>
216 <div class="bq">(calibre AND ebook) NOT gun<br>simple NOT ("high bar" OR hard)</div>
217
218 <div class="h">Phrases near each other</div>
219 <div class="bq">NEAR("people" "in Asia" "try")<br>NEAR("Kovid" "calibre", 30)</div>
220 <p>Here, 30 is the most words allowed between near groups. Defaults to 10 when unspecified.</p>
221
222 <div style="margin-top: 1em"><a href="{fts_url}">Complete syntax reference</a></div>\
223 ''').format(fts_url=fts_url)
224 container.innerHTML = html
225 a = container.querySelector('a[href]')
226 a.setAttribute('target', '_new')
227 a.classList.add('blue-link')
228
229
230 def apply_search_panel_state():
231 q = get_current_query()
232 ftsq = {'query': q.fts_query or '', 'use_stemming': q.fts_use_stemming or 'y', 'restriction': q.restriction if q.restricted is 'y' else ''}
233 component('query').querySelector('input').value = ftsq.query
234 component('related_words').checked = ftsq.use_stemming is 'y'
235 r = component('restrict')
236 r.parentNode.style.display = 'flex' if ftsq.restriction is not '' else 'none'
237 r.querySelector('input').value = q.restriction or ''
238 if not ftsq.query:
239 clear_to_help()
240 return
241 if (current_fts_query.query is not ftsq.query or current_fts_query.use_stemming is not ftsq.use_stemming
242 or current_fts_query.restriction is not ftsq.restriction):
243 make_new_fts_query(ftsq)
244 clear_to_waiting_for_results(_('Searching for {}, please wait…').format(ftsq.query))
245 return
246 show_initial_results()
247
248
249 def open_format(book_id, fmt):
250 snippets = current_fts_query.results?.snippets
251 if snippets:
252 snippets = snippets[book_id]
253 text = ''
254 if snippets and fmt is not 'PDF':
255 for s in snippets:
256 if s.formats.indexOf(fmt) > -1:
257 text = s.text
258 break
259 url = open_book_url(book_id, fmt)
260 if fmt is 'PDF':
261 url = download_url(book_id, fmt, 'inline')
262 w = window.open(url, '_blank')
263 if text:
264 text = str.strip(text, '…').replaceAll('\x1c', '').replaceAll('\x1e', '')
265 w.read_book_initial_open_search_text = {'text': text, 'query': current_fts_query.query}
266
267
268 def reindex_book(book_id):
269 def on_response(end_type, xhr, ev):
270 if end_type is 'abort' or not showing_search_panel():
271 return
272 if end_type is not 'load':
273 if xhr.status is 403:
274 return error_dialog(_('Permission denied'), _(
275 'You do not have permission to re-index books. Only logged in users with write permission are allowed to re-index'), xhr.error_html)
276 return error_dialog(_('Failed to re-index'), _('Re-indexing the book failed. Click "Show details" for more information.'), xhr.error_html)
277 info_dialog(_('Re-indexing scheduled'), _('The book has been scheduled for re-indexing') if book_id else _(
278 'All books have been scheduled for re-indexing')
279 )
280
281 if book_id:
282 ajax_send(f'fts/reindex', {book_id: []}, on_response)
283 else:
284 ajax_send(f'fts/reindex', 'all', on_response)
285
286
287 def reindex_all():
288 question_dialog(_('Are you sure?'), _('Re-indexing all books in the library can take a long time. Are you sure you want to proceed?'),
289 def (yes):
290 if yes:
291 reindex_book()
292 )
293
294
295 def disable_fts_backend():
296 def on_response(end_type, xhr, ev):
297 if end_type is 'abort' or not showing_search_panel():
298 return
299 if end_type is not 'load':
300 if xhr.status is 403:
301 return error_dialog(_('Permission denied'), _(
302 'You do not have permission to disable FTS. Only logged in users with write permission are allowed to disable FTS.'), xhr.error_html)
303 return error_dialog(_('Disabling FTS failed'), _('Disabling FTS failed. Click "Show details" for more information.'), xhr.error_html)
304 library_data.fts_enabled = False
305 info_dialog(
306 _('Full text searching disabled'),
307 _('Full text searching for this library has been disabled. In the future the entire'
308 ' library will have to be re-indexed when re-enabling full text searching.'),
309 on_close=def ():
310 window.setTimeout(home)
311 )
312
313 ajax(f'fts/disable', on_response, bypass_cache=True).send()
314
315
316 def disable_fts():
317 question_dialog(
318 _('Are you sure?'),
319 _('Disabling full text search means that in the future the entire library will have to be'
320 ' re-indexed to use full text search. Are you sure you want to proceed?'),
321 def (yes):
322 if yes:
323 disable_fts_backend()
324 )
325
326
327 def book_result_tile_clicked(ev):
328 result_tile = ev.currentTarget
329 bid = int(result_tile.dataset.bookId)
330 results = current_fts_query.results
331 formats = []
332 for x in results.results:
333 if x.book_id is bid:
334 formats.push(x.format)
335 create_custom_dialog(result_tile.title, def (parent, close_modal):
336 fmc = E.div(
337 style='display: flex; flex-wrap: wrap; margin-top: 1ex'
338 )
339 def open_format_and_close(book_id, fmt):
340 open_format(book_id, fmt)
341 close_modal()
342 def reindex(book_id):
343 close_modal()
344 reindex_book(book_id)
345 for fmt in formats:
346 a = E.a(fmt, class_='blue-link', style='cursor: pointer; margin: 1ex; display: block',
347 href='javascript: void(0)', onclick=open_format_and_close.bind(None, bid, fmt))
348 fmc.appendChild(a)
349 parent.appendChild(E.div(
350 _('Click one of the formats below to open the book at this search result (except PDF which will open at the start):'),
351 fmc
352 ))
353 parent.appendChild(E.div(
354 style='margin-top: 1ex',
355 E.a(_('Open the details page for this book'), class_='blue-link', target='_blank', href=query_as_href({'book_id': str(bid)}, 'book_details'))
356 ))
357 parent.appendChild(E.div(
358 style='margin-top: 1ex',
359 E.a(_('Re-index this book'), class_='blue-link', href='javascript: void(0)', onclick=reindex.bind(None, bid))
360 ))
361 )
362
363
364 def book_result_tile(book_id, title, authors):
365 tile_height, img_max_width = '16ex', '12ex'
366 img = create_image(book_id, THUMBNAIL_MAX_WIDTH, THUMBNAIL_MAX_HEIGHT, def (): pass;)
367 img.style.maxHeight = tile_height
368 img.style.maxWidth = img_max_width
369 tooltip = (title) + ' ' + _('by') + ' ' + (authors)
370 img.alt = _('Cover of') + ' ' + tooltip
371 return E.div(
372 onclick=book_result_tile_clicked,
373 title=tooltip,
374 data_book_id=book_id + '', data_snippets_needed='1',
375 style=f'cursor: pointer; margin-bottom: 1ex; display:flex; height: {tile_height}; max-height: {tile_height}; width: 100%; align-items: stretch',
376 E.div(
377 style=f'margin-right: 1ex; width: {img_max_width}',
378 img
379 ),
380 E.div(
381 style=f'display:flex; flex-direction: column; height: 100%; overflow-y: auto',
382 E.div(E.span(style='font-size: small; font-style: normal; font-weight: 600; opacity: 0.5;', _('loading'), '…'), class_='snippets_container'),
383 )
384 )
385
386
387 def on_snippets_fetched(end_type, xhr, ev):
388 if end_type is 'abort' or not showing_search_panel():
389 return
390 if end_type is not 'load':
391 return error_dialog(_('Failed to search'), _('The search failed. Click "Show details" for more information.'), xhr.error_html)
392 container = component('results')
393 if not container:
394 return
395 try:
396 results = JSON.parse(xhr.responseText)
397 except Exception as err:
398 return error_dialog(_('Server error'), _('Failed to parse search response from server.'), err + '')
399 if results.query_id is not current_fts_query.query_id:
400 return
401 if not current_fts_query.results.snippets:
402 current_fts_query.results.snippets = {}
403 Object.assign(current_fts_query.results.snippets, results.snippets)
404 show_snippets(results.snippets)
405 fetch_snippets()
406
407
408 def render_text(parent, text):
409 in_highlighted = False
410 while text.length > 0:
411 q = '\x1e' if in_highlighted else '\x1c'
412 idx = text.indexOf(q)
413 if idx < 0:
414 idx = text.length
415 chunk = text[:idx]
416 text = text[idx + 1:]
417 if in_highlighted:
418 parent.append(E.span(class_='fts-highlight-match', chunk))
419 in_highlighted = False
420 else:
421 parent.append(E.span(chunk))
422 in_highlighted = True
423
424
425 def show_snippets(snippets):
426 container = component('results')
427 for book_id in Object.keys(snippets):
428 c = container.querySelector(f'[data-book-id="{book_id}"]')
429 if c:
430 v'delete c.dataset.snippetsNeeded'
431 s = c.querySelector('.snippets_container')
432 clear(s)
433 for x in snippets[book_id]:
434 f = ' '.join(x.formats)
435 e = E.div(E.code(
436 style='border: solid 1px currentColor; border-radius: 6px; padding: 0 4px; font-size: smaller',
437 data_formats=f, f)
438 )
439 e.appendChild(E.span(' '))
440 render_text(e, x.text)
441 s.appendChild(e)
442
443
444 def fetch_snippets():
445 container = component('results')
446 if not container:
447 return
448 ids = []
449 for x in container.querySelectorAll('[data-snippets-needed="1"]'):
450 book_id = int(x.dataset.bookId)
451 ids.push(book_id)
452 if ids.length > 1:
453 break
454 if ids.length < 1:
455 return
456 ids = ','.join(ids)
457 q = {}
458 Object.assign(q, current_fts_query)
459 q.results = v'undefined'
460 xhr = ajax(f'fts/snippets/{ids}', on_snippets_fetched, query=q, bypass_cache=True)
461 xhr.send()
462
463
464 def show_initial_results():
465 container = component('results')
466 if not container:
467 return
468 clear(container)
469 results = current_fts_query.results
470 left, total = results.indexing_status['left'], results.indexing_status['total']
471 if left > 0:
472 pc = int(((total - left) / total) * 100)
473 container.appendChild(E.div(
474 style='margin-top: 0.5ex',
475 E.span(_('WARNING:'), style='color: red; font-weight: bold'), '\xa0',
476 _('Indexing of library only {}% complete, search results may be incomplete.').format(pc)
477 ))
478 rc = E.div(style='margin-top: 0.5ex')
479 container.appendChild(rc)
480 mm = results.metadata
481 seen = {}
482 for r in results.results:
483 bid = r['book_id']
484 m = mm[bid]
485 if not seen[bid]:
486 rc.appendChild(book_result_tile(bid, m['title'], m['authors']))
487 seen[bid] = rc.lastChild
488 rc.appendChild(E.hr())
489 if results.results.length < 1:
490 rc.appendChild(E.div(_('No matches found')))
491 fetch_snippets()
492
493
494 def show_panel(visible):
495 c = component(visible)
496 if c:
497 x = c.parentNode.firstChild
498 while x:
499 if x.nodeType is 1 and x.dataset.component:
500 x.style.display = 'block' if x is c else 'none'
501 x = x.nextSibling
502
503
504 def show_search_panel():
505 show_panel('search')
506 apply_search_panel_state()
507
508
509 def show_index_panel():
510 show_panel('index')
511
512
513 def init(container_id):
514 nonlocal overall_container_id
515 overall_container_id = container_id
516 container = document.getElementById(container_id)
517 create_top_bar(container, title=_('Search text of books'), icon='back')
518 container.appendChild(E.div(data_component='index'))
519 container.appendChild(E.div(
520 data_component='search',
521 style='padding:1ex 1em; margin-top: 0.5ex;'
522 ))
523 build_search_page()
524 q = get_current_query()
525 if not q.fts_panel or q.fts_panel is 'search':
526 show_search_panel()
527 elif q.fts_panel is 'index':
528 show_index_panel()
529
530
531 def set_panel_handlers():
532 set_panel_handler('fts', init)