Debian Codegraph @

src/pyj/read_book/search_worker.pyj

raw
1 # vim:fileencoding=utf-8
2 # License: GPL v3 Copyright: 2021, Kovid Goyal <kovid at kovidgoyal.net>
3 from __python__ import bound_methods, hash_literals
4
5 from traceback import format_exception
6
7 from read_book.db import DB
8 from read_book.resources import text_from_serialized_html
9
10 GET_SPINE_FAILED = 1
11 CONNECT_FAILED = 2
12 UNHANDLED_ERROR = 3
13 DB_ERROR = 4
14
15 _RE_ESCAPE = /[\/\\^$*+?.()|[\]{}]/g
16 quote_map = {'"': '"“”', "'": "'‘’"}
17 qpat = /(['"])/g
18 spat = /(\s+)/g
19 invisible_chars = '(?:[\u00ad\u200c\u200d]{0,1})'
20 # Matches each "atom" in an escaped string: an escape sequence (\X) or a
21 # single Unicode code point. The u flag ensures surrogate pairs are kept
22 # together as one code point.
23 _ESCAPED_ATOM = /\\[\s\S]|[\s\S]/gu
24
25
26 def escape(string):
27 return string.replace(_RE_ESCAPE, '\\$&')
28
29
30 def split_string(pat, string):
31 pat.lastIndex = 0
32 return string.split(pat)
33
34
35 def text_to_regex(text):
36 if text and not text.strip():
37 return r'\s+'
38 has_leading = text.lstrip() is not text
39 has_trailing = text.rstrip() is not text
40 ans = []
41 if has_leading:
42 ans.push(r'\s+')
43 for wpart in split_string(spat, text.strip()):
44 if not wpart.strip():
45 ans.push(r'\s+')
46 else:
47 for part in split_string(qpat, wpart):
48 r = quote_map[part]
49 if r:
50 ans.push('[' + r + ']')
51 elif part:
52 atoms = escape(part).match(_ESCAPED_ATOM)
53 if atoms:
54 ans.push(atoms.join(invisible_chars))
55 if has_trailing:
56 ans.push(r'\s+')
57 return ans.join('')
58
59
60 class ToCOffsetMap:
61 def __init__(self, toc_nodes, offset_map, previous_toc_node, parent_map):
62 self.toc_nodes = toc_nodes or []
63 self.offset_map = offset_map or {}
64 self.previous_toc_node = previous_toc_node or None
65 self.parent_map = parent_map or {}
66
67 def toc_nodes_for_offset(self, offset):
68 matches = []
69 for node in self.toc_nodes:
70 q = self.offset_map[node.id]
71 if q?:
72 if q > offset:
73 break
74 matches.push(node)
75 if not matches and self.previous_toc_node:
76 matches.push(self.previous_toc_node)
77 ans = []
78 if matches.length:
79 ancestors = []
80 node = matches[-1]
81 parent = self.parent_map[node.id]
82 while parent?:
83 ancestors.push(parent)
84 parent = self.parent_map[parent.id]
85 if ancestors.length > 1:
86 # last item in ancestors is the root node
87 for v'var i = ancestors.length - 1; i-- > 0;':
88 ans.push(ancestors[i].id)
89 ans.push(node.id)
90 return ans
91
92
93 def toc_offset_map_for_name(name):
94 if wc.toc_offset_map_cache[name]:
95 return wc.toc_offset_map_cache[name]
96 anchor_map = wc.text_cache[name][1]
97 toc_data = wc.current_book.toc_data
98 idx = toc_data.spine_idx_map[name]
99 toc_nodes = toc_data.spine_toc_map[name]
100 if not idx? or idx < 0:
101 wc.toc_offset_map_cache[name] = ToCOffsetMap()
102 return wc.toc_offset_map_cache[name]
103 offset_map = {}
104 for node in toc_nodes:
105 node_id = node.id
106 if node_id?:
107 aid = node.frag
108 offset = anchor_map[aid] or 0
109 offset_map[node_id] = offset
110 prev_toc_node = None
111 for v'var i = idx; i-- > 0;':
112 spine_name = wc.current_book.spine[i]
113 ptn = toc_data.spine_toc_map[spine_name]
114 if ptn?:
115 prev_toc_node = ptn[-1]
116 break
117 wc.toc_offset_map_cache[name] = ToCOffsetMap(toc_nodes, offset_map, prev_toc_node, toc_data.parent_map)
118 return wc.toc_offset_map_cache[name]
119
120
121 def toc_nodes_for_search_result(sr):
122 sidx = sr.spine_idx
123 name = wc.current_book.spine[sidx]
124 if not name:
125 return []
126 tmap = toc_offset_map_for_name(name)
127 return tmap.toc_nodes_for_offset(sr.offset)
128
129
130 class Worker:
131 def __init__(self):
132 self.db = None
133 self.connected_to_db = False
134 self.pending_search = None
135 self.searching = False
136 self.current_query = None
137 self.current_query_id = None
138 self.current_book = None
139 self.regex = None
140 self.result_num = 0
141 self.clear_caches()
142
143 @property
144 def initialize_error_msg(self):
145 return self.db?.initialize_error_msg
146
147 def clear_caches(self):
148 self.text_cache = {}
149 self.toc_offset_map_cache = {}
150
151
152 wc = Worker()
153
154
155 def send_search_complete():
156 self.postMessage({'type': 'search_complete', 'id': wc.current_query_id})
157 wc.current_query = wc.current_query_id = None
158
159
160 def search_in_text_of(name):
161 ctx_size = 75
162 r = wc.regex
163 r.lastIndex = 0
164 haystack = wc.text_cache[name][0] or ''
165 match_counts = {}
166 spine_idx = wc.current_book.spine.indexOf(name)
167 while True:
168 m = r.exec(haystack)
169 if not m:
170 break
171 text = m[0]
172 start, end = m.index, r.lastIndex
173 before = haystack[Math.max(0, start - ctx_size):start]
174 after = haystack[end:end + ctx_size]
175 q = (before or '')[-15:] + text + (after or '')[:15]
176 match_counts[q] = match_counts[q] or 0
177 wc.result_num += 1
178 result = {
179 'file_name': name, 'spine_idx': spine_idx, 'index': match_counts[q], 'offset': start,
180 'text': text, 'before': before, 'after': after, 'mode': wc.current_query.query.mode,
181 'q': q, 'result_num': wc.result_num, 'on_discovery': wc.current_query_id,
182 }
183 result.toc_nodes = toc_nodes_for_search_result(result)
184 self.postMessage({'type': 'search_result', 'id': wc.current_query_id, 'result': result})
185 if wc.current_query.query.only_first_match:
186 break
187 match_counts[q] += 1
188
189
190 def queue_next_spine_item(spine_idx, allow_current_name):
191 spine_idx = spine_idx % wc.current_book.spine.length
192 name = wc.current_book.spine[spine_idx]
193 if wc.current_query.query.only_first_match and wc.result_num > 0:
194 send_search_complete()
195 return
196 if not name or (not allow_current_name and name is wc.current_query.current_name):
197 send_search_complete()
198 return
199 if wc.text_cache[name]:
200 search_in_text_of(name)
201 setTimeout(queue_next_spine_item.bind(None, spine_idx + 1), 0)
202 return
203 query = wc.current_query
204 wc.db.get_book_file(wc.current_book.book_hash, wc.current_book.stored_files, name, got_spine_item.bind(None, query.id, spine_idx))
205
206
207 def got_spine_item(query_id, spine_idx, result):
208 if query_id is not wc.current_query_id:
209 return
210 if result.ok:
211 name = wc.current_book.spine[spine_idx]
212 data = result.result
213 if v'data instanceof Blob': # happens for titlepage which is stored as html for some reason
214 queue_next_spine_item(spine_idx + 1)
215 return
216 wc.text_cache[name] = text_from_serialized_html(data, True)
217 search_in_text_of(name)
218 setTimeout(queue_next_spine_item.bind(None, spine_idx + 1), 0)
219 else:
220 if result.details is 'ENOENT':
221 queue_next_spine_item(spine_idx + 1)
222 else:
223 send_error(GET_SPINE_FAILED, result.details)
224 wc.current_query = wc.current_query_id = None
225
226
227 def regex_for_query(query):
228 expr = query.text
229 flags = 'umg'
230 if not query.case_sensitive:
231 flags += 'i'
232 if query.mode is not 'regex':
233 if query.mode is 'word':
234 words = []
235 for part in expr.split(' '):
236 words.push(r'\b' + text_to_regex(part) + r'\b')
237 expr = words.join(r'\s+')
238 else:
239 expr = text_to_regex(expr)
240 return new RegExp(expr, flags)
241
242
243 def perform_search(query):
244 wc.current_query = query
245 wc.current_query_id = query.id
246 wc.result_num = 0
247 if not wc.current_book or not wc.current_book.spine?.length or not query.query.text:
248 send_search_complete()
249 return
250 wc.regex = regex_for_query(query.query)
251
252 idx = wc.current_book.spine.indexOf(query.current_name)
253 if idx < 0:
254 idx = 0
255 # Ensure current_name is always a valid spine item name so that the search
256 # terminates after going through all spine items exactly once (the search
257 # loop ends when it wraps back around to current_name).
258 wc.current_query.current_name = wc.current_book.spine[idx]
259 queue_next_spine_item(idx, True)
260
261
262 def worker_connection_done():
263 wc.connected_to_db = True
264 if not wc.initialize_error_msg:
265 if wc.pending_search:
266 s = wc.pending_search
267 wc.pending_search = None
268 perform_search(s)
269
270
271 def send_error(code, msg, error):
272 self.postMessage({'type': 'error', 'code': code, 'msg': msg, 'id': wc.current_query_id, 'error': error})
273
274
275 def on_worker_db_error(title, msg, details):
276 send_error(DB_ERROR, msg, {'title': title, 'msg': msg, 'details': details})
277
278
279 def worker_main():
280 wc.db = DB(worker_connection_done, on_worker_db_error)
281
282 self.onmessage = def (e):
283 if e.data.type is 'search':
284 wc.current_query_id = e.data.id
285 if wc.connected_to_db:
286 if wc.initialize_error_msg:
287 send_error(CONNECT_FAILED, wc.initialize_error_msg)
288 else:
289 perform_search(e.data)
290 else:
291 wc.pending_search = e.data
292 elif e.data.type is 'clear_caches':
293 wc.clear_caches()
294 wc.current_book = e.data.book
295
296 self.addEventListener('error', def (e):
297 send_error(UNHANDLED_ERROR, f'{e.lineno}:{e.message}' + '\n\n' + format_exception(e.error).join(''))
298 )