Debian Codegraph @

src/calibre/scraper/qt.py

raw
1 #!/usr/bin/env python
2 # License: GPLv3 Copyright: 2024, Kovid Goyal <kovid at kovidgoyal.net>
3
4 import atexit
5 import json
6 import os
7 import shutil
8 import subprocess
9 import time
10 import weakref
11 from contextlib import suppress
12 from io import BytesIO
13 from queue import Queue
14 from threading import RLock, Thread
15 from urllib.error import URLError
16 from urllib.parse import urlencode
17 from urllib.request import Request
18
19 from calibre.ptempfile import PersistentTemporaryDirectory
20
21
22 class FakeResponse:
23 def __init__(self):
24 self.queue = Queue()
25 self.done = False
26 self.final_url = ''
27 self._reason = ''
28 self._status = None
29 self._headers = []
30 self._data = BytesIO()
31
32 def _wait(self):
33 if self.done:
34 return
35 self.done = True
36 res = self.queue.get()
37 del self.queue
38 if res['action'] == 'input_error':
39 raise Exception(res['error'])
40 self.final_url = res['final_url']
41 self._status = res.get('http_code')
42 self._reason = res.get('http_status_message')
43 if not self._reason:
44 from http.client import responses
45
46 with suppress(KeyError):
47 self._reason = responses[self._status]
48 self._headers = res['headers']
49 if 'error' in res:
50 ex = URLError(res['error'])
51 setattr(ex, 'worth_retry', bool(res.get('worth_retry')))
52 raise ex
53 with suppress(FileNotFoundError):
54 self._data = open(res['output'], 'rb')
55
56 def read(self, *a, **kw):
57 self._wait()
58 ans = self._data.read(*a, **kw)
59 return ans
60
61 def seek(self, *a, **kw):
62 self._wait()
63 return self._data.seek(*a, **kw)
64
65 def tell(self, *a, **kw):
66 return self._data.tell(*a, **kw)
67
68 @property
69 def url(self) -> str:
70 self._wait()
71 return self.final_url
72
73 @property
74 def status(self) -> int | None:
75 self._wait()
76 return self._status
77
78 code = status
79
80 @property
81 def headers(self):
82 self._wait()
83 from email.message import EmailMessage
84
85 ans = EmailMessage()
86 for k, v in self._headers:
87 ans[k] = v
88 return ans
89
90 @property
91 def reason(self) -> str:
92 self._wait()
93 return self._reason or ''
94
95 def getcode(self) -> int | None:
96 return self.status
97
98 def geturl(self):
99 return self.url
100
101 def getinfo(self):
102 return self.headers
103
104 def close(self):
105 self._data.close()
106
107 def __enter__(self):
108 return self
109
110 def __exit__(self, *a):
111 self._data.close()
112
113
114 def shutdown_browser(bref):
115 br = bref()
116 if br is not None:
117 br.shutdown()
118
119
120 class Browser:
121 def __init__(
122 self,
123 user_agent: str = '',
124 headers: tuple[tuple[str, str], ...] = (),
125 verify_ssl_certificates: bool = True,
126 start_worker: bool = False,
127 ):
128 self.tdir = ''
129 self.worker = self.dispatcher = None
130 self.dispatch_map = {}
131 self.verify_ssl_certificates = verify_ssl_certificates
132 self.id_counter = 0
133 self.addheaders: list[tuple[str, str]] = list(headers)
134 self.user_agent = user_agent
135 self.lock = RLock()
136 self.shutting_down = False
137 atexit.register(shutdown_browser, weakref.ref(self))
138 if start_worker:
139 self._ensure_state()
140
141 def _open(self, url_or_request: Request | str, data=None, timeout=None, visit: bool = True):
142 method = 'POST' if data else 'GET'
143 headers = []
144 if hasattr(url_or_request, 'get_method'):
145 assert isinstance(url_or_request, Request)
146 r = url_or_request
147 method = r.get_method()
148 data = data or r.data
149 headers = r.header_items()
150 url = r.full_url
151 else:
152 url = url_or_request
153
154 def has_header(x: str) -> bool:
155 x = x.lower()
156 for h, v in headers:
157 if h.lower() == x:
158 return True
159 return False
160
161 if isinstance(data, dict):
162 headers.append(('Content-Type', 'application/x-www-form-urlencoded'))
163 data = urlencode(data)
164 if isinstance(data, str):
165 data = data.encode('utf-8')
166 if not has_header('Content-Type'):
167 headers.append(('Content-Type', 'text/plain'))
168
169 if not self.is_method_ok(method):
170 raise KeyError(f'The HTTP {method} request method is not supported')
171
172 with self.lock:
173 self._ensure_state()
174 self.id_counter += 1
175 cmd = {
176 'action': 'download',
177 'id': self.id_counter,
178 'url': url,
179 'method': method,
180 'timeout': timeout,
181 'headers': self.addheaders + headers,
182 'visit': visit,
183 }
184 if data:
185 with open(os.path.join(self.tdir, f'i{self.id_counter}'), 'wb') as f:
186 if hasattr(data, 'read'):
187 shutil.copyfileobj(data, f) # type: ignore
188 else:
189 f.write(data) # type: ignore
190 cmd['data_path'] = f.name
191 for k, v in cmd['headers']:
192 if k.lower() == 'content-type':
193 break
194 else:
195 cmd['headers'].append(('Content-Type', 'application/x-www-form-urlencoded'))
196 res = FakeResponse()
197 self.dispatch_map[self.id_counter] = res.queue
198 self._send_command(cmd)
199 return res
200
201 def open(self, url_or_request: Request | str, data=None, timeout=None):
202 return self._open(url_or_request, data, timeout)
203
204 def open_novisit(self, url_or_request: Request, data=None, timeout=None):
205 return self._open(url_or_request, data, timeout, visit=False)
206
207 def is_method_ok(self, method: str) -> bool:
208 return True
209
210 def set_simple_cookie(self, name: str, value: str, domain: str | None = None, path: str | None = '/'):
211 """
212 Set a simple cookie using a name and value. If domain is specified, the cookie is only sent with requests
213 to matching domains, otherwise it is sent with all requests. The leading dot in domain is optional.
214 Similarly, by default all paths match, to restrict to certain path use the path parameter.
215 """
216 c = {'name': name, 'value': value, 'domain': domain, 'path': path}
217 self._send_command({'action': 'set_cookies', 'cookies': [c]})
218
219 set_cookie = set_simple_cookie
220
221 def set_user_agent(self, val: str = '') -> None:
222 self.user_agent = val
223 self._send_command({'action': 'set_user_agent', 'user_agent': val})
224
225 def clone_browser(self):
226 return self
227
228 def _send_command(self, cmd):
229 with self.lock:
230 self._ensure_state()
231 _worker = self.worker
232 assert _worker is not None
233 _w_stdin = _worker.stdin
234 assert _w_stdin is not None
235 _w_stdin.write(json.dumps(cmd).encode())
236 _w_stdin.write(b'\n')
237 _w_stdin.flush()
238
239 def _ensure_state(self):
240 with self.lock:
241 if not self.tdir:
242 self.tdir = PersistentTemporaryDirectory()
243 self.worker = self.run_worker()
244 self.dispatcher = Thread(target=self._dispatch, daemon=True)
245 self.dispatcher.start()
246
247 def run_worker(self) -> subprocess.Popen:
248 return run_worker(self.tdir, self.user_agent, self.verify_ssl_certificates)
249
250 def _dispatch(self):
251 try:
252 _dispatch_worker = self.worker
253 assert _dispatch_worker is not None
254 _dispatch_stdout = _dispatch_worker.stdout
255 assert _dispatch_stdout is not None
256 for line in _dispatch_stdout:
257 cmd = json.loads(line)
258 if cmd.get('action') == 'finished':
259 with self.lock:
260 q = self.dispatch_map.pop(cmd['id'])
261 q.put(cmd)
262 else:
263 raise Exception(f'Unexpected response from backend fetch worker process: {cmd}')
264 except Exception:
265 if not self.shutting_down:
266 import traceback
267
268 traceback.print_exc()
269
270 def shutdown(self):
271 self.shutting_down = True
272 if self.worker:
273 w, self.worker = self.worker, None
274 with suppress(OSError):
275 _w_stdin_s = w.stdin
276 assert _w_stdin_s is not None
277 _w_stdin_s.close()
278 with suppress(OSError):
279 _w_stdout_s = w.stdout
280 assert _w_stdout_s is not None
281 _w_stdout_s.close()
282 give_up_at = time.monotonic() + 1.5
283 while time.monotonic() < give_up_at and w.poll() is None:
284 time.sleep(0.01)
285 if w.poll() is None:
286 w.kill()
287 if self.tdir:
288 with suppress(OSError):
289 shutil.rmtree(self.tdir)
290 self.tdir = ''
291 if self.dispatcher:
292 self.dispatcher.join()
293 self.dispatcher = None
294
295 def __del__(self):
296 self.shutdown()
297
298
299 class WebEngineBrowser(Browser):
300 def run_worker(self) -> subprocess.Popen:
301 return run_worker(self.tdir, self.user_agent, self.verify_ssl_certificates, function='webengine_worker')
302
303
304 def run_worker(tdir: str, user_agent: str, verify_ssl_certificates: bool, function: str = 'worker'):
305 from calibre.utils.ipc.simple_worker import start_pipe_worker
306
307 return start_pipe_worker(f'from calibre.scraper.qt import {function}; {function}({tdir!r}, {user_agent!r}, {verify_ssl_certificates!r})')
308
309
310 def worker(*args):
311 from calibre.gui2 import must_use_qt
312
313 must_use_qt()
314 from .qt_backend import worker
315
316 worker(*args)
317
318
319 def webengine_worker(*args):
320 from calibre.gui2 import must_use_qt
321
322 must_use_qt()
323 from .webengine_backend import worker
324
325 worker(*args)
326
327
328 def develop():
329 import sys
330
331 br = Browser()
332 try:
333 for url in sys.argv[1:]:
334 res = br.open(url)
335 print(url, len(res.read()))
336 finally:
337 del br
338
339
340 if __name__ == '__main__':
341 develop()