| 1 |
#!/usr/bin/env python |
| 2 |
# License: GPLv3 Copyright: 2024, Kovid Goyal <kovid at kovidgoyal.net> |
| 3 |
|
| 4 |
import atexit |
| 5 |
import json |
| 6 |
import os |
| 7 |
import shutil |
| 8 |
import subprocess |
| 9 |
import time |
| 10 |
import weakref |
| 11 |
from contextlib import suppress |
| 12 |
from io import BytesIO |
| 13 |
from queue import Queue |
| 14 |
from threading import RLock, Thread |
| 15 |
from urllib.error import URLError |
| 16 |
from urllib.parse import urlencode |
| 17 |
from urllib.request import Request |
| 18 |
|
| 19 |
from calibre.ptempfile import PersistentTemporaryDirectory |
| 20 |
|
| 21 |
|
| 22 |
class FakeResponse: |
| 23 |
def __init__(self): |
| 24 |
self.queue = Queue() |
| 25 |
self.done = False |
| 26 |
self.final_url = '' |
| 27 |
self._reason = '' |
| 28 |
self._status = None |
| 29 |
self._headers = [] |
| 30 |
self._data = BytesIO() |
| 31 |
|
| 32 |
def _wait(self): |
| 33 |
if self.done: |
| 34 |
return |
| 35 |
self.done = True |
| 36 |
res = self.queue.get() |
| 37 |
del self.queue |
| 38 |
if res['action'] == 'input_error': |
| 39 |
raise Exception(res['error']) |
| 40 |
self.final_url = res['final_url'] |
| 41 |
self._status = res.get('http_code') |
| 42 |
self._reason = res.get('http_status_message') |
| 43 |
if not self._reason: |
| 44 |
from http.client import responses |
| 45 |
|
| 46 |
with suppress(KeyError): |
| 47 |
self._reason = responses[self._status] |
| 48 |
self._headers = res['headers'] |
| 49 |
if 'error' in res: |
| 50 |
ex = URLError(res['error']) |
| 51 |
setattr(ex, 'worth_retry', bool(res.get('worth_retry'))) |
| 52 |
raise ex |
| 53 |
with suppress(FileNotFoundError): |
| 54 |
self._data = open(res['output'], 'rb') |
| 55 |
|
| 56 |
def read(self, *a, **kw): |
| 57 |
self._wait() |
| 58 |
ans = self._data.read(*a, **kw) |
| 59 |
return ans |
| 60 |
|
| 61 |
def seek(self, *a, **kw): |
| 62 |
self._wait() |
| 63 |
return self._data.seek(*a, **kw) |
| 64 |
|
| 65 |
def tell(self, *a, **kw): |
| 66 |
return self._data.tell(*a, **kw) |
| 67 |
|
| 68 |
@property |
| 69 |
def url(self) -> str: |
| 70 |
self._wait() |
| 71 |
return self.final_url |
| 72 |
|
| 73 |
@property |
| 74 |
def status(self) -> int | None: |
| 75 |
self._wait() |
| 76 |
return self._status |
| 77 |
|
| 78 |
code = status |
| 79 |
|
| 80 |
@property |
| 81 |
def headers(self): |
| 82 |
self._wait() |
| 83 |
from email.message import EmailMessage |
| 84 |
|
| 85 |
ans = EmailMessage() |
| 86 |
for k, v in self._headers: |
| 87 |
ans[k] = v |
| 88 |
return ans |
| 89 |
|
| 90 |
@property |
| 91 |
def reason(self) -> str: |
| 92 |
self._wait() |
| 93 |
return self._reason or '' |
| 94 |
|
| 95 |
def getcode(self) -> int | None: |
| 96 |
return self.status |
| 97 |
|
| 98 |
def geturl(self): |
| 99 |
return self.url |
| 100 |
|
| 101 |
def getinfo(self): |
| 102 |
return self.headers |
| 103 |
|
| 104 |
def close(self): |
| 105 |
self._data.close() |
| 106 |
|
| 107 |
def __enter__(self): |
| 108 |
return self |
| 109 |
|
| 110 |
def __exit__(self, *a): |
| 111 |
self._data.close() |
| 112 |
|
| 113 |
|
| 114 |
def shutdown_browser(bref): |
| 115 |
br = bref() |
| 116 |
if br is not None: |
| 117 |
br.shutdown() |
| 118 |
|
| 119 |
|
| 120 |
class Browser: |
| 121 |
def __init__( |
| 122 |
self, |
| 123 |
user_agent: str = '', |
| 124 |
headers: tuple[tuple[str, str], ...] = (), |
| 125 |
verify_ssl_certificates: bool = True, |
| 126 |
start_worker: bool = False, |
| 127 |
): |
| 128 |
self.tdir = '' |
| 129 |
self.worker = self.dispatcher = None |
| 130 |
self.dispatch_map = {} |
| 131 |
self.verify_ssl_certificates = verify_ssl_certificates |
| 132 |
self.id_counter = 0 |
| 133 |
self.addheaders: list[tuple[str, str]] = list(headers) |
| 134 |
self.user_agent = user_agent |
| 135 |
self.lock = RLock() |
| 136 |
self.shutting_down = False |
| 137 |
atexit.register(shutdown_browser, weakref.ref(self)) |
| 138 |
if start_worker: |
| 139 |
self._ensure_state() |
| 140 |
|
| 141 |
def _open(self, url_or_request: Request | str, data=None, timeout=None, visit: bool = True): |
| 142 |
method = 'POST' if data else 'GET' |
| 143 |
headers = [] |
| 144 |
if hasattr(url_or_request, 'get_method'): |
| 145 |
assert isinstance(url_or_request, Request) |
| 146 |
r = url_or_request |
| 147 |
method = r.get_method() |
| 148 |
data = data or r.data |
| 149 |
headers = r.header_items() |
| 150 |
url = r.full_url |
| 151 |
else: |
| 152 |
url = url_or_request |
| 153 |
|
| 154 |
def has_header(x: str) -> bool: |
| 155 |
x = x.lower() |
| 156 |
for h, v in headers: |
| 157 |
if h.lower() == x: |
| 158 |
return True |
| 159 |
return False |
| 160 |
|
| 161 |
if isinstance(data, dict): |
| 162 |
headers.append(('Content-Type', 'application/x-www-form-urlencoded')) |
| 163 |
data = urlencode(data) |
| 164 |
if isinstance(data, str): |
| 165 |
data = data.encode('utf-8') |
| 166 |
if not has_header('Content-Type'): |
| 167 |
headers.append(('Content-Type', 'text/plain')) |
| 168 |
|
| 169 |
if not self.is_method_ok(method): |
| 170 |
raise KeyError(f'The HTTP {method} request method is not supported') |
| 171 |
|
| 172 |
with self.lock: |
| 173 |
self._ensure_state() |
| 174 |
self.id_counter += 1 |
| 175 |
cmd = { |
| 176 |
'action': 'download', |
| 177 |
'id': self.id_counter, |
| 178 |
'url': url, |
| 179 |
'method': method, |
| 180 |
'timeout': timeout, |
| 181 |
'headers': self.addheaders + headers, |
| 182 |
'visit': visit, |
| 183 |
} |
| 184 |
if data: |
| 185 |
with open(os.path.join(self.tdir, f'i{self.id_counter}'), 'wb') as f: |
| 186 |
if hasattr(data, 'read'): |
| 187 |
shutil.copyfileobj(data, f) # type: ignore |
| 188 |
else: |
| 189 |
f.write(data) # type: ignore |
| 190 |
cmd['data_path'] = f.name |
| 191 |
for k, v in cmd['headers']: |
| 192 |
if k.lower() == 'content-type': |
| 193 |
break |
| 194 |
else: |
| 195 |
cmd['headers'].append(('Content-Type', 'application/x-www-form-urlencoded')) |
| 196 |
res = FakeResponse() |
| 197 |
self.dispatch_map[self.id_counter] = res.queue |
| 198 |
self._send_command(cmd) |
| 199 |
return res |
| 200 |
|
| 201 |
def open(self, url_or_request: Request | str, data=None, timeout=None): |
| 202 |
return self._open(url_or_request, data, timeout) |
| 203 |
|
| 204 |
def open_novisit(self, url_or_request: Request, data=None, timeout=None): |
| 205 |
return self._open(url_or_request, data, timeout, visit=False) |
| 206 |
|
| 207 |
def is_method_ok(self, method: str) -> bool: |
| 208 |
return True |
| 209 |
|
| 210 |
def set_simple_cookie(self, name: str, value: str, domain: str | None = None, path: str | None = '/'): |
| 211 |
""" |
| 212 |
Set a simple cookie using a name and value. If domain is specified, the cookie is only sent with requests |
| 213 |
to matching domains, otherwise it is sent with all requests. The leading dot in domain is optional. |
| 214 |
Similarly, by default all paths match, to restrict to certain path use the path parameter. |
| 215 |
""" |
| 216 |
c = {'name': name, 'value': value, 'domain': domain, 'path': path} |
| 217 |
self._send_command({'action': 'set_cookies', 'cookies': [c]}) |
| 218 |
|
| 219 |
set_cookie = set_simple_cookie |
| 220 |
|
| 221 |
def set_user_agent(self, val: str = '') -> None: |
| 222 |
self.user_agent = val |
| 223 |
self._send_command({'action': 'set_user_agent', 'user_agent': val}) |
| 224 |
|
| 225 |
def clone_browser(self): |
| 226 |
return self |
| 227 |
|
| 228 |
def _send_command(self, cmd): |
| 229 |
with self.lock: |
| 230 |
self._ensure_state() |
| 231 |
_worker = self.worker |
| 232 |
assert _worker is not None |
| 233 |
_w_stdin = _worker.stdin |
| 234 |
assert _w_stdin is not None |
| 235 |
_w_stdin.write(json.dumps(cmd).encode()) |
| 236 |
_w_stdin.write(b'\n') |
| 237 |
_w_stdin.flush() |
| 238 |
|
| 239 |
def _ensure_state(self): |
| 240 |
with self.lock: |
| 241 |
if not self.tdir: |
| 242 |
self.tdir = PersistentTemporaryDirectory() |
| 243 |
self.worker = self.run_worker() |
| 244 |
self.dispatcher = Thread(target=self._dispatch, daemon=True) |
| 245 |
self.dispatcher.start() |
| 246 |
|
| 247 |
def run_worker(self) -> subprocess.Popen: |
| 248 |
return run_worker(self.tdir, self.user_agent, self.verify_ssl_certificates) |
| 249 |
|
| 250 |
def _dispatch(self): |
| 251 |
try: |
| 252 |
_dispatch_worker = self.worker |
| 253 |
assert _dispatch_worker is not None |
| 254 |
_dispatch_stdout = _dispatch_worker.stdout |
| 255 |
assert _dispatch_stdout is not None |
| 256 |
for line in _dispatch_stdout: |
| 257 |
cmd = json.loads(line) |
| 258 |
if cmd.get('action') == 'finished': |
| 259 |
with self.lock: |
| 260 |
q = self.dispatch_map.pop(cmd['id']) |
| 261 |
q.put(cmd) |
| 262 |
else: |
| 263 |
raise Exception(f'Unexpected response from backend fetch worker process: {cmd}') |
| 264 |
except Exception: |
| 265 |
if not self.shutting_down: |
| 266 |
import traceback |
| 267 |
|
| 268 |
traceback.print_exc() |
| 269 |
|
| 270 |
def shutdown(self): |
| 271 |
self.shutting_down = True |
| 272 |
if self.worker: |
| 273 |
w, self.worker = self.worker, None |
| 274 |
with suppress(OSError): |
| 275 |
_w_stdin_s = w.stdin |
| 276 |
assert _w_stdin_s is not None |
| 277 |
_w_stdin_s.close() |
| 278 |
with suppress(OSError): |
| 279 |
_w_stdout_s = w.stdout |
| 280 |
assert _w_stdout_s is not None |
| 281 |
_w_stdout_s.close() |
| 282 |
give_up_at = time.monotonic() + 1.5 |
| 283 |
while time.monotonic() < give_up_at and w.poll() is None: |
| 284 |
time.sleep(0.01) |
| 285 |
if w.poll() is None: |
| 286 |
w.kill() |
| 287 |
if self.tdir: |
| 288 |
with suppress(OSError): |
| 289 |
shutil.rmtree(self.tdir) |
| 290 |
self.tdir = '' |
| 291 |
if self.dispatcher: |
| 292 |
self.dispatcher.join() |
| 293 |
self.dispatcher = None |
| 294 |
|
| 295 |
def __del__(self): |
| 296 |
self.shutdown() |
| 297 |
|
| 298 |
|
| 299 |
class WebEngineBrowser(Browser): |
| 300 |
def run_worker(self) -> subprocess.Popen: |
| 301 |
return run_worker(self.tdir, self.user_agent, self.verify_ssl_certificates, function='webengine_worker') |
| 302 |
|
| 303 |
|
| 304 |
def run_worker(tdir: str, user_agent: str, verify_ssl_certificates: bool, function: str = 'worker'): |
| 305 |
from calibre.utils.ipc.simple_worker import start_pipe_worker |
| 306 |
|
| 307 |
return start_pipe_worker(f'from calibre.scraper.qt import {function}; {function}({tdir!r}, {user_agent!r}, {verify_ssl_certificates!r})') |
| 308 |
|
| 309 |
|
| 310 |
def worker(*args): |
| 311 |
from calibre.gui2 import must_use_qt |
| 312 |
|
| 313 |
must_use_qt() |
| 314 |
from .qt_backend import worker |
| 315 |
|
| 316 |
worker(*args) |
| 317 |
|
| 318 |
|
| 319 |
def webengine_worker(*args): |
| 320 |
from calibre.gui2 import must_use_qt |
| 321 |
|
| 322 |
must_use_qt() |
| 323 |
from .webengine_backend import worker |
| 324 |
|
| 325 |
worker(*args) |
| 326 |
|
| 327 |
|
| 328 |
def develop(): |
| 329 |
import sys |
| 330 |
|
| 331 |
br = Browser() |
| 332 |
try: |
| 333 |
for url in sys.argv[1:]: |
| 334 |
res = br.open(url) |
| 335 |
print(url, len(res.read())) |
| 336 |
finally: |
| 337 |
del br |
| 338 |
|
| 339 |
|
| 340 |
if __name__ == '__main__': |
| 341 |
develop() |