| 1 |
#!/usr/bin/env python |
| 2 |
# License: GPLv3 Copyright: 2013, Kovid Goyal <kovid at kovidgoyal.net> |
| 3 |
|
| 4 |
# Imports {{{ |
| 5 |
import argparse |
| 6 |
import ast |
| 7 |
import atexit |
| 8 |
import bz2 |
| 9 |
import errno |
| 10 |
import glob |
| 11 |
import gzip |
| 12 |
import io |
| 13 |
import json |
| 14 |
import os |
| 15 |
import random |
| 16 |
import re |
| 17 |
import socket |
| 18 |
import stat |
| 19 |
import subprocess |
| 20 |
import sys |
| 21 |
import tempfile |
| 22 |
import time |
| 23 |
import zipfile |
| 24 |
import zlib |
| 25 |
from collections import namedtuple |
| 26 |
from contextlib import closing |
| 27 |
from datetime import datetime |
| 28 |
from email.utils import parsedate |
| 29 |
from functools import partial |
| 30 |
from multiprocessing.pool import ThreadPool |
| 31 |
from xml.sax.saxutils import escape, quoteattr |
| 32 |
|
| 33 |
try: |
| 34 |
from html import unescape as u |
| 35 |
except ImportError: |
| 36 |
from HTMLParser import HTMLParser |
| 37 |
|
| 38 |
u = HTMLParser().unescape |
| 39 |
|
| 40 |
try: |
| 41 |
from urllib.parse import parse_qs, urlparse |
| 42 |
except ImportError: |
| 43 |
from urlparse import parse_qs, urlparse |
| 44 |
|
| 45 |
|
| 46 |
try: |
| 47 |
from urllib.error import HTTPError, URLError |
| 48 |
from urllib.request import Request, build_opener, urlopen |
| 49 |
except Exception: |
| 50 |
from urllib2 import HTTPError, Request, URLError, build_opener, urlopen |
| 51 |
# }}} |
| 52 |
|
| 53 |
USER_AGENT = 'calibre mirror' |
| 54 |
MR_URL = 'https://www.mobileread.com/forums/' |
| 55 |
IS_PRODUCTION = os.path.exists('/srv/plugins') |
| 56 |
PLUGINS = 'plugins.json.bz2' |
| 57 |
INDEX = MR_URL + 'showpost.php?p=1362767&postcount=1' |
| 58 |
# INDEX = 'file:///t/raw.html' |
| 59 |
|
| 60 |
IndexEntry = namedtuple('IndexEntry', 'name url donate history uninstall deprecated thread_id category') |
| 61 |
socket.setdefaulttimeout(30) |
| 62 |
|
| 63 |
|
| 64 |
def read(url, get_info=False): # {{{ |
| 65 |
if url.startswith('file://'): |
| 66 |
return urlopen(url).read() |
| 67 |
opener = build_opener() |
| 68 |
opener.addheaders = [ |
| 69 |
('User-Agent', USER_AGENT), |
| 70 |
('Accept-Encoding', 'gzip,deflate'), |
| 71 |
] |
| 72 |
# Sporadic network failures in rackspace, so retry with random sleeps. |
| 73 |
# Also retry on HTTP 429 (rate-limit) with exponential backoff: 5s, 10s, 20s, 40s, 60s, 60s, ... |
| 74 |
for i in range(10): |
| 75 |
try: |
| 76 |
res = opener.open(url) |
| 77 |
break |
| 78 |
except HTTPError as e: |
| 79 |
if e.code != 429 or i == 9: |
| 80 |
raise |
| 81 |
time.sleep(min(5 * (2**i), 60)) |
| 82 |
except URLError as e: |
| 83 |
if not isinstance(e.reason, socket.timeout) or i == 9: |
| 84 |
raise |
| 85 |
time.sleep(random.randint(10, 45)) |
| 86 |
info = res.info() |
| 87 |
encoding = info.get('Content-Encoding') |
| 88 |
raw = res.read() |
| 89 |
res.close() |
| 90 |
if encoding and encoding.lower() in {'gzip', 'x-gzip', 'deflate'}: |
| 91 |
if encoding.lower() == 'deflate': |
| 92 |
raw = zlib.decompress(raw) |
| 93 |
else: |
| 94 |
raw = gzip.GzipFile(fileobj=io.BytesIO(raw)).read() |
| 95 |
if get_info: |
| 96 |
return raw, info |
| 97 |
return raw |
| 98 |
|
| 99 |
|
| 100 |
# }}} |
| 101 |
|
| 102 |
|
| 103 |
def url_to_plugin_id(url, deprecated): |
| 104 |
query = parse_qs(urlparse(url).query) |
| 105 |
ans = (query['t'] if 't' in query else query['p'])[0] |
| 106 |
if deprecated: |
| 107 |
ans += '-deprecated' |
| 108 |
return ans |
| 109 |
|
| 110 |
|
| 111 |
def decode_html(raw_bytes, info=None): |
| 112 |
charset = 'cp1252' |
| 113 |
if info is not None: |
| 114 |
try: |
| 115 |
content_type = info.get('Content-Type', '') |
| 116 |
if 'charset=' in content_type: |
| 117 |
charset = content_type.split('charset=')[-1].strip() |
| 118 |
except Exception: |
| 119 |
pass |
| 120 |
c = charset.lower().replace('_', '-').replace(' ', '') |
| 121 |
if c in ('iso-8859-1', 'latin-1', 'latin1', 'iso8859-1', 'iso_8859-1'): |
| 122 |
charset = 'cp1252' |
| 123 |
try: |
| 124 |
return raw_bytes.decode(charset, 'replace') |
| 125 |
except Exception: |
| 126 |
return raw_bytes.decode('cp1252', 'replace') |
| 127 |
|
| 128 |
|
| 129 |
def parse_index(raw=None): # {{{ |
| 130 |
if raw is None: |
| 131 |
res = read(INDEX, get_info=True) |
| 132 |
if isinstance(res, tuple): |
| 133 |
raw_bytes, info = res |
| 134 |
raw = decode_html(raw_bytes, info) |
| 135 |
else: |
| 136 |
raw = decode_html(res) |
| 137 |
|
| 138 |
dpat = re.compile(r'''(?is)Donate\s*:\s*<a\s+href=['"](.+?)['"]''') |
| 139 |
key_pat = re.compile(r'''(?is)(History|Uninstall)\s*:\s*([^<;]+)[<;]''') |
| 140 |
seen = {} |
| 141 |
dep_start = -1 |
| 142 |
category_offsets = [] |
| 143 |
deprecated_category = 'Deprecated/Renamed/Retired' |
| 144 |
for match in re.finditer(r'''<b>\s*(.+?)\s*Plugins:?\s*</b>''', raw): |
| 145 |
category = match.group(1).strip() |
| 146 |
category_offsets.append((category, match.start())) |
| 147 |
if category == deprecated_category: |
| 148 |
dep_start = match.start() |
| 149 |
if dep_start < 1: |
| 150 |
raise ValueError('Could not find start of deprecated plugins') |
| 151 |
category_offsets = tuple(reversed(category_offsets)) |
| 152 |
|
| 153 |
def category_at(offset): |
| 154 |
for category, q in category_offsets: |
| 155 |
if offset >= q: |
| 156 |
return category |
| 157 |
raise ValueError('Could not find category for offset: ' + str(offset)) |
| 158 |
|
| 159 |
for match in re.finditer(r'''(?is)<li.+?<a\s+href=['"](https://www\.mobileread\.com/forums/showthread\.php\?[pt]=\d+).+?>(.+?)<(.+?)</li>''', raw): |
| 160 |
name, url, rest = u(match.group(2)), u(match.group(1)), match.group(3) |
| 161 |
category = category_at(match.start(2)) |
| 162 |
deprecated = category == deprecated_category |
| 163 |
donate = uninstall = None |
| 164 |
history = False |
| 165 |
m = dpat.search(rest) |
| 166 |
if m is not None: |
| 167 |
donate = u(m.group(1)) |
| 168 |
for m in key_pat.finditer(rest): |
| 169 |
k = m.group(1).lower() |
| 170 |
if k == 'history' and m.group(2).strip().lower() in {'yes', 'true'}: |
| 171 |
history = True |
| 172 |
elif k == 'uninstall': |
| 173 |
uninstall = tuple(x.strip() for x in m.group(2).strip().split(',')) |
| 174 |
|
| 175 |
thread_id = url_to_plugin_id(url, deprecated) |
| 176 |
if thread_id in seen: |
| 177 |
raise ValueError(f'thread_id for {seen[thread_id]} and {name} is the same: {thread_id}') |
| 178 |
seen[thread_id] = name |
| 179 |
entry = IndexEntry(name, url, donate, history, uninstall, deprecated, thread_id, category) |
| 180 |
yield entry |
| 181 |
|
| 182 |
|
| 183 |
# }}} |
| 184 |
|
| 185 |
|
| 186 |
def parse_plugin_zip_url(raw): |
| 187 |
for m in re.finditer(r'''(?is)<a\s+href=['"](attachment.php\?[^'"]+?)['"][^>]*>([^<>]+?\.zip)\s*<''', raw): |
| 188 |
url, name = u(m.group(1)), u(m.group(2).strip()) |
| 189 |
if name.lower().endswith('.zip'): |
| 190 |
return MR_URL + url, name |
| 191 |
return None, None |
| 192 |
|
| 193 |
|
| 194 |
def load_plugins_index(): |
| 195 |
try: |
| 196 |
with open(PLUGINS, 'rb') as f: |
| 197 |
raw = f.read() |
| 198 |
except OSError as err: |
| 199 |
if err.errno == errno.ENOENT: |
| 200 |
return {} |
| 201 |
raise |
| 202 |
return json.loads(bz2.decompress(raw)) |
| 203 |
|
| 204 |
|
| 205 |
# Get metadata from plugin zip file {{{ |
| 206 |
|
| 207 |
|
| 208 |
def convert_node(fields, x, names={}, import_data=None): |
| 209 |
name = x.__class__.__name__ |
| 210 |
|
| 211 |
def conv(x): |
| 212 |
return convert_node(fields, x, names=names, import_data=import_data) |
| 213 |
|
| 214 |
if name == 'Str': |
| 215 |
return x.s.decode('utf-8') if isinstance(x.s, bytes) else x.s |
| 216 |
elif name == 'Num': |
| 217 |
return x.n |
| 218 |
elif name == 'Constant': |
| 219 |
return x.value |
| 220 |
elif name in {'Set', 'List', 'Tuple'}: |
| 221 |
func = {'Set': set, 'List': list, 'Tuple': tuple}[name] |
| 222 |
return func(list(map(conv, x.elts))) |
| 223 |
elif name == 'Dict': |
| 224 |
keys, values = list(map(conv, x.keys)), list(map(conv, x.values)) |
| 225 |
return dict(zip(keys, values)) |
| 226 |
elif name == 'Call': |
| 227 |
if len(x.args) != 1 and len(x.keywords) != 0: |
| 228 |
raise TypeError(f'Unsupported function call for fields: {fields}') |
| 229 |
return tuple(map(conv, x.args))[0] |
| 230 |
elif name == 'Name': |
| 231 |
if x.id not in names: |
| 232 |
if import_data is not None and x.id in import_data[0]: |
| 233 |
return get_import_data(x.id, import_data[0][x.id], *import_data[1:]) |
| 234 |
raise ValueError(f'Could not find name {x.id} for fields: {fields}') |
| 235 |
return names[x.id] |
| 236 |
elif name == 'BinOp': |
| 237 |
if x.right.__class__.__name__ == 'Str': |
| 238 |
return x.right.s.decode('utf-8') if isinstance(x.right.s, bytes) else x.right.s |
| 239 |
if x.right.__class__.__name__ == 'Constant' and isinstance(x.right.value, str): |
| 240 |
return x.right.value |
| 241 |
elif name == 'Attribute': |
| 242 |
return conv(getattr(conv(x.value), x.attr)) |
| 243 |
raise TypeError(f'Unknown datatype {x} for fields: {fields}') |
| 244 |
|
| 245 |
|
| 246 |
Alias = namedtuple('Alias', 'name asname') |
| 247 |
|
| 248 |
|
| 249 |
class Module: |
| 250 |
pass |
| 251 |
|
| 252 |
|
| 253 |
def get_import_data(name, mod, zf, names): |
| 254 |
mod = mod.split('.') |
| 255 |
if mod[0] == 'calibre_plugins': |
| 256 |
mod = mod[2:] |
| 257 |
is_module_import = not mod |
| 258 |
if is_module_import: |
| 259 |
mod = [name] |
| 260 |
mod = '/'.join(mod) + '.py' |
| 261 |
if mod in names: |
| 262 |
raw = zf.open(names[mod]).read() |
| 263 |
module = ast.parse(raw, filename='__init__.py') |
| 264 |
top_level_assigments = [x for x in ast.iter_child_nodes(module) if x.__class__.__name__ == 'Assign'] |
| 265 |
module = Module() |
| 266 |
for node in top_level_assigments: |
| 267 |
targets = {getattr(t, 'id', None) for t in node.targets} |
| 268 |
targets.discard(None) |
| 269 |
for x in targets: |
| 270 |
if is_module_import: |
| 271 |
setattr(module, x, node.value) |
| 272 |
elif x == name: |
| 273 |
return convert_node({x}, node.value) |
| 274 |
if is_module_import: |
| 275 |
return module |
| 276 |
raise ValueError(f'Failed to find name: {name!r} in module: {mod!r}') |
| 277 |
else: |
| 278 |
raise ValueError(f'Failed to find module: {mod!r}') |
| 279 |
|
| 280 |
|
| 281 |
def parse_metadata(raw, namelist, zf): |
| 282 |
module = ast.parse(raw, filename='__init__.py') |
| 283 |
top_level_imports = [x for x in ast.iter_child_nodes(module) if x.__class__.__name__ == 'ImportFrom'] |
| 284 |
top_level_classes = tuple(x for x in ast.iter_child_nodes(module) if x.__class__.__name__ == 'ClassDef') |
| 285 |
top_level_assigments = [x for x in ast.iter_child_nodes(module) if x.__class__.__name__ == 'Assign'] |
| 286 |
defaults = { |
| 287 |
'name': '', |
| 288 |
'description': '', |
| 289 |
'supported_platforms': ['windows', 'osx', 'linux'], |
| 290 |
'version': (1, 0, 0), |
| 291 |
'author': 'Unknown', |
| 292 |
'minimum_calibre_version': (0, 9, 42), |
| 293 |
} |
| 294 |
field_names = set(defaults) |
| 295 |
imported_names = {} |
| 296 |
|
| 297 |
plugin_import_found = set() |
| 298 |
all_imports = [] |
| 299 |
for node in top_level_imports: |
| 300 |
names = getattr(node, 'names', []) |
| 301 |
mod = getattr(node, 'module', None) |
| 302 |
if names and mod: |
| 303 |
names = [Alias(n.name, getattr(n, 'asname', None)) for n in names] |
| 304 |
if ( |
| 305 |
mod |
| 306 |
in { |
| 307 |
'calibre.customize', |
| 308 |
'calibre.customize.conversion', |
| 309 |
'calibre.ebooks.metadata.sources.base', |
| 310 |
'calibre.ebooks.metadata.sources.amazon', |
| 311 |
'calibre.ebooks.metadata.covers', |
| 312 |
'calibre.devices.interface', |
| 313 |
'calibre.ebooks.metadata.fetch', |
| 314 |
'calibre.customize.builtins', |
| 315 |
} |
| 316 |
or re.match(r'calibre\.devices\.[a-z0-9]+\.driver', mod) is not None |
| 317 |
): |
| 318 |
inames = {n.asname or n.name for n in names} |
| 319 |
inames = {x for x in inames if x.lower() != x} |
| 320 |
plugin_import_found |= inames |
| 321 |
else: |
| 322 |
all_imports.append((mod, [n.name for n in names])) |
| 323 |
for alias in names: |
| 324 |
imported_names[alias.asname or alias.name] = mod |
| 325 |
if not plugin_import_found: |
| 326 |
return all_imports |
| 327 |
|
| 328 |
import_data = (imported_names, zf, namelist) |
| 329 |
|
| 330 |
names = {} |
| 331 |
top_level_ann_assigments = [x for x in ast.iter_child_nodes(module) if x.__class__.__name__ == 'AnnAssign'] |
| 332 |
for node in top_level_assigments: |
| 333 |
targets = {getattr(t, 'id', None) for t in node.targets} |
| 334 |
targets.discard(None) |
| 335 |
for x in targets - field_names: |
| 336 |
try: |
| 337 |
val = convert_node({x}, node.value, import_data=import_data) |
| 338 |
except Exception: |
| 339 |
pass |
| 340 |
else: |
| 341 |
names[x] = val |
| 342 |
for node in top_level_ann_assigments: |
| 343 |
if node.value is None: |
| 344 |
continue |
| 345 |
x = getattr(node.target, 'id', None) |
| 346 |
if x is None or x in field_names: |
| 347 |
continue |
| 348 |
try: |
| 349 |
val = convert_node({x}, node.value, import_data=import_data) |
| 350 |
except Exception: |
| 351 |
pass |
| 352 |
else: |
| 353 |
names[x] = val |
| 354 |
|
| 355 |
def parse_class(node): |
| 356 |
class_assigments = [x for x in ast.iter_child_nodes(node) if x.__class__.__name__ == 'Assign'] |
| 357 |
class_ann_assigments = [x for x in ast.iter_child_nodes(node) if x.__class__.__name__ == 'AnnAssign'] |
| 358 |
found = {} |
| 359 |
for node in class_assigments: |
| 360 |
targets = {getattr(t, 'id', None) for t in node.targets} |
| 361 |
targets.discard(None) |
| 362 |
fields = field_names.intersection(targets) |
| 363 |
if fields: |
| 364 |
val = convert_node(fields, node.value, names=names, import_data=import_data) |
| 365 |
for field in fields: |
| 366 |
found[field] = val |
| 367 |
for node in class_ann_assigments: |
| 368 |
if node.value is None: |
| 369 |
continue |
| 370 |
target_id = getattr(node.target, 'id', None) |
| 371 |
if target_id is None: |
| 372 |
continue |
| 373 |
fields = field_names.intersection({target_id}) |
| 374 |
if fields: |
| 375 |
val = convert_node(fields, node.value, names=names, import_data=import_data) |
| 376 |
for field in fields: |
| 377 |
found[field] = val |
| 378 |
return found |
| 379 |
|
| 380 |
if top_level_classes: |
| 381 |
for node in top_level_classes: |
| 382 |
bases = {getattr(x, 'id', None) for x in node.bases} |
| 383 |
if not bases.intersection(plugin_import_found): |
| 384 |
continue |
| 385 |
found = parse_class(node) |
| 386 |
if 'name' in found and 'author' in found: |
| 387 |
defaults.update(found) |
| 388 |
return defaults |
| 389 |
for node in top_level_classes: |
| 390 |
found = parse_class(node) |
| 391 |
if 'name' in found and 'author' in found and 'version' in found: |
| 392 |
defaults.update(found) |
| 393 |
return defaults |
| 394 |
|
| 395 |
raise ValueError('Could not find plugin class') |
| 396 |
|
| 397 |
|
| 398 |
def parse_plugin(raw, names, zf): |
| 399 |
ans = parse_metadata(raw, names, zf) |
| 400 |
if isinstance(ans, dict): |
| 401 |
return ans |
| 402 |
# The plugin is importing its base class from somewhere else, le sigh |
| 403 |
for mod, _ in ans: |
| 404 |
mod = mod.split('.') |
| 405 |
if mod[0] == 'calibre_plugins': |
| 406 |
mod = mod[2:] |
| 407 |
mod = '/'.join(mod) + '.py' |
| 408 |
if mod in names: |
| 409 |
raw = zf.open(names[mod]).read() |
| 410 |
ans = parse_metadata(raw, names, zf) |
| 411 |
if isinstance(ans, dict): |
| 412 |
return ans |
| 413 |
|
| 414 |
raise ValueError('Failed to find plugin class') |
| 415 |
|
| 416 |
|
| 417 |
def get_plugin_init(zf): |
| 418 |
metadata = None |
| 419 |
names = {x.decode('utf-8') if isinstance(x, bytes) else x: x for x in zf.namelist()} |
| 420 |
inits = [x for x in names if x.rpartition('/')[-1] == '__init__.py'] |
| 421 |
inits.sort(key=lambda x: x.count('/')) |
| 422 |
if inits and inits[0] == '__init__.py': |
| 423 |
metadata = names[inits[0]] |
| 424 |
else: |
| 425 |
# Legacy plugin |
| 426 |
for name, val in names.items(): |
| 427 |
if name.endswith('plugin.py'): |
| 428 |
metadata = val |
| 429 |
break |
| 430 |
if metadata is None: |
| 431 |
raise ValueError('No __init__.py found in plugin') |
| 432 |
return zf.open(metadata).read(), names |
| 433 |
|
| 434 |
|
| 435 |
def get_plugin_info(raw_zip): |
| 436 |
with zipfile.ZipFile(io.BytesIO(raw_zip)) as zf: |
| 437 |
raw, names = get_plugin_init(zf) |
| 438 |
try: |
| 439 |
return parse_plugin(raw, names, zf) |
| 440 |
except (SyntaxError, TabError, IndentationError): # fmt: skip |
| 441 |
with tempfile.NamedTemporaryFile(suffix='.zip') as f: |
| 442 |
f.write(raw_zip) |
| 443 |
f.flush() |
| 444 |
res = subprocess.run(['python2', __file__, f.name], stdout=subprocess.PIPE) |
| 445 |
if res.returncode == 0: |
| 446 |
return json.loads(res.stdout) |
| 447 |
raise |
| 448 |
|
| 449 |
|
| 450 |
# }}} |
| 451 |
|
| 452 |
|
| 453 |
def update_plugin_from_entry(plugin, entry): |
| 454 |
plugin['index_name'] = entry.name |
| 455 |
plugin['thread_url'] = entry.url |
| 456 |
plugin['category'] = entry.category |
| 457 |
for x in ('donate', 'history', 'deprecated', 'uninstall', 'thread_id'): |
| 458 |
plugin[x] = getattr(entry, x) |
| 459 |
|
| 460 |
|
| 461 |
def fetch_plugin(old_index, entry): |
| 462 |
lm_map = {plugin['thread_id']: plugin for plugin in old_index.values()} |
| 463 |
res = read(entry.url, get_info=True) |
| 464 |
raw = decode_html(*res) |
| 465 |
url, name = parse_plugin_zip_url(raw) |
| 466 |
if url is None: |
| 467 |
raise ValueError(f'Failed to find zip file URL for entry: {entry!r}') |
| 468 |
plugin = lm_map.get(entry.thread_id, None) |
| 469 |
|
| 470 |
if plugin is not None: |
| 471 |
# Previously downloaded plugin |
| 472 |
lm = datetime(*tuple(map(int, re.split(r'\D', plugin['last_modified'])))[:6]) |
| 473 |
request = Request(url) |
| 474 |
request.get_method = lambda: 'HEAD' |
| 475 |
with closing(urlopen(request)) as response: |
| 476 |
info = response.info() |
| 477 |
slm = datetime(*parsedate(info.get('Last-Modified'))[:6]) |
| 478 |
if lm >= slm: |
| 479 |
# The previously downloaded plugin zip file is up-to-date |
| 480 |
update_plugin_from_entry(plugin, entry) |
| 481 |
return plugin |
| 482 |
|
| 483 |
raw, info = read(url, get_info=True) |
| 484 |
slm = datetime(*parsedate(info.get('Last-Modified'))[:6]) |
| 485 |
plugin = get_plugin_info(raw) |
| 486 |
plugin['last_modified'] = slm.isoformat() |
| 487 |
plugin['file'] = f'staging_{entry.thread_id}.zip' |
| 488 |
plugin['size'] = len(raw) |
| 489 |
plugin['original_url'] = url |
| 490 |
update_plugin_from_entry(plugin, entry) |
| 491 |
with open(plugin['file'], 'wb') as f: |
| 492 |
f.write(raw) |
| 493 |
return plugin |
| 494 |
|
| 495 |
|
| 496 |
def parallel_fetch(old_index, entry): |
| 497 |
try: |
| 498 |
return fetch_plugin(old_index, entry) |
| 499 |
except Exception: |
| 500 |
import traceback |
| 501 |
|
| 502 |
return traceback.format_exc() |
| 503 |
|
| 504 |
|
| 505 |
def log(*args, **kwargs): |
| 506 |
print(*args, **kwargs) |
| 507 |
with open('log', 'a') as f: |
| 508 |
kwargs['file'] = f |
| 509 |
print(*args, **kwargs) |
| 510 |
|
| 511 |
|
| 512 |
def atomic_write(raw, name): |
| 513 |
with tempfile.NamedTemporaryFile(dir=os.getcwd(), delete=False) as f: |
| 514 |
f.write(raw) |
| 515 |
os.fchmod(f.fileno(), stat.S_IREAD | stat.S_IWRITE | stat.S_IRGRP | stat.S_IROTH) |
| 516 |
os.rename(f.name, name) |
| 517 |
|
| 518 |
|
| 519 |
def fetch_plugins(old_index, thread_id=None): |
| 520 |
all_entries = tuple(parse_index()) |
| 521 |
if not all_entries: |
| 522 |
raise SystemExit('Could not find any plugins, probably the markup on the MR index page has changed') |
| 523 |
if thread_id is not None: |
| 524 |
entries = tuple(e for e in all_entries if e.thread_id == thread_id) |
| 525 |
if not entries: |
| 526 |
raise SystemExit(f'No plugin with thread_id {thread_id!r} found in the MR index') |
| 527 |
ans = dict(old_index) |
| 528 |
else: |
| 529 |
entries = all_entries |
| 530 |
ans = {} |
| 531 |
pool = ThreadPool(processes=min(10, len(entries))) |
| 532 |
with closing(pool): |
| 533 |
result = pool.map(partial(parallel_fetch, old_index), entries) |
| 534 |
for entry, plugin in zip(entries, result): |
| 535 |
if isinstance(plugin, dict): |
| 536 |
ans[entry.name] = plugin |
| 537 |
else: |
| 538 |
if entry.name in old_index: |
| 539 |
ans[entry.name] = old_index[entry.name] |
| 540 |
log('Failed to get plugin', entry.name, 'at', datetime.now().isoformat(), 'with error:') |
| 541 |
log(plugin) |
| 542 |
# Move staged files |
| 543 |
for plugin in ans.values(): |
| 544 |
if plugin['file'].startswith('staging_'): |
| 545 |
src = plugin['file'] |
| 546 |
plugin['file'] = src.partition('_')[-1] |
| 547 |
os.rename(src, plugin['file']) |
| 548 |
raw = bz2.compress(json.dumps(ans, sort_keys=True, indent=4, separators=(',', ': ')).encode('utf-8')) |
| 549 |
atomic_write(raw, PLUGINS) |
| 550 |
if thread_id is None: |
| 551 |
# Cleanup any extra .zip files only on full sync |
| 552 |
all_plugin_files = {p['file'] for p in ans.values()} |
| 553 |
extra = set(glob.glob('*.zip')) - all_plugin_files |
| 554 |
for x in extra: |
| 555 |
os.unlink(x) |
| 556 |
return ans |
| 557 |
|
| 558 |
|
| 559 |
def plugin_to_index(plugin, count): |
| 560 |
title = '<h3><img src="plugin-icon.png"><a href={} title="Plugin forum thread">{}</a></h3>'.format(quoteattr(plugin['thread_url']), escape(plugin['name'])) |
| 561 |
released = datetime(*tuple(map(int, re.split(r'\D', plugin['last_modified'])))[:6]).strftime('%e %b, %Y').lstrip() |
| 562 |
details = [ |
| 563 |
'Version: <b>{}</b>'.format(escape('.'.join(map(str, plugin['version'])))), |
| 564 |
f'Released: <b>{escape(released)}</b>', |
| 565 |
'Author: {}'.format(escape(plugin['author'])), |
| 566 |
'calibre: {}'.format(escape('.'.join(map(str, plugin['minimum_calibre_version'])))), |
| 567 |
'Platforms: {}'.format(escape(', '.join(sorted(plugin['supported_platforms']) or ['all']))), |
| 568 |
] |
| 569 |
if plugin['uninstall']: |
| 570 |
details.append('Uninstall: {}'.format(escape(', '.join(plugin['uninstall'])))) |
| 571 |
if plugin['donate']: |
| 572 |
details.append('<a href={} title="Donate">Donate</a>'.format(quoteattr(plugin['donate']))) |
| 573 |
block = [] |
| 574 |
for li in details: |
| 575 |
if li.startswith('calibre:'): |
| 576 |
block.append('<br>') |
| 577 |
block.append(f'<li>{li}</li>') |
| 578 |
block = '<ul>{}</ul>'.format('\n'.join(block)) |
| 579 |
downloads = (f'\xa0<span class="download-count">[{count} total downloads]</span>') if count else '' |
| 580 |
zipfile = '<div class="end"><a href={} title="Download plugin" download={}>Download plugin \u2193</a>{}</div>'.format( |
| 581 |
quoteattr(plugin['file']), quoteattr(plugin['name'] + '.zip'), downloads |
| 582 |
) |
| 583 |
desc = plugin['description'] or '' |
| 584 |
if desc: |
| 585 |
desc = f'<p>{desc}</p>' |
| 586 |
return f'{title}\n{desc}\n{block}\n{zipfile}\n\n' |
| 587 |
|
| 588 |
|
| 589 |
def create_index(index, raw_stats): |
| 590 |
plugins = [] |
| 591 |
stats = {} |
| 592 |
for name in sorted(index): |
| 593 |
plugin = index[name] |
| 594 |
if not plugin['deprecated']: |
| 595 |
count = raw_stats.get(plugin['file'].rpartition('.')[0], 0) |
| 596 |
if count > 0: |
| 597 |
stats[plugin['name']] = count |
| 598 |
plugins.append(plugin_to_index(plugin, count)) |
| 599 |
index = '''\ |
| 600 |
<!DOCTYPE html> |
| 601 |
<html> |
| 602 |
<head><meta charset="utf-8"><title>Index of calibre plugins</title> |
| 603 |
<link rel="icon" type="image/x-icon" href="//calibre-ebook.com/favicon.ico" /> |
| 604 |
<style type="text/css"> |
| 605 |
body { background-color: #eee; } |
| 606 |
a { text-decoration: none } |
| 607 |
a:hover, h3:hover { color: red } |
| 608 |
a:visited { color: blue } |
| 609 |
ul { list-style-type: none; font-size: smaller } |
| 610 |
li { display: inline } |
| 611 |
li+li:before { content: " - " } |
| 612 |
.end { border-bottom: solid 1pt black; padding-bottom: 0.5ex; margin-bottom: 4ex; } |
| 613 |
h1 img, h3 img { vertical-align: middle; margin-right: 0.5em; } |
| 614 |
h1 { text-align: center } |
| 615 |
.download-count { color: gray; font-size: smaller } |
| 616 |
</style> |
| 617 |
</head> |
| 618 |
<body> |
| 619 |
<h1><img src="//manual.calibre-ebook.com/_static/logo.png">Index of calibre plugins</h1> |
| 620 |
<div style="text-align:center"><a href="stats.html">Download counts for all plugins</a></div> |
| 621 |
%s |
| 622 |
</body> |
| 623 |
</html>''' % ('\n'.join(plugins)) # noqa: UP031 |
| 624 |
raw = index.encode('utf-8') |
| 625 |
try: |
| 626 |
with open('index.html', 'rb') as f: |
| 627 |
oraw = f.read() |
| 628 |
except OSError: |
| 629 |
oraw = None |
| 630 |
if raw != oraw: |
| 631 |
atomic_write(raw, 'index.html') |
| 632 |
|
| 633 |
def plugin_stats(x): |
| 634 |
name, count = x |
| 635 |
return f'<tr><td>{escape(name)}</td><td>{count}</td></tr>\n' |
| 636 |
|
| 637 |
pstats = list(map(plugin_stats, sorted(stats.items(), reverse=True, key=lambda x: x[1]))) |
| 638 |
stats = '''\ |
| 639 |
<!DOCTYPE html> |
| 640 |
<html> |
| 641 |
<head><meta charset="utf-8"><title>Stats for calibre plugins</title> |
| 642 |
<link rel="icon" type="image/x-icon" href="//calibre-ebook.com/favicon.ico" /> |
| 643 |
<style type="text/css"> |
| 644 |
body { background-color: #eee; } |
| 645 |
h1 img, h3 img { vertical-align: middle; margin-right: 0.5em; } |
| 646 |
h1 { text-align: center } |
| 647 |
</style> |
| 648 |
</head> |
| 649 |
<body> |
| 650 |
<h1><img src="//manual.calibre-ebook.com/_static/logo.png">Stats for calibre plugins</h1> |
| 651 |
<table> |
| 652 |
<tr><th>Plugin</th><th>Total downloads</th></tr> |
| 653 |
%s |
| 654 |
</table> |
| 655 |
</body> |
| 656 |
</html> |
| 657 |
''' % ('\n'.join(pstats)) # noqa: UP031 |
| 658 |
raw = stats.encode('utf-8') |
| 659 |
try: |
| 660 |
with open('stats.html', 'rb') as f: |
| 661 |
oraw = f.read() |
| 662 |
except OSError: |
| 663 |
oraw = None |
| 664 |
if raw != oraw: |
| 665 |
atomic_write(raw, 'stats.html') |
| 666 |
|
| 667 |
|
| 668 |
_singleinstance = None |
| 669 |
|
| 670 |
|
| 671 |
def singleinstance(): |
| 672 |
global _singleinstance |
| 673 |
s = _singleinstance = socket.socket(socket.AF_UNIX) |
| 674 |
try: |
| 675 |
s.bind(b'\0calibre-plugins-mirror-singleinstance') |
| 676 |
except OSError as err: |
| 677 |
if getattr(err, 'errno', None) == errno.EADDRINUSE: |
| 678 |
return False |
| 679 |
raise |
| 680 |
return True |
| 681 |
|
| 682 |
|
| 683 |
def update_stats(): |
| 684 |
log = olog = 'stats.log' |
| 685 |
if not os.path.exists(log): |
| 686 |
return {} |
| 687 |
stats = {} |
| 688 |
if IS_PRODUCTION: |
| 689 |
try: |
| 690 |
with open('stats.json', 'rb') as f: |
| 691 |
stats = json.load(f) |
| 692 |
except OSError as err: |
| 693 |
if err.errno != errno.ENOENT: |
| 694 |
raise |
| 695 |
if os.geteuid() != 0: |
| 696 |
return stats |
| 697 |
log = 'rotated-' + log |
| 698 |
os.rename(olog, log) |
| 699 |
subprocess.check_call(['/usr/sbin/nginx', '-s', 'reopen']) |
| 700 |
atexit.register(os.remove, log) |
| 701 |
pat = re.compile(rb'GET /(\d+)(?:-deprecated){0,1}\.zip') |
| 702 |
for line in open(log, 'rb'): |
| 703 |
m = pat.search(line) |
| 704 |
if m is not None: |
| 705 |
plugin = m.group(1).decode('utf-8') |
| 706 |
stats[plugin] = stats.get(plugin, 0) + 1 |
| 707 |
data = json.dumps(stats, indent=2) |
| 708 |
if not isinstance(data, bytes): |
| 709 |
data = data.encode('utf-8') |
| 710 |
with open('stats.json', 'wb') as f: |
| 711 |
f.write(data) |
| 712 |
return stats |
| 713 |
|
| 714 |
|
| 715 |
def parse_single_plugin(zipfile_path): |
| 716 |
with zipfile.ZipFile(zipfile_path) as zf: |
| 717 |
raw, names = get_plugin_init(zf) |
| 718 |
ans = parse_plugin(raw, names, zf) |
| 719 |
sys.stdout.write(json.dumps(ans, ensure_ascii=True)) |
| 720 |
|
| 721 |
|
| 722 |
def main(): |
| 723 |
p = argparse.ArgumentParser(description='Mirror calibre plugins from the forums. Or parse a single plugin zip file if specified on the command line') |
| 724 |
p.add_argument('plugin_path', nargs='?', default='', help='Path to plugin zip file to parse') |
| 725 |
WORKDIR = '/srv/plugins' if IS_PRODUCTION else '/t/plugins' |
| 726 |
p.add_argument('-o', '--output-dir', default=WORKDIR, help='Where to place the mirrored plugins. Default is: ' + WORKDIR) |
| 727 |
p.add_argument('--thread-id', default=None, help='Only download/update the plugin with this thread ID (e.g. 1234 or 1234-deprecated)') |
| 728 |
args = p.parse_args() |
| 729 |
if args.plugin_path: |
| 730 |
return parse_single_plugin(args.plugin_path) |
| 731 |
|
| 732 |
os.makedirs(args.output_dir, exist_ok=True) |
| 733 |
os.chdir(args.output_dir) |
| 734 |
if os.geteuid() == 0 and not singleinstance(): |
| 735 |
print('Another instance of plugins-mirror is running', file=sys.stderr) |
| 736 |
raise SystemExit(1) |
| 737 |
open('log', 'w').close() |
| 738 |
stats = update_stats() |
| 739 |
try: |
| 740 |
plugins_index = load_plugins_index() |
| 741 |
plugins_index = fetch_plugins(plugins_index, thread_id=args.thread_id) |
| 742 |
create_index(plugins_index, stats) |
| 743 |
except KeyboardInterrupt: |
| 744 |
raise SystemExit('Exiting on user interrupt') |
| 745 |
except Exception: |
| 746 |
import traceback |
| 747 |
|
| 748 |
log('Failed to run at:', datetime.now().isoformat()) |
| 749 |
log(traceback.format_exc()) |
| 750 |
raise SystemExit(1) |
| 751 |
|
| 752 |
|
| 753 |
# }}} |
| 754 |
|
| 755 |
|
| 756 |
def test_parse_metadata(): # {{{ |
| 757 |
raw = b'''\ |
| 758 |
import os |
| 759 |
from calibre.customize import FileTypePlugin |
| 760 |
# import_placeholder |
| 761 |
|
| 762 |
MV = (0, 7, 53) |
| 763 |
|
| 764 |
class HelloWorld(FileTypePlugin): |
| 765 |
|
| 766 |
name: str = _('name') # Name of the plugin |
| 767 |
description = {1, 2} |
| 768 |
supported_platforms = ['windows', 'osx', 'linux'] # Platforms this plugin will run on |
| 769 |
author = 'Acme Inc.' # The author of this plugin |
| 770 |
version: dict[str, int] = {1:'a', 'b':2} |
| 771 |
file_types = set(['epub', 'mobi']) # The file types that this plugin will be applied to |
| 772 |
on_postprocess = True # Run this plugin after conversion is complete |
| 773 |
minimum_calibre_version = MV |
| 774 |
''' |
| 775 |
vals = { |
| 776 |
'name': 'name', |
| 777 |
'description': {1, 2}, |
| 778 |
'supported_platforms': ['windows', 'osx', 'linux'], |
| 779 |
'author': 'Acme Inc.', |
| 780 |
'version': {1: 'a', 'b': 2}, |
| 781 |
'minimum_calibre_version': (0, 7, 53), |
| 782 |
} |
| 783 |
assert parse_metadata(raw, None, None) == vals |
| 784 |
buf = io.BytesIO() |
| 785 |
with zipfile.ZipFile(buf, 'w') as zf: |
| 786 |
zf.writestr('very/lovely.py', raw.replace(b'MV = (0, 7, 53)', b'from very.ver import MV')) |
| 787 |
zf.writestr('very/ver.py', b'MV = (0, 7, 53)') |
| 788 |
zf.writestr('__init__.py', b'from xxx import yyy\nfrom very.lovely import HelloWorld') |
| 789 |
assert get_plugin_info(buf.getvalue()) == vals |
| 790 |
buf = io.BytesIO() |
| 791 |
with zipfile.ZipFile(buf, 'w') as zf: |
| 792 |
zf.writestr('very/ver.py', b'NAME = "name"\nVERSION = (1,2,3)') |
| 793 |
zf.writestr( |
| 794 |
'__init__.py', |
| 795 |
raw.replace(b"_('name')", b'NAME').replace(b"{1:'a', 'b':2}", b'VERSION').replace(b'# import_placeholder', b'from very.ver import NAME, VERSION'), |
| 796 |
) |
| 797 |
vals['version'] = (1, 2, 3) |
| 798 |
assert get_plugin_info(buf.getvalue()) == vals |
| 799 |
|
| 800 |
# Test: annotated top-level variable used as class attribute value |
| 801 |
raw2 = b'''\ |
| 802 |
from calibre.customize import FileTypePlugin |
| 803 |
|
| 804 |
MV: tuple = (0, 8, 0) |
| 805 |
|
| 806 |
class MyPlugin(FileTypePlugin): |
| 807 |
name: str = 'TestPlugin' |
| 808 |
author: str = 'Test Author' |
| 809 |
version = MV |
| 810 |
minimum_calibre_version = MV |
| 811 |
description = 'A test plugin' |
| 812 |
supported_platforms = ['linux'] |
| 813 |
''' |
| 814 |
vals2 = { |
| 815 |
'name': 'TestPlugin', |
| 816 |
'description': 'A test plugin', |
| 817 |
'supported_platforms': ['linux'], |
| 818 |
'author': 'Test Author', |
| 819 |
'version': (0, 8, 0), |
| 820 |
'minimum_calibre_version': (0, 8, 0), |
| 821 |
} |
| 822 |
assert parse_metadata(raw2, None, None) == vals2 |
| 823 |
|
| 824 |
# Test: annotation without value (bare annotation) is ignored gracefully |
| 825 |
raw3 = b'''\ |
| 826 |
from calibre.customize import FileTypePlugin |
| 827 |
|
| 828 |
class MyPlugin(FileTypePlugin): |
| 829 |
name: str = 'BareAnnot' |
| 830 |
author: str = 'Some Author' |
| 831 |
description: str |
| 832 |
version = (2, 0, 0) |
| 833 |
minimum_calibre_version = (1, 0, 0) |
| 834 |
supported_platforms: list = ['windows', 'linux'] |
| 835 |
''' |
| 836 |
vals3 = { |
| 837 |
'name': 'BareAnnot', |
| 838 |
'description': '', |
| 839 |
'supported_platforms': ['windows', 'linux'], |
| 840 |
'author': 'Some Author', |
| 841 |
'version': (2, 0, 0), |
| 842 |
'minimum_calibre_version': (1, 0, 0), |
| 843 |
} |
| 844 |
assert parse_metadata(raw3, None, None) == vals3 |
| 845 |
|
| 846 |
# Test: multiple classes, only the one inheriting from a plugin base is used |
| 847 |
raw4 = b'''\ |
| 848 |
from calibre.customize import FileTypePlugin |
| 849 |
|
| 850 |
class Helper: |
| 851 |
name = 'should-be-ignored' |
| 852 |
author = 'ignored' |
| 853 |
|
| 854 |
class RealPlugin(FileTypePlugin): |
| 855 |
name = 'RealPlugin' |
| 856 |
author = 'Real Author' |
| 857 |
version = (3, 1, 4) |
| 858 |
minimum_calibre_version = (0, 9, 42) |
| 859 |
supported_platforms = ['windows', 'osx', 'linux'] |
| 860 |
description = 'The real one' |
| 861 |
''' |
| 862 |
vals4 = { |
| 863 |
'name': 'RealPlugin', |
| 864 |
'description': 'The real one', |
| 865 |
'supported_platforms': ['windows', 'osx', 'linux'], |
| 866 |
'author': 'Real Author', |
| 867 |
'version': (3, 1, 4), |
| 868 |
'minimum_calibre_version': (0, 9, 42), |
| 869 |
} |
| 870 |
assert parse_metadata(raw4, None, None) == vals4 |
| 871 |
|
| 872 |
# Test: all class attributes are type-annotated |
| 873 |
raw5 = b'''\ |
| 874 |
from calibre.customize import FileTypePlugin |
| 875 |
|
| 876 |
class FullAnnotPlugin(FileTypePlugin): |
| 877 |
name: str = 'AllAnnotated' |
| 878 |
description: str = 'fully annotated plugin' |
| 879 |
supported_platforms: list = ['osx'] |
| 880 |
author: str = 'Annotated Author' |
| 881 |
version: tuple = (1, 2, 3) |
| 882 |
minimum_calibre_version: tuple = (0, 9, 42) |
| 883 |
''' |
| 884 |
vals5 = { |
| 885 |
'name': 'AllAnnotated', |
| 886 |
'description': 'fully annotated plugin', |
| 887 |
'supported_platforms': ['osx'], |
| 888 |
'author': 'Annotated Author', |
| 889 |
'version': (1, 2, 3), |
| 890 |
'minimum_calibre_version': (0, 9, 42), |
| 891 |
} |
| 892 |
assert parse_metadata(raw5, None, None) == vals5 |
| 893 |
|
| 894 |
|
| 895 |
# }}} |
| 896 |
|
| 897 |
|
| 898 |
if __name__ == '__main__': |
| 899 |
# Run test_parse_metadata() with: |
| 900 |
# python -c 'import runpy; runpy.run_path("setup/plugins_mirror.py")["test_parse_metadata"]()' |
| 901 |
# import pprint |
| 902 |
# pprint.pprint(get_plugin_info(open(sys.argv[-1], 'rb').read())) |
| 903 |
|
| 904 |
# print('\n'.join(map(str, parse_index()))) |
| 905 |
|
| 906 |
main() |