Debian Codegraph @

setup/plugins_mirror.py

raw
1 #!/usr/bin/env python
2 # License: GPLv3 Copyright: 2013, Kovid Goyal <kovid at kovidgoyal.net>
3
4 # Imports {{{
5 import argparse
6 import ast
7 import atexit
8 import bz2
9 import errno
10 import glob
11 import gzip
12 import io
13 import json
14 import os
15 import random
16 import re
17 import socket
18 import stat
19 import subprocess
20 import sys
21 import tempfile
22 import time
23 import zipfile
24 import zlib
25 from collections import namedtuple
26 from contextlib import closing
27 from datetime import datetime
28 from email.utils import parsedate
29 from functools import partial
30 from multiprocessing.pool import ThreadPool
31 from xml.sax.saxutils import escape, quoteattr
32
33 try:
34 from html import unescape as u
35 except ImportError:
36 from HTMLParser import HTMLParser
37
38 u = HTMLParser().unescape
39
40 try:
41 from urllib.parse import parse_qs, urlparse
42 except ImportError:
43 from urlparse import parse_qs, urlparse
44
45
46 try:
47 from urllib.error import HTTPError, URLError
48 from urllib.request import Request, build_opener, urlopen
49 except Exception:
50 from urllib2 import HTTPError, Request, URLError, build_opener, urlopen
51 # }}}
52
53 USER_AGENT = 'calibre mirror'
54 MR_URL = 'https://www.mobileread.com/forums/'
55 IS_PRODUCTION = os.path.exists('/srv/plugins')
56 PLUGINS = 'plugins.json.bz2'
57 INDEX = MR_URL + 'showpost.php?p=1362767&postcount=1'
58 # INDEX = 'file:///t/raw.html'
59
60 IndexEntry = namedtuple('IndexEntry', 'name url donate history uninstall deprecated thread_id category')
61 socket.setdefaulttimeout(30)
62
63
64 def read(url, get_info=False): # {{{
65 if url.startswith('file://'):
66 return urlopen(url).read()
67 opener = build_opener()
68 opener.addheaders = [
69 ('User-Agent', USER_AGENT),
70 ('Accept-Encoding', 'gzip,deflate'),
71 ]
72 # Sporadic network failures in rackspace, so retry with random sleeps.
73 # Also retry on HTTP 429 (rate-limit) with exponential backoff: 5s, 10s, 20s, 40s, 60s, 60s, ...
74 for i in range(10):
75 try:
76 res = opener.open(url)
77 break
78 except HTTPError as e:
79 if e.code != 429 or i == 9:
80 raise
81 time.sleep(min(5 * (2**i), 60))
82 except URLError as e:
83 if not isinstance(e.reason, socket.timeout) or i == 9:
84 raise
85 time.sleep(random.randint(10, 45))
86 info = res.info()
87 encoding = info.get('Content-Encoding')
88 raw = res.read()
89 res.close()
90 if encoding and encoding.lower() in {'gzip', 'x-gzip', 'deflate'}:
91 if encoding.lower() == 'deflate':
92 raw = zlib.decompress(raw)
93 else:
94 raw = gzip.GzipFile(fileobj=io.BytesIO(raw)).read()
95 if get_info:
96 return raw, info
97 return raw
98
99
100 # }}}
101
102
103 def url_to_plugin_id(url, deprecated):
104 query = parse_qs(urlparse(url).query)
105 ans = (query['t'] if 't' in query else query['p'])[0]
106 if deprecated:
107 ans += '-deprecated'
108 return ans
109
110
111 def decode_html(raw_bytes, info=None):
112 charset = 'cp1252'
113 if info is not None:
114 try:
115 content_type = info.get('Content-Type', '')
116 if 'charset=' in content_type:
117 charset = content_type.split('charset=')[-1].strip()
118 except Exception:
119 pass
120 c = charset.lower().replace('_', '-').replace(' ', '')
121 if c in ('iso-8859-1', 'latin-1', 'latin1', 'iso8859-1', 'iso_8859-1'):
122 charset = 'cp1252'
123 try:
124 return raw_bytes.decode(charset, 'replace')
125 except Exception:
126 return raw_bytes.decode('cp1252', 'replace')
127
128
129 def parse_index(raw=None): # {{{
130 if raw is None:
131 res = read(INDEX, get_info=True)
132 if isinstance(res, tuple):
133 raw_bytes, info = res
134 raw = decode_html(raw_bytes, info)
135 else:
136 raw = decode_html(res)
137
138 dpat = re.compile(r'''(?is)Donate\s*:\s*<a\s+href=['"](.+?)['"]''')
139 key_pat = re.compile(r'''(?is)(History|Uninstall)\s*:\s*([^<;]+)[<;]''')
140 seen = {}
141 dep_start = -1
142 category_offsets = []
143 deprecated_category = 'Deprecated/Renamed/Retired'
144 for match in re.finditer(r'''<b>\s*(.+?)\s*Plugins:?\s*</b>''', raw):
145 category = match.group(1).strip()
146 category_offsets.append((category, match.start()))
147 if category == deprecated_category:
148 dep_start = match.start()
149 if dep_start < 1:
150 raise ValueError('Could not find start of deprecated plugins')
151 category_offsets = tuple(reversed(category_offsets))
152
153 def category_at(offset):
154 for category, q in category_offsets:
155 if offset >= q:
156 return category
157 raise ValueError('Could not find category for offset: ' + str(offset))
158
159 for match in re.finditer(r'''(?is)<li.+?<a\s+href=['"](https://www\.mobileread\.com/forums/showthread\.php\?[pt]=\d+).+?>(.+?)<(.+?)</li>''', raw):
160 name, url, rest = u(match.group(2)), u(match.group(1)), match.group(3)
161 category = category_at(match.start(2))
162 deprecated = category == deprecated_category
163 donate = uninstall = None
164 history = False
165 m = dpat.search(rest)
166 if m is not None:
167 donate = u(m.group(1))
168 for m in key_pat.finditer(rest):
169 k = m.group(1).lower()
170 if k == 'history' and m.group(2).strip().lower() in {'yes', 'true'}:
171 history = True
172 elif k == 'uninstall':
173 uninstall = tuple(x.strip() for x in m.group(2).strip().split(','))
174
175 thread_id = url_to_plugin_id(url, deprecated)
176 if thread_id in seen:
177 raise ValueError(f'thread_id for {seen[thread_id]} and {name} is the same: {thread_id}')
178 seen[thread_id] = name
179 entry = IndexEntry(name, url, donate, history, uninstall, deprecated, thread_id, category)
180 yield entry
181
182
183 # }}}
184
185
186 def parse_plugin_zip_url(raw):
187 for m in re.finditer(r'''(?is)<a\s+href=['"](attachment.php\?[^'"]+?)['"][^>]*>([^<>]+?\.zip)\s*<''', raw):
188 url, name = u(m.group(1)), u(m.group(2).strip())
189 if name.lower().endswith('.zip'):
190 return MR_URL + url, name
191 return None, None
192
193
194 def load_plugins_index():
195 try:
196 with open(PLUGINS, 'rb') as f:
197 raw = f.read()
198 except OSError as err:
199 if err.errno == errno.ENOENT:
200 return {}
201 raise
202 return json.loads(bz2.decompress(raw))
203
204
205 # Get metadata from plugin zip file {{{
206
207
208 def convert_node(fields, x, names={}, import_data=None):
209 name = x.__class__.__name__
210
211 def conv(x):
212 return convert_node(fields, x, names=names, import_data=import_data)
213
214 if name == 'Str':
215 return x.s.decode('utf-8') if isinstance(x.s, bytes) else x.s
216 elif name == 'Num':
217 return x.n
218 elif name == 'Constant':
219 return x.value
220 elif name in {'Set', 'List', 'Tuple'}:
221 func = {'Set': set, 'List': list, 'Tuple': tuple}[name]
222 return func(list(map(conv, x.elts)))
223 elif name == 'Dict':
224 keys, values = list(map(conv, x.keys)), list(map(conv, x.values))
225 return dict(zip(keys, values))
226 elif name == 'Call':
227 if len(x.args) != 1 and len(x.keywords) != 0:
228 raise TypeError(f'Unsupported function call for fields: {fields}')
229 return tuple(map(conv, x.args))[0]
230 elif name == 'Name':
231 if x.id not in names:
232 if import_data is not None and x.id in import_data[0]:
233 return get_import_data(x.id, import_data[0][x.id], *import_data[1:])
234 raise ValueError(f'Could not find name {x.id} for fields: {fields}')
235 return names[x.id]
236 elif name == 'BinOp':
237 if x.right.__class__.__name__ == 'Str':
238 return x.right.s.decode('utf-8') if isinstance(x.right.s, bytes) else x.right.s
239 if x.right.__class__.__name__ == 'Constant' and isinstance(x.right.value, str):
240 return x.right.value
241 elif name == 'Attribute':
242 return conv(getattr(conv(x.value), x.attr))
243 raise TypeError(f'Unknown datatype {x} for fields: {fields}')
244
245
246 Alias = namedtuple('Alias', 'name asname')
247
248
249 class Module:
250 pass
251
252
253 def get_import_data(name, mod, zf, names):
254 mod = mod.split('.')
255 if mod[0] == 'calibre_plugins':
256 mod = mod[2:]
257 is_module_import = not mod
258 if is_module_import:
259 mod = [name]
260 mod = '/'.join(mod) + '.py'
261 if mod in names:
262 raw = zf.open(names[mod]).read()
263 module = ast.parse(raw, filename='__init__.py')
264 top_level_assigments = [x for x in ast.iter_child_nodes(module) if x.__class__.__name__ == 'Assign']
265 module = Module()
266 for node in top_level_assigments:
267 targets = {getattr(t, 'id', None) for t in node.targets}
268 targets.discard(None)
269 for x in targets:
270 if is_module_import:
271 setattr(module, x, node.value)
272 elif x == name:
273 return convert_node({x}, node.value)
274 if is_module_import:
275 return module
276 raise ValueError(f'Failed to find name: {name!r} in module: {mod!r}')
277 else:
278 raise ValueError(f'Failed to find module: {mod!r}')
279
280
281 def parse_metadata(raw, namelist, zf):
282 module = ast.parse(raw, filename='__init__.py')
283 top_level_imports = [x for x in ast.iter_child_nodes(module) if x.__class__.__name__ == 'ImportFrom']
284 top_level_classes = tuple(x for x in ast.iter_child_nodes(module) if x.__class__.__name__ == 'ClassDef')
285 top_level_assigments = [x for x in ast.iter_child_nodes(module) if x.__class__.__name__ == 'Assign']
286 defaults = {
287 'name': '',
288 'description': '',
289 'supported_platforms': ['windows', 'osx', 'linux'],
290 'version': (1, 0, 0),
291 'author': 'Unknown',
292 'minimum_calibre_version': (0, 9, 42),
293 }
294 field_names = set(defaults)
295 imported_names = {}
296
297 plugin_import_found = set()
298 all_imports = []
299 for node in top_level_imports:
300 names = getattr(node, 'names', [])
301 mod = getattr(node, 'module', None)
302 if names and mod:
303 names = [Alias(n.name, getattr(n, 'asname', None)) for n in names]
304 if (
305 mod
306 in {
307 'calibre.customize',
308 'calibre.customize.conversion',
309 'calibre.ebooks.metadata.sources.base',
310 'calibre.ebooks.metadata.sources.amazon',
311 'calibre.ebooks.metadata.covers',
312 'calibre.devices.interface',
313 'calibre.ebooks.metadata.fetch',
314 'calibre.customize.builtins',
315 }
316 or re.match(r'calibre\.devices\.[a-z0-9]+\.driver', mod) is not None
317 ):
318 inames = {n.asname or n.name for n in names}
319 inames = {x for x in inames if x.lower() != x}
320 plugin_import_found |= inames
321 else:
322 all_imports.append((mod, [n.name for n in names]))
323 for alias in names:
324 imported_names[alias.asname or alias.name] = mod
325 if not plugin_import_found:
326 return all_imports
327
328 import_data = (imported_names, zf, namelist)
329
330 names = {}
331 top_level_ann_assigments = [x for x in ast.iter_child_nodes(module) if x.__class__.__name__ == 'AnnAssign']
332 for node in top_level_assigments:
333 targets = {getattr(t, 'id', None) for t in node.targets}
334 targets.discard(None)
335 for x in targets - field_names:
336 try:
337 val = convert_node({x}, node.value, import_data=import_data)
338 except Exception:
339 pass
340 else:
341 names[x] = val
342 for node in top_level_ann_assigments:
343 if node.value is None:
344 continue
345 x = getattr(node.target, 'id', None)
346 if x is None or x in field_names:
347 continue
348 try:
349 val = convert_node({x}, node.value, import_data=import_data)
350 except Exception:
351 pass
352 else:
353 names[x] = val
354
355 def parse_class(node):
356 class_assigments = [x for x in ast.iter_child_nodes(node) if x.__class__.__name__ == 'Assign']
357 class_ann_assigments = [x for x in ast.iter_child_nodes(node) if x.__class__.__name__ == 'AnnAssign']
358 found = {}
359 for node in class_assigments:
360 targets = {getattr(t, 'id', None) for t in node.targets}
361 targets.discard(None)
362 fields = field_names.intersection(targets)
363 if fields:
364 val = convert_node(fields, node.value, names=names, import_data=import_data)
365 for field in fields:
366 found[field] = val
367 for node in class_ann_assigments:
368 if node.value is None:
369 continue
370 target_id = getattr(node.target, 'id', None)
371 if target_id is None:
372 continue
373 fields = field_names.intersection({target_id})
374 if fields:
375 val = convert_node(fields, node.value, names=names, import_data=import_data)
376 for field in fields:
377 found[field] = val
378 return found
379
380 if top_level_classes:
381 for node in top_level_classes:
382 bases = {getattr(x, 'id', None) for x in node.bases}
383 if not bases.intersection(plugin_import_found):
384 continue
385 found = parse_class(node)
386 if 'name' in found and 'author' in found:
387 defaults.update(found)
388 return defaults
389 for node in top_level_classes:
390 found = parse_class(node)
391 if 'name' in found and 'author' in found and 'version' in found:
392 defaults.update(found)
393 return defaults
394
395 raise ValueError('Could not find plugin class')
396
397
398 def parse_plugin(raw, names, zf):
399 ans = parse_metadata(raw, names, zf)
400 if isinstance(ans, dict):
401 return ans
402 # The plugin is importing its base class from somewhere else, le sigh
403 for mod, _ in ans:
404 mod = mod.split('.')
405 if mod[0] == 'calibre_plugins':
406 mod = mod[2:]
407 mod = '/'.join(mod) + '.py'
408 if mod in names:
409 raw = zf.open(names[mod]).read()
410 ans = parse_metadata(raw, names, zf)
411 if isinstance(ans, dict):
412 return ans
413
414 raise ValueError('Failed to find plugin class')
415
416
417 def get_plugin_init(zf):
418 metadata = None
419 names = {x.decode('utf-8') if isinstance(x, bytes) else x: x for x in zf.namelist()}
420 inits = [x for x in names if x.rpartition('/')[-1] == '__init__.py']
421 inits.sort(key=lambda x: x.count('/'))
422 if inits and inits[0] == '__init__.py':
423 metadata = names[inits[0]]
424 else:
425 # Legacy plugin
426 for name, val in names.items():
427 if name.endswith('plugin.py'):
428 metadata = val
429 break
430 if metadata is None:
431 raise ValueError('No __init__.py found in plugin')
432 return zf.open(metadata).read(), names
433
434
435 def get_plugin_info(raw_zip):
436 with zipfile.ZipFile(io.BytesIO(raw_zip)) as zf:
437 raw, names = get_plugin_init(zf)
438 try:
439 return parse_plugin(raw, names, zf)
440 except (SyntaxError, TabError, IndentationError): # fmt: skip
441 with tempfile.NamedTemporaryFile(suffix='.zip') as f:
442 f.write(raw_zip)
443 f.flush()
444 res = subprocess.run(['python2', __file__, f.name], stdout=subprocess.PIPE)
445 if res.returncode == 0:
446 return json.loads(res.stdout)
447 raise
448
449
450 # }}}
451
452
453 def update_plugin_from_entry(plugin, entry):
454 plugin['index_name'] = entry.name
455 plugin['thread_url'] = entry.url
456 plugin['category'] = entry.category
457 for x in ('donate', 'history', 'deprecated', 'uninstall', 'thread_id'):
458 plugin[x] = getattr(entry, x)
459
460
461 def fetch_plugin(old_index, entry):
462 lm_map = {plugin['thread_id']: plugin for plugin in old_index.values()}
463 res = read(entry.url, get_info=True)
464 raw = decode_html(*res)
465 url, name = parse_plugin_zip_url(raw)
466 if url is None:
467 raise ValueError(f'Failed to find zip file URL for entry: {entry!r}')
468 plugin = lm_map.get(entry.thread_id, None)
469
470 if plugin is not None:
471 # Previously downloaded plugin
472 lm = datetime(*tuple(map(int, re.split(r'\D', plugin['last_modified'])))[:6])
473 request = Request(url)
474 request.get_method = lambda: 'HEAD'
475 with closing(urlopen(request)) as response:
476 info = response.info()
477 slm = datetime(*parsedate(info.get('Last-Modified'))[:6])
478 if lm >= slm:
479 # The previously downloaded plugin zip file is up-to-date
480 update_plugin_from_entry(plugin, entry)
481 return plugin
482
483 raw, info = read(url, get_info=True)
484 slm = datetime(*parsedate(info.get('Last-Modified'))[:6])
485 plugin = get_plugin_info(raw)
486 plugin['last_modified'] = slm.isoformat()
487 plugin['file'] = f'staging_{entry.thread_id}.zip'
488 plugin['size'] = len(raw)
489 plugin['original_url'] = url
490 update_plugin_from_entry(plugin, entry)
491 with open(plugin['file'], 'wb') as f:
492 f.write(raw)
493 return plugin
494
495
496 def parallel_fetch(old_index, entry):
497 try:
498 return fetch_plugin(old_index, entry)
499 except Exception:
500 import traceback
501
502 return traceback.format_exc()
503
504
505 def log(*args, **kwargs):
506 print(*args, **kwargs)
507 with open('log', 'a') as f:
508 kwargs['file'] = f
509 print(*args, **kwargs)
510
511
512 def atomic_write(raw, name):
513 with tempfile.NamedTemporaryFile(dir=os.getcwd(), delete=False) as f:
514 f.write(raw)
515 os.fchmod(f.fileno(), stat.S_IREAD | stat.S_IWRITE | stat.S_IRGRP | stat.S_IROTH)
516 os.rename(f.name, name)
517
518
519 def fetch_plugins(old_index, thread_id=None):
520 all_entries = tuple(parse_index())
521 if not all_entries:
522 raise SystemExit('Could not find any plugins, probably the markup on the MR index page has changed')
523 if thread_id is not None:
524 entries = tuple(e for e in all_entries if e.thread_id == thread_id)
525 if not entries:
526 raise SystemExit(f'No plugin with thread_id {thread_id!r} found in the MR index')
527 ans = dict(old_index)
528 else:
529 entries = all_entries
530 ans = {}
531 pool = ThreadPool(processes=min(10, len(entries)))
532 with closing(pool):
533 result = pool.map(partial(parallel_fetch, old_index), entries)
534 for entry, plugin in zip(entries, result):
535 if isinstance(plugin, dict):
536 ans[entry.name] = plugin
537 else:
538 if entry.name in old_index:
539 ans[entry.name] = old_index[entry.name]
540 log('Failed to get plugin', entry.name, 'at', datetime.now().isoformat(), 'with error:')
541 log(plugin)
542 # Move staged files
543 for plugin in ans.values():
544 if plugin['file'].startswith('staging_'):
545 src = plugin['file']
546 plugin['file'] = src.partition('_')[-1]
547 os.rename(src, plugin['file'])
548 raw = bz2.compress(json.dumps(ans, sort_keys=True, indent=4, separators=(',', ': ')).encode('utf-8'))
549 atomic_write(raw, PLUGINS)
550 if thread_id is None:
551 # Cleanup any extra .zip files only on full sync
552 all_plugin_files = {p['file'] for p in ans.values()}
553 extra = set(glob.glob('*.zip')) - all_plugin_files
554 for x in extra:
555 os.unlink(x)
556 return ans
557
558
559 def plugin_to_index(plugin, count):
560 title = '<h3><img src="plugin-icon.png"><a href={} title="Plugin forum thread">{}</a></h3>'.format(quoteattr(plugin['thread_url']), escape(plugin['name']))
561 released = datetime(*tuple(map(int, re.split(r'\D', plugin['last_modified'])))[:6]).strftime('%e %b, %Y').lstrip()
562 details = [
563 'Version: <b>{}</b>'.format(escape('.'.join(map(str, plugin['version'])))),
564 f'Released: <b>{escape(released)}</b>',
565 'Author: {}'.format(escape(plugin['author'])),
566 'calibre: {}'.format(escape('.'.join(map(str, plugin['minimum_calibre_version'])))),
567 'Platforms: {}'.format(escape(', '.join(sorted(plugin['supported_platforms']) or ['all']))),
568 ]
569 if plugin['uninstall']:
570 details.append('Uninstall: {}'.format(escape(', '.join(plugin['uninstall']))))
571 if plugin['donate']:
572 details.append('<a href={} title="Donate">Donate</a>'.format(quoteattr(plugin['donate'])))
573 block = []
574 for li in details:
575 if li.startswith('calibre:'):
576 block.append('<br>')
577 block.append(f'<li>{li}</li>')
578 block = '<ul>{}</ul>'.format('\n'.join(block))
579 downloads = (f'\xa0<span class="download-count">[{count} total downloads]</span>') if count else ''
580 zipfile = '<div class="end"><a href={} title="Download plugin" download={}>Download plugin \u2193</a>{}</div>'.format(
581 quoteattr(plugin['file']), quoteattr(plugin['name'] + '.zip'), downloads
582 )
583 desc = plugin['description'] or ''
584 if desc:
585 desc = f'<p>{desc}</p>'
586 return f'{title}\n{desc}\n{block}\n{zipfile}\n\n'
587
588
589 def create_index(index, raw_stats):
590 plugins = []
591 stats = {}
592 for name in sorted(index):
593 plugin = index[name]
594 if not plugin['deprecated']:
595 count = raw_stats.get(plugin['file'].rpartition('.')[0], 0)
596 if count > 0:
597 stats[plugin['name']] = count
598 plugins.append(plugin_to_index(plugin, count))
599 index = '''\
600 <!DOCTYPE html>
601 <html>
602 <head><meta charset="utf-8"><title>Index of calibre plugins</title>
603 <link rel="icon" type="image/x-icon" href="//calibre-ebook.com/favicon.ico" />
604 <style type="text/css">
605 body { background-color: #eee; }
606 a { text-decoration: none }
607 a:hover, h3:hover { color: red }
608 a:visited { color: blue }
609 ul { list-style-type: none; font-size: smaller }
610 li { display: inline }
611 li+li:before { content: " - " }
612 .end { border-bottom: solid 1pt black; padding-bottom: 0.5ex; margin-bottom: 4ex; }
613 h1 img, h3 img { vertical-align: middle; margin-right: 0.5em; }
614 h1 { text-align: center }
615 .download-count { color: gray; font-size: smaller }
616 </style>
617 </head>
618 <body>
619 <h1><img src="//manual.calibre-ebook.com/_static/logo.png">Index of calibre plugins</h1>
620 <div style="text-align:center"><a href="stats.html">Download counts for all plugins</a></div>
621 %s
622 </body>
623 </html>''' % ('\n'.join(plugins)) # noqa: UP031
624 raw = index.encode('utf-8')
625 try:
626 with open('index.html', 'rb') as f:
627 oraw = f.read()
628 except OSError:
629 oraw = None
630 if raw != oraw:
631 atomic_write(raw, 'index.html')
632
633 def plugin_stats(x):
634 name, count = x
635 return f'<tr><td>{escape(name)}</td><td>{count}</td></tr>\n'
636
637 pstats = list(map(plugin_stats, sorted(stats.items(), reverse=True, key=lambda x: x[1])))
638 stats = '''\
639 <!DOCTYPE html>
640 <html>
641 <head><meta charset="utf-8"><title>Stats for calibre plugins</title>
642 <link rel="icon" type="image/x-icon" href="//calibre-ebook.com/favicon.ico" />
643 <style type="text/css">
644 body { background-color: #eee; }
645 h1 img, h3 img { vertical-align: middle; margin-right: 0.5em; }
646 h1 { text-align: center }
647 </style>
648 </head>
649 <body>
650 <h1><img src="//manual.calibre-ebook.com/_static/logo.png">Stats for calibre plugins</h1>
651 <table>
652 <tr><th>Plugin</th><th>Total downloads</th></tr>
653 %s
654 </table>
655 </body>
656 </html>
657 ''' % ('\n'.join(pstats)) # noqa: UP031
658 raw = stats.encode('utf-8')
659 try:
660 with open('stats.html', 'rb') as f:
661 oraw = f.read()
662 except OSError:
663 oraw = None
664 if raw != oraw:
665 atomic_write(raw, 'stats.html')
666
667
668 _singleinstance = None
669
670
671 def singleinstance():
672 global _singleinstance
673 s = _singleinstance = socket.socket(socket.AF_UNIX)
674 try:
675 s.bind(b'\0calibre-plugins-mirror-singleinstance')
676 except OSError as err:
677 if getattr(err, 'errno', None) == errno.EADDRINUSE:
678 return False
679 raise
680 return True
681
682
683 def update_stats():
684 log = olog = 'stats.log'
685 if not os.path.exists(log):
686 return {}
687 stats = {}
688 if IS_PRODUCTION:
689 try:
690 with open('stats.json', 'rb') as f:
691 stats = json.load(f)
692 except OSError as err:
693 if err.errno != errno.ENOENT:
694 raise
695 if os.geteuid() != 0:
696 return stats
697 log = 'rotated-' + log
698 os.rename(olog, log)
699 subprocess.check_call(['/usr/sbin/nginx', '-s', 'reopen'])
700 atexit.register(os.remove, log)
701 pat = re.compile(rb'GET /(\d+)(?:-deprecated){0,1}\.zip')
702 for line in open(log, 'rb'):
703 m = pat.search(line)
704 if m is not None:
705 plugin = m.group(1).decode('utf-8')
706 stats[plugin] = stats.get(plugin, 0) + 1
707 data = json.dumps(stats, indent=2)
708 if not isinstance(data, bytes):
709 data = data.encode('utf-8')
710 with open('stats.json', 'wb') as f:
711 f.write(data)
712 return stats
713
714
715 def parse_single_plugin(zipfile_path):
716 with zipfile.ZipFile(zipfile_path) as zf:
717 raw, names = get_plugin_init(zf)
718 ans = parse_plugin(raw, names, zf)
719 sys.stdout.write(json.dumps(ans, ensure_ascii=True))
720
721
722 def main():
723 p = argparse.ArgumentParser(description='Mirror calibre plugins from the forums. Or parse a single plugin zip file if specified on the command line')
724 p.add_argument('plugin_path', nargs='?', default='', help='Path to plugin zip file to parse')
725 WORKDIR = '/srv/plugins' if IS_PRODUCTION else '/t/plugins'
726 p.add_argument('-o', '--output-dir', default=WORKDIR, help='Where to place the mirrored plugins. Default is: ' + WORKDIR)
727 p.add_argument('--thread-id', default=None, help='Only download/update the plugin with this thread ID (e.g. 1234 or 1234-deprecated)')
728 args = p.parse_args()
729 if args.plugin_path:
730 return parse_single_plugin(args.plugin_path)
731
732 os.makedirs(args.output_dir, exist_ok=True)
733 os.chdir(args.output_dir)
734 if os.geteuid() == 0 and not singleinstance():
735 print('Another instance of plugins-mirror is running', file=sys.stderr)
736 raise SystemExit(1)
737 open('log', 'w').close()
738 stats = update_stats()
739 try:
740 plugins_index = load_plugins_index()
741 plugins_index = fetch_plugins(plugins_index, thread_id=args.thread_id)
742 create_index(plugins_index, stats)
743 except KeyboardInterrupt:
744 raise SystemExit('Exiting on user interrupt')
745 except Exception:
746 import traceback
747
748 log('Failed to run at:', datetime.now().isoformat())
749 log(traceback.format_exc())
750 raise SystemExit(1)
751
752
753 # }}}
754
755
756 def test_parse_metadata(): # {{{
757 raw = b'''\
758 import os
759 from calibre.customize import FileTypePlugin
760 # import_placeholder
761
762 MV = (0, 7, 53)
763
764 class HelloWorld(FileTypePlugin):
765
766 name: str = _('name') # Name of the plugin
767 description = {1, 2}
768 supported_platforms = ['windows', 'osx', 'linux'] # Platforms this plugin will run on
769 author = 'Acme Inc.' # The author of this plugin
770 version: dict[str, int] = {1:'a', 'b':2}
771 file_types = set(['epub', 'mobi']) # The file types that this plugin will be applied to
772 on_postprocess = True # Run this plugin after conversion is complete
773 minimum_calibre_version = MV
774 '''
775 vals = {
776 'name': 'name',
777 'description': {1, 2},
778 'supported_platforms': ['windows', 'osx', 'linux'],
779 'author': 'Acme Inc.',
780 'version': {1: 'a', 'b': 2},
781 'minimum_calibre_version': (0, 7, 53),
782 }
783 assert parse_metadata(raw, None, None) == vals
784 buf = io.BytesIO()
785 with zipfile.ZipFile(buf, 'w') as zf:
786 zf.writestr('very/lovely.py', raw.replace(b'MV = (0, 7, 53)', b'from very.ver import MV'))
787 zf.writestr('very/ver.py', b'MV = (0, 7, 53)')
788 zf.writestr('__init__.py', b'from xxx import yyy\nfrom very.lovely import HelloWorld')
789 assert get_plugin_info(buf.getvalue()) == vals
790 buf = io.BytesIO()
791 with zipfile.ZipFile(buf, 'w') as zf:
792 zf.writestr('very/ver.py', b'NAME = "name"\nVERSION = (1,2,3)')
793 zf.writestr(
794 '__init__.py',
795 raw.replace(b"_('name')", b'NAME').replace(b"{1:'a', 'b':2}", b'VERSION').replace(b'# import_placeholder', b'from very.ver import NAME, VERSION'),
796 )
797 vals['version'] = (1, 2, 3)
798 assert get_plugin_info(buf.getvalue()) == vals
799
800 # Test: annotated top-level variable used as class attribute value
801 raw2 = b'''\
802 from calibre.customize import FileTypePlugin
803
804 MV: tuple = (0, 8, 0)
805
806 class MyPlugin(FileTypePlugin):
807 name: str = 'TestPlugin'
808 author: str = 'Test Author'
809 version = MV
810 minimum_calibre_version = MV
811 description = 'A test plugin'
812 supported_platforms = ['linux']
813 '''
814 vals2 = {
815 'name': 'TestPlugin',
816 'description': 'A test plugin',
817 'supported_platforms': ['linux'],
818 'author': 'Test Author',
819 'version': (0, 8, 0),
820 'minimum_calibre_version': (0, 8, 0),
821 }
822 assert parse_metadata(raw2, None, None) == vals2
823
824 # Test: annotation without value (bare annotation) is ignored gracefully
825 raw3 = b'''\
826 from calibre.customize import FileTypePlugin
827
828 class MyPlugin(FileTypePlugin):
829 name: str = 'BareAnnot'
830 author: str = 'Some Author'
831 description: str
832 version = (2, 0, 0)
833 minimum_calibre_version = (1, 0, 0)
834 supported_platforms: list = ['windows', 'linux']
835 '''
836 vals3 = {
837 'name': 'BareAnnot',
838 'description': '',
839 'supported_platforms': ['windows', 'linux'],
840 'author': 'Some Author',
841 'version': (2, 0, 0),
842 'minimum_calibre_version': (1, 0, 0),
843 }
844 assert parse_metadata(raw3, None, None) == vals3
845
846 # Test: multiple classes, only the one inheriting from a plugin base is used
847 raw4 = b'''\
848 from calibre.customize import FileTypePlugin
849
850 class Helper:
851 name = 'should-be-ignored'
852 author = 'ignored'
853
854 class RealPlugin(FileTypePlugin):
855 name = 'RealPlugin'
856 author = 'Real Author'
857 version = (3, 1, 4)
858 minimum_calibre_version = (0, 9, 42)
859 supported_platforms = ['windows', 'osx', 'linux']
860 description = 'The real one'
861 '''
862 vals4 = {
863 'name': 'RealPlugin',
864 'description': 'The real one',
865 'supported_platforms': ['windows', 'osx', 'linux'],
866 'author': 'Real Author',
867 'version': (3, 1, 4),
868 'minimum_calibre_version': (0, 9, 42),
869 }
870 assert parse_metadata(raw4, None, None) == vals4
871
872 # Test: all class attributes are type-annotated
873 raw5 = b'''\
874 from calibre.customize import FileTypePlugin
875
876 class FullAnnotPlugin(FileTypePlugin):
877 name: str = 'AllAnnotated'
878 description: str = 'fully annotated plugin'
879 supported_platforms: list = ['osx']
880 author: str = 'Annotated Author'
881 version: tuple = (1, 2, 3)
882 minimum_calibre_version: tuple = (0, 9, 42)
883 '''
884 vals5 = {
885 'name': 'AllAnnotated',
886 'description': 'fully annotated plugin',
887 'supported_platforms': ['osx'],
888 'author': 'Annotated Author',
889 'version': (1, 2, 3),
890 'minimum_calibre_version': (0, 9, 42),
891 }
892 assert parse_metadata(raw5, None, None) == vals5
893
894
895 # }}}
896
897
898 if __name__ == '__main__':
899 # Run test_parse_metadata() with:
900 # python -c 'import runpy; runpy.run_path("setup/plugins_mirror.py")["test_parse_metadata"]()'
901 # import pprint
902 # pprint.pprint(get_plugin_info(open(sys.argv[-1], 'rb').read()))
903
904 # print('\n'.join(map(str, parse_index())))
905
906 main()