| 1 |
#!/usr/bin/env python |
| 2 |
|
| 3 |
|
| 4 |
__license__ = 'GPL v3' |
| 5 |
__copyright__ = '2009, Kovid Goyal <[email protected]>' |
| 6 |
__docformat__ = 'restructuredtext en' |
| 7 |
|
| 8 |
import glob |
| 9 |
import json |
| 10 |
import os |
| 11 |
import shutil |
| 12 |
import zipfile |
| 13 |
|
| 14 |
from polyglot.builtins import only_unicode_recursive |
| 15 |
from setup import Command, basenames, download_securely, dump_json |
| 16 |
|
| 17 |
|
| 18 |
def get_opts_from_parser(parser): |
| 19 |
def do_opt(opt): |
| 20 |
yield from opt._long_opts |
| 21 |
yield from opt._short_opts |
| 22 |
|
| 23 |
for o in parser.option_list: |
| 24 |
yield from do_opt(o) |
| 25 |
for g in parser.option_groups: |
| 26 |
for o in g.option_list: |
| 27 |
yield from do_opt(o) |
| 28 |
|
| 29 |
|
| 30 |
class CACerts(Command): # {{{ |
| 31 |
description = 'Get updated mozilla CA certificate bundle' |
| 32 |
CA_PATH = os.path.join(Command.RESOURCES, 'mozilla-ca-certs') |
| 33 |
|
| 34 |
def add_options(self, parser): |
| 35 |
parser.add_option('--path-to-cacerts', help='Path to previously downloaded mozilla-ca-certs.pem') |
| 36 |
|
| 37 |
def run(self, opts): |
| 38 |
import calibre # needed to ensure calibre_extensions is available |
| 39 |
|
| 40 |
_ = calibre |
| 41 |
from calibre_extensions.certgen import create_CA_dir |
| 42 |
|
| 43 |
if opts.path_to_cacerts: |
| 44 |
with open(opts.path_to_cacerts, 'rb') as f: |
| 45 |
raw = f.read() |
| 46 |
else: |
| 47 |
raw = download_securely('https://curl.se/ca/cacert.pem') |
| 48 |
if not raw: |
| 49 |
raise RuntimeError('Failed to download CA cert bundle') |
| 50 |
if os.path.exists(self.CA_PATH): |
| 51 |
shutil.rmtree(self.CA_PATH) |
| 52 |
os.mkdir(self.CA_PATH) |
| 53 |
create_CA_dir(raw, self.CA_PATH) |
| 54 |
self.verify_ca_certs() |
| 55 |
|
| 56 |
def verify_ca_certs(self): |
| 57 |
from calibre.utils.https import get_https_resource_securely |
| 58 |
|
| 59 |
get_https_resource_securely('https://calibre-ebook.com', cadir=self.CA_PATH) |
| 60 |
|
| 61 |
|
| 62 |
# }}} |
| 63 |
|
| 64 |
|
| 65 |
class RecentUAs(Command): # {{{ |
| 66 |
description = 'Get updated list of common browser user agents' |
| 67 |
UA_PATH = os.path.join(Command.RESOURCES, 'user-agent-data.json') |
| 68 |
|
| 69 |
def add_options(self, parser): |
| 70 |
parser.add_option('--path-to-user-agent-data', help='Path to previously downloaded user-agent-data.json') |
| 71 |
|
| 72 |
def run(self, opts): |
| 73 |
from setup.browser_data import get_data |
| 74 |
|
| 75 |
if opts.path_to_user_agent_data: |
| 76 |
shutil.copyfile(opts.path_to_user_agent_data, self.UA_PATH) |
| 77 |
os.chmod(self.UA_PATH, 0o644) |
| 78 |
else: |
| 79 |
data = get_data() |
| 80 |
with open(self.UA_PATH, 'w', encoding='utf-8') as f: |
| 81 |
json.dump(data, f, indent=2, ensure_ascii=False, sort_keys=True) |
| 82 |
|
| 83 |
|
| 84 |
# }}} |
| 85 |
|
| 86 |
|
| 87 |
class RapydScript(Command): # {{{ |
| 88 |
description = 'Compile RapydScript to JavaScript' |
| 89 |
|
| 90 |
def add_options(self, parser): |
| 91 |
parser.add_option('--only-module', default=None, help='Only compile the specified module') |
| 92 |
|
| 93 |
def run(self, opts): |
| 94 |
from calibre.utils.rapydscript import compile_all, compile_editor, compile_srv, compile_viewer |
| 95 |
|
| 96 |
bin_dir = self.j(self.PROJECT_ROOT, '.venv', 'bin') |
| 97 |
if os.path.exists(bin_dir): |
| 98 |
os.environ['PATH'] = bin_dir + ':' + os.environ['PATH'] |
| 99 |
|
| 100 |
if opts.only_module: |
| 101 |
match opts.only_module: |
| 102 |
case 'editor': |
| 103 |
compile_editor() |
| 104 |
case 'srv': |
| 105 |
compile_srv() |
| 106 |
case 'viewer': |
| 107 |
compile_viewer() |
| 108 |
case _: |
| 109 |
raise KeyError(f'Unknown RapydScript --only-module: {opts.only_module}') |
| 110 |
else: |
| 111 |
compile_all() |
| 112 |
|
| 113 |
|
| 114 |
# }}} |
| 115 |
|
| 116 |
|
| 117 |
class Resources(Command): # {{{ |
| 118 |
description = 'Compile various needed calibre resources' |
| 119 |
sub_commands = ['liberation_fonts', 'mathjax', 'rapydscript', 'hyphenation', 'piper_voices'] |
| 120 |
|
| 121 |
def run(self, opts): |
| 122 |
from calibre.utils.serialize import msgpack_dumps |
| 123 |
|
| 124 |
scripts = {} |
| 125 |
for x in ('console', 'gui'): |
| 126 |
for name in basenames[x]: |
| 127 |
if name in ('calibre-complete', 'calibre_postinstall'): |
| 128 |
continue |
| 129 |
scripts[name] = x |
| 130 |
|
| 131 |
dest = self.j(self.RESOURCES, 'scripts.calibre_msgpack') |
| 132 |
if self.newer(dest, self.j(self.SRC, 'calibre', 'linux.py')): |
| 133 |
self.info('\tCreating ' + self.b(dest)) |
| 134 |
with open(dest, 'wb') as f: |
| 135 |
f.write(msgpack_dumps(scripts)) |
| 136 |
|
| 137 |
from calibre.web.feeds.recipes.collection import iterate_over_builtin_recipe_files, serialize_builtin_recipes |
| 138 |
|
| 139 |
files = [x[1] for x in iterate_over_builtin_recipe_files()] |
| 140 |
|
| 141 |
dest = self.j(self.RESOURCES, 'builtin_recipes.xml') |
| 142 |
if self.newer(dest, files): |
| 143 |
self.info('\tCreating builtin_recipes.xml') |
| 144 |
xml = serialize_builtin_recipes() |
| 145 |
with open(dest, 'wb') as f: |
| 146 |
f.write(xml) |
| 147 |
|
| 148 |
recipe_icon_dir = self.a(self.j(self.RESOURCES, '..', 'recipes', 'icons')) |
| 149 |
dest = os.path.splitext(dest)[0] + '.zip' |
| 150 |
files += glob.glob(self.j(recipe_icon_dir, '*.png')) |
| 151 |
if self.newer(dest, files): |
| 152 |
self.info('\tCreating builtin_recipes.zip') |
| 153 |
with zipfile.ZipFile(dest, 'w', zipfile.ZIP_STORED) as zf: |
| 154 |
for n in sorted(files, key=self.b): |
| 155 |
with open(n, 'rb') as f: |
| 156 |
zf.writestr(self.b(n), f.read()) |
| 157 |
|
| 158 |
dest = self.j(self.RESOURCES, 'ebook-convert-complete.calibre_msgpack') |
| 159 |
files = [] |
| 160 |
for x in os.walk(self.j(self.SRC, 'calibre')): |
| 161 |
for f in x[-1]: |
| 162 |
if f.endswith('.py'): |
| 163 |
files.append(self.j(x[0], f)) |
| 164 |
if self.newer(dest, files): |
| 165 |
self.info('\tCreating ' + self.b(dest)) |
| 166 |
complete = {} |
| 167 |
from calibre.ebooks.conversion.plumber import supported_input_formats |
| 168 |
|
| 169 |
complete['input_fmts'] = set(supported_input_formats()) |
| 170 |
from calibre.web.feeds.recipes.collection import get_builtin_recipe_titles |
| 171 |
|
| 172 |
complete['input_recipes'] = [t + '.recipe ' for t in get_builtin_recipe_titles()] |
| 173 |
from calibre.customize.ui import available_output_formats |
| 174 |
|
| 175 |
complete['output'] = set(available_output_formats()) |
| 176 |
from calibre.ebooks.conversion.cli import create_option_parser |
| 177 |
from calibre.utils.logging import Log |
| 178 |
|
| 179 |
log = Log() |
| 180 |
# log.outputs = [] |
| 181 |
for inf in supported_input_formats(): |
| 182 |
if inf in ('zip', 'rar', 'oebzip'): |
| 183 |
continue |
| 184 |
for ouf in available_output_formats(): |
| 185 |
of = ouf if ouf == 'oeb' else 'dummy.' + ouf |
| 186 |
p = create_option_parser(('ec', 'dummy1.' + inf, of, '-h'), log)[0] |
| 187 |
complete[(inf, ouf)] = [x + ' ' for x in get_opts_from_parser(p)] |
| 188 |
|
| 189 |
with open(dest, 'wb') as f: |
| 190 |
f.write(msgpack_dumps(only_unicode_recursive(complete))) |
| 191 |
|
| 192 |
self.info('\tCreating template-functions.json') |
| 193 |
dest = self.j(self.RESOURCES, 'template-functions.json') |
| 194 |
function_dict = {} |
| 195 |
import inspect |
| 196 |
|
| 197 |
from calibre.utils.formatter_functions import formatter_functions |
| 198 |
|
| 199 |
for obj in formatter_functions().get_builtins().values(): |
| 200 |
eval_func = inspect.getmembers(obj, lambda x: inspect.ismethod(x) and x.__name__ == 'evaluate') |
| 201 |
try: |
| 202 |
lines = [l[4:] for l in inspect.getsourcelines(eval_func[0][1])[0]] |
| 203 |
except Exception: |
| 204 |
continue |
| 205 |
lines = ''.join(lines) |
| 206 |
function_dict[obj.name] = lines |
| 207 |
dump_json(function_dict, dest) |
| 208 |
|
| 209 |
self.info('\tCreating editor-functions.json') |
| 210 |
dest = self.j(self.RESOURCES, 'editor-functions.json') |
| 211 |
function_dict = {} |
| 212 |
from calibre.gui2.tweak_book.function_replace import builtin_functions |
| 213 |
|
| 214 |
for func in builtin_functions(): |
| 215 |
try: |
| 216 |
src = ''.join(inspect.getsourcelines(func)[0][1:]) |
| 217 |
except Exception: |
| 218 |
continue |
| 219 |
src = src.replace('def ' + func.__name__, 'def replace') |
| 220 |
imports = [f'from {x.__module__} import {x.__name__}' for x in func.imports] |
| 221 |
if imports: |
| 222 |
src = '\n'.join(imports) + '\n\n' + src |
| 223 |
function_dict[func.name] = src |
| 224 |
dump_json(function_dict, dest) |
| 225 |
self.info('\tCreating user-manual-translation-stats.json') |
| 226 |
d = {} |
| 227 |
for lc, stats in json.load(open(self.j(self.d(self.SRC), 'manual', 'locale', 'completed.json'))).items(): |
| 228 |
total = sum(stats.values()) |
| 229 |
d[lc] = stats['translated'] / float(total) |
| 230 |
dump_json(d, self.j(self.RESOURCES, 'user-manual-translation-stats.json')) |
| 231 |
|
| 232 |
src = self.j(self.SRC, '..', 'Changelog.txt') |
| 233 |
dest = self.j(self.RESOURCES, 'changelog.json') |
| 234 |
if self.newer(dest, [src]): |
| 235 |
self.info('\tCreating changelog.json') |
| 236 |
from setup.changelog import parse |
| 237 |
|
| 238 |
with open(src, encoding='utf-8') as f: |
| 239 |
dump_json(parse(f.read(), parse_dates=False), dest) |
| 240 |
|
| 241 |
def clean(self): |
| 242 |
for x in ('scripts', 'ebook-convert-complete'): |
| 243 |
x = self.j(self.RESOURCES, x + '.pickle') |
| 244 |
if os.path.exists(x): |
| 245 |
os.remove(x) |
| 246 |
for x in ('builtin_recipes.xml', 'builtin_recipes.zip', 'template-functions.json', 'user-manual-translation-stats.json'): |
| 247 |
x = self.j(self.RESOURCES, x) |
| 248 |
if os.path.exists(x): |
| 249 |
os.remove(x) |
| 250 |
|
| 251 |
|
| 252 |
# }}} |