Debian Codegraph @

setup/resources.py

raw
1 #!/usr/bin/env python
2
3
4 __license__ = 'GPL v3'
5 __copyright__ = '2009, Kovid Goyal <[email protected]>'
6 __docformat__ = 'restructuredtext en'
7
8 import glob
9 import json
10 import os
11 import shutil
12 import zipfile
13
14 from polyglot.builtins import only_unicode_recursive
15 from setup import Command, basenames, download_securely, dump_json
16
17
18 def get_opts_from_parser(parser):
19 def do_opt(opt):
20 yield from opt._long_opts
21 yield from opt._short_opts
22
23 for o in parser.option_list:
24 yield from do_opt(o)
25 for g in parser.option_groups:
26 for o in g.option_list:
27 yield from do_opt(o)
28
29
30 class CACerts(Command): # {{{
31 description = 'Get updated mozilla CA certificate bundle'
32 CA_PATH = os.path.join(Command.RESOURCES, 'mozilla-ca-certs')
33
34 def add_options(self, parser):
35 parser.add_option('--path-to-cacerts', help='Path to previously downloaded mozilla-ca-certs.pem')
36
37 def run(self, opts):
38 import calibre # needed to ensure calibre_extensions is available
39
40 _ = calibre
41 from calibre_extensions.certgen import create_CA_dir
42
43 if opts.path_to_cacerts:
44 with open(opts.path_to_cacerts, 'rb') as f:
45 raw = f.read()
46 else:
47 raw = download_securely('https://curl.se/ca/cacert.pem')
48 if not raw:
49 raise RuntimeError('Failed to download CA cert bundle')
50 if os.path.exists(self.CA_PATH):
51 shutil.rmtree(self.CA_PATH)
52 os.mkdir(self.CA_PATH)
53 create_CA_dir(raw, self.CA_PATH)
54 self.verify_ca_certs()
55
56 def verify_ca_certs(self):
57 from calibre.utils.https import get_https_resource_securely
58
59 get_https_resource_securely('https://calibre-ebook.com', cadir=self.CA_PATH)
60
61
62 # }}}
63
64
65 class RecentUAs(Command): # {{{
66 description = 'Get updated list of common browser user agents'
67 UA_PATH = os.path.join(Command.RESOURCES, 'user-agent-data.json')
68
69 def add_options(self, parser):
70 parser.add_option('--path-to-user-agent-data', help='Path to previously downloaded user-agent-data.json')
71
72 def run(self, opts):
73 from setup.browser_data import get_data
74
75 if opts.path_to_user_agent_data:
76 shutil.copyfile(opts.path_to_user_agent_data, self.UA_PATH)
77 os.chmod(self.UA_PATH, 0o644)
78 else:
79 data = get_data()
80 with open(self.UA_PATH, 'w', encoding='utf-8') as f:
81 json.dump(data, f, indent=2, ensure_ascii=False, sort_keys=True)
82
83
84 # }}}
85
86
87 class RapydScript(Command): # {{{
88 description = 'Compile RapydScript to JavaScript'
89
90 def add_options(self, parser):
91 parser.add_option('--only-module', default=None, help='Only compile the specified module')
92
93 def run(self, opts):
94 from calibre.utils.rapydscript import compile_all, compile_editor, compile_srv, compile_viewer
95
96 bin_dir = self.j(self.PROJECT_ROOT, '.venv', 'bin')
97 if os.path.exists(bin_dir):
98 os.environ['PATH'] = bin_dir + ':' + os.environ['PATH']
99
100 if opts.only_module:
101 match opts.only_module:
102 case 'editor':
103 compile_editor()
104 case 'srv':
105 compile_srv()
106 case 'viewer':
107 compile_viewer()
108 case _:
109 raise KeyError(f'Unknown RapydScript --only-module: {opts.only_module}')
110 else:
111 compile_all()
112
113
114 # }}}
115
116
117 class Resources(Command): # {{{
118 description = 'Compile various needed calibre resources'
119 sub_commands = ['liberation_fonts', 'mathjax', 'rapydscript', 'hyphenation', 'piper_voices']
120
121 def run(self, opts):
122 from calibre.utils.serialize import msgpack_dumps
123
124 scripts = {}
125 for x in ('console', 'gui'):
126 for name in basenames[x]:
127 if name in ('calibre-complete', 'calibre_postinstall'):
128 continue
129 scripts[name] = x
130
131 dest = self.j(self.RESOURCES, 'scripts.calibre_msgpack')
132 if self.newer(dest, self.j(self.SRC, 'calibre', 'linux.py')):
133 self.info('\tCreating ' + self.b(dest))
134 with open(dest, 'wb') as f:
135 f.write(msgpack_dumps(scripts))
136
137 from calibre.web.feeds.recipes.collection import iterate_over_builtin_recipe_files, serialize_builtin_recipes
138
139 files = [x[1] for x in iterate_over_builtin_recipe_files()]
140
141 dest = self.j(self.RESOURCES, 'builtin_recipes.xml')
142 if self.newer(dest, files):
143 self.info('\tCreating builtin_recipes.xml')
144 xml = serialize_builtin_recipes()
145 with open(dest, 'wb') as f:
146 f.write(xml)
147
148 recipe_icon_dir = self.a(self.j(self.RESOURCES, '..', 'recipes', 'icons'))
149 dest = os.path.splitext(dest)[0] + '.zip'
150 files += glob.glob(self.j(recipe_icon_dir, '*.png'))
151 if self.newer(dest, files):
152 self.info('\tCreating builtin_recipes.zip')
153 with zipfile.ZipFile(dest, 'w', zipfile.ZIP_STORED) as zf:
154 for n in sorted(files, key=self.b):
155 with open(n, 'rb') as f:
156 zf.writestr(self.b(n), f.read())
157
158 dest = self.j(self.RESOURCES, 'ebook-convert-complete.calibre_msgpack')
159 files = []
160 for x in os.walk(self.j(self.SRC, 'calibre')):
161 for f in x[-1]:
162 if f.endswith('.py'):
163 files.append(self.j(x[0], f))
164 if self.newer(dest, files):
165 self.info('\tCreating ' + self.b(dest))
166 complete = {}
167 from calibre.ebooks.conversion.plumber import supported_input_formats
168
169 complete['input_fmts'] = set(supported_input_formats())
170 from calibre.web.feeds.recipes.collection import get_builtin_recipe_titles
171
172 complete['input_recipes'] = [t + '.recipe ' for t in get_builtin_recipe_titles()]
173 from calibre.customize.ui import available_output_formats
174
175 complete['output'] = set(available_output_formats())
176 from calibre.ebooks.conversion.cli import create_option_parser
177 from calibre.utils.logging import Log
178
179 log = Log()
180 # log.outputs = []
181 for inf in supported_input_formats():
182 if inf in ('zip', 'rar', 'oebzip'):
183 continue
184 for ouf in available_output_formats():
185 of = ouf if ouf == 'oeb' else 'dummy.' + ouf
186 p = create_option_parser(('ec', 'dummy1.' + inf, of, '-h'), log)[0]
187 complete[(inf, ouf)] = [x + ' ' for x in get_opts_from_parser(p)]
188
189 with open(dest, 'wb') as f:
190 f.write(msgpack_dumps(only_unicode_recursive(complete)))
191
192 self.info('\tCreating template-functions.json')
193 dest = self.j(self.RESOURCES, 'template-functions.json')
194 function_dict = {}
195 import inspect
196
197 from calibre.utils.formatter_functions import formatter_functions
198
199 for obj in formatter_functions().get_builtins().values():
200 eval_func = inspect.getmembers(obj, lambda x: inspect.ismethod(x) and x.__name__ == 'evaluate')
201 try:
202 lines = [l[4:] for l in inspect.getsourcelines(eval_func[0][1])[0]]
203 except Exception:
204 continue
205 lines = ''.join(lines)
206 function_dict[obj.name] = lines
207 dump_json(function_dict, dest)
208
209 self.info('\tCreating editor-functions.json')
210 dest = self.j(self.RESOURCES, 'editor-functions.json')
211 function_dict = {}
212 from calibre.gui2.tweak_book.function_replace import builtin_functions
213
214 for func in builtin_functions():
215 try:
216 src = ''.join(inspect.getsourcelines(func)[0][1:])
217 except Exception:
218 continue
219 src = src.replace('def ' + func.__name__, 'def replace')
220 imports = [f'from {x.__module__} import {x.__name__}' for x in func.imports]
221 if imports:
222 src = '\n'.join(imports) + '\n\n' + src
223 function_dict[func.name] = src
224 dump_json(function_dict, dest)
225 self.info('\tCreating user-manual-translation-stats.json')
226 d = {}
227 for lc, stats in json.load(open(self.j(self.d(self.SRC), 'manual', 'locale', 'completed.json'))).items():
228 total = sum(stats.values())
229 d[lc] = stats['translated'] / float(total)
230 dump_json(d, self.j(self.RESOURCES, 'user-manual-translation-stats.json'))
231
232 src = self.j(self.SRC, '..', 'Changelog.txt')
233 dest = self.j(self.RESOURCES, 'changelog.json')
234 if self.newer(dest, [src]):
235 self.info('\tCreating changelog.json')
236 from setup.changelog import parse
237
238 with open(src, encoding='utf-8') as f:
239 dump_json(parse(f.read(), parse_dates=False), dest)
240
241 def clean(self):
242 for x in ('scripts', 'ebook-convert-complete'):
243 x = self.j(self.RESOURCES, x + '.pickle')
244 if os.path.exists(x):
245 os.remove(x)
246 for x in ('builtin_recipes.xml', 'builtin_recipes.zip', 'template-functions.json', 'user-manual-translation-stats.json'):
247 x = self.j(self.RESOURCES, x)
248 if os.path.exists(x):
249 os.remove(x)
250
251
252 # }}}