Debian Codegraph @

src/calibre/srv/convert.py

raw
1 #!/usr/bin/env python
2 # License: GPLv3 Copyright: 2018, Kovid Goyal <kovid at kovidgoyal.net>
3
4 import json as stdlib_json
5 import os
6 import shutil
7 import tempfile
8 from threading import Lock
9
10 from calibre.customize.ui import input_profiles, output_profiles, run_plugins_on_postconvert
11 from calibre.db.errors import NoSuchBook
12 from calibre.srv.changes import formats_added
13 from calibre.srv.errors import BookNotFound, HTTPNotFound
14 from calibre.srv.routes import endpoint, json
15 from calibre.srv.utils import get_library_data
16 from calibre.utils.localization import _
17 from calibre.utils.monotonic import monotonic
18 from calibre.utils.shared_file import share_open
19
20 receive_data_methods = {'GET', 'POST'}
21 conversion_jobs = {}
22 cache_lock = Lock()
23
24 # Conversion options that must never be honoured when they come from a remote
25 # client of the content server, regardless of whether they are advertised by
26 # /conversion/book-data. They either take a filesystem path or URL (which would
27 # let a client read arbitrary server-side files, perform SSRF or manipulate the
28 # filesystem) or deliberately weaken a security guard. cover and
29 # read_metadata_from_opf are additionally set by the server itself in
30 # convert_book(), so a client must never be able to override them.
31 FORBIDDEN_CLIENT_OPTIONS = frozenset({
32 'cover',
33 'read_metadata_from_opf',
34 'debug_pipeline',
35 'allow_local_files_outside_root',
36 'extract_to',
37 })
38
39
40 class JobStatus:
41 def __init__(self, job_id, book_id, tdir, library_id, pathtoebook, conversion_data):
42 self.job_id = job_id
43 self.log = self.traceback = ''
44 self.book_id = book_id
45 # sanitize output_fmt to prevent path traversal
46 output_fmt = conversion_data['output_fmt'].replace('/', '').replace('\\', '').lower()
47 self.output_path = os.path.join(tdir, 'output.' + output_fmt)
48 self.tdir = tdir
49 self.library_id, self.pathtoebook = library_id, pathtoebook
50 self.conversion_data = conversion_data
51 self.running = self.ok = True
52 self.last_check_at = monotonic()
53 self.was_aborted = False
54
55 def cleanup(self):
56 safe_delete_tree(self.tdir)
57 self.log = self.traceback = ''
58
59 @property
60 def current_status(self):
61 try:
62 with share_open(os.path.join(self.tdir, 'status'), 'rb') as f:
63 lines = f.read().decode('utf-8').splitlines()
64 except Exception:
65 lines = ()
66 for line in reversed(lines):
67 if line.endswith('|||'):
68 p, msg = line.partition(':')[::2]
69 percent = float(p)
70 msg = msg[:-3]
71 return percent, msg
72 return 0, ''
73
74
75 def expire_old_jobs():
76 now = monotonic()
77 with cache_lock:
78 remove = [job_id for job_id, job_status in conversion_jobs.items() if now - job_status.last_check_at >= 360]
79 for job_id in remove:
80 job_status = conversion_jobs.pop(job_id)
81 job_status.cleanup()
82
83
84 def safe_delete_file(path):
85 try:
86 os.remove(path)
87 except OSError:
88 pass
89
90
91 def safe_delete_tree(path):
92 try:
93 shutil.rmtree(path, ignore_errors=True)
94 except OSError:
95 pass
96
97
98 def job_done(job):
99 with cache_lock:
100 try:
101 job_status = conversion_jobs[job.job_id]
102 except KeyError:
103 return
104 job_status.running = False
105 if job.failed:
106 job_status.ok = False
107 job_status.log = job.read_log()
108 job_status.was_aborted = job.was_aborted
109 job_status.traceback = job.traceback
110 safe_delete_file(job_status.pathtoebook)
111
112
113 def convert_book(path_to_ebook, opf_path, cover_path, output_fmt, recs):
114 from calibre.customize.conversion import OptionRecommendation
115 from calibre.ebooks.conversion.plumber import Plumber
116 from calibre.utils.logging import Log
117
118 # These are set by the server itself and must take precedence over anything
119 # the client requested. merge_ui_recommendations() will not override an
120 # option that has already reached the HIGH level, so these must be merged
121 # *before* the client supplied recommendations to ensure the client cannot
122 # override them by relying on the ordering of the recommendation list.
123 server_recs = [('verbose', 2, OptionRecommendation.HIGH), ('read_metadata_from_opf', opf_path, OptionRecommendation.HIGH)]
124 if cover_path:
125 server_recs.append(('cover', cover_path, OptionRecommendation.HIGH))
126 log = Log()
127 os.chdir(os.path.dirname(path_to_ebook))
128 status_file = share_open('status', 'wb')
129
130 def notification(percent, msg=''):
131 status_file.write(f'{percent}:{msg}|||\n'.encode())
132 status_file.flush()
133
134 output_path = os.path.abspath('output.' + output_fmt.lower())
135 plumber = Plumber(path_to_ebook, output_path, log, report_progress=notification, override_input_metadata=True)
136 plumber.merge_ui_recommendations(server_recs)
137 plumber.merge_ui_recommendations(recs)
138 plumber.run()
139
140
141 def sanitize_conversion_options(ctx, db, input_fmt, output_fmt, book_id, client_options):
142 """Restrict the conversion options supplied by a (remote) client to the set
143 that the server actually advertises through /conversion/book-data, and drop
144 any option that is dangerous to expose remotely. Without this a client can
145 set *any* conversion option, including ones that take a filesystem path or
146 URL (cover, read_metadata_from_opf, debug_pipeline) or that weaken a security
147 guard (allow_local_files_outside_root)."""
148 allowed = set(get_conversion_options(input_fmt, output_fmt, book_id, db)['options'])
149 ans = {}
150 for name, val in client_options.items():
151 if name in FORBIDDEN_CLIENT_OPTIONS:
152 ctx.log.warn(f'Ignoring conversion option {name!r} from client: not allowed via the network API')
153 continue
154 if name not in allowed:
155 ctx.log.warn(f'Ignoring conversion option {name!r} from client: not exposed by the conversion API')
156 continue
157 ans[name] = val
158 return ans
159
160
161 def queue_job(ctx, rd, library_id, db, fmt, book_id, conversion_data):
162 from calibre.customize.conversion import OptionRecommendation
163 from calibre.ebooks.conversion.config import GuiRecommendations, save_specifics
164 from calibre.ebooks.metadata.opf2 import metadata_to_opf
165
166 tdir = tempfile.mkdtemp(dir=rd.tdir)
167 with tempfile.NamedTemporaryFile(prefix='', suffix=('.' + fmt.lower()), dir=tdir, delete=False) as src_file:
168 db.copy_format_to(book_id, fmt, src_file)
169 with tempfile.NamedTemporaryFile(prefix='', suffix='.jpeg', dir=tdir, delete=False) as cover_file:
170 cover_copied = db.copy_cover_to(book_id, cover_file)
171 cover_path = cover_file.name if cover_copied else None
172 mi = db.get_metadata(book_id)
173 mi.application_id = mi.uuid
174 raw = metadata_to_opf(mi)
175 with tempfile.NamedTemporaryFile(prefix='', suffix='.opf', dir=tdir, delete=False) as opf_file:
176 opf_file.write(raw)
177 recs = GuiRecommendations()
178 recs.update(sanitize_conversion_options(ctx, db, fmt, conversion_data['output_fmt'], book_id, conversion_data['options']))
179 recs['gui_preferred_input_format'] = conversion_data['input_fmt'].lower()
180 save_specifics(db, book_id, recs)
181 recs = [(k, v, OptionRecommendation.HIGH) for k, v in recs.items()]
182
183 job_id = ctx.start_job(
184 f'Convert book {book_id} ({fmt})',
185 'calibre.srv.convert',
186 'convert_book',
187 args=(src_file.name, opf_file.name, cover_path, conversion_data['output_fmt'], recs),
188 job_done_callback=job_done,
189 )
190 expire_old_jobs()
191 with cache_lock:
192 conversion_jobs[job_id] = JobStatus(job_id, book_id, tdir, library_id, src_file.name, conversion_data)
193 return job_id
194
195
196 @endpoint(
197 '/conversion/start/{book_id}',
198 postprocess=json,
199 needs_db_write=True,
200 types={'book_id': int},
201 methods=receive_data_methods,
202 )
203 def start_conversion(ctx, rd, book_id):
204 db, library_id = get_library_data(ctx, rd)[:2]
205 if not ctx.has_id(rd, db, book_id):
206 raise BookNotFound(book_id, db)
207 data = stdlib_json.loads(rd.request_body_file.read())
208 input_fmt = data['input_fmt']
209 job_id = queue_job(ctx, rd, library_id, db, input_fmt, book_id, data)
210 return job_id
211
212
213 @endpoint(
214 '/conversion/status/{job_id}',
215 postprocess=json,
216 needs_db_write=True,
217 types={'job_id': int},
218 methods=receive_data_methods,
219 )
220 def conversion_status(ctx, rd, job_id):
221 with cache_lock:
222 job_status = conversion_jobs.get(job_id)
223 if job_status is None:
224 raise HTTPNotFound(f'No job with id: {job_id}')
225 job_status.last_check_at = monotonic()
226 if job_status.running:
227 percent, msg = job_status.current_status
228 if rd.query.get('abort_job'):
229 ctx.abort_job(job_id)
230 return {'running': True, 'percent': percent, 'msg': msg}
231
232 del conversion_jobs[job_id]
233
234 try:
235 ans = {
236 'running': False,
237 'ok': job_status.ok,
238 'was_aborted': job_status.was_aborted,
239 'traceback': job_status.traceback,
240 'log': job_status.log,
241 }
242 if job_status.ok:
243 db, library_id = get_library_data(ctx, rd)[:2]
244 if library_id != job_status.library_id:
245 raise HTTPNotFound('job library_id does not match')
246 fmt = job_status.output_path.rpartition('.')[-1]
247 try:
248 db.add_format(job_status.book_id, fmt, job_status.output_path)
249 except NoSuchBook:
250 raise HTTPNotFound(f'book_id {job_status.book_id} not found in library')
251 run_plugins_on_postconvert(db, job_status.book_id, fmt)
252 formats_added({job_status.book_id: (fmt,)})
253 ans['size'] = os.path.getsize(job_status.output_path)
254 ans['fmt'] = fmt
255 return ans
256 finally:
257 job_status.cleanup()
258
259
260 def get_conversion_options(input_fmt, output_fmt, book_id, db):
261 from calibre.customize.conversion import OptionRecommendation
262 from calibre.ebooks.conversion.config import OPTIONS, load_defaults, load_specifics, options_for_input_fmt, options_for_output_fmt
263 from calibre.ebooks.conversion.plumber import create_dummy_plumber
264
265 plumber = create_dummy_plumber(input_fmt, output_fmt)
266 specifics = load_specifics(db, book_id)
267 options_dict: dict = {}
268 disabled_set: set = set()
269 defaults_dict: dict = {}
270 help_dict: dict = {}
271 ans = {
272 'options': options_dict,
273 'disabled': disabled_set,
274 'defaults': defaults_dict,
275 'help': help_dict,
276 'input_plugin_name': plumber.input_plugin.commit_name,
277 'output_plugin_name': plumber.output_plugin.commit_name,
278 'input_ui_data': plumber.input_plugin.ui_data,
279 'output_ui_data': plumber.output_plugin.ui_data,
280 }
281
282 def merge_group(group_name, option_names):
283 if not group_name or group_name in ('debug', 'metadata'):
284 return
285 defs = load_defaults(group_name)
286 defs.merge_recommendations(plumber.get_option_by_name, OptionRecommendation.LOW, option_names)
287 specifics.merge_recommendations(plumber.get_option_by_name, OptionRecommendation.HIGH, option_names, only_existing=True)
288 defaults = defs.as_dict()['options']
289 for k in defs:
290 if k in specifics:
291 defs[k] = specifics[k]
292 defs = defs.as_dict()
293 options_dict.update(defs['options'])
294 disabled_set.update(defs['disabled'])
295 defaults_dict.update(defaults)
296 ans['help'] = plumber.get_all_help()
297
298 for group_name, option_names in OPTIONS['pipe'].items():
299 merge_group(group_name, option_names)
300
301 group_name, option_names = options_for_input_fmt(input_fmt)
302 merge_group(group_name, option_names)
303 group_name, option_names = options_for_output_fmt(output_fmt)
304 merge_group(group_name, option_names)
305
306 ans['disabled'] = tuple(disabled_set)
307 return ans
308
309
310 _profiles_cache: dict | None = None
311
312
313 def profiles():
314 global _profiles_cache
315 if _profiles_cache is None:
316
317 def desc(profile):
318 w, h = profile.screen_size
319 if w >= 10000:
320 ss = _('unlimited')
321 else:
322 ss = _('%(width)d x %(height)d pixels') % dict(width=w, height=h)
323 ss = _('Screen size: %s') % ss
324 return {'name': profile.name, 'description': (f'{profile.description} [{ss}]')}
325
326 _profiles_cache = {}
327 _profiles_cache['input'] = {p.short_name: desc(p) for p in input_profiles()}
328 _profiles_cache['output'] = {p.short_name: desc(p) for p in output_profiles()}
329 return _profiles_cache
330
331
332 @endpoint('/conversion/book-data/{book_id}', postprocess=json, types={'book_id': int})
333 def conversion_data(ctx, rd, book_id):
334 from calibre.ebooks.conversion.config import NoSupportedInputFormats, get_input_format_for_book, get_sorted_output_formats
335
336 db = get_library_data(ctx, rd)[0]
337 if not ctx.has_id(rd, db, book_id):
338 raise BookNotFound(book_id, db)
339 try:
340 input_format, input_formats = get_input_format_for_book(db, book_id)
341 except NoSupportedInputFormats:
342 input_formats = []
343 else:
344 if rd.query.get('input_fmt') and rd.query.get('input_fmt').lower() in input_formats:
345 input_format = rd.query.get('input_fmt').lower()
346 if input_format in input_formats:
347 input_formats.remove(input_format)
348 input_formats.insert(0, input_format)
349 input_fmt = input_formats[0] if input_formats else 'epub'
350 output_formats = get_sorted_output_formats(rd.query.get('output_fmt'))
351 ans = {
352 'input_formats': [x.upper() for x in input_formats],
353 'output_formats': output_formats,
354 'profiles': profiles(),
355 'conversion_options': get_conversion_options(input_fmt, output_formats[0], book_id, db),
356 'title': db.field_for('title', book_id),
357 'authors': db.field_for('authors', book_id),
358 'book_id': book_id,
359 }
360 return ans