| 1 |
#!/usr/bin/env python |
| 2 |
# License: GPLv3 Copyright: 2018, Kovid Goyal <kovid at kovidgoyal.net> |
| 3 |
|
| 4 |
import json as stdlib_json |
| 5 |
import os |
| 6 |
import shutil |
| 7 |
import tempfile |
| 8 |
from threading import Lock |
| 9 |
|
| 10 |
from calibre.customize.ui import input_profiles, output_profiles, run_plugins_on_postconvert |
| 11 |
from calibre.db.errors import NoSuchBook |
| 12 |
from calibre.srv.changes import formats_added |
| 13 |
from calibre.srv.errors import BookNotFound, HTTPNotFound |
| 14 |
from calibre.srv.routes import endpoint, json |
| 15 |
from calibre.srv.utils import get_library_data |
| 16 |
from calibre.utils.localization import _ |
| 17 |
from calibre.utils.monotonic import monotonic |
| 18 |
from calibre.utils.shared_file import share_open |
| 19 |
|
| 20 |
receive_data_methods = {'GET', 'POST'} |
| 21 |
conversion_jobs = {} |
| 22 |
cache_lock = Lock() |
| 23 |
|
| 24 |
# Conversion options that must never be honoured when they come from a remote |
| 25 |
# client of the content server, regardless of whether they are advertised by |
| 26 |
# /conversion/book-data. They either take a filesystem path or URL (which would |
| 27 |
# let a client read arbitrary server-side files, perform SSRF or manipulate the |
| 28 |
# filesystem) or deliberately weaken a security guard. cover and |
| 29 |
# read_metadata_from_opf are additionally set by the server itself in |
| 30 |
# convert_book(), so a client must never be able to override them. |
| 31 |
FORBIDDEN_CLIENT_OPTIONS = frozenset({ |
| 32 |
'cover', |
| 33 |
'read_metadata_from_opf', |
| 34 |
'debug_pipeline', |
| 35 |
'allow_local_files_outside_root', |
| 36 |
'extract_to', |
| 37 |
}) |
| 38 |
|
| 39 |
|
| 40 |
class JobStatus: |
| 41 |
def __init__(self, job_id, book_id, tdir, library_id, pathtoebook, conversion_data): |
| 42 |
self.job_id = job_id |
| 43 |
self.log = self.traceback = '' |
| 44 |
self.book_id = book_id |
| 45 |
# sanitize output_fmt to prevent path traversal |
| 46 |
output_fmt = conversion_data['output_fmt'].replace('/', '').replace('\\', '').lower() |
| 47 |
self.output_path = os.path.join(tdir, 'output.' + output_fmt) |
| 48 |
self.tdir = tdir |
| 49 |
self.library_id, self.pathtoebook = library_id, pathtoebook |
| 50 |
self.conversion_data = conversion_data |
| 51 |
self.running = self.ok = True |
| 52 |
self.last_check_at = monotonic() |
| 53 |
self.was_aborted = False |
| 54 |
|
| 55 |
def cleanup(self): |
| 56 |
safe_delete_tree(self.tdir) |
| 57 |
self.log = self.traceback = '' |
| 58 |
|
| 59 |
@property |
| 60 |
def current_status(self): |
| 61 |
try: |
| 62 |
with share_open(os.path.join(self.tdir, 'status'), 'rb') as f: |
| 63 |
lines = f.read().decode('utf-8').splitlines() |
| 64 |
except Exception: |
| 65 |
lines = () |
| 66 |
for line in reversed(lines): |
| 67 |
if line.endswith('|||'): |
| 68 |
p, msg = line.partition(':')[::2] |
| 69 |
percent = float(p) |
| 70 |
msg = msg[:-3] |
| 71 |
return percent, msg |
| 72 |
return 0, '' |
| 73 |
|
| 74 |
|
| 75 |
def expire_old_jobs(): |
| 76 |
now = monotonic() |
| 77 |
with cache_lock: |
| 78 |
remove = [job_id for job_id, job_status in conversion_jobs.items() if now - job_status.last_check_at >= 360] |
| 79 |
for job_id in remove: |
| 80 |
job_status = conversion_jobs.pop(job_id) |
| 81 |
job_status.cleanup() |
| 82 |
|
| 83 |
|
| 84 |
def safe_delete_file(path): |
| 85 |
try: |
| 86 |
os.remove(path) |
| 87 |
except OSError: |
| 88 |
pass |
| 89 |
|
| 90 |
|
| 91 |
def safe_delete_tree(path): |
| 92 |
try: |
| 93 |
shutil.rmtree(path, ignore_errors=True) |
| 94 |
except OSError: |
| 95 |
pass |
| 96 |
|
| 97 |
|
| 98 |
def job_done(job): |
| 99 |
with cache_lock: |
| 100 |
try: |
| 101 |
job_status = conversion_jobs[job.job_id] |
| 102 |
except KeyError: |
| 103 |
return |
| 104 |
job_status.running = False |
| 105 |
if job.failed: |
| 106 |
job_status.ok = False |
| 107 |
job_status.log = job.read_log() |
| 108 |
job_status.was_aborted = job.was_aborted |
| 109 |
job_status.traceback = job.traceback |
| 110 |
safe_delete_file(job_status.pathtoebook) |
| 111 |
|
| 112 |
|
| 113 |
def convert_book(path_to_ebook, opf_path, cover_path, output_fmt, recs): |
| 114 |
from calibre.customize.conversion import OptionRecommendation |
| 115 |
from calibre.ebooks.conversion.plumber import Plumber |
| 116 |
from calibre.utils.logging import Log |
| 117 |
|
| 118 |
# These are set by the server itself and must take precedence over anything |
| 119 |
# the client requested. merge_ui_recommendations() will not override an |
| 120 |
# option that has already reached the HIGH level, so these must be merged |
| 121 |
# *before* the client supplied recommendations to ensure the client cannot |
| 122 |
# override them by relying on the ordering of the recommendation list. |
| 123 |
server_recs = [('verbose', 2, OptionRecommendation.HIGH), ('read_metadata_from_opf', opf_path, OptionRecommendation.HIGH)] |
| 124 |
if cover_path: |
| 125 |
server_recs.append(('cover', cover_path, OptionRecommendation.HIGH)) |
| 126 |
log = Log() |
| 127 |
os.chdir(os.path.dirname(path_to_ebook)) |
| 128 |
status_file = share_open('status', 'wb') |
| 129 |
|
| 130 |
def notification(percent, msg=''): |
| 131 |
status_file.write(f'{percent}:{msg}|||\n'.encode()) |
| 132 |
status_file.flush() |
| 133 |
|
| 134 |
output_path = os.path.abspath('output.' + output_fmt.lower()) |
| 135 |
plumber = Plumber(path_to_ebook, output_path, log, report_progress=notification, override_input_metadata=True) |
| 136 |
plumber.merge_ui_recommendations(server_recs) |
| 137 |
plumber.merge_ui_recommendations(recs) |
| 138 |
plumber.run() |
| 139 |
|
| 140 |
|
| 141 |
def sanitize_conversion_options(ctx, db, input_fmt, output_fmt, book_id, client_options): |
| 142 |
"""Restrict the conversion options supplied by a (remote) client to the set |
| 143 |
that the server actually advertises through /conversion/book-data, and drop |
| 144 |
any option that is dangerous to expose remotely. Without this a client can |
| 145 |
set *any* conversion option, including ones that take a filesystem path or |
| 146 |
URL (cover, read_metadata_from_opf, debug_pipeline) or that weaken a security |
| 147 |
guard (allow_local_files_outside_root).""" |
| 148 |
allowed = set(get_conversion_options(input_fmt, output_fmt, book_id, db)['options']) |
| 149 |
ans = {} |
| 150 |
for name, val in client_options.items(): |
| 151 |
if name in FORBIDDEN_CLIENT_OPTIONS: |
| 152 |
ctx.log.warn(f'Ignoring conversion option {name!r} from client: not allowed via the network API') |
| 153 |
continue |
| 154 |
if name not in allowed: |
| 155 |
ctx.log.warn(f'Ignoring conversion option {name!r} from client: not exposed by the conversion API') |
| 156 |
continue |
| 157 |
ans[name] = val |
| 158 |
return ans |
| 159 |
|
| 160 |
|
| 161 |
def queue_job(ctx, rd, library_id, db, fmt, book_id, conversion_data): |
| 162 |
from calibre.customize.conversion import OptionRecommendation |
| 163 |
from calibre.ebooks.conversion.config import GuiRecommendations, save_specifics |
| 164 |
from calibre.ebooks.metadata.opf2 import metadata_to_opf |
| 165 |
|
| 166 |
tdir = tempfile.mkdtemp(dir=rd.tdir) |
| 167 |
with tempfile.NamedTemporaryFile(prefix='', suffix=('.' + fmt.lower()), dir=tdir, delete=False) as src_file: |
| 168 |
db.copy_format_to(book_id, fmt, src_file) |
| 169 |
with tempfile.NamedTemporaryFile(prefix='', suffix='.jpeg', dir=tdir, delete=False) as cover_file: |
| 170 |
cover_copied = db.copy_cover_to(book_id, cover_file) |
| 171 |
cover_path = cover_file.name if cover_copied else None |
| 172 |
mi = db.get_metadata(book_id) |
| 173 |
mi.application_id = mi.uuid |
| 174 |
raw = metadata_to_opf(mi) |
| 175 |
with tempfile.NamedTemporaryFile(prefix='', suffix='.opf', dir=tdir, delete=False) as opf_file: |
| 176 |
opf_file.write(raw) |
| 177 |
recs = GuiRecommendations() |
| 178 |
recs.update(sanitize_conversion_options(ctx, db, fmt, conversion_data['output_fmt'], book_id, conversion_data['options'])) |
| 179 |
recs['gui_preferred_input_format'] = conversion_data['input_fmt'].lower() |
| 180 |
save_specifics(db, book_id, recs) |
| 181 |
recs = [(k, v, OptionRecommendation.HIGH) for k, v in recs.items()] |
| 182 |
|
| 183 |
job_id = ctx.start_job( |
| 184 |
f'Convert book {book_id} ({fmt})', |
| 185 |
'calibre.srv.convert', |
| 186 |
'convert_book', |
| 187 |
args=(src_file.name, opf_file.name, cover_path, conversion_data['output_fmt'], recs), |
| 188 |
job_done_callback=job_done, |
| 189 |
) |
| 190 |
expire_old_jobs() |
| 191 |
with cache_lock: |
| 192 |
conversion_jobs[job_id] = JobStatus(job_id, book_id, tdir, library_id, src_file.name, conversion_data) |
| 193 |
return job_id |
| 194 |
|
| 195 |
|
| 196 |
@endpoint( |
| 197 |
'/conversion/start/{book_id}', |
| 198 |
postprocess=json, |
| 199 |
needs_db_write=True, |
| 200 |
types={'book_id': int}, |
| 201 |
methods=receive_data_methods, |
| 202 |
) |
| 203 |
def start_conversion(ctx, rd, book_id): |
| 204 |
db, library_id = get_library_data(ctx, rd)[:2] |
| 205 |
if not ctx.has_id(rd, db, book_id): |
| 206 |
raise BookNotFound(book_id, db) |
| 207 |
data = stdlib_json.loads(rd.request_body_file.read()) |
| 208 |
input_fmt = data['input_fmt'] |
| 209 |
job_id = queue_job(ctx, rd, library_id, db, input_fmt, book_id, data) |
| 210 |
return job_id |
| 211 |
|
| 212 |
|
| 213 |
@endpoint( |
| 214 |
'/conversion/status/{job_id}', |
| 215 |
postprocess=json, |
| 216 |
needs_db_write=True, |
| 217 |
types={'job_id': int}, |
| 218 |
methods=receive_data_methods, |
| 219 |
) |
| 220 |
def conversion_status(ctx, rd, job_id): |
| 221 |
with cache_lock: |
| 222 |
job_status = conversion_jobs.get(job_id) |
| 223 |
if job_status is None: |
| 224 |
raise HTTPNotFound(f'No job with id: {job_id}') |
| 225 |
job_status.last_check_at = monotonic() |
| 226 |
if job_status.running: |
| 227 |
percent, msg = job_status.current_status |
| 228 |
if rd.query.get('abort_job'): |
| 229 |
ctx.abort_job(job_id) |
| 230 |
return {'running': True, 'percent': percent, 'msg': msg} |
| 231 |
|
| 232 |
del conversion_jobs[job_id] |
| 233 |
|
| 234 |
try: |
| 235 |
ans = { |
| 236 |
'running': False, |
| 237 |
'ok': job_status.ok, |
| 238 |
'was_aborted': job_status.was_aborted, |
| 239 |
'traceback': job_status.traceback, |
| 240 |
'log': job_status.log, |
| 241 |
} |
| 242 |
if job_status.ok: |
| 243 |
db, library_id = get_library_data(ctx, rd)[:2] |
| 244 |
if library_id != job_status.library_id: |
| 245 |
raise HTTPNotFound('job library_id does not match') |
| 246 |
fmt = job_status.output_path.rpartition('.')[-1] |
| 247 |
try: |
| 248 |
db.add_format(job_status.book_id, fmt, job_status.output_path) |
| 249 |
except NoSuchBook: |
| 250 |
raise HTTPNotFound(f'book_id {job_status.book_id} not found in library') |
| 251 |
run_plugins_on_postconvert(db, job_status.book_id, fmt) |
| 252 |
formats_added({job_status.book_id: (fmt,)}) |
| 253 |
ans['size'] = os.path.getsize(job_status.output_path) |
| 254 |
ans['fmt'] = fmt |
| 255 |
return ans |
| 256 |
finally: |
| 257 |
job_status.cleanup() |
| 258 |
|
| 259 |
|
| 260 |
def get_conversion_options(input_fmt, output_fmt, book_id, db): |
| 261 |
from calibre.customize.conversion import OptionRecommendation |
| 262 |
from calibre.ebooks.conversion.config import OPTIONS, load_defaults, load_specifics, options_for_input_fmt, options_for_output_fmt |
| 263 |
from calibre.ebooks.conversion.plumber import create_dummy_plumber |
| 264 |
|
| 265 |
plumber = create_dummy_plumber(input_fmt, output_fmt) |
| 266 |
specifics = load_specifics(db, book_id) |
| 267 |
options_dict: dict = {} |
| 268 |
disabled_set: set = set() |
| 269 |
defaults_dict: dict = {} |
| 270 |
help_dict: dict = {} |
| 271 |
ans = { |
| 272 |
'options': options_dict, |
| 273 |
'disabled': disabled_set, |
| 274 |
'defaults': defaults_dict, |
| 275 |
'help': help_dict, |
| 276 |
'input_plugin_name': plumber.input_plugin.commit_name, |
| 277 |
'output_plugin_name': plumber.output_plugin.commit_name, |
| 278 |
'input_ui_data': plumber.input_plugin.ui_data, |
| 279 |
'output_ui_data': plumber.output_plugin.ui_data, |
| 280 |
} |
| 281 |
|
| 282 |
def merge_group(group_name, option_names): |
| 283 |
if not group_name or group_name in ('debug', 'metadata'): |
| 284 |
return |
| 285 |
defs = load_defaults(group_name) |
| 286 |
defs.merge_recommendations(plumber.get_option_by_name, OptionRecommendation.LOW, option_names) |
| 287 |
specifics.merge_recommendations(plumber.get_option_by_name, OptionRecommendation.HIGH, option_names, only_existing=True) |
| 288 |
defaults = defs.as_dict()['options'] |
| 289 |
for k in defs: |
| 290 |
if k in specifics: |
| 291 |
defs[k] = specifics[k] |
| 292 |
defs = defs.as_dict() |
| 293 |
options_dict.update(defs['options']) |
| 294 |
disabled_set.update(defs['disabled']) |
| 295 |
defaults_dict.update(defaults) |
| 296 |
ans['help'] = plumber.get_all_help() |
| 297 |
|
| 298 |
for group_name, option_names in OPTIONS['pipe'].items(): |
| 299 |
merge_group(group_name, option_names) |
| 300 |
|
| 301 |
group_name, option_names = options_for_input_fmt(input_fmt) |
| 302 |
merge_group(group_name, option_names) |
| 303 |
group_name, option_names = options_for_output_fmt(output_fmt) |
| 304 |
merge_group(group_name, option_names) |
| 305 |
|
| 306 |
ans['disabled'] = tuple(disabled_set) |
| 307 |
return ans |
| 308 |
|
| 309 |
|
| 310 |
_profiles_cache: dict | None = None |
| 311 |
|
| 312 |
|
| 313 |
def profiles(): |
| 314 |
global _profiles_cache |
| 315 |
if _profiles_cache is None: |
| 316 |
|
| 317 |
def desc(profile): |
| 318 |
w, h = profile.screen_size |
| 319 |
if w >= 10000: |
| 320 |
ss = _('unlimited') |
| 321 |
else: |
| 322 |
ss = _('%(width)d x %(height)d pixels') % dict(width=w, height=h) |
| 323 |
ss = _('Screen size: %s') % ss |
| 324 |
return {'name': profile.name, 'description': (f'{profile.description} [{ss}]')} |
| 325 |
|
| 326 |
_profiles_cache = {} |
| 327 |
_profiles_cache['input'] = {p.short_name: desc(p) for p in input_profiles()} |
| 328 |
_profiles_cache['output'] = {p.short_name: desc(p) for p in output_profiles()} |
| 329 |
return _profiles_cache |
| 330 |
|
| 331 |
|
| 332 |
@endpoint('/conversion/book-data/{book_id}', postprocess=json, types={'book_id': int}) |
| 333 |
def conversion_data(ctx, rd, book_id): |
| 334 |
from calibre.ebooks.conversion.config import NoSupportedInputFormats, get_input_format_for_book, get_sorted_output_formats |
| 335 |
|
| 336 |
db = get_library_data(ctx, rd)[0] |
| 337 |
if not ctx.has_id(rd, db, book_id): |
| 338 |
raise BookNotFound(book_id, db) |
| 339 |
try: |
| 340 |
input_format, input_formats = get_input_format_for_book(db, book_id) |
| 341 |
except NoSupportedInputFormats: |
| 342 |
input_formats = [] |
| 343 |
else: |
| 344 |
if rd.query.get('input_fmt') and rd.query.get('input_fmt').lower() in input_formats: |
| 345 |
input_format = rd.query.get('input_fmt').lower() |
| 346 |
if input_format in input_formats: |
| 347 |
input_formats.remove(input_format) |
| 348 |
input_formats.insert(0, input_format) |
| 349 |
input_fmt = input_formats[0] if input_formats else 'epub' |
| 350 |
output_formats = get_sorted_output_formats(rd.query.get('output_fmt')) |
| 351 |
ans = { |
| 352 |
'input_formats': [x.upper() for x in input_formats], |
| 353 |
'output_formats': output_formats, |
| 354 |
'profiles': profiles(), |
| 355 |
'conversion_options': get_conversion_options(input_fmt, output_formats[0], book_id, db), |
| 356 |
'title': db.field_for('title', book_id), |
| 357 |
'authors': db.field_for('authors', book_id), |
| 358 |
'book_id': book_id, |
| 359 |
} |
| 360 |
return ans |