| 1 |
#!/usr/bin/env python |
| 2 |
# vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:ai |
| 3 |
# License: GPLv3 Copyright: 2008, Kovid Goyal <kovid at kovidgoyal.net> |
| 4 |
|
| 5 |
import os |
| 6 |
import re |
| 7 |
from functools import lru_cache, partial |
| 8 |
from tempfile import TemporaryDirectory |
| 9 |
|
| 10 |
from epub import EPUBHelpBuilder |
| 11 |
from sphinx.util.console import bold |
| 12 |
from sphinx.util.logging import getLogger |
| 13 |
|
| 14 |
from calibre.linux import cli_index_strings, entry_points |
| 15 |
from calibre.utils.localization import _ |
| 16 |
|
| 17 |
|
| 18 |
def info(*a): |
| 19 |
getLogger(__name__).info(*a) |
| 20 |
|
| 21 |
|
| 22 |
def warn(*a): |
| 23 |
getLogger(__name__).warning(*a) |
| 24 |
|
| 25 |
|
| 26 |
include_pat = re.compile(r'^.. include:: (\S+.rst)', re.M) |
| 27 |
|
| 28 |
|
| 29 |
@lru_cache(2) |
| 30 |
def formatter_funcs(): |
| 31 |
from calibre.db.legacy import LibraryDatabase |
| 32 |
from calibre.utils.ffml_processor import FFMLProcessor |
| 33 |
from calibre.utils.formatter_functions import formatter_functions |
| 34 |
|
| 35 |
ans = {'doc': {}, 'sum': {}} |
| 36 |
with TemporaryDirectory() as tdir: |
| 37 |
db = LibraryDatabase(tdir) # needed to load formatter_funcs |
| 38 |
ffml = FFMLProcessor() |
| 39 |
all_funcs = formatter_functions().get_builtins() |
| 40 |
for func_name, func in all_funcs.items(): |
| 41 |
# indent the text since :ffdoc: is used inside lists |
| 42 |
# if we need no indent we can create a new role like |
| 43 |
# :ffdoc-no-indent: |
| 44 |
text = ffml.document_to_rst(func.doc, func_name, indent=1) |
| 45 |
ans['doc'][func_name] = text.strip() |
| 46 |
text = ffml.document_to_summary_rst(func.doc, func_name, indent=1) |
| 47 |
ans['sum'][func_name] = text.strip() |
| 48 |
db.close() |
| 49 |
del db |
| 50 |
return ans |
| 51 |
|
| 52 |
|
| 53 |
def ffdoc(language, m): |
| 54 |
func_name = m.group(1) |
| 55 |
try: |
| 56 |
return formatter_funcs()['doc'][func_name] |
| 57 |
except Exception as e: |
| 58 |
if language in ('en', 'eng'): |
| 59 |
raise |
| 60 |
warn(f'Failed to process template language docs for in the {language} language with error: {e}') |
| 61 |
return 'INVALID TRANSLATION' |
| 62 |
|
| 63 |
|
| 64 |
def ffsum(language, m): |
| 65 |
func_name = m.group(1) |
| 66 |
try: |
| 67 |
return formatter_funcs()['sum'][func_name] |
| 68 |
except Exception as e: |
| 69 |
if language in ('en', 'eng'): |
| 70 |
raise |
| 71 |
warn(f'Failed to process template language summary docs for in the {language} language with error: {e}') |
| 72 |
return 'INVALID TRANSLATION' |
| 73 |
|
| 74 |
|
| 75 |
def source_read_handler(app, docname, source): |
| 76 |
src = source[0] |
| 77 |
if app.builder.name == 'gettext': |
| 78 |
if docname == 'template_lang': |
| 79 |
src = re.sub(r':(ffdoc|ffsum):`(.+?)`', ' ', src) # ffdoc and ffsum should not be translated |
| 80 |
else: |
| 81 |
if app.config.language != 'en': |
| 82 |
src = re.sub(r'(\s+generated/)en/', r'\1' + app.config.language + '/', src) |
| 83 |
if docname == 'template_lang': |
| 84 |
try: |
| 85 |
src = re.sub(r':ffdoc:`(.+?)`', partial(ffdoc, app.config.language), src) |
| 86 |
src = re.sub(r':ffsum:`(.+?)`', partial(ffsum, app.config.language), src) |
| 87 |
except Exception: |
| 88 |
import traceback |
| 89 |
|
| 90 |
traceback.print_exc() |
| 91 |
raise |
| 92 |
# Sphinx does not call source_read_handle for the .. include directive |
| 93 |
for m in reversed(tuple(include_pat.finditer(src))): |
| 94 |
included_doc_name = m.group(1).lstrip('/') |
| 95 |
ss = [open(included_doc_name, 'rb').read().decode('utf-8')] |
| 96 |
source_read_handler(app, included_doc_name.partition('.')[0], ss) |
| 97 |
src = src[: m.start()] + ss[0] + src[m.end() :] |
| 98 |
source[0] = src |
| 99 |
|
| 100 |
|
| 101 |
CLI_INDEX = ''' |
| 102 |
.. _cli: |
| 103 |
|
| 104 |
%s |
| 105 |
========================================================= |
| 106 |
|
| 107 |
.. image:: ../../images/cli.png |
| 108 |
|
| 109 |
.. note:: |
| 110 |
%s |
| 111 |
|
| 112 |
%s |
| 113 |
-------------------------------------- |
| 114 |
|
| 115 |
.. toctree:: |
| 116 |
:maxdepth: 1 |
| 117 |
|
| 118 |
{documented} |
| 119 |
|
| 120 |
%s |
| 121 |
---------------------------------------- |
| 122 |
|
| 123 |
{undocumented} |
| 124 |
|
| 125 |
%s |
| 126 |
''' |
| 127 |
|
| 128 |
CLI_PREAMBLE = '''\ |
| 129 |
.. _{cmdref}: |
| 130 |
|
| 131 |
.. raw:: html |
| 132 |
|
| 133 |
<style>code {{font-size: 1em; background-color: transparent; font-family: sans-serif }}</style> |
| 134 |
|
| 135 |
``{cmd}`` |
| 136 |
=================================================================== |
| 137 |
|
| 138 |
.. code-block:: none |
| 139 |
|
| 140 |
{cmdline} |
| 141 |
|
| 142 |
{usage} |
| 143 |
''' |
| 144 |
|
| 145 |
|
| 146 |
def titlecase(language, x): |
| 147 |
if x and language == 'en': |
| 148 |
from calibre.utils.titlecase import titlecase as tc |
| 149 |
|
| 150 |
x = tc(x) |
| 151 |
return x |
| 152 |
|
| 153 |
|
| 154 |
def generate_calibredb_help(preamble, language): |
| 155 |
from calibre.db.cli.main import COMMANDS, get_parser, option_parser_for |
| 156 |
|
| 157 |
preamble = preamble[: preamble.find('\n\n\n', preamble.find('code-block'))] |
| 158 |
preamble += '\n\n' |
| 159 |
preamble += _('''\ |
| 160 |
:command:`calibredb` is the command line interface to the calibre database. It has |
| 161 |
several sub-commands, documented below. |
| 162 |
|
| 163 |
:command:`calibredb` can be used to manipulate either a calibre database |
| 164 |
specified by path or a calibre :guilabel:`Content server` running either on |
| 165 |
the local machine or over the internet. You can start a calibre |
| 166 |
:guilabel:`Content server` using either the :command:`calibre-server` |
| 167 |
program or in the main calibre program click :guilabel:`Connect/share -> |
| 168 |
Start Content server`. Since :command:`calibredb` can make changes to your |
| 169 |
calibre libraries, you must setup authentication on the server first. There |
| 170 |
are two ways to do that: |
| 171 |
|
| 172 |
* If you plan to connect only to a server running on the same computer, |
| 173 |
you can simply use the ``--enable-local-write`` option of the |
| 174 |
Content server, to allow any program, including calibredb, running on |
| 175 |
the local computer to make changes to your calibre data. When running |
| 176 |
the server from the main calibre program, this option is in |
| 177 |
:guilabel:`Preferences->Sharing over the net->Advanced`. Be careful though, |
| 178 |
because some browsers, primarily Firefox, allow arbitrary websites to also |
| 179 |
access local servers. If you use Firefox, it is better to setup username |
| 180 |
and password, as described below. Local Network Access restriction has |
| 181 |
still not been implemented as of Firefox v149. |
| 182 |
|
| 183 |
* If you want to enable access over the internet, then you should setup |
| 184 |
user accounts on the server and use the :option:`--username` and :option:`--password` |
| 185 |
options to :command:`calibredb` to give it access. You can setup |
| 186 |
user authentication for :command:`calibre-server` by using the ``--enable-auth`` |
| 187 |
option and using ``--manage-users`` to create the user accounts. |
| 188 |
If you are running the server from the main calibre program, use |
| 189 |
:guilabel:`Preferences->Sharing over the net->Require username/password`. |
| 190 |
|
| 191 |
To connect to a running Content server, pass the URL of the server to the |
| 192 |
:option:`--with-library` option, see the documentation of that option for |
| 193 |
details and examples. |
| 194 |
''') |
| 195 |
|
| 196 |
global_parser = get_parser('') |
| 197 |
groups = [] |
| 198 |
for grp in global_parser.option_groups: |
| 199 |
groups.append((titlecase(language, grp.title), grp.description, grp.option_list)) |
| 200 |
|
| 201 |
global_options = '\n'.join(render_options('calibredb', groups, False, False)) |
| 202 |
|
| 203 |
lines = [] |
| 204 |
for cmd in COMMANDS: |
| 205 |
parser = option_parser_for(cmd)() |
| 206 |
lines += [f'.. _calibredb-{language}-{cmd}:', ''] |
| 207 |
lines += [cmd, '~' * 20, ''] |
| 208 |
usage = parser.usage.strip() |
| 209 |
usage = usage.replace('%prog', 'calibredb').splitlines() |
| 210 |
cmdline = ' ' + usage[0] |
| 211 |
usage = usage[1:] |
| 212 |
usage = [re.sub(rf'({cmd})([^a-zA-Z0-9])', r':command:`\1`\2', i) for i in usage] |
| 213 |
lines += ['.. code-block:: none', '', cmdline, ''] |
| 214 |
lines += usage |
| 215 |
groups = [(None, None, parser.option_list)] |
| 216 |
lines += [''] |
| 217 |
lines += render_options('calibredb ' + cmd, groups, False) |
| 218 |
lines += [''] |
| 219 |
for group in parser.option_groups: |
| 220 |
if not getattr(group, 'is_global_options', False): |
| 221 |
lines.extend( |
| 222 |
render_options( |
| 223 |
'calibredb_' + cmd, [[titlecase(language, group.title), group.description, group.option_list]], False, False, header_level='^' |
| 224 |
) |
| 225 |
) |
| 226 |
lines += [''] |
| 227 |
|
| 228 |
raw = preamble + '\n\n' + '.. contents::\n :local:' + '\n\n' + global_options + '\n\n' + '\n'.join(lines) |
| 229 |
update_cli_doc('calibredb', raw, language) |
| 230 |
|
| 231 |
|
| 232 |
def generate_ebook_convert_help(preamble, app): |
| 233 |
from calibre.customize.ui import input_format_plugins, output_format_plugins |
| 234 |
from calibre.ebooks.conversion.cli import create_option_parser, manual_index_strings |
| 235 |
from calibre.utils.logging import default_log |
| 236 |
|
| 237 |
preamble = re.sub(r'http.*\.html', ':ref:`conversion`', preamble) |
| 238 |
|
| 239 |
raw = preamble + '\n\n' + manual_index_strings() % 'ebook-convert myfile.input_format myfile.output_format -h' |
| 240 |
parser, plumber = create_option_parser(['ebook-convert', 'dummyi.mobi', 'dummyo.epub', '-h'], default_log) |
| 241 |
groups = [(None, None, parser.option_list)] |
| 242 |
for grp in parser.option_groups: |
| 243 |
if grp.title not in {'INPUT OPTIONS', 'OUTPUT OPTIONS'}: |
| 244 |
groups.append((titlecase(app, grp.title), grp.description, grp.option_list)) |
| 245 |
options = '\n'.join(render_options('ebook-convert', groups, False)) |
| 246 |
|
| 247 |
raw += '\n\n.. contents::\n :local:' |
| 248 |
|
| 249 |
raw += '\n\n' + options |
| 250 |
for pl in sorted(input_format_plugins(), key=lambda x: x.name): |
| 251 |
parser, plumber = create_option_parser(['ebook-convert', 'dummyi.' + min(pl.file_types), 'dummyo.epub', '-h'], default_log) |
| 252 |
groups = [(pl.name + ' Options', '', g.option_list) for g in parser.option_groups if g.title == 'INPUT OPTIONS'] |
| 253 |
prog = 'ebook-convert-' + (pl.name.lower().replace(' ', '-')) |
| 254 |
raw += '\n\n' + '\n'.join(render_options(prog, groups, False, True)) |
| 255 |
for pl in sorted(output_format_plugins(), key=lambda x: x.name): |
| 256 |
parser, plumber = create_option_parser(['ebook-convert', 'd.epub', 'dummyi.' + pl.file_type, '-h'], default_log) |
| 257 |
groups = [(pl.name + ' Options', '', g.option_list) for g in parser.option_groups if g.title == 'OUTPUT OPTIONS'] |
| 258 |
prog = 'ebook-convert-' + (pl.name.lower().replace(' ', '-')) |
| 259 |
raw += '\n\n' + '\n'.join(render_options(prog, groups, False, True)) |
| 260 |
|
| 261 |
update_cli_doc('ebook-convert', raw, app) |
| 262 |
|
| 263 |
|
| 264 |
def update_cli_doc(name, raw, language): |
| 265 |
if isinstance(raw, bytes): |
| 266 |
raw = raw.decode('utf-8') |
| 267 |
path = f'generated/{language}/{name}.rst' |
| 268 |
old_raw = open(path, encoding='utf-8').read() if os.path.exists(path) else '' |
| 269 |
if not os.path.exists(path) or old_raw != raw: |
| 270 |
import difflib |
| 271 |
|
| 272 |
print(path, 'has changed') |
| 273 |
if old_raw: |
| 274 |
lines = difflib.unified_diff(old_raw.splitlines(), raw.splitlines(), path, path) |
| 275 |
for line in lines: |
| 276 |
print(line) |
| 277 |
info('creating ' + os.path.splitext(os.path.basename(path))[0]) |
| 278 |
p = os.path.dirname(path) |
| 279 |
if p and not os.path.exists(p): |
| 280 |
os.makedirs(p) |
| 281 |
open(path, 'wb').write(raw.encode('utf-8')) |
| 282 |
|
| 283 |
|
| 284 |
def render_options(cmd, groups, options_header=True, add_program=True, header_level='~'): |
| 285 |
lines = [''] |
| 286 |
if options_header: |
| 287 |
lines = [_('[options]'), '-' * 40, ''] |
| 288 |
if add_program: |
| 289 |
lines += ['.. program:: ' + cmd, ''] |
| 290 |
for title, desc, options in groups: |
| 291 |
if title: |
| 292 |
lines.extend([title, header_level * (len(title) + 4)]) |
| 293 |
lines.append('') |
| 294 |
if desc: |
| 295 |
lines.extend([desc, '']) |
| 296 |
for opt in sorted(options, key=lambda x: x.get_opt_string()): |
| 297 |
help = opt.help or '' |
| 298 |
help = help.replace('\n', ' ').replace('*', '\\*').replace('%default', str(opt.default)) |
| 299 |
help = help.replace('"', r'\ ``"``\ ') |
| 300 |
help = help.replace("'", r"\ ``'``\ ") |
| 301 |
help = mark_options(help) |
| 302 |
opt_strings = (x.strip() for x in tuple(opt._long_opts or ()) + tuple(opt._short_opts or ())) |
| 303 |
opt = '.. option:: ' + ', '.join(opt_strings) |
| 304 |
lines.extend([opt, '', ' ' + help, '']) |
| 305 |
return lines |
| 306 |
|
| 307 |
|
| 308 |
def mark_options(raw): |
| 309 |
raw = re.sub(r'(\s+)--(\s+)', r'\1``--``\2', raw) |
| 310 |
|
| 311 |
def sub(m): |
| 312 |
opt = m.group() |
| 313 |
a, b = opt.partition('=')[::2] |
| 314 |
if a in ('--option1', '--option2'): |
| 315 |
return '``' + m.group() + '``' |
| 316 |
a = ':option:`' + a + '`' |
| 317 |
b = (' = ``' + b + '``') if b else '' |
| 318 |
return a + b |
| 319 |
|
| 320 |
raw = re.sub(r'(--[|()a-zA-Z0-9_=,-]+)', sub, raw) |
| 321 |
return raw |
| 322 |
|
| 323 |
|
| 324 |
def get_cli_docs(): |
| 325 |
documented_cmds = [] |
| 326 |
undocumented_cmds = [] |
| 327 |
for script in entry_points['console_scripts'] + entry_points['gui_scripts']: |
| 328 |
module = script[script.index('=') + 1 : script.index(':')].strip() |
| 329 |
cmd = script[: script.index('=')].strip() |
| 330 |
if cmd in ('calibre-complete', 'calibre-parallel'): |
| 331 |
continue |
| 332 |
module = __import__(module, fromlist=[module.split('.')[-1]]) |
| 333 |
if hasattr(module, 'option_parser'): |
| 334 |
try: |
| 335 |
documented_cmds.append((cmd, getattr(module, 'option_parser')())) |
| 336 |
except TypeError: |
| 337 |
documented_cmds.append((cmd, getattr(module, 'option_parser')(cmd))) |
| 338 |
else: |
| 339 |
undocumented_cmds.append(cmd) |
| 340 |
return documented_cmds, undocumented_cmds |
| 341 |
|
| 342 |
|
| 343 |
def cli_docs(language): |
| 344 |
info(bold('creating CLI documentation...')) |
| 345 |
documented_cmds, undocumented_cmds = get_cli_docs() |
| 346 |
|
| 347 |
documented_cmds.sort(key=lambda x: x[0]) |
| 348 |
undocumented_cmds.sort() |
| 349 |
|
| 350 |
documented = [' ' * 4 + c[0] for c in documented_cmds] |
| 351 |
undocumented = [' * ' + c for c in undocumented_cmds] |
| 352 |
|
| 353 |
raw = (CLI_INDEX % cli_index_strings()[:5]).format(documented='\n'.join(documented), undocumented='\n'.join(undocumented)) |
| 354 |
if not os.path.exists('cli'): |
| 355 |
os.makedirs('cli') |
| 356 |
update_cli_doc('cli-index', raw, language) |
| 357 |
|
| 358 |
for cmd, parser in documented_cmds: |
| 359 |
usage = [mark_options(i) for i in parser.usage.replace('%prog', cmd).splitlines()] |
| 360 |
cmdline = usage[0] |
| 361 |
usage = usage[1:] |
| 362 |
usage = [i.replace(cmd, f':command:`{cmd}`') for i in usage] |
| 363 |
usage = '\n'.join(usage) |
| 364 |
preamble = CLI_PREAMBLE.format(cmd=cmd, cmdref=cmd + '-' + language, cmdline=cmdline, usage=usage) |
| 365 |
if cmd == 'ebook-convert': |
| 366 |
generate_ebook_convert_help(preamble, language) |
| 367 |
elif cmd == 'calibredb': |
| 368 |
generate_calibredb_help(preamble, language) |
| 369 |
else: |
| 370 |
groups = [(None, None, parser.option_list)] |
| 371 |
for grp in parser.option_groups: |
| 372 |
groups.append((grp.title, grp.description, grp.option_list)) |
| 373 |
raw = preamble |
| 374 |
lines = render_options(cmd, groups) |
| 375 |
raw += '\n' + '\n'.join(lines) |
| 376 |
update_cli_doc(cmd, raw, language) |
| 377 |
|
| 378 |
|
| 379 |
def generate_docs(language): |
| 380 |
cli_docs(language) |
| 381 |
template_docs(language) |
| 382 |
|
| 383 |
|
| 384 |
def template_docs(language): |
| 385 |
from template_ref_generate import generate_template_language_help |
| 386 |
|
| 387 |
raw = generate_template_language_help(language, getLogger(__name__)) |
| 388 |
update_cli_doc('template_ref', raw, language) |
| 389 |
|
| 390 |
|
| 391 |
def localized_path(app, langcode, pagename): |
| 392 |
href = app.builder.get_target_uri(pagename) |
| 393 |
href = re.sub(r'generated/[a-z]+/', f'generated/{langcode}/', href) |
| 394 |
prefix = '/' |
| 395 |
if langcode != 'en': |
| 396 |
prefix += langcode + '/' |
| 397 |
return prefix + href |
| 398 |
|
| 399 |
|
| 400 |
def add_html_context(app, pagename, templatename, context, *args): |
| 401 |
context['localized_path'] = partial(localized_path, app) |
| 402 |
context['change_language_text'] = cli_index_strings()[5] |
| 403 |
context['search_box_text'] = cli_index_strings()[6] |
| 404 |
|
| 405 |
|
| 406 |
def guilabel_role(typ, rawtext, text, *args, **kwargs): |
| 407 |
from sphinx.roles import GUILabel |
| 408 |
|
| 409 |
text = text.replace('->', '\N{THIN SPACE}\N{RIGHTWARDS ARROW}\N{THIN SPACE}') |
| 410 |
return GUILabel()(typ, rawtext, text, *args, **kwargs) |
| 411 |
|
| 412 |
|
| 413 |
def setup_man_pages(app): |
| 414 |
documented_cmds = get_cli_docs()[0] |
| 415 |
man_pages = [] |
| 416 |
for cmd, _option_parser in documented_cmds: |
| 417 |
path = f'generated/{app.config.language}/{cmd}' |
| 418 |
man_pages.append((path, cmd, cmd, 'Kovid Goyal', 1)) |
| 419 |
app.config['man_pages'] = man_pages |
| 420 |
|
| 421 |
|
| 422 |
def setup(app): |
| 423 |
from docutils.parsers.rst import roles |
| 424 |
|
| 425 |
setup_man_pages(app) |
| 426 |
generate_docs(app.config.language) |
| 427 |
app.add_css_file('custom.css') |
| 428 |
app.add_builder(EPUBHelpBuilder) |
| 429 |
app.connect('source-read', source_read_handler) |
| 430 |
app.connect('html-page-context', add_html_context) |
| 431 |
roles.register_local_role('guilabel', guilabel_role) |