Debian Codegraph @

manual/custom.py

raw
1 #!/usr/bin/env python
2 # vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:ai
3 # License: GPLv3 Copyright: 2008, Kovid Goyal <kovid at kovidgoyal.net>
4
5 import os
6 import re
7 from functools import lru_cache, partial
8 from tempfile import TemporaryDirectory
9
10 from epub import EPUBHelpBuilder
11 from sphinx.util.console import bold
12 from sphinx.util.logging import getLogger
13
14 from calibre.linux import cli_index_strings, entry_points
15 from calibre.utils.localization import _
16
17
18 def info(*a):
19 getLogger(__name__).info(*a)
20
21
22 def warn(*a):
23 getLogger(__name__).warning(*a)
24
25
26 include_pat = re.compile(r'^.. include:: (\S+.rst)', re.M)
27
28
29 @lru_cache(2)
30 def formatter_funcs():
31 from calibre.db.legacy import LibraryDatabase
32 from calibre.utils.ffml_processor import FFMLProcessor
33 from calibre.utils.formatter_functions import formatter_functions
34
35 ans = {'doc': {}, 'sum': {}}
36 with TemporaryDirectory() as tdir:
37 db = LibraryDatabase(tdir) # needed to load formatter_funcs
38 ffml = FFMLProcessor()
39 all_funcs = formatter_functions().get_builtins()
40 for func_name, func in all_funcs.items():
41 # indent the text since :ffdoc: is used inside lists
42 # if we need no indent we can create a new role like
43 # :ffdoc-no-indent:
44 text = ffml.document_to_rst(func.doc, func_name, indent=1)
45 ans['doc'][func_name] = text.strip()
46 text = ffml.document_to_summary_rst(func.doc, func_name, indent=1)
47 ans['sum'][func_name] = text.strip()
48 db.close()
49 del db
50 return ans
51
52
53 def ffdoc(language, m):
54 func_name = m.group(1)
55 try:
56 return formatter_funcs()['doc'][func_name]
57 except Exception as e:
58 if language in ('en', 'eng'):
59 raise
60 warn(f'Failed to process template language docs for in the {language} language with error: {e}')
61 return 'INVALID TRANSLATION'
62
63
64 def ffsum(language, m):
65 func_name = m.group(1)
66 try:
67 return formatter_funcs()['sum'][func_name]
68 except Exception as e:
69 if language in ('en', 'eng'):
70 raise
71 warn(f'Failed to process template language summary docs for in the {language} language with error: {e}')
72 return 'INVALID TRANSLATION'
73
74
75 def source_read_handler(app, docname, source):
76 src = source[0]
77 if app.builder.name == 'gettext':
78 if docname == 'template_lang':
79 src = re.sub(r':(ffdoc|ffsum):`(.+?)`', ' ', src) # ffdoc and ffsum should not be translated
80 else:
81 if app.config.language != 'en':
82 src = re.sub(r'(\s+generated/)en/', r'\1' + app.config.language + '/', src)
83 if docname == 'template_lang':
84 try:
85 src = re.sub(r':ffdoc:`(.+?)`', partial(ffdoc, app.config.language), src)
86 src = re.sub(r':ffsum:`(.+?)`', partial(ffsum, app.config.language), src)
87 except Exception:
88 import traceback
89
90 traceback.print_exc()
91 raise
92 # Sphinx does not call source_read_handle for the .. include directive
93 for m in reversed(tuple(include_pat.finditer(src))):
94 included_doc_name = m.group(1).lstrip('/')
95 ss = [open(included_doc_name, 'rb').read().decode('utf-8')]
96 source_read_handler(app, included_doc_name.partition('.')[0], ss)
97 src = src[: m.start()] + ss[0] + src[m.end() :]
98 source[0] = src
99
100
101 CLI_INDEX = '''
102 .. _cli:
103
104 %s
105 =========================================================
106
107 .. image:: ../../images/cli.png
108
109 .. note::
110 %s
111
112 %s
113 --------------------------------------
114
115 .. toctree::
116 :maxdepth: 1
117
118 {documented}
119
120 %s
121 ----------------------------------------
122
123 {undocumented}
124
125 %s
126 '''
127
128 CLI_PREAMBLE = '''\
129 .. _{cmdref}:
130
131 .. raw:: html
132
133 <style>code {{font-size: 1em; background-color: transparent; font-family: sans-serif }}</style>
134
135 ``{cmd}``
136 ===================================================================
137
138 .. code-block:: none
139
140 {cmdline}
141
142 {usage}
143 '''
144
145
146 def titlecase(language, x):
147 if x and language == 'en':
148 from calibre.utils.titlecase import titlecase as tc
149
150 x = tc(x)
151 return x
152
153
154 def generate_calibredb_help(preamble, language):
155 from calibre.db.cli.main import COMMANDS, get_parser, option_parser_for
156
157 preamble = preamble[: preamble.find('\n\n\n', preamble.find('code-block'))]
158 preamble += '\n\n'
159 preamble += _('''\
160 :command:`calibredb` is the command line interface to the calibre database. It has
161 several sub-commands, documented below.
162
163 :command:`calibredb` can be used to manipulate either a calibre database
164 specified by path or a calibre :guilabel:`Content server` running either on
165 the local machine or over the internet. You can start a calibre
166 :guilabel:`Content server` using either the :command:`calibre-server`
167 program or in the main calibre program click :guilabel:`Connect/share ->
168 Start Content server`. Since :command:`calibredb` can make changes to your
169 calibre libraries, you must setup authentication on the server first. There
170 are two ways to do that:
171
172 * If you plan to connect only to a server running on the same computer,
173 you can simply use the ``--enable-local-write`` option of the
174 Content server, to allow any program, including calibredb, running on
175 the local computer to make changes to your calibre data. When running
176 the server from the main calibre program, this option is in
177 :guilabel:`Preferences->Sharing over the net->Advanced`. Be careful though,
178 because some browsers, primarily Firefox, allow arbitrary websites to also
179 access local servers. If you use Firefox, it is better to setup username
180 and password, as described below. Local Network Access restriction has
181 still not been implemented as of Firefox v149.
182
183 * If you want to enable access over the internet, then you should setup
184 user accounts on the server and use the :option:`--username` and :option:`--password`
185 options to :command:`calibredb` to give it access. You can setup
186 user authentication for :command:`calibre-server` by using the ``--enable-auth``
187 option and using ``--manage-users`` to create the user accounts.
188 If you are running the server from the main calibre program, use
189 :guilabel:`Preferences->Sharing over the net->Require username/password`.
190
191 To connect to a running Content server, pass the URL of the server to the
192 :option:`--with-library` option, see the documentation of that option for
193 details and examples.
194 ''')
195
196 global_parser = get_parser('')
197 groups = []
198 for grp in global_parser.option_groups:
199 groups.append((titlecase(language, grp.title), grp.description, grp.option_list))
200
201 global_options = '\n'.join(render_options('calibredb', groups, False, False))
202
203 lines = []
204 for cmd in COMMANDS:
205 parser = option_parser_for(cmd)()
206 lines += [f'.. _calibredb-{language}-{cmd}:', '']
207 lines += [cmd, '~' * 20, '']
208 usage = parser.usage.strip()
209 usage = usage.replace('%prog', 'calibredb').splitlines()
210 cmdline = ' ' + usage[0]
211 usage = usage[1:]
212 usage = [re.sub(rf'({cmd})([^a-zA-Z0-9])', r':command:`\1`\2', i) for i in usage]
213 lines += ['.. code-block:: none', '', cmdline, '']
214 lines += usage
215 groups = [(None, None, parser.option_list)]
216 lines += ['']
217 lines += render_options('calibredb ' + cmd, groups, False)
218 lines += ['']
219 for group in parser.option_groups:
220 if not getattr(group, 'is_global_options', False):
221 lines.extend(
222 render_options(
223 'calibredb_' + cmd, [[titlecase(language, group.title), group.description, group.option_list]], False, False, header_level='^'
224 )
225 )
226 lines += ['']
227
228 raw = preamble + '\n\n' + '.. contents::\n :local:' + '\n\n' + global_options + '\n\n' + '\n'.join(lines)
229 update_cli_doc('calibredb', raw, language)
230
231
232 def generate_ebook_convert_help(preamble, app):
233 from calibre.customize.ui import input_format_plugins, output_format_plugins
234 from calibre.ebooks.conversion.cli import create_option_parser, manual_index_strings
235 from calibre.utils.logging import default_log
236
237 preamble = re.sub(r'http.*\.html', ':ref:`conversion`', preamble)
238
239 raw = preamble + '\n\n' + manual_index_strings() % 'ebook-convert myfile.input_format myfile.output_format -h'
240 parser, plumber = create_option_parser(['ebook-convert', 'dummyi.mobi', 'dummyo.epub', '-h'], default_log)
241 groups = [(None, None, parser.option_list)]
242 for grp in parser.option_groups:
243 if grp.title not in {'INPUT OPTIONS', 'OUTPUT OPTIONS'}:
244 groups.append((titlecase(app, grp.title), grp.description, grp.option_list))
245 options = '\n'.join(render_options('ebook-convert', groups, False))
246
247 raw += '\n\n.. contents::\n :local:'
248
249 raw += '\n\n' + options
250 for pl in sorted(input_format_plugins(), key=lambda x: x.name):
251 parser, plumber = create_option_parser(['ebook-convert', 'dummyi.' + min(pl.file_types), 'dummyo.epub', '-h'], default_log)
252 groups = [(pl.name + ' Options', '', g.option_list) for g in parser.option_groups if g.title == 'INPUT OPTIONS']
253 prog = 'ebook-convert-' + (pl.name.lower().replace(' ', '-'))
254 raw += '\n\n' + '\n'.join(render_options(prog, groups, False, True))
255 for pl in sorted(output_format_plugins(), key=lambda x: x.name):
256 parser, plumber = create_option_parser(['ebook-convert', 'd.epub', 'dummyi.' + pl.file_type, '-h'], default_log)
257 groups = [(pl.name + ' Options', '', g.option_list) for g in parser.option_groups if g.title == 'OUTPUT OPTIONS']
258 prog = 'ebook-convert-' + (pl.name.lower().replace(' ', '-'))
259 raw += '\n\n' + '\n'.join(render_options(prog, groups, False, True))
260
261 update_cli_doc('ebook-convert', raw, app)
262
263
264 def update_cli_doc(name, raw, language):
265 if isinstance(raw, bytes):
266 raw = raw.decode('utf-8')
267 path = f'generated/{language}/{name}.rst'
268 old_raw = open(path, encoding='utf-8').read() if os.path.exists(path) else ''
269 if not os.path.exists(path) or old_raw != raw:
270 import difflib
271
272 print(path, 'has changed')
273 if old_raw:
274 lines = difflib.unified_diff(old_raw.splitlines(), raw.splitlines(), path, path)
275 for line in lines:
276 print(line)
277 info('creating ' + os.path.splitext(os.path.basename(path))[0])
278 p = os.path.dirname(path)
279 if p and not os.path.exists(p):
280 os.makedirs(p)
281 open(path, 'wb').write(raw.encode('utf-8'))
282
283
284 def render_options(cmd, groups, options_header=True, add_program=True, header_level='~'):
285 lines = ['']
286 if options_header:
287 lines = [_('[options]'), '-' * 40, '']
288 if add_program:
289 lines += ['.. program:: ' + cmd, '']
290 for title, desc, options in groups:
291 if title:
292 lines.extend([title, header_level * (len(title) + 4)])
293 lines.append('')
294 if desc:
295 lines.extend([desc, ''])
296 for opt in sorted(options, key=lambda x: x.get_opt_string()):
297 help = opt.help or ''
298 help = help.replace('\n', ' ').replace('*', '\\*').replace('%default', str(opt.default))
299 help = help.replace('"', r'\ ``"``\ ')
300 help = help.replace("'", r"\ ``'``\ ")
301 help = mark_options(help)
302 opt_strings = (x.strip() for x in tuple(opt._long_opts or ()) + tuple(opt._short_opts or ()))
303 opt = '.. option:: ' + ', '.join(opt_strings)
304 lines.extend([opt, '', ' ' + help, ''])
305 return lines
306
307
308 def mark_options(raw):
309 raw = re.sub(r'(\s+)--(\s+)', r'\1``--``\2', raw)
310
311 def sub(m):
312 opt = m.group()
313 a, b = opt.partition('=')[::2]
314 if a in ('--option1', '--option2'):
315 return '``' + m.group() + '``'
316 a = ':option:`' + a + '`'
317 b = (' = ``' + b + '``') if b else ''
318 return a + b
319
320 raw = re.sub(r'(--[|()a-zA-Z0-9_=,-]+)', sub, raw)
321 return raw
322
323
324 def get_cli_docs():
325 documented_cmds = []
326 undocumented_cmds = []
327 for script in entry_points['console_scripts'] + entry_points['gui_scripts']:
328 module = script[script.index('=') + 1 : script.index(':')].strip()
329 cmd = script[: script.index('=')].strip()
330 if cmd in ('calibre-complete', 'calibre-parallel'):
331 continue
332 module = __import__(module, fromlist=[module.split('.')[-1]])
333 if hasattr(module, 'option_parser'):
334 try:
335 documented_cmds.append((cmd, getattr(module, 'option_parser')()))
336 except TypeError:
337 documented_cmds.append((cmd, getattr(module, 'option_parser')(cmd)))
338 else:
339 undocumented_cmds.append(cmd)
340 return documented_cmds, undocumented_cmds
341
342
343 def cli_docs(language):
344 info(bold('creating CLI documentation...'))
345 documented_cmds, undocumented_cmds = get_cli_docs()
346
347 documented_cmds.sort(key=lambda x: x[0])
348 undocumented_cmds.sort()
349
350 documented = [' ' * 4 + c[0] for c in documented_cmds]
351 undocumented = [' * ' + c for c in undocumented_cmds]
352
353 raw = (CLI_INDEX % cli_index_strings()[:5]).format(documented='\n'.join(documented), undocumented='\n'.join(undocumented))
354 if not os.path.exists('cli'):
355 os.makedirs('cli')
356 update_cli_doc('cli-index', raw, language)
357
358 for cmd, parser in documented_cmds:
359 usage = [mark_options(i) for i in parser.usage.replace('%prog', cmd).splitlines()]
360 cmdline = usage[0]
361 usage = usage[1:]
362 usage = [i.replace(cmd, f':command:`{cmd}`') for i in usage]
363 usage = '\n'.join(usage)
364 preamble = CLI_PREAMBLE.format(cmd=cmd, cmdref=cmd + '-' + language, cmdline=cmdline, usage=usage)
365 if cmd == 'ebook-convert':
366 generate_ebook_convert_help(preamble, language)
367 elif cmd == 'calibredb':
368 generate_calibredb_help(preamble, language)
369 else:
370 groups = [(None, None, parser.option_list)]
371 for grp in parser.option_groups:
372 groups.append((grp.title, grp.description, grp.option_list))
373 raw = preamble
374 lines = render_options(cmd, groups)
375 raw += '\n' + '\n'.join(lines)
376 update_cli_doc(cmd, raw, language)
377
378
379 def generate_docs(language):
380 cli_docs(language)
381 template_docs(language)
382
383
384 def template_docs(language):
385 from template_ref_generate import generate_template_language_help
386
387 raw = generate_template_language_help(language, getLogger(__name__))
388 update_cli_doc('template_ref', raw, language)
389
390
391 def localized_path(app, langcode, pagename):
392 href = app.builder.get_target_uri(pagename)
393 href = re.sub(r'generated/[a-z]+/', f'generated/{langcode}/', href)
394 prefix = '/'
395 if langcode != 'en':
396 prefix += langcode + '/'
397 return prefix + href
398
399
400 def add_html_context(app, pagename, templatename, context, *args):
401 context['localized_path'] = partial(localized_path, app)
402 context['change_language_text'] = cli_index_strings()[5]
403 context['search_box_text'] = cli_index_strings()[6]
404
405
406 def guilabel_role(typ, rawtext, text, *args, **kwargs):
407 from sphinx.roles import GUILabel
408
409 text = text.replace('->', '\N{THIN SPACE}\N{RIGHTWARDS ARROW}\N{THIN SPACE}')
410 return GUILabel()(typ, rawtext, text, *args, **kwargs)
411
412
413 def setup_man_pages(app):
414 documented_cmds = get_cli_docs()[0]
415 man_pages = []
416 for cmd, _option_parser in documented_cmds:
417 path = f'generated/{app.config.language}/{cmd}'
418 man_pages.append((path, cmd, cmd, 'Kovid Goyal', 1))
419 app.config['man_pages'] = man_pages
420
421
422 def setup(app):
423 from docutils.parsers.rst import roles
424
425 setup_man_pages(app)
426 generate_docs(app.config.language)
427 app.add_css_file('custom.css')
428 app.add_builder(EPUBHelpBuilder)
429 app.connect('source-read', source_read_handler)
430 app.connect('html-page-context', add_html_context)
431 roles.register_local_role('guilabel', guilabel_role)