Debian Codegraph @

utils/pylit/pylit.py

raw ◀ ▶
Source reconstructed at patch state 0 of 1 — not the installed (fully patched) source.
1 Debian patch touch this file
1 #!/usr/bin/env python
2 # -*- coding: iso-8859-1 -*-
3
4 # pylit.py
5 # ********
6 # Literate programming with reStructuredText
7 # ++++++++++++++++++++++++++++++++++++++++++
8 #
9 # :Date: $Date$
10 # :Revision: $Revision$
11 # :URL: $URL$
12 # :Copyright: © 2005, 2007 Günter Milde.
13 # Released without warranty under the terms of the
14 # GNU General Public License (v. 2 or later)
15 #
16 # ::
17
18 from __future__ import print_function
19
20 """pylit: bidirectional text <-> code converter
21
22 Convert between a *text document* with embedded code
23 and *source code* with embedded documentation.
24 """
25
26 # .. contents::
27 #
28 # Frontmatter
29 # ===========
30 #
31 # Changelog
32 # ---------
33 #
34 # .. class:: borderless
35 #
36 # ====== ========== ===========================================================
37 # 0.1 2005-06-29 Initial version.
38 # 0.1.1 2005-06-30 First literate version.
39 # 0.1.2 2005-07-01 Object orientated script using generators.
40 # 0.1.3 2005-07-10 Two state machine (later added 'header' state).
41 # 0.2b 2006-12-04 Start of work on version 0.2 (code restructuring).
42 # 0.2 2007-01-23 Published at http://pylit.berlios.de.
43 # 0.2.1 2007-01-25 Outsourced non-core documentation to the PyLit pages.
44 # 0.2.2 2007-01-26 New behaviour of `diff` function.
45 # 0.2.3 2007-01-29 New `header` methods after suggestion by Riccardo Murri.
46 # 0.2.4 2007-01-31 Raise Error if code indent is too small.
47 # 0.2.5 2007-02-05 New command line option --comment-string.
48 # 0.2.6 2007-02-09 Add section with open questions,
49 # Code2Text: let only blank lines (no comment str)
50 # separate text and code,
51 # fix `Code2Text.header`.
52 # 0.2.7 2007-02-19 Simplify `Code2Text.header`,
53 # new `iter_strip` method replacing a lot of ``if``-s.
54 # 0.2.8 2007-02-22 Set `mtime` of outfile to the one of infile.
55 # 0.3 2007-02-27 New `Code2Text` converter after an idea by Riccardo Murri,
56 # explicit `option_defaults` dict for easier customisation.
57 # 0.3.1 2007-03-02 Expand hard-tabs to prevent errors in indentation,
58 # `Text2Code` now also works on blocks,
59 # removed dependency on SimpleStates module.
60 # 0.3.2 2007-03-06 Bug fix: do not set `language` in `option_defaults`
61 # renamed `code_languages` to `languages`.
62 # 0.3.3 2007-03-16 New language css,
63 # option_defaults -> defaults = optparse.Values(),
64 # simpler PylitOptions: don't store parsed values,
65 # don't parse at initialisation,
66 # OptionValues: return `None` for non-existing attributes,
67 # removed -infile and -outfile, use positional arguments.
68 # 0.3.4 2007-03-19 Documentation update,
69 # separate `execute` function.
70 # 2007-03-21 Code cleanup in `Text2Code.__iter__`.
71 # 0.3.5 2007-03-23 Removed "css" from known languages after learning that
72 # there is no C++ style "// " comment string in CSS2.
73 # 0.3.6 2007-04-24 Documentation update.
74 # 0.4 2007-05-18 Implement Converter.__iter__ as stack of iterator
75 # generators. Iterating over a converter instance now
76 # yields lines instead of blocks.
77 # Provide "hooks" for pre- and postprocessing filters.
78 # Rename states to reduce confusion with formats:
79 # "text" -> "documentation", "code" -> "code_block".
80 # 0.4.1 2007-05-22 Converter.__iter__: cleanup and reorganisation,
81 # rename parent class Converter -> TextCodeConverter.
82 # 0.4.2 2007-05-23 Merged Text2Code.converter and Code2Text.converter into
83 # TextCodeConverter.converter.
84 # 0.4.3 2007-05-30 Replaced use of defaults.code_extensions with
85 # values.languages.keys().
86 # Removed spurious `print` statement in code_block_handler.
87 # Added basic support for 'c' and 'css' languages
88 # with `dumb_c_preprocessor`_ and `dumb_c_postprocessor`_.
89 # 0.5 2007-06-06 Moved `collect_blocks`_ out of `TextCodeConverter`_,
90 # bug fix: collect all trailing blank lines into a block.
91 # Expand tabs with `expandtabs_filter`_.
92 # 0.6 2007-06-20 Configurable code-block marker (default ``::``)
93 # 0.6.1 2007-06-28 Bug fix: reset self.code_block_marker_missing.
94 # 0.7 2007-12-12 prepending an empty string to sys.path in run_doctest()
95 # to allow imports from the current working dir.
96 # 0.7.1 2008-01-07 If outfile does not exist, do a round-trip conversion
97 # and report differences (as with outfile=='-').
98 # 0.7.2 2008-01-28 Do not add missing code-block separators with
99 # `doctest_run` on the code source. Keeps lines consistent.
100 # 0.7.3 2008-04-07 Use value of code_block_marker for insertion of missing
101 # transition marker in Code2Text.code_block_handler
102 # Add "shell" to defaults.languages
103 # 0.7.4 2008-06-23 Add "latex" to defaults.languages
104 # 0.7.5 2009-05-14 Bugfix: ignore blank lines in test for end of code block
105 # 0.7.6 2009-12-15 language-dependent code-block markers (after a
106 # `feature request and patch by jrioux`_),
107 # use DefaultDict for language-dependent defaults,
108 # new defaults setting `add_missing_marker`_.
109 # 0.7.7 2010-06-23 New command line option --codeindent.
110 # 0.7.8 2011-03-30 bugfix: do not overwrite custom `add_missing_marker` value,
111 # allow directive options following the 'code' directive.
112 # 0.7.9 2011-04-05 Decode doctest string if 'magic comment' gives encoding.
113 # ====== ========== ===========================================================
114 #
115 # ::
116
117 _version = "0.7.9"
118
119 __docformat__ = 'restructuredtext'
120
121
122 # Introduction
123 # ------------
124 #
125 # PyLit is a bidirectional converter between two formats of a computer
126 # program source:
127 #
128 # * a (reStructured) text document with program code embedded in
129 # *code blocks*, and
130 # * a compilable (or executable) code source with *documentation*
131 # embedded in comment blocks
132 #
133 #
134 # Requirements
135 # ------------
136 #
137 # ::
138
139 import os, sys, io
140 import re, optparse
141
142
143 # DefaultDict
144 # ~~~~~~~~~~~
145 # As `collections.defaultdict` is only introduced in Python 2.5, we
146 # define a simplified version of the dictionary with default from
147 # http://code.activestate.com/recipes/389639/
148 # ::
149
150 class DefaultDict(dict):
151 """Minimalistic Dictionary with default value."""
152 def __init__(self, default=None, *args, **kwargs):
153 self.update(dict(*args, **kwargs))
154 self.default = default
155
156 def __getitem__(self, key):
157 return self.get(key, self.default)
158
159
160 # Defaults
161 # ========
162 #
163 # The `defaults` object provides a central repository for default
164 # values and their customisation. ::
165
166 defaults = optparse.Values()
167
168 # It is used for
169 #
170 # * the initialisation of data arguments in TextCodeConverter_ and
171 # PylitOptions_
172 #
173 # * completion of command line options in `PylitOptions.complete_values`_.
174 #
175 # This allows the easy creation of back-ends that customise the
176 # defaults and then call `main`_ e.g.:
177 #
178 # >>> import pylit
179 # >>> pylit.defaults.comment_string = "## "
180 # >>> pylit.defaults.codeindent = 4
181 # >>> pylit.main()
182 #
183 # The following default values are defined in pylit.py:
184 #
185 # languages
186 # ---------
187 #
188 # Mapping of code file extensions to code language::
189
190 defaults.languages = DefaultDict("python", # fallback language
191 {".c": "c",
192 ".cc": "c++",
193 ".cpp": "c++",
194 ".css": "css",
195 ".py": "python",
196 ".sh": "shell",
197 ".sl": "slang",
198 ".sty": "latex",
199 ".tex": "latex",
200 ".ufl": "python"
201 })
202
203 # Will be overridden by the ``--language`` command line option.
204 #
205 # The first argument is the fallback language, used if there is no
206 # matching extension (e.g. if pylit is used as filter) and no
207 # ``--language`` is specified. It can be changed programmatically by
208 # assignment to the ``.default`` attribute, e.g.
209 #
210 # >>> defaults.languages.default='c++'
211 #
212 #
213 # .. _text_extension:
214 #
215 # text_extensions
216 # ---------------
217 #
218 # List of known extensions of (reStructured) text files. The first
219 # extension in this list is used by the `_get_outfile_name`_ method to
220 # generate a text output filename::
221
222 defaults.text_extensions = [".txt", ".rst"]
223
224
225 # comment_strings
226 # ---------------
227 #
228 # Comment strings for known languages. Used in Code2Text_ to recognise
229 # text blocks and in Text2Code_ to format text blocks as comments.
230 # Defaults to ``'# '``.
231 #
232 # **Comment strings include trailing whitespace.** ::
233
234 defaults.comment_strings = DefaultDict('# ',
235 {"css": '// ',
236 "c": '// ',
237 "c++": '// ',
238 "latex": '% ',
239 "python": '# ',
240 "shell": '# ',
241 "slang": '% '
242 })
243
244
245 # header_string
246 # -------------
247 #
248 # Marker string for a header code block in the text source. No trailing
249 # whitespace needed as indented code follows.
250 # Must be a valid rst directive that accepts code on the same line, e.g.
251 # ``'..admonition::'``.
252 #
253 # Default is a comment marker::
254
255 defaults.header_string = '..'
256
257
258 # .. _code_block_marker:
259 #
260 # code_block_markers
261 # ------------------
262 #
263 # Markup at the end of a documentation block.
264 # Default is Docutils' marker for a `literal block`_::
265
266 defaults.code_block_markers = DefaultDict('::')
267 defaults.code_block_markers["c++"] = u".. code-block:: cpp"
268 #defaults.code_block_markers['python'] = '.. code-block:: python'
269
270 # The `code_block_marker` string is `inserted into a regular expression`_.
271 # Language-specific markers can be defined programmatically, e.g. in a
272 # wrapper script.
273 #
274 # In a document where code examples are only one of several uses of
275 # literal blocks, it is more appropriate to single out the source code
276 # ,e.g. with the double colon at a separate line ("expanded form")
277 #
278 # ``defaults.code_block_marker.default = ':: *'``
279 #
280 # or a dedicated ``.. code-block::`` directive [#]_
281 #
282 # ``defaults.code_block_marker['c++'] = '.. code-block:: *c++'``
283 #
284 # The latter form also allows code in different languages kept together
285 # in one literate source file.
286 #
287 # .. [#] The ``.. code-block::`` directive is not (yet) supported by
288 # standard Docutils. It is provided by several add-ons, including
289 # the `code-block directive`_ project in the Docutils Sandbox and
290 # Sphinx_.
291 #
292 #
293 # strip
294 # -----
295 #
296 # Export to the output format stripping documentation or code blocks::
297
298 defaults.strip = False
299
300 # strip_marker
301 # ------------
302 #
303 # Strip literal marker from the end of documentation blocks when
304 # converting to code format. Makes the code more concise but looses the
305 # synchronisation of line numbers in text and code formats. Can also be used
306 # (together with the auto-completion of the code-text conversion) to change
307 # the `code_block_marker`::
308
309 defaults.strip_marker = False
310
311 # add_missing_marker
312 # ------------------
313 #
314 # When converting from code format to text format, add a `code_block_marker`
315 # at the end of documentation blocks if it is missing::
316
317 defaults.add_missing_marker = True
318
319 # Keep this at ``True``, if you want to re-convert to code format later!
320 #
321 #
322 # .. _defaults.preprocessors:
323 #
324 # preprocessors
325 # -------------
326 #
327 # Preprocess the data with language-specific filters_
328 # Set below in Filters_::
329
330 defaults.preprocessors = {}
331
332 # .. _defaults.postprocessors:
333 #
334 # postprocessors
335 # --------------
336 #
337 # Postprocess the data with language-specific filters_::
338
339 defaults.postprocessors = {}
340
341 # .. _defaults.codeindent:
342 #
343 # codeindent
344 # ----------
345 #
346 # Number of spaces to indent code blocks in `Code2Text.code_block_handler`_::
347
348 defaults.codeindent = 2
349
350 # In `Text2Code.code_block_handler`_, the codeindent is determined by the
351 # first recognised code line (header or first indented literal block
352 # of the text source).
353 #
354 # overwrite
355 # ---------
356 #
357 # What to do if the outfile already exists? (ignored if `outfile` == '-')::
358
359 defaults.overwrite = 'yes'
360
361 # Recognised values:
362 #
363 # :'yes': overwrite eventually existing `outfile`,
364 # :'update': fail if the `outfile` is newer than `infile`,
365 # :'no': fail if `outfile` exists.
366 #
367 #
368 # Extensions
369 # ==========
370 #
371 # Try to import optional extensions::
372
373 try:
374 import pylit_elisp
375 except ImportError:
376 pass
377
378
379 # Converter Classes
380 # =================
381 #
382 # The converter classes implement a simple state machine to separate and
383 # transform documentation and code blocks. For this task, only a very limited
384 # parsing is needed. PyLit's parser assumes:
385 #
386 # * `indented literal blocks`_ in a text source are code blocks.
387 #
388 # * comment blocks in a code source where every line starts with a matching
389 # comment string are documentation blocks.
390 #
391 # TextCodeConverter
392 # -----------------
393 # ::
394
395 class TextCodeConverter(object):
396 """Parent class for the converters `Text2Code` and `Code2Text`.
397 """
398
399 # The parent class defines data attributes and functions used in both
400 # `Text2Code`_ converting a text source to executable code source, and
401 # `Code2Text`_ converting commented code to a text source.
402 #
403 # Data attributes
404 # ~~~~~~~~~~~~~~~
405 #
406 # Class default values are fetched from the `defaults`_ object and can be
407 # overridden by matching keyword arguments during class instantiation. This
408 # also works with keyword arguments to `get_converter`_ and `main`_, as these
409 # functions pass on unused keyword args to the instantiation of a converter
410 # class. ::
411
412 language = defaults.languages.default
413 comment_strings = defaults.comment_strings
414 comment_string = "" # set in __init__ (if empty)
415 codeindent = defaults.codeindent
416 header_string = defaults.header_string
417 code_block_markers = defaults.code_block_markers
418 code_block_marker = "" # set in __init__ (if empty)
419 strip = defaults.strip
420 strip_marker = defaults.strip_marker
421 add_missing_marker = defaults.add_missing_marker
422 directive_option_regexp = re.compile(r' +:(\w|[-._+:])+:( |$)')
423 state = "" # type of current block, see `TextCodeConverter.convert`_
424
425 # Interface methods
426 # ~~~~~~~~~~~~~~~~~
427 #
428 # .. _TextCodeConverter.__init__:
429 #
430 # __init__
431 # """"""""
432 #
433 # Initialising sets the `data` attribute, an iterable object yielding lines of
434 # the source to convert. [#]_
435 #
436 # .. [#] The most common choice of data is a `file` object with the text
437 # or code source.
438 #
439 # To convert a string into a suitable object, use its splitlines method
440 # like ``"2 lines\nof source".splitlines(True)``.
441 #
442 #
443 # Additional keyword arguments are stored as instance variables,
444 # overwriting the class defaults::
445
446 def __init__(self, data, **keyw):
447 """data -- iterable data object
448 (list, file, generator, string, ...)
449 **keyw -- remaining keyword arguments are
450 stored as data-attributes
451 """
452 self.data = data
453 self.__dict__.update(keyw)
454
455 # If empty, `code_block_marker` and `comment_string` are set according
456 # to the `language`::
457
458 if not self.code_block_marker:
459 self.code_block_marker = self.code_block_markers[self.language]
460 if not self.comment_string:
461 self.comment_string = self.comment_strings[self.language]
462 self.stripped_comment_string = self.comment_string.rstrip()
463
464 # Pre- and postprocessing filters are set (with
465 # `TextCodeConverter.get_filter`_)::
466
467 self.preprocessor = self.get_filter("preprocessors", self.language)
468 self.postprocessor = self.get_filter("postprocessors", self.language)
469
470 # .. _inserted into a regular expression:
471 #
472 # Finally, a regular_expression for the `code_block_marker` is compiled
473 # to find valid cases of `code_block_marker` in a given line and return
474 # the groups: ``\1 prefix, \2 code_block_marker, \3 remainder`` ::
475
476 marker = self.code_block_marker
477 if marker == '::':
478 # the default marker may occur at the end of a text line
479 self.marker_regexp = re.compile('^( *(?!\.\.).*)(::)([ \n]*)$')
480 else:
481 # marker must be on a separate line
482 self.marker_regexp = re.compile('^( *)(%s)(.*\n?)$' % marker)
483
484 # .. _TextCodeConverter.__iter__:
485 #
486 # __iter__
487 # """"""""
488 #
489 # Return an iterator for the instance. Iteration yields lines of converted
490 # data.
491 #
492 # The iterator is a chain of iterators acting on `self.data` that does
493 #
494 # * preprocessing
495 # * text<->code format conversion
496 # * postprocessing
497 #
498 # Pre- and postprocessing are only performed, if filters for the current
499 # language are registered in `defaults.preprocessors`_ and|or
500 # `defaults.postprocessors`_. The filters must accept an iterable as first
501 # argument and yield the processed input data line-wise.
502 # ::
503
504 def __iter__(self):
505 """Iterate over input data source and yield converted lines
506 """
507 return self.postprocessor(self.convert(self.preprocessor(self.data)))
508
509
510 # .. _TextCodeConverter.__call__:
511 #
512 # __call__
513 # """"""""
514 # The special `__call__` method allows the use of class instances as callable
515 # objects. It returns the converted data as list of lines::
516
517 def __call__(self):
518 """Iterate over state-machine and return results as list of lines"""
519 return [line for line in self]
520
521
522 # .. _TextCodeConverter.__str__:
523 #
524 # __str__
525 # """""""
526 # Return converted data as string::
527
528 def __str__(self):
529 return "".join(self())
530
531
532 # Helpers and convenience methods
533 # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
534 #
535 # .. _TextCodeConverter.convert:
536 #
537 # convert
538 # """""""
539 #
540 # The `convert` method generates an iterator that does the actual code <-->
541 # text format conversion. The converted data is yielded line-wise and the
542 # instance's `status` argument indicates whether the current line is "header",
543 # "documentation", or "code_block"::
544
545 def convert(self, lines):
546 """Iterate over lines of a program document and convert
547 between "text" and "code" format
548 """
549
550 # Initialise internal data arguments. (Done here, so that every new iteration
551 # re-initialises them.)
552 #
553 # `state`
554 # the "type" of the currently processed block of lines. One of
555 #
556 # :"": initial state: check for header,
557 # :"header": leading code block: strip `header_string`,
558 # :"documentation": documentation part: comment out,
559 # :"code_block": literal blocks containing source code: unindent.
560 #
561 # ::
562
563 self.state = ""
564
565 # `_codeindent`
566 # * Do not confuse the internal attribute `_codeindent` with the configurable
567 # `codeindent` (without the leading underscore).
568 # * `_codeindent` is set in `Text2Code.code_block_handler`_ to the indent of
569 # first non-blank "code_block" line and stripped from all "code_block" lines
570 # in the text-to-code conversion,
571 # * `codeindent` is set in `__init__` to `defaults.codeindent`_ and added to
572 # "code_block" lines in the code-to-text conversion.
573 #
574 # ::
575
576 self._codeindent = 0
577
578 # `_textindent`
579 # * set by `Text2Code.documentation_handler`_ to the minimal indent of a
580 # documentation block,
581 # * used in `Text2Code.set_state`_ to find the end of a code block.
582 #
583 # ::
584
585 self._textindent = 0
586
587 # `_add_code_block_marker`
588 # If the last paragraph of a documentation block does not end with a
589 # code_block_marker_, it should be added (otherwise, the back-conversion
590 # fails.).
591 #
592 # `_add_code_block_marker` is set by `Code2Text.documentation_handler`_
593 # and evaluated by `Code2Text.code_block_handler`_, because the
594 # documentation_handler does not know whether the next block will be
595 # documentation (with no need for a code_block_marker) or a code block.
596 #
597 # ::
598
599 self._add_code_block_marker = False
600
601
602
603 # Determine the state of the block and convert with the matching "handler"::
604
605 for block in collect_blocks(expandtabs_filter(lines)):
606 self.set_state(block)
607 for line in getattr(self, self.state+"_handler")(block):
608 yield line
609
610
611 # .. _TextCodeConverter.get_filter:
612 #
613 # get_filter
614 # """"""""""
615 # ::
616
617 def get_filter(self, filter_set, language):
618 """Return language specific filter"""
619 if self.__class__ == Text2Code:
620 key = "text2"+language
621 elif self.__class__ == Code2Text:
622 key = language+"2text"
623 else:
624 key = ""
625 try:
626 return getattr(defaults, filter_set)[key]
627 except (AttributeError, KeyError):
628 # print("there is no %r filter in %r"%(key, filter_set))
629 pass
630 return identity_filter
631
632
633 # get_indent
634 # """"""""""
635 # Return the number of leading spaces in `line`::
636
637 def get_indent(self, line):
638 """Return the indentation of `string`.
639 """
640 return len(line) - len(line.lstrip())
641
642
643 # Text2Code
644 # ---------
645 #
646 # The `Text2Code` converter separates *code-blocks* [#]_ from *documentation*.
647 # Code blocks are unindented, documentation is commented (or filtered, if the
648 # ``strip`` option is True).
649 #
650 # .. [#] Only `indented literal blocks`_ are considered code-blocks. `quoted
651 # literal blocks`_, `parsed-literal blocks`_, and `doctest blocks`_ are
652 # treated as part of the documentation. This allows the inclusion of
653 # examples:
654 #
655 # >>> 23 + 3
656 # 26
657 #
658 # Mark that there is no double colon before the doctest block in the
659 # text source.
660 #
661 # The class inherits the interface and helper functions from
662 # TextCodeConverter_ and adds functions specific to the text-to-code format
663 # conversion::
664
665 class Text2Code(TextCodeConverter):
666 """Convert a (reStructured) text source to code source
667 """
668
669 # .. _Text2Code.set_state:
670 #
671 # set_state
672 # ~~~~~~~~~
673 # ::
674
675 def set_state(self, block):
676 """Determine state of `block`. Set `self.state`
677 """
678
679 # `set_state` is used inside an iteration. Hence, if we are out of data, a
680 # StopItertion exception should be raised::
681
682 if not block:
683 raise StopIteration
684
685 # The new state depends on the active state (from the last block) and
686 # features of the current block. It is either "header", "documentation", or
687 # "code_block".
688 #
689 # If the current state is "" (first block), check for
690 # the `header_string` indicating a leading code block::
691
692 if self.state == "":
693 # print("set state for %r"%block)
694 if block[0].startswith(self.header_string):
695 self.state = "header"
696 else:
697 self.state = "documentation"
698
699 # If the current state is "documentation", the next block is also
700 # documentation. The end of a documentation part is detected in the
701 # `Text2Code.documentation_handler`_::
702
703 # elif self.state == "documentation":
704 # self.state = "documentation"
705
706 # A "code_block" ends with the first less indented, non-blank line.
707 # `_textindent` is set by the documentation handler to the indent of the
708 # preceding documentation block::
709
710 elif self.state in ["code_block", "header"]:
711 indents = [self.get_indent(line) for line in block
712 if line.rstrip()]
713 # print("set_state:", indents, self._textindent)
714 if indents and min(indents) <= self._textindent:
715 self.state = 'documentation'
716 else:
717 self.state = 'code_block'
718
719 # TODO: (or not to do?) insert blank line before the first line with too-small
720 # codeindent using self.ensure_trailing_blank_line(lines, line) (would need
721 # split and push-back of the documentation part)?
722 #
723 # .. _Text2Code.header_handler:
724 #
725 # header_handler
726 # ~~~~~~~~~~~~~~
727 #
728 # Sometimes code needs to remain on the first line(s) of the document to be
729 # valid. The most common example is the "shebang" line that tells a POSIX
730 # shell how to process an executable file::
731
732 #!/usr/bin/env python
733
734 # In Python, the special comment to indicate the encoding, e.g.
735 # ``# -*- coding: iso-8859-1 -*-``, must occur before any other comment
736 # or code too.
737 #
738 # If we want to keep the line numbers in sync for text and code source, the
739 # reStructured Text markup for these header lines must start at the same line
740 # as the first header line. Therefore, header lines could not be marked as
741 # literal block (this would require the ``::`` and an empty line above the
742 # code_block).
743 #
744 # OTOH, a comment may start at the same line as the comment marker and it
745 # includes subsequent indented lines. Comments are visible in the reStructured
746 # Text source but hidden in the pretty-printed output.
747 #
748 # With a header converted to comment in the text source, everything before
749 # the first documentation block (i.e. before the first paragraph using the
750 # matching comment string) will be hidden away (in HTML or PDF output).
751 #
752 # This seems a good compromise, the advantages
753 #
754 # * line numbers are kept
755 # * the "normal" code_block conversion rules (indent/unindent by `codeindent` apply
756 # * greater flexibility: you can hide a repeating header in a project
757 # consisting of many source files.
758 #
759 # set off the disadvantages
760 #
761 # - it may come as surprise if a part of the file is not "printed",
762 # - one more syntax element to learn for rst newbies to start with pylit,
763 # (however, starting from the code source, this will be auto-generated)
764 #
765 # In the case that there is no matching comment at all, the complete code
766 # source will become a comment -- however, in this case it is not very likely
767 # the source is a literate document anyway.
768 #
769 # If needed for the documentation, it is possible to quote the header in (or
770 # after) the first documentation block, e.g. as `parsed literal`.
771 # ::
772
773 def header_handler(self, lines):
774 """Format leading code block"""
775 # strip header string from first line
776 lines[0] = lines[0].replace(self.header_string, "", 1)
777 # yield remaining lines formatted as code-block
778 for line in self.code_block_handler(lines):
779 yield line
780
781
782 # .. _Text2Code.documentation_handler:
783 #
784 # documentation_handler
785 # ~~~~~~~~~~~~~~~~~~~~~
786 #
787 # The 'documentation' handler processes everything that is not recognised as
788 # "code_block". Documentation is quoted with `self.comment_string`
789 # (or filtered with `--strip=True`).
790 #
791 # If end-of-documentation marker is detected,
792 #
793 # * set state to 'code_block'
794 # * set `self._textindent` (needed by `Text2Code.set_state`_ to find the
795 # next "documentation" block)
796 #
797 # ::
798
799 def documentation_handler(self, lines):
800 """Convert documentation blocks from text to code format
801 """
802 for line in lines:
803 # test lines following the code-block marker for false positives
804 if (self.state == "code_block" and line.rstrip()
805 and not self.directive_option_regexp.search(line)):
806 self.state = "documentation"
807 # test for end of documentation block
808 if self.marker_regexp.search(line):
809 self.state = "code_block"
810 self._textindent = self.get_indent(line)
811 # yield lines
812 if self.strip:
813 continue
814 # do not comment blank lines preceding a code block
815 if self.state == "code_block" and not line.rstrip():
816 yield line
817 else:
818 yield self.comment_string + line
819
820
821
822
823 # .. _Text2Code.code_block_handler:
824 #
825 # code_block_handler
826 # ~~~~~~~~~~~~~~~~~~
827 #
828 # The "code_block" handler is called with an indented literal block. It
829 # removes leading whitespace up to the indentation of the first code line in
830 # the file (this deviation from Docutils behaviour allows indented blocks of
831 # Python code). ::
832
833 def code_block_handler(self, block):
834 """Convert indented literal blocks to source code format
835 """
836
837 # If still unset, determine the indentation of code blocks from first non-blank
838 # code line::
839
840 if self._codeindent == 0:
841 self._codeindent = self.get_indent(block[0])
842
843 # Yield unindented lines after check whether we can safely unindent. If the
844 # line is less indented then `_codeindent`, something got wrong. ::
845
846 for line in block:
847 if line.lstrip() and self.get_indent(line) < self._codeindent:
848 raise ValueError("code block contains line less indented "
849 "than %d spaces \n%r" % (self._codeindent, block))
850 yield line.replace(" "*self._codeindent, "", 1)
851
852
853 # Code2Text
854 # ---------
855 #
856 # The `Code2Text` converter does the opposite of `Text2Code`_ -- it processes
857 # a source in "code format" (i.e. in a programming language), extracts
858 # documentation from comment blocks, and puts program code in literal blocks.
859 #
860 # The class inherits the interface and helper functions from
861 # TextCodeConverter_ and adds functions specific to the text-to-code format
862 # conversion::
863
864 class Code2Text(TextCodeConverter):
865 """Convert code source to text source
866 """
867
868 # set_state
869 # ~~~~~~~~~
870 #
871 # Check if block is "header", "documentation", or "code_block":
872 #
873 # A paragraph is "documentation", if every non-blank line starts with a
874 # matching comment string (including whitespace except for commented blank
875 # lines) ::
876
877 def set_state(self, block):
878 """Determine state of `block`."""
879 for line in block:
880 # skip documentation lines (commented, blank or blank comment)
881 if (line.startswith(self.comment_string)
882 or not line.rstrip()
883 or line.rstrip() == self.comment_string.rstrip()
884 ):
885 continue
886 # non-commented line found:
887 if self.state == "":
888 self.state = "header"
889 else:
890 self.state = "code_block"
891 break
892 else:
893 # no code line found
894 # keep state if the block is just a blank line
895 # if len(block) == 1 and self._is_blank_codeline(line):
896 # return
897 self.state = "documentation"
898
899
900 # header_handler
901 # ~~~~~~~~~~~~~~
902 #
903 # Handle a leading code block. (See `Text2Code.header_handler`_ for a
904 # discussion of the "header" state.) ::
905
906 def header_handler(self, lines):
907 """Format leading code block"""
908 if self.strip == True:
909 return
910 # get iterator over the lines that formats them as code-block
911 lines = iter(self.code_block_handler(lines))
912 # prepend header string to first line
913 yield self.header_string + lines.next()
914 # yield remaining lines
915 for line in lines:
916 yield line
917
918 # .. _Code2Text.documentation_handler:
919 #
920 # documentation_handler
921 # ~~~~~~~~~~~~~~~~~~~~~
922 #
923 # The *documentation state* handler converts a comment to a documentation
924 # block by stripping the leading `comment string` from every line::
925
926 def documentation_handler(self, block):
927 """Uncomment documentation blocks in source code
928 """
929
930 # Strip comment strings::
931
932 lines = [self.uncomment_line(line) for line in block]
933
934 # If the code block is stripped, the literal marker would lead to an
935 # error when the text is converted with Docutils. Strip it as well. ::
936
937 if self.strip or self.strip_marker:
938 self.strip_code_block_marker(lines)
939
940 # Otherwise, check for the `code_block_marker`_ at the end of the
941 # documentation block (skipping directive options that might follow it)::
942
943 elif self.add_missing_marker:
944 for line in lines[::-1]:
945 if self.marker_regexp.search(line):
946 self._add_code_block_marker = False
947 break
948 if (line.rstrip() and
949 not self.directive_option_regexp.search(line)):
950 self._add_code_block_marker = True
951 break
952 else:
953 self._add_code_block_marker = True
954
955 # Yield lines::
956
957 for line in lines:
958 yield line
959
960 # uncomment_line
961 # ~~~~~~~~~~~~~~
962 #
963 # Return documentation line after stripping comment string. Consider the
964 # case that a blank line has a comment string without trailing whitespace::
965
966 def uncomment_line(self, line):
967 """Return uncommented documentation line"""
968 line = line.replace(self.comment_string, "", 1)
969 if line.rstrip() == self.stripped_comment_string:
970 line = line.replace(self.stripped_comment_string, "", 1)
971 return line
972
973 # .. _Code2Text.code_block_handler:
974 #
975 # code_block_handler
976 # ~~~~~~~~~~~~~~~~~~
977 #
978 # The `code_block` handler returns the code block as indented literal
979 # block (or filters it, if ``self.strip == True``). The amount of the code
980 # indentation is controlled by `self.codeindent` (default 2). ::
981
982 def code_block_handler(self, lines):
983 """Covert code blocks to text format (indent or strip)
984 """
985 if self.strip == True:
986 return
987 # eventually insert transition marker
988 if self._add_code_block_marker:
989 self.state = "documentation"
990 yield self.code_block_marker + "\n"
991 yield "\n"
992 self._add_code_block_marker = False
993 self.state = "code_block"
994 for line in lines:
995 yield " "*self.codeindent + line
996
997
998
999 # strip_code_block_marker
1000 # ~~~~~~~~~~~~~~~~~~~~~~~
1001 #
1002 # Replace the literal marker with the equivalent of Docutils replace rules
1003 #
1004 # * strip ``::``-line (and preceding blank line) if on a line on its own
1005 # * strip ``::`` if it is preceded by whitespace.
1006 # * convert ``::`` to a single colon if preceded by text
1007 #
1008 # `lines` is a list of documentation lines (with a trailing blank line).
1009 # It is modified in-place::
1010
1011 def strip_code_block_marker(self, lines):
1012 try:
1013 line = lines[-2]
1014 except IndexError:
1015 return # just one line (no trailing blank line)
1016
1017 # match with regexp: `match` is None or has groups
1018 # \1 leading text, \2 code_block_marker, \3 remainder
1019 match = self.marker_regexp.search(line)
1020
1021 if not match: # no code_block_marker present
1022 return
1023 if not match.group(1): # `code_block_marker` on an extra line
1024 del(lines[-2])
1025 # delete preceding line if it is blank
1026 if len(lines) >= 2 and not lines[-2].lstrip():
1027 del(lines[-2])
1028 elif match.group(1).rstrip() < match.group(1):
1029 # '::' follows whitespace
1030 lines[-2] = match.group(1).rstrip() + match.group(3)
1031 else: # '::' follows text
1032 lines[-2] = match.group(1).rstrip() + ':' + match.group(3)
1033
1034 # Filters
1035 # =======
1036 #
1037 # Filters allow pre- and post-processing of the data to bring it in a format
1038 # suitable for the "normal" text<->code conversion. An example is conversion
1039 # of `C` ``/*`` ``*/`` comments into C++ ``//`` comments (and back).
1040 # Another example is the conversion of `C` ``/*`` ``*/`` comments into C++
1041 # ``//`` comments (and back).
1042 #
1043 # Filters are generator functions that return an iterator acting on a
1044 # `data` iterable and yielding processed `data` lines.
1045 #
1046 # identity_filter
1047 # ---------------
1048 #
1049 # The most basic filter is the identity filter, that returns its argument as
1050 # iterator::
1051
1052 def identity_filter(data):
1053 """Return data iterator without any processing"""
1054 return iter(data)
1055
1056 # expandtabs_filter
1057 # -----------------
1058 #
1059 # Expand hard-tabs in every line of `data` (cf. `str.expandtabs`).
1060 #
1061 # This filter is applied to the input data by `TextCodeConverter.convert`_ as
1062 # hard tabs can lead to errors when the indentation is changed. ::
1063
1064 def expandtabs_filter(data):
1065 """Yield data tokens with hard-tabs expanded"""
1066 for line in data:
1067 yield line.expandtabs()
1068
1069
1070 # collect_blocks
1071 # --------------
1072 #
1073 # A filter to aggregate "paragraphs" (blocks separated by blank
1074 # lines). Yields lists of lines::
1075
1076 def collect_blocks(lines):
1077 """collect lines in a list
1078
1079 yield list for each paragraph, i.e. block of lines separated by a
1080 blank line (whitespace only).
1081
1082 Trailing blank lines are collected as well.
1083 """
1084 blank_line_reached = False
1085 block = []
1086 for line in lines:
1087 if blank_line_reached and line.rstrip():
1088 yield block
1089 blank_line_reached = False
1090 block = [line]
1091 continue
1092 if not line.rstrip():
1093 blank_line_reached = True
1094 block.append(line)
1095 yield block
1096
1097
1098
1099 # dumb_c_preprocessor
1100 # -------------------
1101 #
1102 # This is a basic filter to convert `C` to `C++` comments. Works line-wise and
1103 # only converts lines that
1104 #
1105 # * start with "/\* " and end with " \*/" (followed by whitespace only)
1106 #
1107 # A more sophisticated version would also
1108 #
1109 # * convert multi-line comments
1110 #
1111 # + Keep indentation or strip 3 leading spaces?
1112 #
1113 # * account for nested comments
1114 #
1115 # * only convert comments that are separated from code by a blank line
1116 #
1117 # ::
1118
1119 def dumb_c_preprocessor(data):
1120 """change `C` ``/* `` `` */`` comments into C++ ``// `` comments"""
1121 comment_string = defaults.comment_strings["c++"]
1122 boc_string = "/* "
1123 eoc_string = " */"
1124 for line in data:
1125 if (line.startswith(boc_string)
1126 and line.rstrip().endswith(eoc_string)
1127 ):
1128 line = line.replace(boc_string, comment_string, 1)
1129 line = "".join(line.rsplit(eoc_string, 1))
1130 yield line
1131
1132 # Unfortunately, the `replace` method of strings does not support negative
1133 # numbers for the `count` argument:
1134 #
1135 # >>> "foo */ baz */ bar".replace(" */", "", -1) == "foo */ baz bar"
1136 # False
1137 #
1138 # However, there is the `rsplit` method, that can be used together with `join`:
1139 #
1140 # >>> "".join("foo */ baz */ bar".rsplit(" */", 1)) == "foo */ baz bar"
1141 # True
1142 #
1143 # dumb_c_postprocessor
1144 # --------------------
1145 #
1146 # Undo the preparations by the dumb_c_preprocessor and re-insert valid comment
1147 # delimiters ::
1148
1149 def dumb_c_postprocessor(data):
1150 """change C++ ``// `` comments into `C` ``/* `` `` */`` comments"""
1151 comment_string = defaults.comment_strings["c++"]
1152 boc_string = "/* "
1153 eoc_string = " */"
1154 for line in data:
1155 if line.rstrip() == comment_string.rstrip():
1156 line = line.replace(comment_string, "", 1)
1157 elif line.startswith(comment_string):
1158 line = line.replace(comment_string, boc_string, 1)
1159 line = line.rstrip() + eoc_string + "\n"
1160 yield line
1161
1162
1163 # register filters
1164 # ----------------
1165 #
1166 # ::
1167
1168 defaults.preprocessors['c2text'] = dumb_c_preprocessor
1169 defaults.preprocessors['css2text'] = dumb_c_preprocessor
1170 defaults.postprocessors['text2c'] = dumb_c_postprocessor
1171 defaults.postprocessors['text2css'] = dumb_c_postprocessor
1172
1173
1174 # Command line use
1175 # ================
1176 #
1177 # Using this script from the command line will convert a file according to its
1178 # extension. This default can be overridden by a couple of options.
1179 #
1180 # Dual source handling
1181 # --------------------
1182 #
1183 # How to determine which source is up-to-date?
1184 # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
1185 #
1186 # - set modification date of `outfile` to the one of `infile`
1187 #
1188 # Points out that the source files are 'synchronised'.
1189 #
1190 # * Are there problems to expect from "backdating" a file? Which?
1191 #
1192 # Looking at http://www.unix.com/showthread.php?t=20526, it seems
1193 # perfectly legal to set `mtime` (while leaving `ctime`) as `mtime` is a
1194 # description of the "actuality" of the data in the file.
1195 #
1196 # * Should this become a default or an option?
1197 #
1198 # - alternatively move input file to a backup copy (with option: `--replace`)
1199 #
1200 # - check modification date before overwriting
1201 # (with option: `--overwrite=update`)
1202 #
1203 # - check modification date before editing (implemented as `Jed editor`_
1204 # function `pylit_check()` in `pylit.sl`_)
1205 #
1206 # .. _Jed editor: http://www.jedsoft.org/jed/
1207 # .. _pylit.sl: http://jedmodes.sourceforge.net/mode/pylit/
1208 #
1209 # Recognised Filename Extensions
1210 # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
1211 #
1212 # Instead of defining a new extension for "pylit" literate programs,
1213 # by default ``.txt`` will be appended for the text source and stripped by
1214 # the conversion to the code source. I.e. for a Python program foo:
1215 #
1216 # * the code source is called ``foo.py``
1217 # * the text source is called ``foo.py.txt``
1218 # * the html rendering is called ``foo.py.html``
1219 #
1220 #
1221 # OptionValues
1222 # ------------
1223 #
1224 # The following class adds `as_dict`_, `complete`_ and `__getattr__`_
1225 # methods to `optparse.Values`::
1226
1227 class OptionValues(optparse.Values):
1228
1229 # .. _OptionValues.as_dict:
1230 #
1231 # as_dict
1232 # ~~~~~~~
1233 #
1234 # For use as keyword arguments, it is handy to have the options in a
1235 # dictionary. `as_dict` returns a copy of the instances object dictionary::
1236
1237 def as_dict(self):
1238 """Return options as dictionary object"""
1239 return self.__dict__.copy()
1240
1241 # .. _OptionValues.complete:
1242 #
1243 # complete
1244 # ~~~~~~~~
1245 #
1246 # ::
1247
1248 def complete(self, **keyw):
1249 """
1250 Complete the option values with keyword arguments.
1251
1252 Do not overwrite existing values. Only use arguments that do not
1253 have a corresponding attribute in `self`,
1254 """
1255 for key in keyw:
1256 if not self.__dict__.__contains__(key):
1257 setattr(self, key, keyw[key])
1258
1259 # .. _OptionValues.__getattr__:
1260 #
1261 # __getattr__
1262 # ~~~~~~~~~~~
1263 #
1264 # To replace calls using ``options.ensure_value("OPTION", None)`` with the
1265 # more concise ``options.OPTION``, we define `__getattr__` [#]_ ::
1266
1267 def __getattr__(self, name):
1268 """Return default value for non existing options"""
1269 return None
1270
1271
1272 # .. [#] The special method `__getattr__` is only called when an attribute
1273 # look-up has not found the attribute in the usual places (i.e. it is
1274 # not an instance attribute nor is it found in the class tree for
1275 # self).
1276 #
1277 #
1278 # PylitOptions
1279 # ------------
1280 #
1281 # The `PylitOptions` class comprises an option parser and methods for parsing
1282 # and completion of command line options::
1283
1284 class PylitOptions(object):
1285 """Storage and handling of command line options for pylit"""
1286
1287 # Instantiation
1288 # ~~~~~~~~~~~~~
1289 #
1290 # ::
1291
1292 def __init__(self):
1293 """Set up an `OptionParser` instance for pylit command line options
1294
1295 """
1296 p = optparse.OptionParser(usage=main.__doc__, version=_version)
1297
1298 # Conversion settings
1299
1300 p.add_option("-c", "--code2txt", dest="txt2code", action="store_false",
1301 help="convert code source to text source")
1302 p.add_option("-t", "--txt2code", action="store_true",
1303 help="convert text source to code source")
1304 p.add_option("--language",
1305 choices = list(defaults.languages.values()),
1306 help="use LANGUAGE native comment style")
1307 p.add_option("--comment-string", dest="comment_string",
1308 help="documentation block marker in code source "
1309 "(including trailing whitespace, "
1310 "default: language dependent)")
1311 p.add_option("-m", "--code-block-marker", dest="code_block_marker",
1312 help="syntax token starting a code block. (default '::')")
1313 p.add_option("--codeindent", type="int",
1314 help="Number of spaces to indent code blocks with "
1315 "text2code (default %d)" % defaults.codeindent)
1316
1317 # Output file handling
1318
1319 p.add_option("--overwrite", action="store",
1320 choices = ["yes", "update", "no"],
1321 help="overwrite output file (default 'update')")
1322 p.add_option("--replace", action="store_true",
1323 help="move infile to a backup copy (appending '~')")
1324 p.add_option("-s", "--strip", action="store_true",
1325 help='"export" by stripping documentation or code')
1326
1327 # Special actions
1328
1329 p.add_option("-d", "--diff", action="store_true",
1330 help="test for differences to existing file")
1331 p.add_option("--doctest", action="store_true",
1332 help="run doctest.testfile() on the text version")
1333 p.add_option("-e", "--execute", action="store_true",
1334 help="execute code (Python only)")
1335
1336 self.parser = p
1337
1338 # .. _PylitOptions.parse_args:
1339 #
1340 # parse_args
1341 # ~~~~~~~~~~
1342 #
1343 # The `parse_args` method calls the `optparse.OptionParser` on command
1344 # line or provided args and returns the result as `PylitOptions.Values`
1345 # instance. Defaults can be provided as keyword arguments::
1346
1347 def parse_args(self, args=sys.argv[1:], **keyw):
1348 """parse command line arguments using `optparse.OptionParser`
1349
1350 parse_args(args, **keyw) -> OptionValues instance
1351
1352 args -- list of command line arguments.
1353 keyw -- keyword arguments or dictionary of option defaults
1354 """
1355 # parse arguments
1356 (values, args) = self.parser.parse_args(args, OptionValues(keyw))
1357 # Convert FILE and OUTFILE positional args to option values
1358 # (other positional arguments are ignored)
1359 try:
1360 values.infile = args[0]
1361 values.outfile = args[1]
1362 except IndexError:
1363 pass
1364
1365 return values
1366
1367 # .. _PylitOptions.complete_values:
1368 #
1369 # complete_values
1370 # ~~~~~~~~~~~~~~~
1371 #
1372 # Complete an OptionValues instance `values`. Use module-level defaults and
1373 # context information to set missing option values to sensible defaults (if
1374 # possible) ::
1375
1376 def complete_values(self, values):
1377 """complete option values with module and context sensible defaults
1378
1379 x.complete_values(values) -> values
1380 values -- OptionValues instance
1381 """
1382
1383 # Complete with module-level defaults_::
1384
1385 values.complete(**defaults.__dict__)
1386
1387 # Ensure infile is a string::
1388
1389 values.ensure_value("infile", "")
1390
1391 # Guess conversion direction from `infile` filename::
1392
1393 if values.txt2code is None:
1394 in_extension = os.path.splitext(values.infile)[1]
1395 if in_extension in values.text_extensions:
1396 values.txt2code = True
1397 elif in_extension in values.languages.keys():
1398 values.txt2code = False
1399
1400 # Auto-determine the output file name::
1401
1402 values.ensure_value("outfile", self._get_outfile_name(values))
1403
1404 # Second try: Guess conversion direction from outfile filename::
1405
1406 if values.txt2code is None:
1407 out_extension = os.path.splitext(values.outfile)[1]
1408 values.txt2code = not (out_extension in values.text_extensions)
1409
1410 # Set the language of the code::
1411
1412 if values.txt2code is True:
1413 code_extension = os.path.splitext(values.outfile)[1]
1414 elif values.txt2code is False:
1415 code_extension = os.path.splitext(values.infile)[1]
1416 values.ensure_value("language", values.languages[code_extension])
1417
1418 return values
1419
1420 # _get_outfile_name
1421 # ~~~~~~~~~~~~~~~~~
1422 #
1423 # Construct a matching filename for the output file. The output filename is
1424 # constructed from `infile` by the following rules:
1425 #
1426 # * '-' (stdin) results in '-' (stdout)
1427 # * strip the `text_extension`_ (txt2code) or
1428 # * add the `text_extension`_ (code2txt)
1429 # * fallback: if no guess can be made, add ".out"
1430 #
1431 # .. TODO: use values.outfile_extension if it exists?
1432 #
1433 # ::
1434
1435 def _get_outfile_name(self, values):
1436 """Return a matching output filename for `infile`
1437 """
1438 # if input is stdin, default output is stdout
1439 if values.infile == '-':
1440 return '-'
1441
1442 # Derive from `infile` name: strip or add text extension
1443 (base, ext) = os.path.splitext(values.infile)
1444 if ext in values.text_extensions:
1445 return base # strip
1446 if ext in values.languages.keys() or values.txt2code == False:
1447 return values.infile + values.text_extensions[0] # add
1448 # give up
1449 return values.infile + ".out"
1450
1451 # .. _PylitOptions.__call__:
1452 #
1453 # __call__
1454 # ~~~~~~~~
1455 #
1456 # The special `__call__` method allows to use PylitOptions instances as
1457 # *callables*: Calling an instance parses the argument list to extract option
1458 # values and completes them based on "context-sensitive defaults". Keyword
1459 # arguments are passed to `PylitOptions.parse_args`_ as default values. ::
1460
1461 def __call__(self, args=sys.argv[1:], **keyw):
1462 """parse and complete command line args return option values
1463 """
1464 values = self.parse_args(args, **keyw)
1465 return self.complete_values(values)
1466
1467
1468
1469 # Helper functions
1470 # ----------------
1471 #
1472 # open_streams
1473 # ~~~~~~~~~~~~
1474 #
1475 # Return file objects for in- and output. If the input path is missing,
1476 # write usage and abort. (An alternative would be to use stdin as default.
1477 # However, this leaves the uninitiated user with a non-responding application
1478 # if (s)he just tries the script without any arguments) ::
1479
1480 def open_streams(infile = '-', outfile = '-', overwrite='update', **keyw):
1481 """Open and return the input and output stream
1482
1483 open_streams(infile, outfile) -> (in_stream, out_stream)
1484
1485 in_stream -- open(infile) or sys.stdin
1486 out_stream -- open(outfile) or sys.stdout
1487 overwrite -- 'yes': overwrite eventually existing `outfile`,
1488 'update': fail if the `outfile` is newer than `infile`,
1489 'no': fail if `outfile` exists.
1490
1491 Irrelevant if `outfile` == '-'.
1492 """
1493 if not infile:
1494 strerror = "Missing input file name ('-' for stdin; -h for help)"
1495 raise IOError((2, strerror, infile))
1496 if infile == '-':
1497 in_stream = sys.stdin
1498 else:
1499 in_stream = io.open(infile, 'r', encoding='utf-8')
1500
1501 if outfile == '-':
1502 out_stream = sys.stdout
1503 elif overwrite == 'no' and os.path.exists(outfile):
1504 raise IOError((1, "Output file exists!", outfile))
1505 elif overwrite == 'update' and is_newer(outfile, infile):
1506 raise IOError((1, "Output file is newer than input file!", outfile))
1507 else:
1508 out_stream = io.open(outfile, 'w', encoding='utf-8')
1509 return (in_stream, out_stream)
1510
1511 # is_newer
1512 # ~~~~~~~~
1513 #
1514 # ::
1515
1516 def is_newer(path1, path2):
1517 """Check if `path1` is newer than `path2` (using mtime)
1518
1519 Compare modification time of files at path1 and path2.
1520
1521 Non-existing files are considered oldest: Return False if path1 does not
1522 exist and True if path2 does not exist.
1523
1524 Return None for equal modification time. (This evaluates to False in a
1525 Boolean context but allows a test for equality.)
1526
1527 """
1528 try:
1529 mtime1 = os.path.getmtime(path1)
1530 except OSError:
1531 mtime1 = -1
1532 try:
1533 mtime2 = os.path.getmtime(path2)
1534 except OSError:
1535 mtime2 = -1
1536 # print("mtime1", mtime1, path1, "\n", "mtime2", mtime2, path2)
1537
1538 if mtime1 == mtime2:
1539 return None
1540 return mtime1 > mtime2
1541
1542
1543 # get_converter
1544 # ~~~~~~~~~~~~~
1545 #
1546 # Get an instance of the converter state machine::
1547
1548 def get_converter(data, txt2code=True, **keyw):
1549 if txt2code:
1550 return Text2Code(data, **keyw)
1551 else:
1552 return Code2Text(data, **keyw)
1553
1554
1555 # Use cases
1556 # ---------
1557 #
1558 # run_doctest
1559 # ~~~~~~~~~~~
1560 # ::
1561
1562 def run_doctest(infile="-", txt2code=True,
1563 globs={}, verbose=False, optionflags=0, **keyw):
1564 """run doctest on the text source
1565 """
1566
1567 # Allow imports from the current working dir by prepending an empty string to
1568 # sys.path (see doc of sys.path())::
1569
1570 sys.path.insert(0, '')
1571
1572 # Import classes from the doctest module::
1573
1574 from doctest import DocTestParser, DocTestRunner
1575
1576 # Read in source. Make sure it is in text format, as tests in comments are not
1577 # found by doctest::
1578
1579 (data, out_stream) = open_streams(infile, "-")
1580 if txt2code is False:
1581 keyw.update({'add_missing_marker': False})
1582 converter = Code2Text(data, **keyw)
1583 docstring = str(converter)
1584 else:
1585 docstring = data.read()
1586
1587 # decode doc string if there is a "magic comment" in the first or second line
1588 # (http://docs.python.org/reference/lexical_analysis.html#encoding-declarations)
1589 # ::
1590
1591 firstlines = ' '.join(docstring.splitlines()[:2])
1592 match = re.search('coding[=:]\s*([-\w.]+)', firstlines)
1593 if match:
1594 docencoding = match.group(1)
1595 docstring = docstring.decode(docencoding)
1596
1597 # Use the doctest Advanced API to run all doctests in the source text::
1598
1599 test = DocTestParser().get_doctest(docstring, globs, name="",
1600 filename=infile, lineno=0)
1601 runner = DocTestRunner(verbose, optionflags)
1602 runner.run(test)
1603 runner.summarize
1604 # give feedback also if no failures occurred
1605 if not runner.failures:
1606 print("%d failures in %d tests"%(runner.failures, runner.tries))
1607 return runner.failures, runner.tries
1608
1609
1610 # diff
1611 # ~~~~
1612 #
1613 # ::
1614
1615 def diff(infile='-', outfile='-', txt2code=True, **keyw):
1616 """Report differences between converted infile and existing outfile
1617
1618 If outfile does not exist or is '-', do a round-trip conversion and
1619 report differences.
1620 """
1621
1622 import difflib
1623
1624 instream = open(infile)
1625 # for diffing, we need a copy of the data as list::
1626 data = instream.readlines()
1627 # convert
1628 converter = get_converter(data, txt2code, **keyw)
1629 new = converter()
1630
1631 if outfile != '-' and os.path.exists(outfile):
1632 outstream = open(outfile)
1633 old = outstream.readlines()
1634 oldname = outfile
1635 newname = "<conversion of %s>"%infile
1636 else:
1637 old = data
1638 oldname = infile
1639 # back-convert the output data
1640 converter = get_converter(new, not txt2code)
1641 new = converter()
1642 newname = "<round-conversion of %s>"%infile
1643
1644 # find and print the differences
1645 is_different = False
1646 # print(type(old), old)
1647 # print(type(new), new)
1648 delta = difflib.unified_diff(old, new,
1649 # delta = difflib.unified_diff(["heute\n", "schon\n"], ["heute\n", "noch\n"],
1650 fromfile=oldname, tofile=newname)
1651 for line in delta:
1652 is_different = True
1653 print(line, end="")
1654 if not is_different:
1655 print(oldname)
1656 print(newname)
1657 print("no differences found")
1658 return is_different
1659
1660
1661 # execute
1662 # ~~~~~~~
1663 #
1664 # Works only for python code.
1665 #
1666 # Does not work with `eval`, as code is not just one expression. ::
1667
1668 def execute(infile="-", txt2code=True, **keyw):
1669 """Execute the input file. Convert first, if it is a text source.
1670 """
1671
1672 data = open(infile)
1673 if txt2code:
1674 data = str(Text2Code(data, **keyw))
1675 # print("executing " + options.infile)
1676 exec(data)
1677
1678
1679 # main
1680 # ----
1681 #
1682 # If this script is called from the command line, the `main` function will
1683 # convert the input (file or stdin) between text and code formats.
1684 #
1685 # Option default values for the conversion can be given as keyword arguments
1686 # to `main`_. The option defaults will be updated by command line options and
1687 # extended with "intelligent guesses" by `PylitOptions`_ and passed on to
1688 # helper functions and the converter instantiation.
1689 #
1690 # This allows easy customisation for programmatic use -- just call `main`
1691 # with the appropriate keyword options, e.g. ``pylit.main(comment_string="## ")``
1692 #
1693 # ::
1694
1695 def main(args=sys.argv[1:], **defaults):
1696 """%prog [options] INFILE [OUTFILE]
1697
1698 Convert between (reStructured) text source with embedded code,
1699 and code source with embedded documentation (comment blocks)
1700
1701 The special filename '-' stands for standard in and output.
1702 """
1703
1704 # Parse and complete the options::
1705
1706 options = PylitOptions()(args, **defaults)
1707 # print("infile", repr(options.infile))
1708
1709 # Special actions with early return::
1710
1711 if options.doctest:
1712 return run_doctest(**options.as_dict())
1713
1714 if options.diff:
1715 return diff(**options.as_dict())
1716
1717 if options.execute:
1718 return execute(**options.as_dict())
1719
1720 # Open in- and output streams::
1721
1722 try:
1723 (data, out_stream) = open_streams(**options.as_dict())
1724 except IOError as ex:
1725 print("IOError: %s %s" % (ex.filename, ex.strerror))
1726 sys.exit(ex.errno)
1727
1728 # Get a converter instance::
1729
1730 converter = get_converter(data, **options.as_dict())
1731
1732 # Convert and write to out_stream::
1733
1734 if sys.version_info[0] == 2:
1735 out_stream.write(unicode(converter))
1736 else:
1737 out_stream.write(str(converter))
1738
1739 if out_stream is not sys.stdout:
1740 print("extract written to", out_stream.name)
1741 out_stream.close()
1742
1743 # If input and output are from files, set the modification time (`mtime`) of
1744 # the output file to the one of the input file to indicate that the contained
1745 # information is equal. [#]_ ::
1746
1747 try:
1748 os.utime(options.outfile, (os.path.getatime(options.outfile),
1749 os.path.getmtime(options.infile))
1750 )
1751 except OSError:
1752 pass
1753
1754 ## print("mtime", os.path.getmtime(options.infile), options.infile)
1755 ## print("mtime", os.path.getmtime(options.outfile), options.outfile)
1756
1757
1758 # .. [#] Make sure the corresponding file object (here `out_stream`) is
1759 # closed, as otherwise the change will be overwritten when `close` is
1760 # called afterwards (either explicitly or at program exit).
1761 #
1762 #
1763 # Rename the infile to a backup copy if ``--replace`` is set::
1764
1765 if options.replace:
1766 os.rename(options.infile, options.infile + "~")
1767
1768
1769 # Run main, if called from the command line::
1770
1771 if __name__ == '__main__':
1772 main()
1773
1774
1775 # Open questions
1776 # ==============
1777 #
1778 # Open questions and ideas for further development
1779 #
1780 # Clean code
1781 # ----------
1782 #
1783 # * can we gain from using "shutils" over "os.path" and "os"?
1784 # * use pylint or pyChecker to enforce a consistent style?
1785 #
1786 # Options
1787 # -------
1788 #
1789 # * Use templates for the "intelligent guesses" (with Python syntax for string
1790 # replacement with dicts: ``"hello %(what)s" % {'what': 'world'}``)
1791 #
1792 # * Is it sensible to offer the `header_string` option also as command line
1793 # option?
1794 #
1795 # treatment of blank lines
1796 # ------------------------
1797 #
1798 # Alternatives: Keep blank lines blank
1799 #
1800 # - "never" (current setting) -> "visually merges" all documentation
1801 # if there is no interjacent code
1802 #
1803 # - "always" -> disrupts documentation blocks,
1804 #
1805 # - "if empty" (no whitespace). Comment if there is whitespace.
1806 #
1807 # This would allow non-obstructing markup but unfortunately this is (in
1808 # most editors) also non-visible markup.
1809 #
1810 # + "if double" (if there is more than one consecutive blank line)
1811 #
1812 # With this handling, the "visual gap" remains in both, text and code
1813 # source.
1814 #
1815 #
1816 # Parsing Problems
1817 # ----------------
1818 #
1819 # * Ignore "matching comments" in literal strings?
1820 #
1821 # Too complicated: Would need a specific detection algorithm for every
1822 # language that supports multi-line literal strings (C++, PHP, Python)
1823 #
1824 # * Warn if a comment in code will become documentation after round-trip?
1825 #
1826 #
1827 # docstrings in code blocks
1828 # -------------------------
1829 #
1830 # * How to handle docstrings in code blocks? (it would be nice to convert them
1831 # to rst-text if ``__docformat__ == restructuredtext``)
1832 #
1833 # TODO: Ask at Docutils users|developers
1834 #
1835 # Plug-ins
1836 # --------
1837 #
1838 # Specify a path for user additions and plug-ins. This would require to
1839 # convert Pylit from a pure module to a package...
1840 #
1841 # 6.4.3 Packages in Multiple Directories
1842 #
1843 # Packages support one more special attribute, __path__. This is initialized
1844 # to be a list containing the name of the directory holding the package's
1845 # __init__.py before the code in that file is executed. This
1846 # variable can be modified; doing so affects future searches for modules and
1847 # subpackages contained in the package.
1848 #
1849 # While this feature is not often needed, it can be used to extend the set
1850 # of modules found in a package.
1851 #
1852 #
1853 # .. References
1854 #
1855 # .. _Docutils: http://docutils.sourceforge.net/
1856 # .. _Sphinx: http://sphinx.pocoo.org
1857 # .. _Pygments: http://pygments.org/
1858 # .. _code-block directive:
1859 # http://docutils.sourceforge.net/sandbox/code-block-directive/
1860 # .. _literal block:
1861 # .. _literal blocks:
1862 # http://docutils.sf.net/docs/ref/rst/restructuredtext.html#literal-blocks
1863 # .. _indented literal block:
1864 # .. _indented literal blocks:
1865 # http://docutils.sf.net/docs/ref/rst/restructuredtext.html#indented-literal-blocks
1866 # .. _quoted literal block:
1867 # .. _quoted literal blocks:
1868 # http://docutils.sf.net/docs/ref/rst/restructuredtext.html#quoted-literal-blocks
1869 # .. _parsed-literal blocks:
1870 # http://docutils.sf.net/docs/ref/rst/directives.html#parsed-literal-block
1871 # .. _doctest block:
1872 # .. _doctest blocks:
1873 # http://docutils.sf.net/docs/ref/rst/restructuredtext.html#doctest-blocks
1874 #
1875 # .. _feature request and patch by jrioux:
1876 # http://developer.berlios.de/feature/?func=detailfeature&feature_id=4890&group_id=7974