| 1 |
#!/usr/bin/env python |
| 2 |
# -*- coding: iso-8859-1 -*- |
| 3 |
|
| 4 |
# pylit.py |
| 5 |
# ******** |
| 6 |
# Literate programming with reStructuredText |
| 7 |
# ++++++++++++++++++++++++++++++++++++++++++ |
| 8 |
# |
| 9 |
# :Date: $Date$ |
| 10 |
# :Revision: $Revision$ |
| 11 |
# :URL: $URL$ |
| 12 |
# :Copyright: © 2005, 2007 Günter Milde. |
| 13 |
# Released without warranty under the terms of the |
| 14 |
# GNU General Public License (v. 2 or later) |
| 15 |
# |
| 16 |
# :: |
| 17 |
|
| 18 |
from __future__ import print_function |
| 19 |
|
| 20 |
"""pylit: bidirectional text <-> code converter |
| 21 |
|
| 22 |
Convert between a *text document* with embedded code |
| 23 |
and *source code* with embedded documentation. |
| 24 |
""" |
| 25 |
|
| 26 |
# .. contents:: |
| 27 |
# |
| 28 |
# Frontmatter |
| 29 |
# =========== |
| 30 |
# |
| 31 |
# Changelog |
| 32 |
# --------- |
| 33 |
# |
| 34 |
# .. class:: borderless |
| 35 |
# |
| 36 |
# ====== ========== =========================================================== |
| 37 |
# 0.1 2005-06-29 Initial version. |
| 38 |
# 0.1.1 2005-06-30 First literate version. |
| 39 |
# 0.1.2 2005-07-01 Object orientated script using generators. |
| 40 |
# 0.1.3 2005-07-10 Two state machine (later added 'header' state). |
| 41 |
# 0.2b 2006-12-04 Start of work on version 0.2 (code restructuring). |
| 42 |
# 0.2 2007-01-23 Published at http://pylit.berlios.de. |
| 43 |
# 0.2.1 2007-01-25 Outsourced non-core documentation to the PyLit pages. |
| 44 |
# 0.2.2 2007-01-26 New behaviour of `diff` function. |
| 45 |
# 0.2.3 2007-01-29 New `header` methods after suggestion by Riccardo Murri. |
| 46 |
# 0.2.4 2007-01-31 Raise Error if code indent is too small. |
| 47 |
# 0.2.5 2007-02-05 New command line option --comment-string. |
| 48 |
# 0.2.6 2007-02-09 Add section with open questions, |
| 49 |
# Code2Text: let only blank lines (no comment str) |
| 50 |
# separate text and code, |
| 51 |
# fix `Code2Text.header`. |
| 52 |
# 0.2.7 2007-02-19 Simplify `Code2Text.header`, |
| 53 |
# new `iter_strip` method replacing a lot of ``if``-s. |
| 54 |
# 0.2.8 2007-02-22 Set `mtime` of outfile to the one of infile. |
| 55 |
# 0.3 2007-02-27 New `Code2Text` converter after an idea by Riccardo Murri, |
| 56 |
# explicit `option_defaults` dict for easier customisation. |
| 57 |
# 0.3.1 2007-03-02 Expand hard-tabs to prevent errors in indentation, |
| 58 |
# `Text2Code` now also works on blocks, |
| 59 |
# removed dependency on SimpleStates module. |
| 60 |
# 0.3.2 2007-03-06 Bug fix: do not set `language` in `option_defaults` |
| 61 |
# renamed `code_languages` to `languages`. |
| 62 |
# 0.3.3 2007-03-16 New language css, |
| 63 |
# option_defaults -> defaults = optparse.Values(), |
| 64 |
# simpler PylitOptions: don't store parsed values, |
| 65 |
# don't parse at initialisation, |
| 66 |
# OptionValues: return `None` for non-existing attributes, |
| 67 |
# removed -infile and -outfile, use positional arguments. |
| 68 |
# 0.3.4 2007-03-19 Documentation update, |
| 69 |
# separate `execute` function. |
| 70 |
# 2007-03-21 Code cleanup in `Text2Code.__iter__`. |
| 71 |
# 0.3.5 2007-03-23 Removed "css" from known languages after learning that |
| 72 |
# there is no C++ style "// " comment string in CSS2. |
| 73 |
# 0.3.6 2007-04-24 Documentation update. |
| 74 |
# 0.4 2007-05-18 Implement Converter.__iter__ as stack of iterator |
| 75 |
# generators. Iterating over a converter instance now |
| 76 |
# yields lines instead of blocks. |
| 77 |
# Provide "hooks" for pre- and postprocessing filters. |
| 78 |
# Rename states to reduce confusion with formats: |
| 79 |
# "text" -> "documentation", "code" -> "code_block". |
| 80 |
# 0.4.1 2007-05-22 Converter.__iter__: cleanup and reorganisation, |
| 81 |
# rename parent class Converter -> TextCodeConverter. |
| 82 |
# 0.4.2 2007-05-23 Merged Text2Code.converter and Code2Text.converter into |
| 83 |
# TextCodeConverter.converter. |
| 84 |
# 0.4.3 2007-05-30 Replaced use of defaults.code_extensions with |
| 85 |
# values.languages.keys(). |
| 86 |
# Removed spurious `print` statement in code_block_handler. |
| 87 |
# Added basic support for 'c' and 'css' languages |
| 88 |
# with `dumb_c_preprocessor`_ and `dumb_c_postprocessor`_. |
| 89 |
# 0.5 2007-06-06 Moved `collect_blocks`_ out of `TextCodeConverter`_, |
| 90 |
# bug fix: collect all trailing blank lines into a block. |
| 91 |
# Expand tabs with `expandtabs_filter`_. |
| 92 |
# 0.6 2007-06-20 Configurable code-block marker (default ``::``) |
| 93 |
# 0.6.1 2007-06-28 Bug fix: reset self.code_block_marker_missing. |
| 94 |
# 0.7 2007-12-12 prepending an empty string to sys.path in run_doctest() |
| 95 |
# to allow imports from the current working dir. |
| 96 |
# 0.7.1 2008-01-07 If outfile does not exist, do a round-trip conversion |
| 97 |
# and report differences (as with outfile=='-'). |
| 98 |
# 0.7.2 2008-01-28 Do not add missing code-block separators with |
| 99 |
# `doctest_run` on the code source. Keeps lines consistent. |
| 100 |
# 0.7.3 2008-04-07 Use value of code_block_marker for insertion of missing |
| 101 |
# transition marker in Code2Text.code_block_handler |
| 102 |
# Add "shell" to defaults.languages |
| 103 |
# 0.7.4 2008-06-23 Add "latex" to defaults.languages |
| 104 |
# 0.7.5 2009-05-14 Bugfix: ignore blank lines in test for end of code block |
| 105 |
# 0.7.6 2009-12-15 language-dependent code-block markers (after a |
| 106 |
# `feature request and patch by jrioux`_), |
| 107 |
# use DefaultDict for language-dependent defaults, |
| 108 |
# new defaults setting `add_missing_marker`_. |
| 109 |
# 0.7.7 2010-06-23 New command line option --codeindent. |
| 110 |
# 0.7.8 2011-03-30 bugfix: do not overwrite custom `add_missing_marker` value, |
| 111 |
# allow directive options following the 'code' directive. |
| 112 |
# 0.7.9 2011-04-05 Decode doctest string if 'magic comment' gives encoding. |
| 113 |
# ====== ========== =========================================================== |
| 114 |
# |
| 115 |
# :: |
| 116 |
|
| 117 |
_version = "0.7.9" |
| 118 |
|
| 119 |
__docformat__ = 'restructuredtext' |
| 120 |
|
| 121 |
|
| 122 |
# Introduction |
| 123 |
# ------------ |
| 124 |
# |
| 125 |
# PyLit is a bidirectional converter between two formats of a computer |
| 126 |
# program source: |
| 127 |
# |
| 128 |
# * a (reStructured) text document with program code embedded in |
| 129 |
# *code blocks*, and |
| 130 |
# * a compilable (or executable) code source with *documentation* |
| 131 |
# embedded in comment blocks |
| 132 |
# |
| 133 |
# |
| 134 |
# Requirements |
| 135 |
# ------------ |
| 136 |
# |
| 137 |
# :: |
| 138 |
|
| 139 |
import os, sys, io |
| 140 |
import re, optparse |
| 141 |
|
| 142 |
|
| 143 |
# DefaultDict |
| 144 |
# ~~~~~~~~~~~ |
| 145 |
# As `collections.defaultdict` is only introduced in Python 2.5, we |
| 146 |
# define a simplified version of the dictionary with default from |
| 147 |
# http://code.activestate.com/recipes/389639/ |
| 148 |
# :: |
| 149 |
|
| 150 |
class DefaultDict(dict): |
| 151 |
"""Minimalistic Dictionary with default value.""" |
| 152 |
def __init__(self, default=None, *args, **kwargs): |
| 153 |
self.update(dict(*args, **kwargs)) |
| 154 |
self.default = default |
| 155 |
|
| 156 |
def __getitem__(self, key): |
| 157 |
return self.get(key, self.default) |
| 158 |
|
| 159 |
|
| 160 |
# Defaults |
| 161 |
# ======== |
| 162 |
# |
| 163 |
# The `defaults` object provides a central repository for default |
| 164 |
# values and their customisation. :: |
| 165 |
|
| 166 |
defaults = optparse.Values() |
| 167 |
|
| 168 |
# It is used for |
| 169 |
# |
| 170 |
# * the initialisation of data arguments in TextCodeConverter_ and |
| 171 |
# PylitOptions_ |
| 172 |
# |
| 173 |
# * completion of command line options in `PylitOptions.complete_values`_. |
| 174 |
# |
| 175 |
# This allows the easy creation of back-ends that customise the |
| 176 |
# defaults and then call `main`_ e.g.: |
| 177 |
# |
| 178 |
# >>> import pylit |
| 179 |
# >>> pylit.defaults.comment_string = "## " |
| 180 |
# >>> pylit.defaults.codeindent = 4 |
| 181 |
# >>> pylit.main() |
| 182 |
# |
| 183 |
# The following default values are defined in pylit.py: |
| 184 |
# |
| 185 |
# languages |
| 186 |
# --------- |
| 187 |
# |
| 188 |
# Mapping of code file extensions to code language:: |
| 189 |
|
| 190 |
defaults.languages = DefaultDict("python", # fallback language |
| 191 |
{".c": "c", |
| 192 |
".cc": "c++", |
| 193 |
".cpp": "c++", |
| 194 |
".css": "css", |
| 195 |
".py": "python", |
| 196 |
".sh": "shell", |
| 197 |
".sl": "slang", |
| 198 |
".sty": "latex", |
| 199 |
".tex": "latex", |
| 200 |
".ufl": "python" |
| 201 |
}) |
| 202 |
|
| 203 |
# Will be overridden by the ``--language`` command line option. |
| 204 |
# |
| 205 |
# The first argument is the fallback language, used if there is no |
| 206 |
# matching extension (e.g. if pylit is used as filter) and no |
| 207 |
# ``--language`` is specified. It can be changed programmatically by |
| 208 |
# assignment to the ``.default`` attribute, e.g. |
| 209 |
# |
| 210 |
# >>> defaults.languages.default='c++' |
| 211 |
# |
| 212 |
# |
| 213 |
# .. _text_extension: |
| 214 |
# |
| 215 |
# text_extensions |
| 216 |
# --------------- |
| 217 |
# |
| 218 |
# List of known extensions of (reStructured) text files. The first |
| 219 |
# extension in this list is used by the `_get_outfile_name`_ method to |
| 220 |
# generate a text output filename:: |
| 221 |
|
| 222 |
defaults.text_extensions = [".txt", ".rst"] |
| 223 |
|
| 224 |
|
| 225 |
# comment_strings |
| 226 |
# --------------- |
| 227 |
# |
| 228 |
# Comment strings for known languages. Used in Code2Text_ to recognise |
| 229 |
# text blocks and in Text2Code_ to format text blocks as comments. |
| 230 |
# Defaults to ``'# '``. |
| 231 |
# |
| 232 |
# **Comment strings include trailing whitespace.** :: |
| 233 |
|
| 234 |
defaults.comment_strings = DefaultDict('# ', |
| 235 |
{"css": '// ', |
| 236 |
"c": '// ', |
| 237 |
"c++": '// ', |
| 238 |
"latex": '% ', |
| 239 |
"python": '# ', |
| 240 |
"shell": '# ', |
| 241 |
"slang": '% ' |
| 242 |
}) |
| 243 |
|
| 244 |
|
| 245 |
# header_string |
| 246 |
# ------------- |
| 247 |
# |
| 248 |
# Marker string for a header code block in the text source. No trailing |
| 249 |
# whitespace needed as indented code follows. |
| 250 |
# Must be a valid rst directive that accepts code on the same line, e.g. |
| 251 |
# ``'..admonition::'``. |
| 252 |
# |
| 253 |
# Default is a comment marker:: |
| 254 |
|
| 255 |
defaults.header_string = '..' |
| 256 |
|
| 257 |
|
| 258 |
# .. _code_block_marker: |
| 259 |
# |
| 260 |
# code_block_markers |
| 261 |
# ------------------ |
| 262 |
# |
| 263 |
# Markup at the end of a documentation block. |
| 264 |
# Default is Docutils' marker for a `literal block`_:: |
| 265 |
|
| 266 |
defaults.code_block_markers = DefaultDict('::') |
| 267 |
defaults.code_block_markers["c++"] = u".. code-block:: cpp" |
| 268 |
#defaults.code_block_markers['python'] = '.. code-block:: python' |
| 269 |
|
| 270 |
# The `code_block_marker` string is `inserted into a regular expression`_. |
| 271 |
# Language-specific markers can be defined programmatically, e.g. in a |
| 272 |
# wrapper script. |
| 273 |
# |
| 274 |
# In a document where code examples are only one of several uses of |
| 275 |
# literal blocks, it is more appropriate to single out the source code |
| 276 |
# ,e.g. with the double colon at a separate line ("expanded form") |
| 277 |
# |
| 278 |
# ``defaults.code_block_marker.default = ':: *'`` |
| 279 |
# |
| 280 |
# or a dedicated ``.. code-block::`` directive [#]_ |
| 281 |
# |
| 282 |
# ``defaults.code_block_marker['c++'] = '.. code-block:: *c++'`` |
| 283 |
# |
| 284 |
# The latter form also allows code in different languages kept together |
| 285 |
# in one literate source file. |
| 286 |
# |
| 287 |
# .. [#] The ``.. code-block::`` directive is not (yet) supported by |
| 288 |
# standard Docutils. It is provided by several add-ons, including |
| 289 |
# the `code-block directive`_ project in the Docutils Sandbox and |
| 290 |
# Sphinx_. |
| 291 |
# |
| 292 |
# |
| 293 |
# strip |
| 294 |
# ----- |
| 295 |
# |
| 296 |
# Export to the output format stripping documentation or code blocks:: |
| 297 |
|
| 298 |
defaults.strip = False |
| 299 |
|
| 300 |
# strip_marker |
| 301 |
# ------------ |
| 302 |
# |
| 303 |
# Strip literal marker from the end of documentation blocks when |
| 304 |
# converting to code format. Makes the code more concise but looses the |
| 305 |
# synchronisation of line numbers in text and code formats. Can also be used |
| 306 |
# (together with the auto-completion of the code-text conversion) to change |
| 307 |
# the `code_block_marker`:: |
| 308 |
|
| 309 |
defaults.strip_marker = False |
| 310 |
|
| 311 |
# add_missing_marker |
| 312 |
# ------------------ |
| 313 |
# |
| 314 |
# When converting from code format to text format, add a `code_block_marker` |
| 315 |
# at the end of documentation blocks if it is missing:: |
| 316 |
|
| 317 |
defaults.add_missing_marker = True |
| 318 |
|
| 319 |
# Keep this at ``True``, if you want to re-convert to code format later! |
| 320 |
# |
| 321 |
# |
| 322 |
# .. _defaults.preprocessors: |
| 323 |
# |
| 324 |
# preprocessors |
| 325 |
# ------------- |
| 326 |
# |
| 327 |
# Preprocess the data with language-specific filters_ |
| 328 |
# Set below in Filters_:: |
| 329 |
|
| 330 |
defaults.preprocessors = {} |
| 331 |
|
| 332 |
# .. _defaults.postprocessors: |
| 333 |
# |
| 334 |
# postprocessors |
| 335 |
# -------------- |
| 336 |
# |
| 337 |
# Postprocess the data with language-specific filters_:: |
| 338 |
|
| 339 |
defaults.postprocessors = {} |
| 340 |
|
| 341 |
# .. _defaults.codeindent: |
| 342 |
# |
| 343 |
# codeindent |
| 344 |
# ---------- |
| 345 |
# |
| 346 |
# Number of spaces to indent code blocks in `Code2Text.code_block_handler`_:: |
| 347 |
|
| 348 |
defaults.codeindent = 2 |
| 349 |
|
| 350 |
# In `Text2Code.code_block_handler`_, the codeindent is determined by the |
| 351 |
# first recognised code line (header or first indented literal block |
| 352 |
# of the text source). |
| 353 |
# |
| 354 |
# overwrite |
| 355 |
# --------- |
| 356 |
# |
| 357 |
# What to do if the outfile already exists? (ignored if `outfile` == '-'):: |
| 358 |
|
| 359 |
defaults.overwrite = 'yes' |
| 360 |
|
| 361 |
# Recognised values: |
| 362 |
# |
| 363 |
# :'yes': overwrite eventually existing `outfile`, |
| 364 |
# :'update': fail if the `outfile` is newer than `infile`, |
| 365 |
# :'no': fail if `outfile` exists. |
| 366 |
# |
| 367 |
# |
| 368 |
# Extensions |
| 369 |
# ========== |
| 370 |
# |
| 371 |
# Try to import optional extensions:: |
| 372 |
|
| 373 |
try: |
| 374 |
import pylit_elisp |
| 375 |
except ImportError: |
| 376 |
pass |
| 377 |
|
| 378 |
|
| 379 |
# Converter Classes |
| 380 |
# ================= |
| 381 |
# |
| 382 |
# The converter classes implement a simple state machine to separate and |
| 383 |
# transform documentation and code blocks. For this task, only a very limited |
| 384 |
# parsing is needed. PyLit's parser assumes: |
| 385 |
# |
| 386 |
# * `indented literal blocks`_ in a text source are code blocks. |
| 387 |
# |
| 388 |
# * comment blocks in a code source where every line starts with a matching |
| 389 |
# comment string are documentation blocks. |
| 390 |
# |
| 391 |
# TextCodeConverter |
| 392 |
# ----------------- |
| 393 |
# :: |
| 394 |
|
| 395 |
class TextCodeConverter(object): |
| 396 |
"""Parent class for the converters `Text2Code` and `Code2Text`. |
| 397 |
""" |
| 398 |
|
| 399 |
# The parent class defines data attributes and functions used in both |
| 400 |
# `Text2Code`_ converting a text source to executable code source, and |
| 401 |
# `Code2Text`_ converting commented code to a text source. |
| 402 |
# |
| 403 |
# Data attributes |
| 404 |
# ~~~~~~~~~~~~~~~ |
| 405 |
# |
| 406 |
# Class default values are fetched from the `defaults`_ object and can be |
| 407 |
# overridden by matching keyword arguments during class instantiation. This |
| 408 |
# also works with keyword arguments to `get_converter`_ and `main`_, as these |
| 409 |
# functions pass on unused keyword args to the instantiation of a converter |
| 410 |
# class. :: |
| 411 |
|
| 412 |
language = defaults.languages.default |
| 413 |
comment_strings = defaults.comment_strings |
| 414 |
comment_string = "" # set in __init__ (if empty) |
| 415 |
codeindent = defaults.codeindent |
| 416 |
header_string = defaults.header_string |
| 417 |
code_block_markers = defaults.code_block_markers |
| 418 |
code_block_marker = "" # set in __init__ (if empty) |
| 419 |
strip = defaults.strip |
| 420 |
strip_marker = defaults.strip_marker |
| 421 |
add_missing_marker = defaults.add_missing_marker |
| 422 |
directive_option_regexp = re.compile(r' +:(\w|[-._+:])+:( |$)') |
| 423 |
state = "" # type of current block, see `TextCodeConverter.convert`_ |
| 424 |
|
| 425 |
# Interface methods |
| 426 |
# ~~~~~~~~~~~~~~~~~ |
| 427 |
# |
| 428 |
# .. _TextCodeConverter.__init__: |
| 429 |
# |
| 430 |
# __init__ |
| 431 |
# """""""" |
| 432 |
# |
| 433 |
# Initialising sets the `data` attribute, an iterable object yielding lines of |
| 434 |
# the source to convert. [#]_ |
| 435 |
# |
| 436 |
# .. [#] The most common choice of data is a `file` object with the text |
| 437 |
# or code source. |
| 438 |
# |
| 439 |
# To convert a string into a suitable object, use its splitlines method |
| 440 |
# like ``"2 lines\nof source".splitlines(True)``. |
| 441 |
# |
| 442 |
# |
| 443 |
# Additional keyword arguments are stored as instance variables, |
| 444 |
# overwriting the class defaults:: |
| 445 |
|
| 446 |
def __init__(self, data, **keyw): |
| 447 |
"""data -- iterable data object |
| 448 |
(list, file, generator, string, ...) |
| 449 |
**keyw -- remaining keyword arguments are |
| 450 |
stored as data-attributes |
| 451 |
""" |
| 452 |
self.data = data |
| 453 |
self.__dict__.update(keyw) |
| 454 |
|
| 455 |
# If empty, `code_block_marker` and `comment_string` are set according |
| 456 |
# to the `language`:: |
| 457 |
|
| 458 |
if not self.code_block_marker: |
| 459 |
self.code_block_marker = self.code_block_markers[self.language] |
| 460 |
if not self.comment_string: |
| 461 |
self.comment_string = self.comment_strings[self.language] |
| 462 |
self.stripped_comment_string = self.comment_string.rstrip() |
| 463 |
|
| 464 |
# Pre- and postprocessing filters are set (with |
| 465 |
# `TextCodeConverter.get_filter`_):: |
| 466 |
|
| 467 |
self.preprocessor = self.get_filter("preprocessors", self.language) |
| 468 |
self.postprocessor = self.get_filter("postprocessors", self.language) |
| 469 |
|
| 470 |
# .. _inserted into a regular expression: |
| 471 |
# |
| 472 |
# Finally, a regular_expression for the `code_block_marker` is compiled |
| 473 |
# to find valid cases of `code_block_marker` in a given line and return |
| 474 |
# the groups: ``\1 prefix, \2 code_block_marker, \3 remainder`` :: |
| 475 |
|
| 476 |
marker = self.code_block_marker |
| 477 |
if marker == '::': |
| 478 |
# the default marker may occur at the end of a text line |
| 479 |
self.marker_regexp = re.compile('^( *(?!\.\.).*)(::)([ \n]*)$') |
| 480 |
else: |
| 481 |
# marker must be on a separate line |
| 482 |
self.marker_regexp = re.compile('^( *)(%s)(.*\n?)$' % marker) |
| 483 |
|
| 484 |
# .. _TextCodeConverter.__iter__: |
| 485 |
# |
| 486 |
# __iter__ |
| 487 |
# """""""" |
| 488 |
# |
| 489 |
# Return an iterator for the instance. Iteration yields lines of converted |
| 490 |
# data. |
| 491 |
# |
| 492 |
# The iterator is a chain of iterators acting on `self.data` that does |
| 493 |
# |
| 494 |
# * preprocessing |
| 495 |
# * text<->code format conversion |
| 496 |
# * postprocessing |
| 497 |
# |
| 498 |
# Pre- and postprocessing are only performed, if filters for the current |
| 499 |
# language are registered in `defaults.preprocessors`_ and|or |
| 500 |
# `defaults.postprocessors`_. The filters must accept an iterable as first |
| 501 |
# argument and yield the processed input data line-wise. |
| 502 |
# :: |
| 503 |
|
| 504 |
def __iter__(self): |
| 505 |
"""Iterate over input data source and yield converted lines |
| 506 |
""" |
| 507 |
return self.postprocessor(self.convert(self.preprocessor(self.data))) |
| 508 |
|
| 509 |
|
| 510 |
# .. _TextCodeConverter.__call__: |
| 511 |
# |
| 512 |
# __call__ |
| 513 |
# """""""" |
| 514 |
# The special `__call__` method allows the use of class instances as callable |
| 515 |
# objects. It returns the converted data as list of lines:: |
| 516 |
|
| 517 |
def __call__(self): |
| 518 |
"""Iterate over state-machine and return results as list of lines""" |
| 519 |
return [line for line in self] |
| 520 |
|
| 521 |
|
| 522 |
# .. _TextCodeConverter.__str__: |
| 523 |
# |
| 524 |
# __str__ |
| 525 |
# """"""" |
| 526 |
# Return converted data as string:: |
| 527 |
|
| 528 |
def __str__(self): |
| 529 |
return "".join(self()) |
| 530 |
|
| 531 |
|
| 532 |
# Helpers and convenience methods |
| 533 |
# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ |
| 534 |
# |
| 535 |
# .. _TextCodeConverter.convert: |
| 536 |
# |
| 537 |
# convert |
| 538 |
# """"""" |
| 539 |
# |
| 540 |
# The `convert` method generates an iterator that does the actual code <--> |
| 541 |
# text format conversion. The converted data is yielded line-wise and the |
| 542 |
# instance's `status` argument indicates whether the current line is "header", |
| 543 |
# "documentation", or "code_block":: |
| 544 |
|
| 545 |
def convert(self, lines): |
| 546 |
"""Iterate over lines of a program document and convert |
| 547 |
between "text" and "code" format |
| 548 |
""" |
| 549 |
|
| 550 |
# Initialise internal data arguments. (Done here, so that every new iteration |
| 551 |
# re-initialises them.) |
| 552 |
# |
| 553 |
# `state` |
| 554 |
# the "type" of the currently processed block of lines. One of |
| 555 |
# |
| 556 |
# :"": initial state: check for header, |
| 557 |
# :"header": leading code block: strip `header_string`, |
| 558 |
# :"documentation": documentation part: comment out, |
| 559 |
# :"code_block": literal blocks containing source code: unindent. |
| 560 |
# |
| 561 |
# :: |
| 562 |
|
| 563 |
self.state = "" |
| 564 |
|
| 565 |
# `_codeindent` |
| 566 |
# * Do not confuse the internal attribute `_codeindent` with the configurable |
| 567 |
# `codeindent` (without the leading underscore). |
| 568 |
# * `_codeindent` is set in `Text2Code.code_block_handler`_ to the indent of |
| 569 |
# first non-blank "code_block" line and stripped from all "code_block" lines |
| 570 |
# in the text-to-code conversion, |
| 571 |
# * `codeindent` is set in `__init__` to `defaults.codeindent`_ and added to |
| 572 |
# "code_block" lines in the code-to-text conversion. |
| 573 |
# |
| 574 |
# :: |
| 575 |
|
| 576 |
self._codeindent = 0 |
| 577 |
|
| 578 |
# `_textindent` |
| 579 |
# * set by `Text2Code.documentation_handler`_ to the minimal indent of a |
| 580 |
# documentation block, |
| 581 |
# * used in `Text2Code.set_state`_ to find the end of a code block. |
| 582 |
# |
| 583 |
# :: |
| 584 |
|
| 585 |
self._textindent = 0 |
| 586 |
|
| 587 |
# `_add_code_block_marker` |
| 588 |
# If the last paragraph of a documentation block does not end with a |
| 589 |
# code_block_marker_, it should be added (otherwise, the back-conversion |
| 590 |
# fails.). |
| 591 |
# |
| 592 |
# `_add_code_block_marker` is set by `Code2Text.documentation_handler`_ |
| 593 |
# and evaluated by `Code2Text.code_block_handler`_, because the |
| 594 |
# documentation_handler does not know whether the next block will be |
| 595 |
# documentation (with no need for a code_block_marker) or a code block. |
| 596 |
# |
| 597 |
# :: |
| 598 |
|
| 599 |
self._add_code_block_marker = False |
| 600 |
|
| 601 |
|
| 602 |
|
| 603 |
# Determine the state of the block and convert with the matching "handler":: |
| 604 |
|
| 605 |
for block in collect_blocks(expandtabs_filter(lines)): |
| 606 |
self.set_state(block) |
| 607 |
for line in getattr(self, self.state+"_handler")(block): |
| 608 |
yield line |
| 609 |
|
| 610 |
|
| 611 |
# .. _TextCodeConverter.get_filter: |
| 612 |
# |
| 613 |
# get_filter |
| 614 |
# """""""""" |
| 615 |
# :: |
| 616 |
|
| 617 |
def get_filter(self, filter_set, language): |
| 618 |
"""Return language specific filter""" |
| 619 |
if self.__class__ == Text2Code: |
| 620 |
key = "text2"+language |
| 621 |
elif self.__class__ == Code2Text: |
| 622 |
key = language+"2text" |
| 623 |
else: |
| 624 |
key = "" |
| 625 |
try: |
| 626 |
return getattr(defaults, filter_set)[key] |
| 627 |
except (AttributeError, KeyError): |
| 628 |
# print("there is no %r filter in %r"%(key, filter_set)) |
| 629 |
pass |
| 630 |
return identity_filter |
| 631 |
|
| 632 |
|
| 633 |
# get_indent |
| 634 |
# """""""""" |
| 635 |
# Return the number of leading spaces in `line`:: |
| 636 |
|
| 637 |
def get_indent(self, line): |
| 638 |
"""Return the indentation of `string`. |
| 639 |
""" |
| 640 |
return len(line) - len(line.lstrip()) |
| 641 |
|
| 642 |
|
| 643 |
# Text2Code |
| 644 |
# --------- |
| 645 |
# |
| 646 |
# The `Text2Code` converter separates *code-blocks* [#]_ from *documentation*. |
| 647 |
# Code blocks are unindented, documentation is commented (or filtered, if the |
| 648 |
# ``strip`` option is True). |
| 649 |
# |
| 650 |
# .. [#] Only `indented literal blocks`_ are considered code-blocks. `quoted |
| 651 |
# literal blocks`_, `parsed-literal blocks`_, and `doctest blocks`_ are |
| 652 |
# treated as part of the documentation. This allows the inclusion of |
| 653 |
# examples: |
| 654 |
# |
| 655 |
# >>> 23 + 3 |
| 656 |
# 26 |
| 657 |
# |
| 658 |
# Mark that there is no double colon before the doctest block in the |
| 659 |
# text source. |
| 660 |
# |
| 661 |
# The class inherits the interface and helper functions from |
| 662 |
# TextCodeConverter_ and adds functions specific to the text-to-code format |
| 663 |
# conversion:: |
| 664 |
|
| 665 |
class Text2Code(TextCodeConverter): |
| 666 |
"""Convert a (reStructured) text source to code source |
| 667 |
""" |
| 668 |
|
| 669 |
# .. _Text2Code.set_state: |
| 670 |
# |
| 671 |
# set_state |
| 672 |
# ~~~~~~~~~ |
| 673 |
# :: |
| 674 |
|
| 675 |
def set_state(self, block): |
| 676 |
"""Determine state of `block`. Set `self.state` |
| 677 |
""" |
| 678 |
|
| 679 |
# `set_state` is used inside an iteration. Hence, if we are out of data, a |
| 680 |
# StopItertion exception should be raised:: |
| 681 |
|
| 682 |
if not block: |
| 683 |
raise StopIteration |
| 684 |
|
| 685 |
# The new state depends on the active state (from the last block) and |
| 686 |
# features of the current block. It is either "header", "documentation", or |
| 687 |
# "code_block". |
| 688 |
# |
| 689 |
# If the current state is "" (first block), check for |
| 690 |
# the `header_string` indicating a leading code block:: |
| 691 |
|
| 692 |
if self.state == "": |
| 693 |
# print("set state for %r"%block) |
| 694 |
if block[0].startswith(self.header_string): |
| 695 |
self.state = "header" |
| 696 |
else: |
| 697 |
self.state = "documentation" |
| 698 |
|
| 699 |
# If the current state is "documentation", the next block is also |
| 700 |
# documentation. The end of a documentation part is detected in the |
| 701 |
# `Text2Code.documentation_handler`_:: |
| 702 |
|
| 703 |
# elif self.state == "documentation": |
| 704 |
# self.state = "documentation" |
| 705 |
|
| 706 |
# A "code_block" ends with the first less indented, non-blank line. |
| 707 |
# `_textindent` is set by the documentation handler to the indent of the |
| 708 |
# preceding documentation block:: |
| 709 |
|
| 710 |
elif self.state in ["code_block", "header"]: |
| 711 |
indents = [self.get_indent(line) for line in block |
| 712 |
if line.rstrip()] |
| 713 |
# print("set_state:", indents, self._textindent) |
| 714 |
if indents and min(indents) <= self._textindent: |
| 715 |
self.state = 'documentation' |
| 716 |
else: |
| 717 |
self.state = 'code_block' |
| 718 |
|
| 719 |
# TODO: (or not to do?) insert blank line before the first line with too-small |
| 720 |
# codeindent using self.ensure_trailing_blank_line(lines, line) (would need |
| 721 |
# split and push-back of the documentation part)? |
| 722 |
# |
| 723 |
# .. _Text2Code.header_handler: |
| 724 |
# |
| 725 |
# header_handler |
| 726 |
# ~~~~~~~~~~~~~~ |
| 727 |
# |
| 728 |
# Sometimes code needs to remain on the first line(s) of the document to be |
| 729 |
# valid. The most common example is the "shebang" line that tells a POSIX |
| 730 |
# shell how to process an executable file:: |
| 731 |
|
| 732 |
#!/usr/bin/env python |
| 733 |
|
| 734 |
# In Python, the special comment to indicate the encoding, e.g. |
| 735 |
# ``# -*- coding: iso-8859-1 -*-``, must occur before any other comment |
| 736 |
# or code too. |
| 737 |
# |
| 738 |
# If we want to keep the line numbers in sync for text and code source, the |
| 739 |
# reStructured Text markup for these header lines must start at the same line |
| 740 |
# as the first header line. Therefore, header lines could not be marked as |
| 741 |
# literal block (this would require the ``::`` and an empty line above the |
| 742 |
# code_block). |
| 743 |
# |
| 744 |
# OTOH, a comment may start at the same line as the comment marker and it |
| 745 |
# includes subsequent indented lines. Comments are visible in the reStructured |
| 746 |
# Text source but hidden in the pretty-printed output. |
| 747 |
# |
| 748 |
# With a header converted to comment in the text source, everything before |
| 749 |
# the first documentation block (i.e. before the first paragraph using the |
| 750 |
# matching comment string) will be hidden away (in HTML or PDF output). |
| 751 |
# |
| 752 |
# This seems a good compromise, the advantages |
| 753 |
# |
| 754 |
# * line numbers are kept |
| 755 |
# * the "normal" code_block conversion rules (indent/unindent by `codeindent` apply |
| 756 |
# * greater flexibility: you can hide a repeating header in a project |
| 757 |
# consisting of many source files. |
| 758 |
# |
| 759 |
# set off the disadvantages |
| 760 |
# |
| 761 |
# - it may come as surprise if a part of the file is not "printed", |
| 762 |
# - one more syntax element to learn for rst newbies to start with pylit, |
| 763 |
# (however, starting from the code source, this will be auto-generated) |
| 764 |
# |
| 765 |
# In the case that there is no matching comment at all, the complete code |
| 766 |
# source will become a comment -- however, in this case it is not very likely |
| 767 |
# the source is a literate document anyway. |
| 768 |
# |
| 769 |
# If needed for the documentation, it is possible to quote the header in (or |
| 770 |
# after) the first documentation block, e.g. as `parsed literal`. |
| 771 |
# :: |
| 772 |
|
| 773 |
def header_handler(self, lines): |
| 774 |
"""Format leading code block""" |
| 775 |
# strip header string from first line |
| 776 |
lines[0] = lines[0].replace(self.header_string, "", 1) |
| 777 |
# yield remaining lines formatted as code-block |
| 778 |
for line in self.code_block_handler(lines): |
| 779 |
yield line |
| 780 |
|
| 781 |
|
| 782 |
# .. _Text2Code.documentation_handler: |
| 783 |
# |
| 784 |
# documentation_handler |
| 785 |
# ~~~~~~~~~~~~~~~~~~~~~ |
| 786 |
# |
| 787 |
# The 'documentation' handler processes everything that is not recognised as |
| 788 |
# "code_block". Documentation is quoted with `self.comment_string` |
| 789 |
# (or filtered with `--strip=True`). |
| 790 |
# |
| 791 |
# If end-of-documentation marker is detected, |
| 792 |
# |
| 793 |
# * set state to 'code_block' |
| 794 |
# * set `self._textindent` (needed by `Text2Code.set_state`_ to find the |
| 795 |
# next "documentation" block) |
| 796 |
# |
| 797 |
# :: |
| 798 |
|
| 799 |
def documentation_handler(self, lines): |
| 800 |
"""Convert documentation blocks from text to code format |
| 801 |
""" |
| 802 |
for line in lines: |
| 803 |
# test lines following the code-block marker for false positives |
| 804 |
if (self.state == "code_block" and line.rstrip() |
| 805 |
and not self.directive_option_regexp.search(line)): |
| 806 |
self.state = "documentation" |
| 807 |
# test for end of documentation block |
| 808 |
if self.marker_regexp.search(line): |
| 809 |
self.state = "code_block" |
| 810 |
self._textindent = self.get_indent(line) |
| 811 |
# yield lines |
| 812 |
if self.strip: |
| 813 |
continue |
| 814 |
# do not comment blank lines preceding a code block |
| 815 |
if self.state == "code_block" and not line.rstrip(): |
| 816 |
yield line |
| 817 |
else: |
| 818 |
yield self.comment_string + line |
| 819 |
|
| 820 |
|
| 821 |
|
| 822 |
|
| 823 |
# .. _Text2Code.code_block_handler: |
| 824 |
# |
| 825 |
# code_block_handler |
| 826 |
# ~~~~~~~~~~~~~~~~~~ |
| 827 |
# |
| 828 |
# The "code_block" handler is called with an indented literal block. It |
| 829 |
# removes leading whitespace up to the indentation of the first code line in |
| 830 |
# the file (this deviation from Docutils behaviour allows indented blocks of |
| 831 |
# Python code). :: |
| 832 |
|
| 833 |
def code_block_handler(self, block): |
| 834 |
"""Convert indented literal blocks to source code format |
| 835 |
""" |
| 836 |
|
| 837 |
# If still unset, determine the indentation of code blocks from first non-blank |
| 838 |
# code line:: |
| 839 |
|
| 840 |
if self._codeindent == 0: |
| 841 |
self._codeindent = self.get_indent(block[0]) |
| 842 |
|
| 843 |
# Yield unindented lines after check whether we can safely unindent. If the |
| 844 |
# line is less indented then `_codeindent`, something got wrong. :: |
| 845 |
|
| 846 |
for line in block: |
| 847 |
if line.lstrip() and self.get_indent(line) < self._codeindent: |
| 848 |
raise ValueError("code block contains line less indented " |
| 849 |
"than %d spaces \n%r" % (self._codeindent, block)) |
| 850 |
yield line.replace(" "*self._codeindent, "", 1) |
| 851 |
|
| 852 |
|
| 853 |
# Code2Text |
| 854 |
# --------- |
| 855 |
# |
| 856 |
# The `Code2Text` converter does the opposite of `Text2Code`_ -- it processes |
| 857 |
# a source in "code format" (i.e. in a programming language), extracts |
| 858 |
# documentation from comment blocks, and puts program code in literal blocks. |
| 859 |
# |
| 860 |
# The class inherits the interface and helper functions from |
| 861 |
# TextCodeConverter_ and adds functions specific to the text-to-code format |
| 862 |
# conversion:: |
| 863 |
|
| 864 |
class Code2Text(TextCodeConverter): |
| 865 |
"""Convert code source to text source |
| 866 |
""" |
| 867 |
|
| 868 |
# set_state |
| 869 |
# ~~~~~~~~~ |
| 870 |
# |
| 871 |
# Check if block is "header", "documentation", or "code_block": |
| 872 |
# |
| 873 |
# A paragraph is "documentation", if every non-blank line starts with a |
| 874 |
# matching comment string (including whitespace except for commented blank |
| 875 |
# lines) :: |
| 876 |
|
| 877 |
def set_state(self, block): |
| 878 |
"""Determine state of `block`.""" |
| 879 |
for line in block: |
| 880 |
# skip documentation lines (commented, blank or blank comment) |
| 881 |
if (line.startswith(self.comment_string) |
| 882 |
or not line.rstrip() |
| 883 |
or line.rstrip() == self.comment_string.rstrip() |
| 884 |
): |
| 885 |
continue |
| 886 |
# non-commented line found: |
| 887 |
if self.state == "": |
| 888 |
self.state = "header" |
| 889 |
else: |
| 890 |
self.state = "code_block" |
| 891 |
break |
| 892 |
else: |
| 893 |
# no code line found |
| 894 |
# keep state if the block is just a blank line |
| 895 |
# if len(block) == 1 and self._is_blank_codeline(line): |
| 896 |
# return |
| 897 |
self.state = "documentation" |
| 898 |
|
| 899 |
|
| 900 |
# header_handler |
| 901 |
# ~~~~~~~~~~~~~~ |
| 902 |
# |
| 903 |
# Handle a leading code block. (See `Text2Code.header_handler`_ for a |
| 904 |
# discussion of the "header" state.) :: |
| 905 |
|
| 906 |
def header_handler(self, lines): |
| 907 |
"""Format leading code block""" |
| 908 |
if self.strip == True: |
| 909 |
return |
| 910 |
# get iterator over the lines that formats them as code-block |
| 911 |
lines = iter(self.code_block_handler(lines)) |
| 912 |
# prepend header string to first line |
| 913 |
yield self.header_string + lines.next() |
| 914 |
# yield remaining lines |
| 915 |
for line in lines: |
| 916 |
yield line |
| 917 |
|
| 918 |
# .. _Code2Text.documentation_handler: |
| 919 |
# |
| 920 |
# documentation_handler |
| 921 |
# ~~~~~~~~~~~~~~~~~~~~~ |
| 922 |
# |
| 923 |
# The *documentation state* handler converts a comment to a documentation |
| 924 |
# block by stripping the leading `comment string` from every line:: |
| 925 |
|
| 926 |
def documentation_handler(self, block): |
| 927 |
"""Uncomment documentation blocks in source code |
| 928 |
""" |
| 929 |
|
| 930 |
# Strip comment strings:: |
| 931 |
|
| 932 |
lines = [self.uncomment_line(line) for line in block] |
| 933 |
|
| 934 |
# If the code block is stripped, the literal marker would lead to an |
| 935 |
# error when the text is converted with Docutils. Strip it as well. :: |
| 936 |
|
| 937 |
if self.strip or self.strip_marker: |
| 938 |
self.strip_code_block_marker(lines) |
| 939 |
|
| 940 |
# Otherwise, check for the `code_block_marker`_ at the end of the |
| 941 |
# documentation block (skipping directive options that might follow it):: |
| 942 |
|
| 943 |
elif self.add_missing_marker: |
| 944 |
for line in lines[::-1]: |
| 945 |
if self.marker_regexp.search(line): |
| 946 |
self._add_code_block_marker = False |
| 947 |
break |
| 948 |
if (line.rstrip() and |
| 949 |
not self.directive_option_regexp.search(line)): |
| 950 |
self._add_code_block_marker = True |
| 951 |
break |
| 952 |
else: |
| 953 |
self._add_code_block_marker = True |
| 954 |
|
| 955 |
# Yield lines:: |
| 956 |
|
| 957 |
for line in lines: |
| 958 |
yield line |
| 959 |
|
| 960 |
# uncomment_line |
| 961 |
# ~~~~~~~~~~~~~~ |
| 962 |
# |
| 963 |
# Return documentation line after stripping comment string. Consider the |
| 964 |
# case that a blank line has a comment string without trailing whitespace:: |
| 965 |
|
| 966 |
def uncomment_line(self, line): |
| 967 |
"""Return uncommented documentation line""" |
| 968 |
line = line.replace(self.comment_string, "", 1) |
| 969 |
if line.rstrip() == self.stripped_comment_string: |
| 970 |
line = line.replace(self.stripped_comment_string, "", 1) |
| 971 |
return line |
| 972 |
|
| 973 |
# .. _Code2Text.code_block_handler: |
| 974 |
# |
| 975 |
# code_block_handler |
| 976 |
# ~~~~~~~~~~~~~~~~~~ |
| 977 |
# |
| 978 |
# The `code_block` handler returns the code block as indented literal |
| 979 |
# block (or filters it, if ``self.strip == True``). The amount of the code |
| 980 |
# indentation is controlled by `self.codeindent` (default 2). :: |
| 981 |
|
| 982 |
def code_block_handler(self, lines): |
| 983 |
"""Covert code blocks to text format (indent or strip) |
| 984 |
""" |
| 985 |
if self.strip == True: |
| 986 |
return |
| 987 |
# eventually insert transition marker |
| 988 |
if self._add_code_block_marker: |
| 989 |
self.state = "documentation" |
| 990 |
yield self.code_block_marker + "\n" |
| 991 |
yield "\n" |
| 992 |
self._add_code_block_marker = False |
| 993 |
self.state = "code_block" |
| 994 |
for line in lines: |
| 995 |
yield " "*self.codeindent + line |
| 996 |
|
| 997 |
|
| 998 |
|
| 999 |
# strip_code_block_marker |
| 1000 |
# ~~~~~~~~~~~~~~~~~~~~~~~ |
| 1001 |
# |
| 1002 |
# Replace the literal marker with the equivalent of Docutils replace rules |
| 1003 |
# |
| 1004 |
# * strip ``::``-line (and preceding blank line) if on a line on its own |
| 1005 |
# * strip ``::`` if it is preceded by whitespace. |
| 1006 |
# * convert ``::`` to a single colon if preceded by text |
| 1007 |
# |
| 1008 |
# `lines` is a list of documentation lines (with a trailing blank line). |
| 1009 |
# It is modified in-place:: |
| 1010 |
|
| 1011 |
def strip_code_block_marker(self, lines): |
| 1012 |
try: |
| 1013 |
line = lines[-2] |
| 1014 |
except IndexError: |
| 1015 |
return # just one line (no trailing blank line) |
| 1016 |
|
| 1017 |
# match with regexp: `match` is None or has groups |
| 1018 |
# \1 leading text, \2 code_block_marker, \3 remainder |
| 1019 |
match = self.marker_regexp.search(line) |
| 1020 |
|
| 1021 |
if not match: # no code_block_marker present |
| 1022 |
return |
| 1023 |
if not match.group(1): # `code_block_marker` on an extra line |
| 1024 |
del(lines[-2]) |
| 1025 |
# delete preceding line if it is blank |
| 1026 |
if len(lines) >= 2 and not lines[-2].lstrip(): |
| 1027 |
del(lines[-2]) |
| 1028 |
elif match.group(1).rstrip() < match.group(1): |
| 1029 |
# '::' follows whitespace |
| 1030 |
lines[-2] = match.group(1).rstrip() + match.group(3) |
| 1031 |
else: # '::' follows text |
| 1032 |
lines[-2] = match.group(1).rstrip() + ':' + match.group(3) |
| 1033 |
|
| 1034 |
# Filters |
| 1035 |
# ======= |
| 1036 |
# |
| 1037 |
# Filters allow pre- and post-processing of the data to bring it in a format |
| 1038 |
# suitable for the "normal" text<->code conversion. An example is conversion |
| 1039 |
# of `C` ``/*`` ``*/`` comments into C++ ``//`` comments (and back). |
| 1040 |
# Another example is the conversion of `C` ``/*`` ``*/`` comments into C++ |
| 1041 |
# ``//`` comments (and back). |
| 1042 |
# |
| 1043 |
# Filters are generator functions that return an iterator acting on a |
| 1044 |
# `data` iterable and yielding processed `data` lines. |
| 1045 |
# |
| 1046 |
# identity_filter |
| 1047 |
# --------------- |
| 1048 |
# |
| 1049 |
# The most basic filter is the identity filter, that returns its argument as |
| 1050 |
# iterator:: |
| 1051 |
|
| 1052 |
def identity_filter(data): |
| 1053 |
"""Return data iterator without any processing""" |
| 1054 |
return iter(data) |
| 1055 |
|
| 1056 |
# expandtabs_filter |
| 1057 |
# ----------------- |
| 1058 |
# |
| 1059 |
# Expand hard-tabs in every line of `data` (cf. `str.expandtabs`). |
| 1060 |
# |
| 1061 |
# This filter is applied to the input data by `TextCodeConverter.convert`_ as |
| 1062 |
# hard tabs can lead to errors when the indentation is changed. :: |
| 1063 |
|
| 1064 |
def expandtabs_filter(data): |
| 1065 |
"""Yield data tokens with hard-tabs expanded""" |
| 1066 |
for line in data: |
| 1067 |
yield line.expandtabs() |
| 1068 |
|
| 1069 |
|
| 1070 |
# collect_blocks |
| 1071 |
# -------------- |
| 1072 |
# |
| 1073 |
# A filter to aggregate "paragraphs" (blocks separated by blank |
| 1074 |
# lines). Yields lists of lines:: |
| 1075 |
|
| 1076 |
def collect_blocks(lines): |
| 1077 |
"""collect lines in a list |
| 1078 |
|
| 1079 |
yield list for each paragraph, i.e. block of lines separated by a |
| 1080 |
blank line (whitespace only). |
| 1081 |
|
| 1082 |
Trailing blank lines are collected as well. |
| 1083 |
""" |
| 1084 |
blank_line_reached = False |
| 1085 |
block = [] |
| 1086 |
for line in lines: |
| 1087 |
if blank_line_reached and line.rstrip(): |
| 1088 |
yield block |
| 1089 |
blank_line_reached = False |
| 1090 |
block = [line] |
| 1091 |
continue |
| 1092 |
if not line.rstrip(): |
| 1093 |
blank_line_reached = True |
| 1094 |
block.append(line) |
| 1095 |
yield block |
| 1096 |
|
| 1097 |
|
| 1098 |
|
| 1099 |
# dumb_c_preprocessor |
| 1100 |
# ------------------- |
| 1101 |
# |
| 1102 |
# This is a basic filter to convert `C` to `C++` comments. Works line-wise and |
| 1103 |
# only converts lines that |
| 1104 |
# |
| 1105 |
# * start with "/\* " and end with " \*/" (followed by whitespace only) |
| 1106 |
# |
| 1107 |
# A more sophisticated version would also |
| 1108 |
# |
| 1109 |
# * convert multi-line comments |
| 1110 |
# |
| 1111 |
# + Keep indentation or strip 3 leading spaces? |
| 1112 |
# |
| 1113 |
# * account for nested comments |
| 1114 |
# |
| 1115 |
# * only convert comments that are separated from code by a blank line |
| 1116 |
# |
| 1117 |
# :: |
| 1118 |
|
| 1119 |
def dumb_c_preprocessor(data): |
| 1120 |
"""change `C` ``/* `` `` */`` comments into C++ ``// `` comments""" |
| 1121 |
comment_string = defaults.comment_strings["c++"] |
| 1122 |
boc_string = "/* " |
| 1123 |
eoc_string = " */" |
| 1124 |
for line in data: |
| 1125 |
if (line.startswith(boc_string) |
| 1126 |
and line.rstrip().endswith(eoc_string) |
| 1127 |
): |
| 1128 |
line = line.replace(boc_string, comment_string, 1) |
| 1129 |
line = "".join(line.rsplit(eoc_string, 1)) |
| 1130 |
yield line |
| 1131 |
|
| 1132 |
# Unfortunately, the `replace` method of strings does not support negative |
| 1133 |
# numbers for the `count` argument: |
| 1134 |
# |
| 1135 |
# >>> "foo */ baz */ bar".replace(" */", "", -1) == "foo */ baz bar" |
| 1136 |
# False |
| 1137 |
# |
| 1138 |
# However, there is the `rsplit` method, that can be used together with `join`: |
| 1139 |
# |
| 1140 |
# >>> "".join("foo */ baz */ bar".rsplit(" */", 1)) == "foo */ baz bar" |
| 1141 |
# True |
| 1142 |
# |
| 1143 |
# dumb_c_postprocessor |
| 1144 |
# -------------------- |
| 1145 |
# |
| 1146 |
# Undo the preparations by the dumb_c_preprocessor and re-insert valid comment |
| 1147 |
# delimiters :: |
| 1148 |
|
| 1149 |
def dumb_c_postprocessor(data): |
| 1150 |
"""change C++ ``// `` comments into `C` ``/* `` `` */`` comments""" |
| 1151 |
comment_string = defaults.comment_strings["c++"] |
| 1152 |
boc_string = "/* " |
| 1153 |
eoc_string = " */" |
| 1154 |
for line in data: |
| 1155 |
if line.rstrip() == comment_string.rstrip(): |
| 1156 |
line = line.replace(comment_string, "", 1) |
| 1157 |
elif line.startswith(comment_string): |
| 1158 |
line = line.replace(comment_string, boc_string, 1) |
| 1159 |
line = line.rstrip() + eoc_string + "\n" |
| 1160 |
yield line |
| 1161 |
|
| 1162 |
|
| 1163 |
# register filters |
| 1164 |
# ---------------- |
| 1165 |
# |
| 1166 |
# :: |
| 1167 |
|
| 1168 |
defaults.preprocessors['c2text'] = dumb_c_preprocessor |
| 1169 |
defaults.preprocessors['css2text'] = dumb_c_preprocessor |
| 1170 |
defaults.postprocessors['text2c'] = dumb_c_postprocessor |
| 1171 |
defaults.postprocessors['text2css'] = dumb_c_postprocessor |
| 1172 |
|
| 1173 |
|
| 1174 |
# Command line use |
| 1175 |
# ================ |
| 1176 |
# |
| 1177 |
# Using this script from the command line will convert a file according to its |
| 1178 |
# extension. This default can be overridden by a couple of options. |
| 1179 |
# |
| 1180 |
# Dual source handling |
| 1181 |
# -------------------- |
| 1182 |
# |
| 1183 |
# How to determine which source is up-to-date? |
| 1184 |
# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ |
| 1185 |
# |
| 1186 |
# - set modification date of `outfile` to the one of `infile` |
| 1187 |
# |
| 1188 |
# Points out that the source files are 'synchronised'. |
| 1189 |
# |
| 1190 |
# * Are there problems to expect from "backdating" a file? Which? |
| 1191 |
# |
| 1192 |
# Looking at http://www.unix.com/showthread.php?t=20526, it seems |
| 1193 |
# perfectly legal to set `mtime` (while leaving `ctime`) as `mtime` is a |
| 1194 |
# description of the "actuality" of the data in the file. |
| 1195 |
# |
| 1196 |
# * Should this become a default or an option? |
| 1197 |
# |
| 1198 |
# - alternatively move input file to a backup copy (with option: `--replace`) |
| 1199 |
# |
| 1200 |
# - check modification date before overwriting |
| 1201 |
# (with option: `--overwrite=update`) |
| 1202 |
# |
| 1203 |
# - check modification date before editing (implemented as `Jed editor`_ |
| 1204 |
# function `pylit_check()` in `pylit.sl`_) |
| 1205 |
# |
| 1206 |
# .. _Jed editor: http://www.jedsoft.org/jed/ |
| 1207 |
# .. _pylit.sl: http://jedmodes.sourceforge.net/mode/pylit/ |
| 1208 |
# |
| 1209 |
# Recognised Filename Extensions |
| 1210 |
# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ |
| 1211 |
# |
| 1212 |
# Instead of defining a new extension for "pylit" literate programs, |
| 1213 |
# by default ``.txt`` will be appended for the text source and stripped by |
| 1214 |
# the conversion to the code source. I.e. for a Python program foo: |
| 1215 |
# |
| 1216 |
# * the code source is called ``foo.py`` |
| 1217 |
# * the text source is called ``foo.py.txt`` |
| 1218 |
# * the html rendering is called ``foo.py.html`` |
| 1219 |
# |
| 1220 |
# |
| 1221 |
# OptionValues |
| 1222 |
# ------------ |
| 1223 |
# |
| 1224 |
# The following class adds `as_dict`_, `complete`_ and `__getattr__`_ |
| 1225 |
# methods to `optparse.Values`:: |
| 1226 |
|
| 1227 |
class OptionValues(optparse.Values): |
| 1228 |
|
| 1229 |
# .. _OptionValues.as_dict: |
| 1230 |
# |
| 1231 |
# as_dict |
| 1232 |
# ~~~~~~~ |
| 1233 |
# |
| 1234 |
# For use as keyword arguments, it is handy to have the options in a |
| 1235 |
# dictionary. `as_dict` returns a copy of the instances object dictionary:: |
| 1236 |
|
| 1237 |
def as_dict(self): |
| 1238 |
"""Return options as dictionary object""" |
| 1239 |
return self.__dict__.copy() |
| 1240 |
|
| 1241 |
# .. _OptionValues.complete: |
| 1242 |
# |
| 1243 |
# complete |
| 1244 |
# ~~~~~~~~ |
| 1245 |
# |
| 1246 |
# :: |
| 1247 |
|
| 1248 |
def complete(self, **keyw): |
| 1249 |
""" |
| 1250 |
Complete the option values with keyword arguments. |
| 1251 |
|
| 1252 |
Do not overwrite existing values. Only use arguments that do not |
| 1253 |
have a corresponding attribute in `self`, |
| 1254 |
""" |
| 1255 |
for key in keyw: |
| 1256 |
if not self.__dict__.__contains__(key): |
| 1257 |
setattr(self, key, keyw[key]) |
| 1258 |
|
| 1259 |
# .. _OptionValues.__getattr__: |
| 1260 |
# |
| 1261 |
# __getattr__ |
| 1262 |
# ~~~~~~~~~~~ |
| 1263 |
# |
| 1264 |
# To replace calls using ``options.ensure_value("OPTION", None)`` with the |
| 1265 |
# more concise ``options.OPTION``, we define `__getattr__` [#]_ :: |
| 1266 |
|
| 1267 |
def __getattr__(self, name): |
| 1268 |
"""Return default value for non existing options""" |
| 1269 |
return None |
| 1270 |
|
| 1271 |
|
| 1272 |
# .. [#] The special method `__getattr__` is only called when an attribute |
| 1273 |
# look-up has not found the attribute in the usual places (i.e. it is |
| 1274 |
# not an instance attribute nor is it found in the class tree for |
| 1275 |
# self). |
| 1276 |
# |
| 1277 |
# |
| 1278 |
# PylitOptions |
| 1279 |
# ------------ |
| 1280 |
# |
| 1281 |
# The `PylitOptions` class comprises an option parser and methods for parsing |
| 1282 |
# and completion of command line options:: |
| 1283 |
|
| 1284 |
class PylitOptions(object): |
| 1285 |
"""Storage and handling of command line options for pylit""" |
| 1286 |
|
| 1287 |
# Instantiation |
| 1288 |
# ~~~~~~~~~~~~~ |
| 1289 |
# |
| 1290 |
# :: |
| 1291 |
|
| 1292 |
def __init__(self): |
| 1293 |
"""Set up an `OptionParser` instance for pylit command line options |
| 1294 |
|
| 1295 |
""" |
| 1296 |
p = optparse.OptionParser(usage=main.__doc__, version=_version) |
| 1297 |
|
| 1298 |
# Conversion settings |
| 1299 |
|
| 1300 |
p.add_option("-c", "--code2txt", dest="txt2code", action="store_false", |
| 1301 |
help="convert code source to text source") |
| 1302 |
p.add_option("-t", "--txt2code", action="store_true", |
| 1303 |
help="convert text source to code source") |
| 1304 |
p.add_option("--language", |
| 1305 |
choices = list(defaults.languages.values()), |
| 1306 |
help="use LANGUAGE native comment style") |
| 1307 |
p.add_option("--comment-string", dest="comment_string", |
| 1308 |
help="documentation block marker in code source " |
| 1309 |
"(including trailing whitespace, " |
| 1310 |
"default: language dependent)") |
| 1311 |
p.add_option("-m", "--code-block-marker", dest="code_block_marker", |
| 1312 |
help="syntax token starting a code block. (default '::')") |
| 1313 |
p.add_option("--codeindent", type="int", |
| 1314 |
help="Number of spaces to indent code blocks with " |
| 1315 |
"text2code (default %d)" % defaults.codeindent) |
| 1316 |
|
| 1317 |
# Output file handling |
| 1318 |
|
| 1319 |
p.add_option("--overwrite", action="store", |
| 1320 |
choices = ["yes", "update", "no"], |
| 1321 |
help="overwrite output file (default 'update')") |
| 1322 |
p.add_option("--replace", action="store_true", |
| 1323 |
help="move infile to a backup copy (appending '~')") |
| 1324 |
p.add_option("-s", "--strip", action="store_true", |
| 1325 |
help='"export" by stripping documentation or code') |
| 1326 |
|
| 1327 |
# Special actions |
| 1328 |
|
| 1329 |
p.add_option("-d", "--diff", action="store_true", |
| 1330 |
help="test for differences to existing file") |
| 1331 |
p.add_option("--doctest", action="store_true", |
| 1332 |
help="run doctest.testfile() on the text version") |
| 1333 |
p.add_option("-e", "--execute", action="store_true", |
| 1334 |
help="execute code (Python only)") |
| 1335 |
|
| 1336 |
self.parser = p |
| 1337 |
|
| 1338 |
# .. _PylitOptions.parse_args: |
| 1339 |
# |
| 1340 |
# parse_args |
| 1341 |
# ~~~~~~~~~~ |
| 1342 |
# |
| 1343 |
# The `parse_args` method calls the `optparse.OptionParser` on command |
| 1344 |
# line or provided args and returns the result as `PylitOptions.Values` |
| 1345 |
# instance. Defaults can be provided as keyword arguments:: |
| 1346 |
|
| 1347 |
def parse_args(self, args=sys.argv[1:], **keyw): |
| 1348 |
"""parse command line arguments using `optparse.OptionParser` |
| 1349 |
|
| 1350 |
parse_args(args, **keyw) -> OptionValues instance |
| 1351 |
|
| 1352 |
args -- list of command line arguments. |
| 1353 |
keyw -- keyword arguments or dictionary of option defaults |
| 1354 |
""" |
| 1355 |
# parse arguments |
| 1356 |
(values, args) = self.parser.parse_args(args, OptionValues(keyw)) |
| 1357 |
# Convert FILE and OUTFILE positional args to option values |
| 1358 |
# (other positional arguments are ignored) |
| 1359 |
try: |
| 1360 |
values.infile = args[0] |
| 1361 |
values.outfile = args[1] |
| 1362 |
except IndexError: |
| 1363 |
pass |
| 1364 |
|
| 1365 |
return values |
| 1366 |
|
| 1367 |
# .. _PylitOptions.complete_values: |
| 1368 |
# |
| 1369 |
# complete_values |
| 1370 |
# ~~~~~~~~~~~~~~~ |
| 1371 |
# |
| 1372 |
# Complete an OptionValues instance `values`. Use module-level defaults and |
| 1373 |
# context information to set missing option values to sensible defaults (if |
| 1374 |
# possible) :: |
| 1375 |
|
| 1376 |
def complete_values(self, values): |
| 1377 |
"""complete option values with module and context sensible defaults |
| 1378 |
|
| 1379 |
x.complete_values(values) -> values |
| 1380 |
values -- OptionValues instance |
| 1381 |
""" |
| 1382 |
|
| 1383 |
# Complete with module-level defaults_:: |
| 1384 |
|
| 1385 |
values.complete(**defaults.__dict__) |
| 1386 |
|
| 1387 |
# Ensure infile is a string:: |
| 1388 |
|
| 1389 |
values.ensure_value("infile", "") |
| 1390 |
|
| 1391 |
# Guess conversion direction from `infile` filename:: |
| 1392 |
|
| 1393 |
if values.txt2code is None: |
| 1394 |
in_extension = os.path.splitext(values.infile)[1] |
| 1395 |
if in_extension in values.text_extensions: |
| 1396 |
values.txt2code = True |
| 1397 |
elif in_extension in values.languages.keys(): |
| 1398 |
values.txt2code = False |
| 1399 |
|
| 1400 |
# Auto-determine the output file name:: |
| 1401 |
|
| 1402 |
values.ensure_value("outfile", self._get_outfile_name(values)) |
| 1403 |
|
| 1404 |
# Second try: Guess conversion direction from outfile filename:: |
| 1405 |
|
| 1406 |
if values.txt2code is None: |
| 1407 |
out_extension = os.path.splitext(values.outfile)[1] |
| 1408 |
values.txt2code = not (out_extension in values.text_extensions) |
| 1409 |
|
| 1410 |
# Set the language of the code:: |
| 1411 |
|
| 1412 |
if values.txt2code is True: |
| 1413 |
code_extension = os.path.splitext(values.outfile)[1] |
| 1414 |
elif values.txt2code is False: |
| 1415 |
code_extension = os.path.splitext(values.infile)[1] |
| 1416 |
values.ensure_value("language", values.languages[code_extension]) |
| 1417 |
|
| 1418 |
return values |
| 1419 |
|
| 1420 |
# _get_outfile_name |
| 1421 |
# ~~~~~~~~~~~~~~~~~ |
| 1422 |
# |
| 1423 |
# Construct a matching filename for the output file. The output filename is |
| 1424 |
# constructed from `infile` by the following rules: |
| 1425 |
# |
| 1426 |
# * '-' (stdin) results in '-' (stdout) |
| 1427 |
# * strip the `text_extension`_ (txt2code) or |
| 1428 |
# * add the `text_extension`_ (code2txt) |
| 1429 |
# * fallback: if no guess can be made, add ".out" |
| 1430 |
# |
| 1431 |
# .. TODO: use values.outfile_extension if it exists? |
| 1432 |
# |
| 1433 |
# :: |
| 1434 |
|
| 1435 |
def _get_outfile_name(self, values): |
| 1436 |
"""Return a matching output filename for `infile` |
| 1437 |
""" |
| 1438 |
# if input is stdin, default output is stdout |
| 1439 |
if values.infile == '-': |
| 1440 |
return '-' |
| 1441 |
|
| 1442 |
# Derive from `infile` name: strip or add text extension |
| 1443 |
(base, ext) = os.path.splitext(values.infile) |
| 1444 |
if ext in values.text_extensions: |
| 1445 |
return base # strip |
| 1446 |
if ext in values.languages.keys() or values.txt2code == False: |
| 1447 |
return values.infile + values.text_extensions[0] # add |
| 1448 |
# give up |
| 1449 |
return values.infile + ".out" |
| 1450 |
|
| 1451 |
# .. _PylitOptions.__call__: |
| 1452 |
# |
| 1453 |
# __call__ |
| 1454 |
# ~~~~~~~~ |
| 1455 |
# |
| 1456 |
# The special `__call__` method allows to use PylitOptions instances as |
| 1457 |
# *callables*: Calling an instance parses the argument list to extract option |
| 1458 |
# values and completes them based on "context-sensitive defaults". Keyword |
| 1459 |
# arguments are passed to `PylitOptions.parse_args`_ as default values. :: |
| 1460 |
|
| 1461 |
def __call__(self, args=sys.argv[1:], **keyw): |
| 1462 |
"""parse and complete command line args return option values |
| 1463 |
""" |
| 1464 |
values = self.parse_args(args, **keyw) |
| 1465 |
return self.complete_values(values) |
| 1466 |
|
| 1467 |
|
| 1468 |
|
| 1469 |
# Helper functions |
| 1470 |
# ---------------- |
| 1471 |
# |
| 1472 |
# open_streams |
| 1473 |
# ~~~~~~~~~~~~ |
| 1474 |
# |
| 1475 |
# Return file objects for in- and output. If the input path is missing, |
| 1476 |
# write usage and abort. (An alternative would be to use stdin as default. |
| 1477 |
# However, this leaves the uninitiated user with a non-responding application |
| 1478 |
# if (s)he just tries the script without any arguments) :: |
| 1479 |
|
| 1480 |
def open_streams(infile = '-', outfile = '-', overwrite='update', **keyw): |
| 1481 |
"""Open and return the input and output stream |
| 1482 |
|
| 1483 |
open_streams(infile, outfile) -> (in_stream, out_stream) |
| 1484 |
|
| 1485 |
in_stream -- open(infile) or sys.stdin |
| 1486 |
out_stream -- open(outfile) or sys.stdout |
| 1487 |
overwrite -- 'yes': overwrite eventually existing `outfile`, |
| 1488 |
'update': fail if the `outfile` is newer than `infile`, |
| 1489 |
'no': fail if `outfile` exists. |
| 1490 |
|
| 1491 |
Irrelevant if `outfile` == '-'. |
| 1492 |
""" |
| 1493 |
if not infile: |
| 1494 |
strerror = "Missing input file name ('-' for stdin; -h for help)" |
| 1495 |
raise IOError((2, strerror, infile)) |
| 1496 |
if infile == '-': |
| 1497 |
in_stream = sys.stdin |
| 1498 |
else: |
| 1499 |
in_stream = io.open(infile, 'r', encoding='utf-8') |
| 1500 |
|
| 1501 |
if outfile == '-': |
| 1502 |
out_stream = sys.stdout |
| 1503 |
elif overwrite == 'no' and os.path.exists(outfile): |
| 1504 |
raise IOError((1, "Output file exists!", outfile)) |
| 1505 |
elif overwrite == 'update' and is_newer(outfile, infile): |
| 1506 |
raise IOError((1, "Output file is newer than input file!", outfile)) |
| 1507 |
else: |
| 1508 |
out_stream = io.open(outfile, 'w', encoding='utf-8') |
| 1509 |
return (in_stream, out_stream) |
| 1510 |
|
| 1511 |
# is_newer |
| 1512 |
# ~~~~~~~~ |
| 1513 |
# |
| 1514 |
# :: |
| 1515 |
|
| 1516 |
def is_newer(path1, path2): |
| 1517 |
"""Check if `path1` is newer than `path2` (using mtime) |
| 1518 |
|
| 1519 |
Compare modification time of files at path1 and path2. |
| 1520 |
|
| 1521 |
Non-existing files are considered oldest: Return False if path1 does not |
| 1522 |
exist and True if path2 does not exist. |
| 1523 |
|
| 1524 |
Return None for equal modification time. (This evaluates to False in a |
| 1525 |
Boolean context but allows a test for equality.) |
| 1526 |
|
| 1527 |
""" |
| 1528 |
try: |
| 1529 |
mtime1 = os.path.getmtime(path1) |
| 1530 |
except OSError: |
| 1531 |
mtime1 = -1 |
| 1532 |
try: |
| 1533 |
mtime2 = os.path.getmtime(path2) |
| 1534 |
except OSError: |
| 1535 |
mtime2 = -1 |
| 1536 |
# print("mtime1", mtime1, path1, "\n", "mtime2", mtime2, path2) |
| 1537 |
|
| 1538 |
if mtime1 == mtime2: |
| 1539 |
return None |
| 1540 |
return mtime1 > mtime2 |
| 1541 |
|
| 1542 |
|
| 1543 |
# get_converter |
| 1544 |
# ~~~~~~~~~~~~~ |
| 1545 |
# |
| 1546 |
# Get an instance of the converter state machine:: |
| 1547 |
|
| 1548 |
def get_converter(data, txt2code=True, **keyw): |
| 1549 |
if txt2code: |
| 1550 |
return Text2Code(data, **keyw) |
| 1551 |
else: |
| 1552 |
return Code2Text(data, **keyw) |
| 1553 |
|
| 1554 |
|
| 1555 |
# Use cases |
| 1556 |
# --------- |
| 1557 |
# |
| 1558 |
# run_doctest |
| 1559 |
# ~~~~~~~~~~~ |
| 1560 |
# :: |
| 1561 |
|
| 1562 |
def run_doctest(infile="-", txt2code=True, |
| 1563 |
globs={}, verbose=False, optionflags=0, **keyw): |
| 1564 |
"""run doctest on the text source |
| 1565 |
""" |
| 1566 |
|
| 1567 |
# Allow imports from the current working dir by prepending an empty string to |
| 1568 |
# sys.path (see doc of sys.path()):: |
| 1569 |
|
| 1570 |
sys.path.insert(0, '') |
| 1571 |
|
| 1572 |
# Import classes from the doctest module:: |
| 1573 |
|
| 1574 |
from doctest import DocTestParser, DocTestRunner |
| 1575 |
|
| 1576 |
# Read in source. Make sure it is in text format, as tests in comments are not |
| 1577 |
# found by doctest:: |
| 1578 |
|
| 1579 |
(data, out_stream) = open_streams(infile, "-") |
| 1580 |
if txt2code is False: |
| 1581 |
keyw.update({'add_missing_marker': False}) |
| 1582 |
converter = Code2Text(data, **keyw) |
| 1583 |
docstring = str(converter) |
| 1584 |
else: |
| 1585 |
docstring = data.read() |
| 1586 |
|
| 1587 |
# decode doc string if there is a "magic comment" in the first or second line |
| 1588 |
# (http://docs.python.org/reference/lexical_analysis.html#encoding-declarations) |
| 1589 |
# :: |
| 1590 |
|
| 1591 |
firstlines = ' '.join(docstring.splitlines()[:2]) |
| 1592 |
match = re.search('coding[=:]\s*([-\w.]+)', firstlines) |
| 1593 |
if match: |
| 1594 |
docencoding = match.group(1) |
| 1595 |
docstring = docstring.decode(docencoding) |
| 1596 |
|
| 1597 |
# Use the doctest Advanced API to run all doctests in the source text:: |
| 1598 |
|
| 1599 |
test = DocTestParser().get_doctest(docstring, globs, name="", |
| 1600 |
filename=infile, lineno=0) |
| 1601 |
runner = DocTestRunner(verbose, optionflags) |
| 1602 |
runner.run(test) |
| 1603 |
runner.summarize |
| 1604 |
# give feedback also if no failures occurred |
| 1605 |
if not runner.failures: |
| 1606 |
print("%d failures in %d tests"%(runner.failures, runner.tries)) |
| 1607 |
return runner.failures, runner.tries |
| 1608 |
|
| 1609 |
|
| 1610 |
# diff |
| 1611 |
# ~~~~ |
| 1612 |
# |
| 1613 |
# :: |
| 1614 |
|
| 1615 |
def diff(infile='-', outfile='-', txt2code=True, **keyw): |
| 1616 |
"""Report differences between converted infile and existing outfile |
| 1617 |
|
| 1618 |
If outfile does not exist or is '-', do a round-trip conversion and |
| 1619 |
report differences. |
| 1620 |
""" |
| 1621 |
|
| 1622 |
import difflib |
| 1623 |
|
| 1624 |
instream = open(infile) |
| 1625 |
# for diffing, we need a copy of the data as list:: |
| 1626 |
data = instream.readlines() |
| 1627 |
# convert |
| 1628 |
converter = get_converter(data, txt2code, **keyw) |
| 1629 |
new = converter() |
| 1630 |
|
| 1631 |
if outfile != '-' and os.path.exists(outfile): |
| 1632 |
outstream = open(outfile) |
| 1633 |
old = outstream.readlines() |
| 1634 |
oldname = outfile |
| 1635 |
newname = "<conversion of %s>"%infile |
| 1636 |
else: |
| 1637 |
old = data |
| 1638 |
oldname = infile |
| 1639 |
# back-convert the output data |
| 1640 |
converter = get_converter(new, not txt2code) |
| 1641 |
new = converter() |
| 1642 |
newname = "<round-conversion of %s>"%infile |
| 1643 |
|
| 1644 |
# find and print the differences |
| 1645 |
is_different = False |
| 1646 |
# print(type(old), old) |
| 1647 |
# print(type(new), new) |
| 1648 |
delta = difflib.unified_diff(old, new, |
| 1649 |
# delta = difflib.unified_diff(["heute\n", "schon\n"], ["heute\n", "noch\n"], |
| 1650 |
fromfile=oldname, tofile=newname) |
| 1651 |
for line in delta: |
| 1652 |
is_different = True |
| 1653 |
print(line, end="") |
| 1654 |
if not is_different: |
| 1655 |
print(oldname) |
| 1656 |
print(newname) |
| 1657 |
print("no differences found") |
| 1658 |
return is_different |
| 1659 |
|
| 1660 |
|
| 1661 |
# execute |
| 1662 |
# ~~~~~~~ |
| 1663 |
# |
| 1664 |
# Works only for python code. |
| 1665 |
# |
| 1666 |
# Does not work with `eval`, as code is not just one expression. :: |
| 1667 |
|
| 1668 |
def execute(infile="-", txt2code=True, **keyw): |
| 1669 |
"""Execute the input file. Convert first, if it is a text source. |
| 1670 |
""" |
| 1671 |
|
| 1672 |
data = open(infile) |
| 1673 |
if txt2code: |
| 1674 |
data = str(Text2Code(data, **keyw)) |
| 1675 |
# print("executing " + options.infile) |
| 1676 |
exec(data) |
| 1677 |
|
| 1678 |
|
| 1679 |
# main |
| 1680 |
# ---- |
| 1681 |
# |
| 1682 |
# If this script is called from the command line, the `main` function will |
| 1683 |
# convert the input (file or stdin) between text and code formats. |
| 1684 |
# |
| 1685 |
# Option default values for the conversion can be given as keyword arguments |
| 1686 |
# to `main`_. The option defaults will be updated by command line options and |
| 1687 |
# extended with "intelligent guesses" by `PylitOptions`_ and passed on to |
| 1688 |
# helper functions and the converter instantiation. |
| 1689 |
# |
| 1690 |
# This allows easy customisation for programmatic use -- just call `main` |
| 1691 |
# with the appropriate keyword options, e.g. ``pylit.main(comment_string="## ")`` |
| 1692 |
# |
| 1693 |
# :: |
| 1694 |
|
| 1695 |
def main(args=sys.argv[1:], **defaults): |
| 1696 |
"""%prog [options] INFILE [OUTFILE] |
| 1697 |
|
| 1698 |
Convert between (reStructured) text source with embedded code, |
| 1699 |
and code source with embedded documentation (comment blocks) |
| 1700 |
|
| 1701 |
The special filename '-' stands for standard in and output. |
| 1702 |
""" |
| 1703 |
|
| 1704 |
# Parse and complete the options:: |
| 1705 |
|
| 1706 |
options = PylitOptions()(args, **defaults) |
| 1707 |
# print("infile", repr(options.infile)) |
| 1708 |
|
| 1709 |
# Special actions with early return:: |
| 1710 |
|
| 1711 |
if options.doctest: |
| 1712 |
return run_doctest(**options.as_dict()) |
| 1713 |
|
| 1714 |
if options.diff: |
| 1715 |
return diff(**options.as_dict()) |
| 1716 |
|
| 1717 |
if options.execute: |
| 1718 |
return execute(**options.as_dict()) |
| 1719 |
|
| 1720 |
# Open in- and output streams:: |
| 1721 |
|
| 1722 |
try: |
| 1723 |
(data, out_stream) = open_streams(**options.as_dict()) |
| 1724 |
except IOError as ex: |
| 1725 |
print("IOError: %s %s" % (ex.filename, ex.strerror)) |
| 1726 |
sys.exit(ex.errno) |
| 1727 |
|
| 1728 |
# Get a converter instance:: |
| 1729 |
|
| 1730 |
converter = get_converter(data, **options.as_dict()) |
| 1731 |
|
| 1732 |
# Convert and write to out_stream:: |
| 1733 |
|
| 1734 |
if sys.version_info[0] == 2: |
| 1735 |
out_stream.write(unicode(converter)) |
| 1736 |
else: |
| 1737 |
out_stream.write(str(converter)) |
| 1738 |
|
| 1739 |
if out_stream is not sys.stdout: |
| 1740 |
print("extract written to", out_stream.name) |
| 1741 |
out_stream.close() |
| 1742 |
|
| 1743 |
# If input and output are from files, set the modification time (`mtime`) of |
| 1744 |
# the output file to the one of the input file to indicate that the contained |
| 1745 |
# information is equal. [#]_ :: |
| 1746 |
|
| 1747 |
try: |
| 1748 |
os.utime(options.outfile, (os.path.getatime(options.outfile), |
| 1749 |
os.path.getmtime(options.infile)) |
| 1750 |
) |
| 1751 |
except OSError: |
| 1752 |
pass |
| 1753 |
|
| 1754 |
## print("mtime", os.path.getmtime(options.infile), options.infile) |
| 1755 |
## print("mtime", os.path.getmtime(options.outfile), options.outfile) |
| 1756 |
|
| 1757 |
|
| 1758 |
# .. [#] Make sure the corresponding file object (here `out_stream`) is |
| 1759 |
# closed, as otherwise the change will be overwritten when `close` is |
| 1760 |
# called afterwards (either explicitly or at program exit). |
| 1761 |
# |
| 1762 |
# |
| 1763 |
# Rename the infile to a backup copy if ``--replace`` is set:: |
| 1764 |
|
| 1765 |
if options.replace: |
| 1766 |
os.rename(options.infile, options.infile + "~") |
| 1767 |
|
| 1768 |
|
| 1769 |
# Run main, if called from the command line:: |
| 1770 |
|
| 1771 |
if __name__ == '__main__': |
| 1772 |
main() |
| 1773 |
|
| 1774 |
|
| 1775 |
# Open questions |
| 1776 |
# ============== |
| 1777 |
# |
| 1778 |
# Open questions and ideas for further development |
| 1779 |
# |
| 1780 |
# Clean code |
| 1781 |
# ---------- |
| 1782 |
# |
| 1783 |
# * can we gain from using "shutils" over "os.path" and "os"? |
| 1784 |
# * use pylint or pyChecker to enforce a consistent style? |
| 1785 |
# |
| 1786 |
# Options |
| 1787 |
# ------- |
| 1788 |
# |
| 1789 |
# * Use templates for the "intelligent guesses" (with Python syntax for string |
| 1790 |
# replacement with dicts: ``"hello %(what)s" % {'what': 'world'}``) |
| 1791 |
# |
| 1792 |
# * Is it sensible to offer the `header_string` option also as command line |
| 1793 |
# option? |
| 1794 |
# |
| 1795 |
# treatment of blank lines |
| 1796 |
# ------------------------ |
| 1797 |
# |
| 1798 |
# Alternatives: Keep blank lines blank |
| 1799 |
# |
| 1800 |
# - "never" (current setting) -> "visually merges" all documentation |
| 1801 |
# if there is no interjacent code |
| 1802 |
# |
| 1803 |
# - "always" -> disrupts documentation blocks, |
| 1804 |
# |
| 1805 |
# - "if empty" (no whitespace). Comment if there is whitespace. |
| 1806 |
# |
| 1807 |
# This would allow non-obstructing markup but unfortunately this is (in |
| 1808 |
# most editors) also non-visible markup. |
| 1809 |
# |
| 1810 |
# + "if double" (if there is more than one consecutive blank line) |
| 1811 |
# |
| 1812 |
# With this handling, the "visual gap" remains in both, text and code |
| 1813 |
# source. |
| 1814 |
# |
| 1815 |
# |
| 1816 |
# Parsing Problems |
| 1817 |
# ---------------- |
| 1818 |
# |
| 1819 |
# * Ignore "matching comments" in literal strings? |
| 1820 |
# |
| 1821 |
# Too complicated: Would need a specific detection algorithm for every |
| 1822 |
# language that supports multi-line literal strings (C++, PHP, Python) |
| 1823 |
# |
| 1824 |
# * Warn if a comment in code will become documentation after round-trip? |
| 1825 |
# |
| 1826 |
# |
| 1827 |
# docstrings in code blocks |
| 1828 |
# ------------------------- |
| 1829 |
# |
| 1830 |
# * How to handle docstrings in code blocks? (it would be nice to convert them |
| 1831 |
# to rst-text if ``__docformat__ == restructuredtext``) |
| 1832 |
# |
| 1833 |
# TODO: Ask at Docutils users|developers |
| 1834 |
# |
| 1835 |
# Plug-ins |
| 1836 |
# -------- |
| 1837 |
# |
| 1838 |
# Specify a path for user additions and plug-ins. This would require to |
| 1839 |
# convert Pylit from a pure module to a package... |
| 1840 |
# |
| 1841 |
# 6.4.3 Packages in Multiple Directories |
| 1842 |
# |
| 1843 |
# Packages support one more special attribute, __path__. This is initialized |
| 1844 |
# to be a list containing the name of the directory holding the package's |
| 1845 |
# __init__.py before the code in that file is executed. This |
| 1846 |
# variable can be modified; doing so affects future searches for modules and |
| 1847 |
# subpackages contained in the package. |
| 1848 |
# |
| 1849 |
# While this feature is not often needed, it can be used to extend the set |
| 1850 |
# of modules found in a package. |
| 1851 |
# |
| 1852 |
# |
| 1853 |
# .. References |
| 1854 |
# |
| 1855 |
# .. _Docutils: http://docutils.sourceforge.net/ |
| 1856 |
# .. _Sphinx: http://sphinx.pocoo.org |
| 1857 |
# .. _Pygments: http://pygments.org/ |
| 1858 |
# .. _code-block directive: |
| 1859 |
# http://docutils.sourceforge.net/sandbox/code-block-directive/ |
| 1860 |
# .. _literal block: |
| 1861 |
# .. _literal blocks: |
| 1862 |
# http://docutils.sf.net/docs/ref/rst/restructuredtext.html#literal-blocks |
| 1863 |
# .. _indented literal block: |
| 1864 |
# .. _indented literal blocks: |
| 1865 |
# http://docutils.sf.net/docs/ref/rst/restructuredtext.html#indented-literal-blocks |
| 1866 |
# .. _quoted literal block: |
| 1867 |
# .. _quoted literal blocks: |
| 1868 |
# http://docutils.sf.net/docs/ref/rst/restructuredtext.html#quoted-literal-blocks |
| 1869 |
# .. _parsed-literal blocks: |
| 1870 |
# http://docutils.sf.net/docs/ref/rst/directives.html#parsed-literal-block |
| 1871 |
# .. _doctest block: |
| 1872 |
# .. _doctest blocks: |
| 1873 |
# http://docutils.sf.net/docs/ref/rst/restructuredtext.html#doctest-blocks |
| 1874 |
# |
| 1875 |
# .. _feature request and patch by jrioux: |
| 1876 |
# http://developer.berlios.de/feature/?func=detailfeature&feature_id=4890&group_id=7974 |