| 1 |
# Copyright (C) 2002 Python Software Foundation |
| 2 |
# Author: Ben Gertzfield, Barry Warsaw |
| 3 |
# Contact: [email protected] |
| 4 |
|
| 5 |
"""Header encoding and decoding functionality.""" |
| 6 |
|
| 7 |
__all__ = [ |
| 8 |
'Header', |
| 9 |
'decode_header', |
| 10 |
'make_header', |
| 11 |
] |
| 12 |
|
| 13 |
import re |
| 14 |
import binascii |
| 15 |
|
| 16 |
import email.quoprimime |
| 17 |
import email.base64mime |
| 18 |
|
| 19 |
from email.errors import HeaderParseError |
| 20 |
from email import charset as _charset |
| 21 |
Charset = _charset.Charset |
| 22 |
|
| 23 |
NL = '\n' |
| 24 |
SPACE = ' ' |
| 25 |
BSPACE = b' ' |
| 26 |
SPACE8 = ' ' * 8 |
| 27 |
EMPTYSTRING = '' |
| 28 |
MAXLINELEN = 78 |
| 29 |
FWS = ' \t' |
| 30 |
|
| 31 |
USASCII = Charset('us-ascii') |
| 32 |
UTF8 = Charset('utf-8') |
| 33 |
|
| 34 |
# Match encoded-word strings in the form =?charset?q?Hello_World?= |
| 35 |
ecre = re.compile(r''' |
| 36 |
=\? # literal =? |
| 37 |
(?P<charset>[^?]*?) # non-greedy up to the next ? is the charset |
| 38 |
\? # literal ? |
| 39 |
(?P<encoding>[qQbB]) # either a "q" or a "b", case insensitive |
| 40 |
\? # literal ? |
| 41 |
(?P<encoded>.*?) # non-greedy up to the next ?= is the encoded string |
| 42 |
\?= # literal ?= |
| 43 |
''', re.VERBOSE | re.MULTILINE) |
| 44 |
|
| 45 |
# Field name regexp, including trailing colon, but not separating whitespace, |
| 46 |
# according to RFC 2822. Character range is from tilde to exclamation mark. |
| 47 |
# For use with .match() |
| 48 |
fcre = re.compile(r'[\041-\176]+:$') |
| 49 |
|
| 50 |
# Find a header embedded in a putative header value. Used to check for |
| 51 |
# header injection attack. |
| 52 |
_embedded_header = re.compile(r'\n[^ \t]+:') |
| 53 |
|
| 54 |
|
| 55 |
# Helpers |
| 56 |
_max_append = email.quoprimime._max_append |
| 57 |
|
| 58 |
|
| 59 |
def decode_header(header): |
| 60 |
"""Decode a message header value without converting charset. |
| 61 |
|
| 62 |
For historical reasons, this function may return either: |
| 63 |
|
| 64 |
1. A list of length 1 containing a pair (str, None). |
| 65 |
2. A list of (bytes, charset) pairs containing each of the decoded |
| 66 |
parts of the header. Charset is None for non-encoded parts of the header, |
| 67 |
otherwise a lower-case string containing the name of the character set |
| 68 |
specified in the encoded string. |
| 69 |
|
| 70 |
header may be a string that may or may not contain RFC2047 encoded words, |
| 71 |
or it may be a Header object. |
| 72 |
|
| 73 |
An email.errors.HeaderParseError may be raised when certain decoding error |
| 74 |
occurs (e.g. a base64 decoding exception). |
| 75 |
|
| 76 |
This function exists for backwards compatibility only. For new code, we |
| 77 |
recommend using email.headerregistry.HeaderRegistry instead. |
| 78 |
""" |
| 79 |
# If it is a Header object, we can just return the encoded chunks. |
| 80 |
if hasattr(header, '_chunks'): |
| 81 |
return [(_charset._encode(string, str(charset)), str(charset)) |
| 82 |
for string, charset in header._chunks] |
| 83 |
# If no encoding, just return the header with no charset. |
| 84 |
if not ecre.search(header): |
| 85 |
return [(header, None)] |
| 86 |
# First step is to parse all the encoded parts into triplets of the form |
| 87 |
# (encoded_string, encoding, charset). For unencoded strings, the last |
| 88 |
# two parts will be None. |
| 89 |
words = [] |
| 90 |
for line in header.splitlines(): |
| 91 |
parts = ecre.split(line) |
| 92 |
first = True |
| 93 |
while parts: |
| 94 |
unencoded = parts.pop(0) |
| 95 |
if first: |
| 96 |
unencoded = unencoded.lstrip() |
| 97 |
first = False |
| 98 |
if unencoded: |
| 99 |
words.append((unencoded, None, None)) |
| 100 |
if parts: |
| 101 |
charset = parts.pop(0).lower() |
| 102 |
encoding = parts.pop(0).lower() |
| 103 |
encoded = parts.pop(0) |
| 104 |
words.append((encoded, encoding, charset)) |
| 105 |
# Now loop over words and remove words that consist of whitespace |
| 106 |
# between two encoded strings. |
| 107 |
droplist = [] |
| 108 |
for n, w in enumerate(words): |
| 109 |
if n>1 and w[1] and words[n-2][1] and words[n-1][0].isspace(): |
| 110 |
droplist.append(n-1) |
| 111 |
for d in reversed(droplist): |
| 112 |
del words[d] |
| 113 |
|
| 114 |
# The next step is to decode each encoded word by applying the reverse |
| 115 |
# base64 or quopri transformation. decoded_words is now a list of the |
| 116 |
# form (decoded_word, charset). |
| 117 |
decoded_words = [] |
| 118 |
for encoded_string, encoding, charset in words: |
| 119 |
if encoding is None: |
| 120 |
# This is an unencoded word. |
| 121 |
decoded_words.append((encoded_string, charset)) |
| 122 |
elif encoding == 'q': |
| 123 |
word = email.quoprimime.header_decode(encoded_string) |
| 124 |
decoded_words.append((word, charset)) |
| 125 |
elif encoding == 'b': |
| 126 |
paderr = len(encoded_string) % 4 # Postel's law: add missing padding |
| 127 |
if paderr: |
| 128 |
encoded_string += '==='[:4 - paderr] |
| 129 |
try: |
| 130 |
word = email.base64mime.decode(encoded_string) |
| 131 |
except binascii.Error: |
| 132 |
raise HeaderParseError('Base64 decoding error') |
| 133 |
else: |
| 134 |
decoded_words.append((word, charset)) |
| 135 |
else: |
| 136 |
raise AssertionError('Unexpected encoding: ' + encoding) |
| 137 |
# Now convert all words to bytes and collapse consecutive runs of |
| 138 |
# similarly encoded words. |
| 139 |
collapsed = [] |
| 140 |
last_word = last_charset = None |
| 141 |
for word, charset in decoded_words: |
| 142 |
if isinstance(word, str): |
| 143 |
word = bytes(word, 'raw-unicode-escape') |
| 144 |
if last_word is None: |
| 145 |
last_word = word |
| 146 |
last_charset = charset |
| 147 |
elif charset != last_charset: |
| 148 |
collapsed.append((last_word, last_charset)) |
| 149 |
last_word = word |
| 150 |
last_charset = charset |
| 151 |
elif last_charset is None: |
| 152 |
last_word += BSPACE + word |
| 153 |
else: |
| 154 |
last_word += word |
| 155 |
collapsed.append((last_word, last_charset)) |
| 156 |
return collapsed |
| 157 |
|
| 158 |
|
| 159 |
def make_header(decoded_seq, maxlinelen=None, header_name=None, |
| 160 |
continuation_ws=' '): |
| 161 |
"""Create a Header from a sequence of pairs as returned by decode_header() |
| 162 |
|
| 163 |
decode_header() takes a header value string and returns a sequence of |
| 164 |
pairs of the format (decoded_string, charset) where charset is the string |
| 165 |
name of the character set. |
| 166 |
|
| 167 |
This function takes one of those sequence of pairs and returns a Header |
| 168 |
instance. Optional maxlinelen, header_name, and continuation_ws are as in |
| 169 |
the Header constructor. |
| 170 |
|
| 171 |
This function exists for backwards compatibility only, and is not |
| 172 |
recommended for use in new code. |
| 173 |
""" |
| 174 |
h = Header(maxlinelen=maxlinelen, header_name=header_name, |
| 175 |
continuation_ws=continuation_ws) |
| 176 |
for s, charset in decoded_seq: |
| 177 |
# None means us-ascii but we can simply pass it on to h.append() |
| 178 |
if charset is not None and not isinstance(charset, Charset): |
| 179 |
charset = Charset(charset) |
| 180 |
h.append(s, charset) |
| 181 |
return h |
| 182 |
|
| 183 |
|
| 184 |
class Header: |
| 185 |
def __init__(self, s=None, charset=None, |
| 186 |
maxlinelen=None, header_name=None, |
| 187 |
continuation_ws=' ', errors='strict'): |
| 188 |
"""Create a MIME-compliant header that can contain many character sets. |
| 189 |
|
| 190 |
Optional s is the initial header value. If None, the initial header |
| 191 |
value is not set. You can later append to the header with .append() |
| 192 |
method calls. s may be a byte string or a Unicode string, but see the |
| 193 |
.append() documentation for semantics. |
| 194 |
|
| 195 |
Optional charset serves two purposes: it has the same meaning as the |
| 196 |
charset argument to the .append() method. It also sets the default |
| 197 |
character set for all subsequent .append() calls that omit the charset |
| 198 |
argument. If charset is not provided in the constructor, the us-ascii |
| 199 |
charset is used both as s's initial charset and as the default for |
| 200 |
subsequent .append() calls. |
| 201 |
|
| 202 |
The maximum line length can be specified explicitly via maxlinelen. For |
| 203 |
splitting the first line to a shorter value (to account for the field |
| 204 |
header which isn't included in s, e.g. 'Subject') pass in the name of |
| 205 |
the field in header_name. The default maxlinelen is 78 as recommended |
| 206 |
by RFC 2822. |
| 207 |
|
| 208 |
continuation_ws must be RFC 2822 compliant folding whitespace (usually |
| 209 |
either a space or a hard tab) which will be prepended to continuation |
| 210 |
lines. |
| 211 |
|
| 212 |
errors is passed through to the .append() call. |
| 213 |
""" |
| 214 |
if charset is None: |
| 215 |
charset = USASCII |
| 216 |
elif not isinstance(charset, Charset): |
| 217 |
charset = Charset(charset) |
| 218 |
self._charset = charset |
| 219 |
self._continuation_ws = continuation_ws |
| 220 |
self._chunks = [] |
| 221 |
if s is not None: |
| 222 |
self.append(s, charset, errors) |
| 223 |
if maxlinelen is None: |
| 224 |
maxlinelen = MAXLINELEN |
| 225 |
self._maxlinelen = maxlinelen |
| 226 |
if header_name is None: |
| 227 |
self._headerlen = 0 |
| 228 |
else: |
| 229 |
# Take the separating colon and space into account. |
| 230 |
self._headerlen = len(header_name) + 2 |
| 231 |
|
| 232 |
def __str__(self): |
| 233 |
"""Return the string value of the header.""" |
| 234 |
self._normalize() |
| 235 |
uchunks = [] |
| 236 |
lastcs = None |
| 237 |
lastspace = None |
| 238 |
for string, charset in self._chunks: |
| 239 |
# We must preserve spaces between encoded and non-encoded word |
| 240 |
# boundaries, which means for us we need to add a space when we go |
| 241 |
# from a charset to None/us-ascii, or from None/us-ascii to a |
| 242 |
# charset. Only do this for the second and subsequent chunks. |
| 243 |
# Don't add a space if the None/us-ascii string already has |
| 244 |
# a space (trailing or leading depending on transition) |
| 245 |
nextcs = charset |
| 246 |
if nextcs == _charset.UNKNOWN8BIT: |
| 247 |
original_bytes = string.encode('ascii', 'surrogateescape') |
| 248 |
string = original_bytes.decode('ascii', 'replace') |
| 249 |
if uchunks: |
| 250 |
hasspace = string and self._nonctext(string[0]) |
| 251 |
if lastcs not in (None, 'us-ascii'): |
| 252 |
if nextcs in (None, 'us-ascii') and not hasspace: |
| 253 |
uchunks.append(SPACE) |
| 254 |
nextcs = None |
| 255 |
elif nextcs not in (None, 'us-ascii') and not lastspace: |
| 256 |
uchunks.append(SPACE) |
| 257 |
lastspace = string and self._nonctext(string[-1]) |
| 258 |
lastcs = nextcs |
| 259 |
uchunks.append(string) |
| 260 |
return EMPTYSTRING.join(uchunks) |
| 261 |
|
| 262 |
# Rich comparison operators for equality only. BAW: does it make sense to |
| 263 |
# have or explicitly disable <, <=, >, >= operators? |
| 264 |
def __eq__(self, other): |
| 265 |
# other may be a Header or a string. Both are fine so coerce |
| 266 |
# ourselves to a unicode (of the unencoded header value), swap the |
| 267 |
# args and do another comparison. |
| 268 |
return other == str(self) |
| 269 |
|
| 270 |
def append(self, s, charset=None, errors='strict'): |
| 271 |
"""Append a string to the MIME header. |
| 272 |
|
| 273 |
Optional charset, if given, should be a Charset instance or the name |
| 274 |
of a character set (which will be converted to a Charset instance). A |
| 275 |
value of None (the default) means that the charset given in the |
| 276 |
constructor is used. |
| 277 |
|
| 278 |
s may be a byte string or a Unicode string. If it is a byte string |
| 279 |
(i.e. isinstance(s, str) is false), then charset is the encoding of |
| 280 |
that byte string, and a UnicodeError will be raised if the string |
| 281 |
cannot be decoded with that charset. If s is a Unicode string, then |
| 282 |
charset is a hint specifying the character set of the characters in |
| 283 |
the string. In either case, when producing an RFC 2822 compliant |
| 284 |
header using RFC 2047 rules, the string will be encoded using the |
| 285 |
output codec of the charset. If the string cannot be encoded to the |
| 286 |
output codec, a UnicodeError will be raised. |
| 287 |
|
| 288 |
Optional 'errors' is passed as the errors argument to the decode |
| 289 |
call if s is a byte string. |
| 290 |
""" |
| 291 |
if charset is None: |
| 292 |
charset = self._charset |
| 293 |
elif not isinstance(charset, Charset): |
| 294 |
charset = Charset(charset) |
| 295 |
if not isinstance(s, str): |
| 296 |
input_charset = charset.input_codec or 'us-ascii' |
| 297 |
if input_charset == _charset.UNKNOWN8BIT: |
| 298 |
s = s.decode('us-ascii', 'surrogateescape') |
| 299 |
else: |
| 300 |
s = s.decode(input_charset, errors) |
| 301 |
# Ensure that the bytes we're storing can be decoded to the output |
| 302 |
# character set, otherwise an early error is raised. |
| 303 |
output_charset = charset.output_codec or 'us-ascii' |
| 304 |
if output_charset != _charset.UNKNOWN8BIT: |
| 305 |
try: |
| 306 |
s.encode(output_charset, errors) |
| 307 |
except UnicodeEncodeError: |
| 308 |
if output_charset!='us-ascii': |
| 309 |
raise |
| 310 |
charset = UTF8 |
| 311 |
self._chunks.append((s, charset)) |
| 312 |
|
| 313 |
def _nonctext(self, s): |
| 314 |
"""True if string s is not a ctext character of RFC822. |
| 315 |
""" |
| 316 |
return s.isspace() or s in ('(', ')', '\\') |
| 317 |
|
| 318 |
def encode(self, splitchars=';, \t', maxlinelen=None, linesep='\n'): |
| 319 |
r"""Encode a message header into an RFC-compliant format. |
| 320 |
|
| 321 |
There are many issues involved in converting a given string for use in |
| 322 |
an email header. Only certain character sets are readable in most |
| 323 |
email clients, and as header strings can only contain a subset of |
| 324 |
7-bit ASCII, care must be taken to properly convert and encode (with |
| 325 |
Base64 or quoted-printable) header strings. In addition, there is a |
| 326 |
75-character length limit on any given encoded header field, so |
| 327 |
line-wrapping must be performed, even with double-byte character sets. |
| 328 |
|
| 329 |
Optional maxlinelen specifies the maximum length of each generated |
| 330 |
line, exclusive of the linesep string. Individual lines may be longer |
| 331 |
than maxlinelen if a folding point cannot be found. The first line |
| 332 |
will be shorter by the length of the header name plus ": " if a header |
| 333 |
name was specified at Header construction time. The default value for |
| 334 |
maxlinelen is determined at header construction time. |
| 335 |
|
| 336 |
Optional splitchars is a string containing characters which should be |
| 337 |
given extra weight by the splitting algorithm during normal header |
| 338 |
wrapping. This is in very rough support of RFC 2822's 'higher level |
| 339 |
syntactic breaks': split points preceded by a splitchar are preferred |
| 340 |
during line splitting, with the characters preferred in the order in |
| 341 |
which they appear in the string. Space and tab may be included in the |
| 342 |
string to indicate whether preference should be given to one over the |
| 343 |
other as a split point when other split chars do not appear in the line |
| 344 |
being split. Splitchars does not affect RFC 2047 encoded lines. |
| 345 |
|
| 346 |
Optional linesep is a string to be used to separate the lines of |
| 347 |
the value. The default value is the most useful for typical |
| 348 |
Python applications, but it can be set to \r\n to produce RFC-compliant |
| 349 |
line separators when needed. |
| 350 |
""" |
| 351 |
self._normalize() |
| 352 |
if maxlinelen is None: |
| 353 |
maxlinelen = self._maxlinelen |
| 354 |
# A maxlinelen of 0 means don't wrap. For all practical purposes, |
| 355 |
# choosing a huge number here accomplishes that and makes the |
| 356 |
# _ValueFormatter algorithm much simpler. |
| 357 |
if maxlinelen == 0: |
| 358 |
maxlinelen = 1000000 |
| 359 |
formatter = _ValueFormatter(self._headerlen, maxlinelen, |
| 360 |
self._continuation_ws, splitchars) |
| 361 |
lastcs = None |
| 362 |
hasspace = lastspace = None |
| 363 |
for string, charset in self._chunks: |
| 364 |
if hasspace is not None: |
| 365 |
hasspace = string and self._nonctext(string[0]) |
| 366 |
if lastcs not in (None, 'us-ascii'): |
| 367 |
if not hasspace or charset not in (None, 'us-ascii'): |
| 368 |
formatter.add_transition() |
| 369 |
elif charset not in (None, 'us-ascii') and not lastspace: |
| 370 |
formatter.add_transition() |
| 371 |
lastspace = string and self._nonctext(string[-1]) |
| 372 |
lastcs = charset |
| 373 |
hasspace = False |
| 374 |
lines = string.splitlines() |
| 375 |
if lines: |
| 376 |
formatter.feed('', lines[0], charset) |
| 377 |
else: |
| 378 |
formatter.feed('', '', charset) |
| 379 |
for line in lines[1:]: |
| 380 |
formatter.newline() |
| 381 |
if charset.header_encoding is not None: |
| 382 |
formatter.feed(self._continuation_ws, ' ' + line.lstrip(), |
| 383 |
charset) |
| 384 |
else: |
| 385 |
sline = line.lstrip() |
| 386 |
fws = line[:len(line)-len(sline)] |
| 387 |
formatter.feed(fws, sline, charset) |
| 388 |
if len(lines) > 1: |
| 389 |
formatter.newline() |
| 390 |
if self._chunks: |
| 391 |
formatter.add_transition() |
| 392 |
value = formatter._str(linesep) |
| 393 |
if _embedded_header.search(value): |
| 394 |
raise HeaderParseError("header value appears to contain " |
| 395 |
"an embedded header: {!r}".format(value)) |
| 396 |
return value |
| 397 |
|
| 398 |
def _normalize(self): |
| 399 |
# Step 1: Normalize the chunks so that all runs of identical charsets |
| 400 |
# get collapsed into a single unicode string. |
| 401 |
chunks = [] |
| 402 |
last_charset = None |
| 403 |
last_chunk = [] |
| 404 |
for string, charset in self._chunks: |
| 405 |
if charset == last_charset: |
| 406 |
last_chunk.append(string) |
| 407 |
else: |
| 408 |
if last_charset is not None: |
| 409 |
chunks.append((SPACE.join(last_chunk), last_charset)) |
| 410 |
last_chunk = [string] |
| 411 |
last_charset = charset |
| 412 |
if last_chunk: |
| 413 |
chunks.append((SPACE.join(last_chunk), last_charset)) |
| 414 |
self._chunks = chunks |
| 415 |
|
| 416 |
|
| 417 |
class _ValueFormatter: |
| 418 |
def __init__(self, headerlen, maxlen, continuation_ws, splitchars): |
| 419 |
self._maxlen = maxlen |
| 420 |
self._continuation_ws = continuation_ws |
| 421 |
self._continuation_ws_len = len(continuation_ws) |
| 422 |
self._splitchars = splitchars |
| 423 |
self._lines = [] |
| 424 |
self._current_line = _Accumulator(headerlen) |
| 425 |
|
| 426 |
def _str(self, linesep): |
| 427 |
self.newline() |
| 428 |
return linesep.join(self._lines) |
| 429 |
|
| 430 |
def __str__(self): |
| 431 |
return self._str(NL) |
| 432 |
|
| 433 |
def newline(self): |
| 434 |
end_of_line = self._current_line.pop() |
| 435 |
if end_of_line != (' ', ''): |
| 436 |
self._current_line.push(*end_of_line) |
| 437 |
if len(self._current_line) > 0: |
| 438 |
if self._current_line.is_onlyws() and self._lines: |
| 439 |
self._lines[-1] += str(self._current_line) |
| 440 |
else: |
| 441 |
self._lines.append(str(self._current_line)) |
| 442 |
self._current_line.reset() |
| 443 |
|
| 444 |
def add_transition(self): |
| 445 |
self._current_line.push(' ', '') |
| 446 |
|
| 447 |
def feed(self, fws, string, charset): |
| 448 |
# If the charset has no header encoding (i.e. it is an ASCII encoding) |
| 449 |
# then we must split the header at the "highest level syntactic break" |
| 450 |
# possible. Note that we don't have a lot of smarts about field |
| 451 |
# syntax; we just try to break on semi-colons, then commas, then |
| 452 |
# whitespace. Eventually, this should be pluggable. |
| 453 |
if charset.header_encoding is None: |
| 454 |
self._ascii_split(fws, string, self._splitchars) |
| 455 |
return |
| 456 |
# Otherwise, we're doing either a Base64 or a quoted-printable |
| 457 |
# encoding which means we don't need to split the line on syntactic |
| 458 |
# breaks. We can basically just find enough characters to fit on the |
| 459 |
# current line, minus the RFC 2047 chrome. What makes this trickier |
| 460 |
# though is that we have to split at octet boundaries, not character |
| 461 |
# boundaries but it's only safe to split at character boundaries so at |
| 462 |
# best we can only get close. |
| 463 |
encoded_lines = charset.header_encode_lines(string, self._maxlengths()) |
| 464 |
# The first element extends the current line, but if it's None then |
| 465 |
# nothing more fit on the current line so start a new line. |
| 466 |
try: |
| 467 |
first_line = encoded_lines.pop(0) |
| 468 |
except IndexError: |
| 469 |
# There are no encoded lines, so we're done. |
| 470 |
return |
| 471 |
if first_line is not None: |
| 472 |
self._append_chunk(fws, first_line) |
| 473 |
try: |
| 474 |
last_line = encoded_lines.pop() |
| 475 |
except IndexError: |
| 476 |
# There was only one line. |
| 477 |
return |
| 478 |
self.newline() |
| 479 |
self._current_line.push(self._continuation_ws, last_line) |
| 480 |
# Everything else are full lines in themselves. |
| 481 |
for line in encoded_lines: |
| 482 |
self._lines.append(self._continuation_ws + line) |
| 483 |
|
| 484 |
def _maxlengths(self): |
| 485 |
# The first line's length. |
| 486 |
yield self._maxlen - len(self._current_line) |
| 487 |
while True: |
| 488 |
yield self._maxlen - self._continuation_ws_len |
| 489 |
|
| 490 |
def _ascii_split(self, fws, string, splitchars): |
| 491 |
# The RFC 2822 header folding algorithm is simple in principle but |
| 492 |
# complex in practice. Lines may be folded any place where "folding |
| 493 |
# white space" appears by inserting a linesep character in front of the |
| 494 |
# FWS. The complication is that not all spaces or tabs qualify as FWS, |
| 495 |
# and we are also supposed to prefer to break at "higher level |
| 496 |
# syntactic breaks". We can't do either of these without intimate |
| 497 |
# knowledge of the structure of structured headers, which we don't have |
| 498 |
# here. So the best we can do here is prefer to break at the specified |
| 499 |
# splitchars, and hope that we don't choose any spaces or tabs that |
| 500 |
# aren't legal FWS. (This is at least better than the old algorithm, |
| 501 |
# where we would sometimes *introduce* FWS after a splitchar, or the |
| 502 |
# algorithm before that, where we would turn all white space runs into |
| 503 |
# single spaces or tabs.) |
| 504 |
parts = re.split("(["+FWS+"]+)", fws+string) |
| 505 |
if parts[0]: |
| 506 |
parts[:0] = [''] |
| 507 |
else: |
| 508 |
parts.pop(0) |
| 509 |
for fws, part in zip(*[iter(parts)]*2): |
| 510 |
self._append_chunk(fws, part) |
| 511 |
|
| 512 |
def _append_chunk(self, fws, string): |
| 513 |
self._current_line.push(fws, string) |
| 514 |
if len(self._current_line) > self._maxlen: |
| 515 |
# Find the best split point, working backward from the end. |
| 516 |
# There might be none, on a long first line. |
| 517 |
for ch in self._splitchars: |
| 518 |
for i in range(self._current_line.part_count()-1, 0, -1): |
| 519 |
if ch.isspace(): |
| 520 |
fws = self._current_line[i][0] |
| 521 |
if fws and fws[0]==ch: |
| 522 |
break |
| 523 |
prevpart = self._current_line[i-1][1] |
| 524 |
if prevpart and prevpart[-1]==ch: |
| 525 |
break |
| 526 |
else: |
| 527 |
continue |
| 528 |
break |
| 529 |
else: |
| 530 |
fws, part = self._current_line.pop() |
| 531 |
if self._current_line._initial_size > 0: |
| 532 |
# There will be a header, so leave it on a line by itself. |
| 533 |
self.newline() |
| 534 |
if not fws: |
| 535 |
# We don't use continuation_ws here because the whitespace |
| 536 |
# after a header should always be a space. |
| 537 |
fws = ' ' |
| 538 |
self._current_line.push(fws, part) |
| 539 |
return |
| 540 |
remainder = self._current_line.pop_from(i) |
| 541 |
self._lines.append(str(self._current_line)) |
| 542 |
self._current_line.reset(remainder) |
| 543 |
|
| 544 |
|
| 545 |
class _Accumulator(list): |
| 546 |
|
| 547 |
def __init__(self, initial_size=0): |
| 548 |
self._initial_size = initial_size |
| 549 |
super().__init__() |
| 550 |
|
| 551 |
def push(self, fws, string): |
| 552 |
self.append((fws, string)) |
| 553 |
|
| 554 |
def pop_from(self, i=0): |
| 555 |
popped = self[i:] |
| 556 |
self[i:] = [] |
| 557 |
return popped |
| 558 |
|
| 559 |
def pop(self): |
| 560 |
if self.part_count()==0: |
| 561 |
return ('', '') |
| 562 |
return super().pop() |
| 563 |
|
| 564 |
def __len__(self): |
| 565 |
return sum((len(fws)+len(part) for fws, part in self), |
| 566 |
self._initial_size) |
| 567 |
|
| 568 |
def __str__(self): |
| 569 |
return EMPTYSTRING.join((EMPTYSTRING.join((fws, part)) |
| 570 |
for fws, part in self)) |
| 571 |
|
| 572 |
def reset(self, startval=None): |
| 573 |
if startval is None: |
| 574 |
startval = [] |
| 575 |
self[:] = startval |
| 576 |
self._initial_size = 0 |
| 577 |
|
| 578 |
def is_onlyws(self): |
| 579 |
return self._initial_size==0 and (not self or str(self).isspace()) |
| 580 |
|
| 581 |
def part_count(self): |
| 582 |
return super().__len__() |