Debian Codegraph @

Lib/email/header.py

raw
1 # Copyright (C) 2002 Python Software Foundation
2 # Author: Ben Gertzfield, Barry Warsaw
3 # Contact: [email protected]
4
5 """Header encoding and decoding functionality."""
6
7 __all__ = [
8 'Header',
9 'decode_header',
10 'make_header',
11 ]
12
13 import re
14 import binascii
15
16 import email.quoprimime
17 import email.base64mime
18
19 from email.errors import HeaderParseError
20 from email import charset as _charset
21 Charset = _charset.Charset
22
23 NL = '\n'
24 SPACE = ' '
25 BSPACE = b' '
26 SPACE8 = ' ' * 8
27 EMPTYSTRING = ''
28 MAXLINELEN = 78
29 FWS = ' \t'
30
31 USASCII = Charset('us-ascii')
32 UTF8 = Charset('utf-8')
33
34 # Match encoded-word strings in the form =?charset?q?Hello_World?=
35 ecre = re.compile(r'''
36 =\? # literal =?
37 (?P<charset>[^?]*?) # non-greedy up to the next ? is the charset
38 \? # literal ?
39 (?P<encoding>[qQbB]) # either a "q" or a "b", case insensitive
40 \? # literal ?
41 (?P<encoded>.*?) # non-greedy up to the next ?= is the encoded string
42 \?= # literal ?=
43 ''', re.VERBOSE | re.MULTILINE)
44
45 # Field name regexp, including trailing colon, but not separating whitespace,
46 # according to RFC 2822. Character range is from tilde to exclamation mark.
47 # For use with .match()
48 fcre = re.compile(r'[\041-\176]+:$')
49
50 # Find a header embedded in a putative header value. Used to check for
51 # header injection attack.
52 _embedded_header = re.compile(r'\n[^ \t]+:')
53
54
55 # Helpers
56 _max_append = email.quoprimime._max_append
57
58
59 def decode_header(header):
60 """Decode a message header value without converting charset.
61
62 For historical reasons, this function may return either:
63
64 1. A list of length 1 containing a pair (str, None).
65 2. A list of (bytes, charset) pairs containing each of the decoded
66 parts of the header. Charset is None for non-encoded parts of the header,
67 otherwise a lower-case string containing the name of the character set
68 specified in the encoded string.
69
70 header may be a string that may or may not contain RFC2047 encoded words,
71 or it may be a Header object.
72
73 An email.errors.HeaderParseError may be raised when certain decoding error
74 occurs (e.g. a base64 decoding exception).
75
76 This function exists for backwards compatibility only. For new code, we
77 recommend using email.headerregistry.HeaderRegistry instead.
78 """
79 # If it is a Header object, we can just return the encoded chunks.
80 if hasattr(header, '_chunks'):
81 return [(_charset._encode(string, str(charset)), str(charset))
82 for string, charset in header._chunks]
83 # If no encoding, just return the header with no charset.
84 if not ecre.search(header):
85 return [(header, None)]
86 # First step is to parse all the encoded parts into triplets of the form
87 # (encoded_string, encoding, charset). For unencoded strings, the last
88 # two parts will be None.
89 words = []
90 for line in header.splitlines():
91 parts = ecre.split(line)
92 first = True
93 while parts:
94 unencoded = parts.pop(0)
95 if first:
96 unencoded = unencoded.lstrip()
97 first = False
98 if unencoded:
99 words.append((unencoded, None, None))
100 if parts:
101 charset = parts.pop(0).lower()
102 encoding = parts.pop(0).lower()
103 encoded = parts.pop(0)
104 words.append((encoded, encoding, charset))
105 # Now loop over words and remove words that consist of whitespace
106 # between two encoded strings.
107 droplist = []
108 for n, w in enumerate(words):
109 if n>1 and w[1] and words[n-2][1] and words[n-1][0].isspace():
110 droplist.append(n-1)
111 for d in reversed(droplist):
112 del words[d]
113
114 # The next step is to decode each encoded word by applying the reverse
115 # base64 or quopri transformation. decoded_words is now a list of the
116 # form (decoded_word, charset).
117 decoded_words = []
118 for encoded_string, encoding, charset in words:
119 if encoding is None:
120 # This is an unencoded word.
121 decoded_words.append((encoded_string, charset))
122 elif encoding == 'q':
123 word = email.quoprimime.header_decode(encoded_string)
124 decoded_words.append((word, charset))
125 elif encoding == 'b':
126 paderr = len(encoded_string) % 4 # Postel's law: add missing padding
127 if paderr:
128 encoded_string += '==='[:4 - paderr]
129 try:
130 word = email.base64mime.decode(encoded_string)
131 except binascii.Error:
132 raise HeaderParseError('Base64 decoding error')
133 else:
134 decoded_words.append((word, charset))
135 else:
136 raise AssertionError('Unexpected encoding: ' + encoding)
137 # Now convert all words to bytes and collapse consecutive runs of
138 # similarly encoded words.
139 collapsed = []
140 last_word = last_charset = None
141 for word, charset in decoded_words:
142 if isinstance(word, str):
143 word = bytes(word, 'raw-unicode-escape')
144 if last_word is None:
145 last_word = word
146 last_charset = charset
147 elif charset != last_charset:
148 collapsed.append((last_word, last_charset))
149 last_word = word
150 last_charset = charset
151 elif last_charset is None:
152 last_word += BSPACE + word
153 else:
154 last_word += word
155 collapsed.append((last_word, last_charset))
156 return collapsed
157
158
159 def make_header(decoded_seq, maxlinelen=None, header_name=None,
160 continuation_ws=' '):
161 """Create a Header from a sequence of pairs as returned by decode_header()
162
163 decode_header() takes a header value string and returns a sequence of
164 pairs of the format (decoded_string, charset) where charset is the string
165 name of the character set.
166
167 This function takes one of those sequence of pairs and returns a Header
168 instance. Optional maxlinelen, header_name, and continuation_ws are as in
169 the Header constructor.
170
171 This function exists for backwards compatibility only, and is not
172 recommended for use in new code.
173 """
174 h = Header(maxlinelen=maxlinelen, header_name=header_name,
175 continuation_ws=continuation_ws)
176 for s, charset in decoded_seq:
177 # None means us-ascii but we can simply pass it on to h.append()
178 if charset is not None and not isinstance(charset, Charset):
179 charset = Charset(charset)
180 h.append(s, charset)
181 return h
182
183
184 class Header:
185 def __init__(self, s=None, charset=None,
186 maxlinelen=None, header_name=None,
187 continuation_ws=' ', errors='strict'):
188 """Create a MIME-compliant header that can contain many character sets.
189
190 Optional s is the initial header value. If None, the initial header
191 value is not set. You can later append to the header with .append()
192 method calls. s may be a byte string or a Unicode string, but see the
193 .append() documentation for semantics.
194
195 Optional charset serves two purposes: it has the same meaning as the
196 charset argument to the .append() method. It also sets the default
197 character set for all subsequent .append() calls that omit the charset
198 argument. If charset is not provided in the constructor, the us-ascii
199 charset is used both as s's initial charset and as the default for
200 subsequent .append() calls.
201
202 The maximum line length can be specified explicitly via maxlinelen. For
203 splitting the first line to a shorter value (to account for the field
204 header which isn't included in s, e.g. 'Subject') pass in the name of
205 the field in header_name. The default maxlinelen is 78 as recommended
206 by RFC 2822.
207
208 continuation_ws must be RFC 2822 compliant folding whitespace (usually
209 either a space or a hard tab) which will be prepended to continuation
210 lines.
211
212 errors is passed through to the .append() call.
213 """
214 if charset is None:
215 charset = USASCII
216 elif not isinstance(charset, Charset):
217 charset = Charset(charset)
218 self._charset = charset
219 self._continuation_ws = continuation_ws
220 self._chunks = []
221 if s is not None:
222 self.append(s, charset, errors)
223 if maxlinelen is None:
224 maxlinelen = MAXLINELEN
225 self._maxlinelen = maxlinelen
226 if header_name is None:
227 self._headerlen = 0
228 else:
229 # Take the separating colon and space into account.
230 self._headerlen = len(header_name) + 2
231
232 def __str__(self):
233 """Return the string value of the header."""
234 self._normalize()
235 uchunks = []
236 lastcs = None
237 lastspace = None
238 for string, charset in self._chunks:
239 # We must preserve spaces between encoded and non-encoded word
240 # boundaries, which means for us we need to add a space when we go
241 # from a charset to None/us-ascii, or from None/us-ascii to a
242 # charset. Only do this for the second and subsequent chunks.
243 # Don't add a space if the None/us-ascii string already has
244 # a space (trailing or leading depending on transition)
245 nextcs = charset
246 if nextcs == _charset.UNKNOWN8BIT:
247 original_bytes = string.encode('ascii', 'surrogateescape')
248 string = original_bytes.decode('ascii', 'replace')
249 if uchunks:
250 hasspace = string and self._nonctext(string[0])
251 if lastcs not in (None, 'us-ascii'):
252 if nextcs in (None, 'us-ascii') and not hasspace:
253 uchunks.append(SPACE)
254 nextcs = None
255 elif nextcs not in (None, 'us-ascii') and not lastspace:
256 uchunks.append(SPACE)
257 lastspace = string and self._nonctext(string[-1])
258 lastcs = nextcs
259 uchunks.append(string)
260 return EMPTYSTRING.join(uchunks)
261
262 # Rich comparison operators for equality only. BAW: does it make sense to
263 # have or explicitly disable <, <=, >, >= operators?
264 def __eq__(self, other):
265 # other may be a Header or a string. Both are fine so coerce
266 # ourselves to a unicode (of the unencoded header value), swap the
267 # args and do another comparison.
268 return other == str(self)
269
270 def append(self, s, charset=None, errors='strict'):
271 """Append a string to the MIME header.
272
273 Optional charset, if given, should be a Charset instance or the name
274 of a character set (which will be converted to a Charset instance). A
275 value of None (the default) means that the charset given in the
276 constructor is used.
277
278 s may be a byte string or a Unicode string. If it is a byte string
279 (i.e. isinstance(s, str) is false), then charset is the encoding of
280 that byte string, and a UnicodeError will be raised if the string
281 cannot be decoded with that charset. If s is a Unicode string, then
282 charset is a hint specifying the character set of the characters in
283 the string. In either case, when producing an RFC 2822 compliant
284 header using RFC 2047 rules, the string will be encoded using the
285 output codec of the charset. If the string cannot be encoded to the
286 output codec, a UnicodeError will be raised.
287
288 Optional 'errors' is passed as the errors argument to the decode
289 call if s is a byte string.
290 """
291 if charset is None:
292 charset = self._charset
293 elif not isinstance(charset, Charset):
294 charset = Charset(charset)
295 if not isinstance(s, str):
296 input_charset = charset.input_codec or 'us-ascii'
297 if input_charset == _charset.UNKNOWN8BIT:
298 s = s.decode('us-ascii', 'surrogateescape')
299 else:
300 s = s.decode(input_charset, errors)
301 # Ensure that the bytes we're storing can be decoded to the output
302 # character set, otherwise an early error is raised.
303 output_charset = charset.output_codec or 'us-ascii'
304 if output_charset != _charset.UNKNOWN8BIT:
305 try:
306 s.encode(output_charset, errors)
307 except UnicodeEncodeError:
308 if output_charset!='us-ascii':
309 raise
310 charset = UTF8
311 self._chunks.append((s, charset))
312
313 def _nonctext(self, s):
314 """True if string s is not a ctext character of RFC822.
315 """
316 return s.isspace() or s in ('(', ')', '\\')
317
318 def encode(self, splitchars=';, \t', maxlinelen=None, linesep='\n'):
319 r"""Encode a message header into an RFC-compliant format.
320
321 There are many issues involved in converting a given string for use in
322 an email header. Only certain character sets are readable in most
323 email clients, and as header strings can only contain a subset of
324 7-bit ASCII, care must be taken to properly convert and encode (with
325 Base64 or quoted-printable) header strings. In addition, there is a
326 75-character length limit on any given encoded header field, so
327 line-wrapping must be performed, even with double-byte character sets.
328
329 Optional maxlinelen specifies the maximum length of each generated
330 line, exclusive of the linesep string. Individual lines may be longer
331 than maxlinelen if a folding point cannot be found. The first line
332 will be shorter by the length of the header name plus ": " if a header
333 name was specified at Header construction time. The default value for
334 maxlinelen is determined at header construction time.
335
336 Optional splitchars is a string containing characters which should be
337 given extra weight by the splitting algorithm during normal header
338 wrapping. This is in very rough support of RFC 2822's 'higher level
339 syntactic breaks': split points preceded by a splitchar are preferred
340 during line splitting, with the characters preferred in the order in
341 which they appear in the string. Space and tab may be included in the
342 string to indicate whether preference should be given to one over the
343 other as a split point when other split chars do not appear in the line
344 being split. Splitchars does not affect RFC 2047 encoded lines.
345
346 Optional linesep is a string to be used to separate the lines of
347 the value. The default value is the most useful for typical
348 Python applications, but it can be set to \r\n to produce RFC-compliant
349 line separators when needed.
350 """
351 self._normalize()
352 if maxlinelen is None:
353 maxlinelen = self._maxlinelen
354 # A maxlinelen of 0 means don't wrap. For all practical purposes,
355 # choosing a huge number here accomplishes that and makes the
356 # _ValueFormatter algorithm much simpler.
357 if maxlinelen == 0:
358 maxlinelen = 1000000
359 formatter = _ValueFormatter(self._headerlen, maxlinelen,
360 self._continuation_ws, splitchars)
361 lastcs = None
362 hasspace = lastspace = None
363 for string, charset in self._chunks:
364 if hasspace is not None:
365 hasspace = string and self._nonctext(string[0])
366 if lastcs not in (None, 'us-ascii'):
367 if not hasspace or charset not in (None, 'us-ascii'):
368 formatter.add_transition()
369 elif charset not in (None, 'us-ascii') and not lastspace:
370 formatter.add_transition()
371 lastspace = string and self._nonctext(string[-1])
372 lastcs = charset
373 hasspace = False
374 lines = string.splitlines()
375 if lines:
376 formatter.feed('', lines[0], charset)
377 else:
378 formatter.feed('', '', charset)
379 for line in lines[1:]:
380 formatter.newline()
381 if charset.header_encoding is not None:
382 formatter.feed(self._continuation_ws, ' ' + line.lstrip(),
383 charset)
384 else:
385 sline = line.lstrip()
386 fws = line[:len(line)-len(sline)]
387 formatter.feed(fws, sline, charset)
388 if len(lines) > 1:
389 formatter.newline()
390 if self._chunks:
391 formatter.add_transition()
392 value = formatter._str(linesep)
393 if _embedded_header.search(value):
394 raise HeaderParseError("header value appears to contain "
395 "an embedded header: {!r}".format(value))
396 return value
397
398 def _normalize(self):
399 # Step 1: Normalize the chunks so that all runs of identical charsets
400 # get collapsed into a single unicode string.
401 chunks = []
402 last_charset = None
403 last_chunk = []
404 for string, charset in self._chunks:
405 if charset == last_charset:
406 last_chunk.append(string)
407 else:
408 if last_charset is not None:
409 chunks.append((SPACE.join(last_chunk), last_charset))
410 last_chunk = [string]
411 last_charset = charset
412 if last_chunk:
413 chunks.append((SPACE.join(last_chunk), last_charset))
414 self._chunks = chunks
415
416
417 class _ValueFormatter:
418 def __init__(self, headerlen, maxlen, continuation_ws, splitchars):
419 self._maxlen = maxlen
420 self._continuation_ws = continuation_ws
421 self._continuation_ws_len = len(continuation_ws)
422 self._splitchars = splitchars
423 self._lines = []
424 self._current_line = _Accumulator(headerlen)
425
426 def _str(self, linesep):
427 self.newline()
428 return linesep.join(self._lines)
429
430 def __str__(self):
431 return self._str(NL)
432
433 def newline(self):
434 end_of_line = self._current_line.pop()
435 if end_of_line != (' ', ''):
436 self._current_line.push(*end_of_line)
437 if len(self._current_line) > 0:
438 if self._current_line.is_onlyws() and self._lines:
439 self._lines[-1] += str(self._current_line)
440 else:
441 self._lines.append(str(self._current_line))
442 self._current_line.reset()
443
444 def add_transition(self):
445 self._current_line.push(' ', '')
446
447 def feed(self, fws, string, charset):
448 # If the charset has no header encoding (i.e. it is an ASCII encoding)
449 # then we must split the header at the "highest level syntactic break"
450 # possible. Note that we don't have a lot of smarts about field
451 # syntax; we just try to break on semi-colons, then commas, then
452 # whitespace. Eventually, this should be pluggable.
453 if charset.header_encoding is None:
454 self._ascii_split(fws, string, self._splitchars)
455 return
456 # Otherwise, we're doing either a Base64 or a quoted-printable
457 # encoding which means we don't need to split the line on syntactic
458 # breaks. We can basically just find enough characters to fit on the
459 # current line, minus the RFC 2047 chrome. What makes this trickier
460 # though is that we have to split at octet boundaries, not character
461 # boundaries but it's only safe to split at character boundaries so at
462 # best we can only get close.
463 encoded_lines = charset.header_encode_lines(string, self._maxlengths())
464 # The first element extends the current line, but if it's None then
465 # nothing more fit on the current line so start a new line.
466 try:
467 first_line = encoded_lines.pop(0)
468 except IndexError:
469 # There are no encoded lines, so we're done.
470 return
471 if first_line is not None:
472 self._append_chunk(fws, first_line)
473 try:
474 last_line = encoded_lines.pop()
475 except IndexError:
476 # There was only one line.
477 return
478 self.newline()
479 self._current_line.push(self._continuation_ws, last_line)
480 # Everything else are full lines in themselves.
481 for line in encoded_lines:
482 self._lines.append(self._continuation_ws + line)
483
484 def _maxlengths(self):
485 # The first line's length.
486 yield self._maxlen - len(self._current_line)
487 while True:
488 yield self._maxlen - self._continuation_ws_len
489
490 def _ascii_split(self, fws, string, splitchars):
491 # The RFC 2822 header folding algorithm is simple in principle but
492 # complex in practice. Lines may be folded any place where "folding
493 # white space" appears by inserting a linesep character in front of the
494 # FWS. The complication is that not all spaces or tabs qualify as FWS,
495 # and we are also supposed to prefer to break at "higher level
496 # syntactic breaks". We can't do either of these without intimate
497 # knowledge of the structure of structured headers, which we don't have
498 # here. So the best we can do here is prefer to break at the specified
499 # splitchars, and hope that we don't choose any spaces or tabs that
500 # aren't legal FWS. (This is at least better than the old algorithm,
501 # where we would sometimes *introduce* FWS after a splitchar, or the
502 # algorithm before that, where we would turn all white space runs into
503 # single spaces or tabs.)
504 parts = re.split("(["+FWS+"]+)", fws+string)
505 if parts[0]:
506 parts[:0] = ['']
507 else:
508 parts.pop(0)
509 for fws, part in zip(*[iter(parts)]*2):
510 self._append_chunk(fws, part)
511
512 def _append_chunk(self, fws, string):
513 self._current_line.push(fws, string)
514 if len(self._current_line) > self._maxlen:
515 # Find the best split point, working backward from the end.
516 # There might be none, on a long first line.
517 for ch in self._splitchars:
518 for i in range(self._current_line.part_count()-1, 0, -1):
519 if ch.isspace():
520 fws = self._current_line[i][0]
521 if fws and fws[0]==ch:
522 break
523 prevpart = self._current_line[i-1][1]
524 if prevpart and prevpart[-1]==ch:
525 break
526 else:
527 continue
528 break
529 else:
530 fws, part = self._current_line.pop()
531 if self._current_line._initial_size > 0:
532 # There will be a header, so leave it on a line by itself.
533 self.newline()
534 if not fws:
535 # We don't use continuation_ws here because the whitespace
536 # after a header should always be a space.
537 fws = ' '
538 self._current_line.push(fws, part)
539 return
540 remainder = self._current_line.pop_from(i)
541 self._lines.append(str(self._current_line))
542 self._current_line.reset(remainder)
543
544
545 class _Accumulator(list):
546
547 def __init__(self, initial_size=0):
548 self._initial_size = initial_size
549 super().__init__()
550
551 def push(self, fws, string):
552 self.append((fws, string))
553
554 def pop_from(self, i=0):
555 popped = self[i:]
556 self[i:] = []
557 return popped
558
559 def pop(self):
560 if self.part_count()==0:
561 return ('', '')
562 return super().pop()
563
564 def __len__(self):
565 return sum((len(fws)+len(part) for fws, part in self),
566 self._initial_size)
567
568 def __str__(self):
569 return EMPTYSTRING.join((EMPTYSTRING.join((fws, part))
570 for fws, part in self))
571
572 def reset(self, startval=None):
573 if startval is None:
574 startval = []
575 self[:] = startval
576 self._initial_size = 0
577
578 def is_onlyws(self):
579 return self._initial_size==0 and (not self or str(self).isspace())
580
581 def part_count(self):
582 return super().__len__()