Debian Codegraph @

Lib/email/generator.py

raw
1 # Copyright (C) 2001 Python Software Foundation
2 # Author: Barry Warsaw
3 # Contact: [email protected]
4
5 """Classes to generate plain text from a message object tree."""
6
7 __all__ = ['Generator', 'DecodedGenerator', 'BytesGenerator']
8
9 import re
10 import sys
11 import time
12 import random
13
14 from copy import deepcopy
15 from io import StringIO, BytesIO
16 from email.utils import _has_surrogates
17 from email.errors import HeaderWriteError
18
19 UNDERSCORE = '_'
20 NL = '\n' # XXX: no longer used by the code below.
21
22 NLCRE = re.compile(r'\r\n|\r|\n')
23 fcre = re.compile(r'^From ', re.MULTILINE)
24 NEWLINE_WITHOUT_FWSP = re.compile(r'\r\n[^ \t]|\r[^ \n\t]|\n[^ \t]')
25 NEWLINE_WITHOUT_FWSP_BYTES = re.compile(br'\r\n[^ \t]|\r[^ \n\t]|\n[^ \t]')
26
27
28 class Generator:
29 """Generates output from a Message object tree.
30
31 This basic generator writes the message to the given file object as plain
32 text.
33 """
34 #
35 # Public interface
36 #
37
38 def __init__(self, outfp, mangle_from_=None, maxheaderlen=None, *,
39 policy=None):
40 """Create the generator for message flattening.
41
42 outfp is the output file-like object for writing the message to. It
43 must have a write() method.
44
45 Optional mangle_from_ is a flag that, when True (the default if policy
46 is not set), escapes From_ lines in the body of the message by putting
47 a '>' in front of them.
48
49 Optional maxheaderlen specifies the longest length for a non-continued
50 header. When a header line is longer (in characters, with tabs
51 expanded to 8 spaces) than maxheaderlen, the header will split as
52 defined in the Header class. Set maxheaderlen to zero to disable
53 header wrapping. The default is 78, as recommended (but not required)
54 by RFC 5322 section 2.1.1.
55
56 The policy keyword specifies a policy object that controls a number of
57 aspects of the generator's operation. If no policy is specified,
58 the policy associated with the Message object passed to the
59 flatten method is used.
60
61 """
62
63 if mangle_from_ is None:
64 mangle_from_ = True if policy is None else policy.mangle_from_
65 self._fp = outfp
66 self._mangle_from_ = mangle_from_
67 self.maxheaderlen = maxheaderlen
68 self.policy = policy
69
70 def write(self, s):
71 # Just delegate to the file object
72 self._fp.write(s)
73
74 def flatten(self, msg, unixfrom=False, linesep=None):
75 r"""Print the message object tree rooted at msg to the output file
76 specified when the Generator instance was created.
77
78 unixfrom is a flag that forces the printing of a Unix From_ delimiter
79 before the first object in the message tree. If the original message
80 has no From_ delimiter, a 'standard' one is crafted. By default, this
81 is False to inhibit the printing of any From_ delimiter.
82
83 Note that for subobjects, no From_ line is printed.
84
85 linesep specifies the characters used to indicate a new line in
86 the output. The default value is determined by the policy specified
87 when the Generator instance was created or, if none was specified,
88 from the policy associated with the msg.
89
90 """
91 # We use the _XXX constants for operating on data that comes directly
92 # from the msg, and _encoded_XXX constants for operating on data that
93 # has already been converted (to bytes in the BytesGenerator) and
94 # inserted into a temporary buffer.
95 policy = msg.policy if self.policy is None else self.policy
96 if linesep is not None:
97 policy = policy.clone(linesep=linesep)
98 if self.maxheaderlen is not None:
99 policy = policy.clone(max_line_length=self.maxheaderlen)
100 self._NL = policy.linesep
101 self._encoded_NL = self._encode(self._NL)
102 self._EMPTY = ''
103 self._encoded_EMPTY = self._encode(self._EMPTY)
104 # Because we use clone (below) when we recursively process message
105 # subparts, and because clone uses the computed policy (not None),
106 # submessages will automatically get set to the computed policy when
107 # they are processed by this code.
108 old_gen_policy = self.policy
109 old_msg_policy = msg.policy
110 try:
111 self.policy = policy
112 msg.policy = policy
113 if unixfrom:
114 ufrom = msg.get_unixfrom()
115 if not ufrom:
116 ufrom = 'From nobody ' + time.ctime(time.time())
117 self.write(ufrom + self._NL)
118 self._write(msg)
119 finally:
120 self.policy = old_gen_policy
121 msg.policy = old_msg_policy
122
123 def clone(self, fp):
124 """Clone this generator with the exact same options."""
125 return self.__class__(fp,
126 self._mangle_from_,
127 None, # Use policy setting, which we've adjusted
128 policy=self.policy)
129
130 #
131 # Protected interface - undocumented ;/
132 #
133
134 # Note that we use 'self.write' when what we are writing is coming from
135 # the source, and self._fp.write when what we are writing is coming from a
136 # buffer (because the Bytes subclass has already had a chance to transform
137 # the data in its write method in that case). This is an entirely
138 # pragmatic split determined by experiment; we could be more general by
139 # always using write and having the Bytes subclass write method detect when
140 # it has already transformed the input; but, since this whole thing is a
141 # hack anyway this seems good enough.
142
143 def _new_buffer(self):
144 # BytesGenerator overrides this to return BytesIO.
145 return StringIO()
146
147 def _encode(self, s):
148 # BytesGenerator overrides this to encode strings to bytes.
149 return s
150
151 def _write_lines(self, lines):
152 # We have to transform the line endings.
153 if not lines:
154 return
155 lines = NLCRE.split(lines)
156 for line in lines[:-1]:
157 self.write(line)
158 self.write(self._NL)
159 if lines[-1]:
160 self.write(lines[-1])
161 # XXX logic tells me this else should be needed, but the tests fail
162 # with it and pass without it. (NLCRE.split ends with a blank element
163 # if and only if there was a trailing newline.)
164 #else:
165 # self.write(self._NL)
166
167 def _write(self, msg):
168 # We can't write the headers yet because of the following scenario:
169 # say a multipart message includes the boundary string somewhere in
170 # its body. We'd have to calculate the new boundary /before/ we write
171 # the headers so that we can write the correct Content-Type:
172 # parameter.
173 #
174 # The way we do this, so as to make the _handle_*() methods simpler,
175 # is to cache any subpart writes into a buffer. Then we write the
176 # headers and the buffer contents. That way, subpart handlers can
177 # Do The Right Thing, and can still modify the Content-Type: header if
178 # necessary.
179 oldfp = self._fp
180 try:
181 self._munge_cte = None
182 self._fp = sfp = self._new_buffer()
183 self._dispatch(msg)
184 finally:
185 self._fp = oldfp
186 munge_cte = self._munge_cte
187 del self._munge_cte
188 # If we munged the cte, copy the message again and re-fix the CTE.
189 if munge_cte:
190 msg = deepcopy(msg)
191 # Preserve the header order if the CTE header already exists.
192 if msg.get('content-transfer-encoding') is None:
193 msg['Content-Transfer-Encoding'] = munge_cte[0]
194 else:
195 msg.replace_header('content-transfer-encoding', munge_cte[0])
196 msg.replace_header('content-type', munge_cte[1])
197 # Write the headers. First we see if the message object wants to
198 # handle that itself. If not, we'll do it generically.
199 meth = getattr(msg, '_write_headers', None)
200 if meth is None:
201 self._write_headers(msg)
202 else:
203 meth(self)
204 self._fp.write(sfp.getvalue())
205
206 def _dispatch(self, msg):
207 # Get the Content-Type: for the message, then try to dispatch to
208 # self._handle_<maintype>_<subtype>(). If there's no handler for the
209 # full MIME type, then dispatch to self._handle_<maintype>(). If
210 # that's missing too, then dispatch to self._writeBody().
211 main = msg.get_content_maintype()
212 sub = msg.get_content_subtype()
213 specific = UNDERSCORE.join((main, sub)).replace('-', '_')
214 meth = getattr(self, '_handle_' + specific, None)
215 if meth is None:
216 generic = main.replace('-', '_')
217 meth = getattr(self, '_handle_' + generic, None)
218 if meth is None:
219 meth = self._writeBody
220 meth(msg)
221
222 #
223 # Default handlers
224 #
225
226 def _write_headers(self, msg):
227 for h, v in msg.raw_items():
228 folded = self.policy.fold(h, v)
229 if self.policy.verify_generated_headers:
230 linesep = self.policy.linesep
231 if not folded.endswith(linesep):
232 raise HeaderWriteError(
233 f'folded header does not end with {linesep!r}: {folded!r}')
234 if NEWLINE_WITHOUT_FWSP.search(folded.removesuffix(linesep)):
235 raise HeaderWriteError(
236 f'folded header contains newline: {folded!r}')
237 self.write(folded)
238 # A blank line always separates headers from body
239 self.write(self._NL)
240
241 #
242 # Handlers for writing types and subtypes
243 #
244
245 def _handle_text(self, msg):
246 payload = msg.get_payload()
247 if payload is None:
248 return
249 if not isinstance(payload, str):
250 raise TypeError('string payload expected: %s' % type(payload))
251 if _has_surrogates(msg._payload):
252 charset = msg.get_param('charset')
253 if charset is not None:
254 # XXX: This copy stuff is an ugly hack to avoid modifying the
255 # existing message.
256 msg = deepcopy(msg)
257 del msg['content-transfer-encoding']
258 msg.set_payload(msg._payload, charset)
259 payload = msg.get_payload()
260 self._munge_cte = (msg['content-transfer-encoding'],
261 msg['content-type'])
262 if self._mangle_from_:
263 payload = fcre.sub('>From ', payload)
264 self._write_lines(payload)
265
266 # Default body handler
267 _writeBody = _handle_text
268
269 def _handle_multipart(self, msg):
270 # The trick here is to write out each part separately, merge them all
271 # together, and then make sure that the boundary we've chosen isn't
272 # present in the payload.
273 msgtexts = []
274 subparts = msg.get_payload()
275 if subparts is None:
276 subparts = []
277 elif isinstance(subparts, str):
278 # e.g. a non-strict parse of a message with no starting boundary.
279 self.write(subparts)
280 return
281 elif not isinstance(subparts, list):
282 # Scalar payload
283 subparts = [subparts]
284 for part in subparts:
285 s = self._new_buffer()
286 g = self.clone(s)
287 g.flatten(part, unixfrom=False, linesep=self._NL)
288 msgtexts.append(s.getvalue())
289 # BAW: What about boundaries that are wrapped in double-quotes?
290 boundary = msg.get_boundary()
291 if not boundary:
292 # Create a boundary that doesn't appear in any of the
293 # message texts.
294 alltext = self._encoded_NL.join(msgtexts)
295 boundary = self._make_boundary(alltext)
296 msg.set_boundary(boundary)
297 # If there's a preamble, write it out, with a trailing CRLF
298 if msg.preamble is not None:
299 if self._mangle_from_:
300 preamble = fcre.sub('>From ', msg.preamble)
301 else:
302 preamble = msg.preamble
303 self._write_lines(preamble)
304 self.write(self._NL)
305 # dash-boundary transport-padding CRLF
306 self.write('--' + boundary + self._NL)
307 # body-part
308 if msgtexts:
309 self._fp.write(msgtexts.pop(0))
310 # *encapsulation
311 # --> delimiter transport-padding
312 # --> CRLF body-part
313 for body_part in msgtexts:
314 # delimiter transport-padding CRLF
315 self.write(self._NL + '--' + boundary + self._NL)
316 # body-part
317 self._fp.write(body_part)
318 # close-delimiter transport-padding
319 self.write(self._NL + '--' + boundary + '--' + self._NL)
320 if msg.epilogue is not None:
321 if self._mangle_from_:
322 epilogue = fcre.sub('>From ', msg.epilogue)
323 else:
324 epilogue = msg.epilogue
325 self._write_lines(epilogue)
326
327 def _handle_multipart_signed(self, msg):
328 # The contents of signed parts has to stay unmodified in order to keep
329 # the signature intact per RFC1847 2.1, so we disable header wrapping.
330 # RDM: This isn't enough to completely preserve the part, but it helps.
331 p = self.policy
332 self.policy = p.clone(max_line_length=0)
333 try:
334 self._handle_multipart(msg)
335 finally:
336 self.policy = p
337
338 def _handle_message_delivery_status(self, msg):
339 # We can't just write the headers directly to self's file object
340 # because this will leave an extra newline between the last header
341 # block and the boundary. Sigh.
342 blocks = []
343 for part in msg.get_payload():
344 s = self._new_buffer()
345 g = self.clone(s)
346 g.flatten(part, unixfrom=False, linesep=self._NL)
347 text = s.getvalue()
348 lines = text.split(self._encoded_NL)
349 # Strip off the unnecessary trailing empty line
350 if lines and lines[-1] == self._encoded_EMPTY:
351 blocks.append(self._encoded_NL.join(lines[:-1]))
352 else:
353 blocks.append(text)
354 # Now join all the blocks with an empty line. This has the lovely
355 # effect of separating each block with an empty line, but not adding
356 # an extra one after the last one.
357 self._fp.write(self._encoded_NL.join(blocks))
358
359 def _handle_message(self, msg):
360 s = self._new_buffer()
361 g = self.clone(s)
362 # The payload of a message/rfc822 part should be a multipart sequence
363 # of length 1. The zeroth element of the list should be the Message
364 # object for the subpart. Extract that object, stringify it, and
365 # write it out.
366 # Except, it turns out, when it's a string instead, which happens when
367 # and only when HeaderParser is used on a message of mime type
368 # message/rfc822. Such messages are generated by, for example,
369 # Groupwise when forwarding unadorned messages. (Issue 7970.) So
370 # in that case we just emit the string body.
371 payload = msg._payload
372 if isinstance(payload, list):
373 g.flatten(msg.get_payload(0), unixfrom=False, linesep=self._NL)
374 payload = s.getvalue()
375 else:
376 payload = self._encode(payload)
377 self._fp.write(payload)
378
379 # This used to be a module level function; we use a classmethod for this
380 # and _compile_re so we can continue to provide the module level function
381 # for backward compatibility by doing
382 # _make_boundary = Generator._make_boundary
383 # at the end of the module. It *is* internal, so we could drop that...
384 @classmethod
385 def _make_boundary(cls, text=None):
386 # Craft a random boundary. If text is given, ensure that the chosen
387 # boundary doesn't appear in the text.
388 token = random.randrange(sys.maxsize)
389 boundary = ('=' * 15) + (_fmt % token) + '=='
390 if text is None:
391 return boundary
392 b = boundary
393 counter = 0
394 while True:
395 cre = cls._compile_re('^--' + re.escape(b) + '(--)?\r?$', re.MULTILINE)
396 if not cre.search(text):
397 break
398 b = boundary + '.' + str(counter)
399 counter += 1
400 return b
401
402 @classmethod
403 def _compile_re(cls, s, flags):
404 return re.compile(s, flags)
405
406
407 class BytesGenerator(Generator):
408 """Generates a bytes version of a Message object tree.
409
410 Functionally identical to the base Generator except that the output is
411 bytes and not string. When surrogates were used in the input to encode
412 bytes, these are decoded back to bytes for output. If the policy has
413 cte_type set to 7bit, then the message is transformed such that the
414 non-ASCII bytes are properly content transfer encoded, using the charset
415 unknown-8bit.
416
417 The outfp object must accept bytes in its write method.
418 """
419
420 def write(self, s):
421 self._fp.write(s.encode('ascii', 'surrogateescape'))
422
423 def _new_buffer(self):
424 return BytesIO()
425
426 def _encode(self, s):
427 return s.encode('ascii')
428
429 def _write_headers(self, msg):
430 # This is almost the same as the string version, except for handling
431 # strings with 8bit bytes.
432 for h, v in msg.raw_items():
433 folded = self.policy.fold_binary(h, v)
434 if self.policy.verify_generated_headers:
435 linesep = self.policy.linesep.encode()
436 if not folded.endswith(linesep):
437 raise HeaderWriteError(
438 f'folded header does not end with {linesep!r}: {folded!r}')
439 if NEWLINE_WITHOUT_FWSP_BYTES.search(folded.removesuffix(linesep)):
440 raise HeaderWriteError(
441 f'folded header contains newline: {folded!r}')
442 self._fp.write(folded)
443 # A blank line always separates headers from body
444 self.write(self._NL)
445
446 def _handle_text(self, msg):
447 # If the string has surrogates the original source was bytes, so
448 # just write it back out.
449 if msg._payload is None:
450 return
451 if _has_surrogates(msg._payload) and not self.policy.cte_type=='7bit':
452 if self._mangle_from_:
453 msg._payload = fcre.sub(">From ", msg._payload)
454 self._write_lines(msg._payload)
455 else:
456 super(BytesGenerator,self)._handle_text(msg)
457
458 # Default body handler
459 _writeBody = _handle_text
460
461 @classmethod
462 def _compile_re(cls, s, flags):
463 return re.compile(s.encode('ascii'), flags)
464
465
466 _FMT = '[Non-text (%(type)s) part of message omitted, filename %(filename)s]'
467
468 class DecodedGenerator(Generator):
469 """Generates a text representation of a message.
470
471 Like the Generator base class, except that non-text parts are substituted
472 with a format string representing the part.
473 """
474 def __init__(self, outfp, mangle_from_=None, maxheaderlen=None, fmt=None, *,
475 policy=None):
476 """Like Generator.__init__() except that an additional optional
477 argument is allowed.
478
479 Walks through all subparts of a message. If the subpart is of main
480 type 'text', then it prints the decoded payload of the subpart.
481
482 Otherwise, fmt is a format string that is used instead of the message
483 payload. fmt is expanded with the following keywords (in
484 %(keyword)s format):
485
486 type : Full MIME type of the non-text part
487 maintype : Main MIME type of the non-text part
488 subtype : Sub-MIME type of the non-text part
489 filename : Filename of the non-text part
490 description: Description associated with the non-text part
491 encoding : Content transfer encoding of the non-text part
492
493 The default value for fmt is None, meaning
494
495 [Non-text (%(type)s) part of message omitted, filename %(filename)s]
496 """
497 Generator.__init__(self, outfp, mangle_from_, maxheaderlen,
498 policy=policy)
499 if fmt is None:
500 self._fmt = _FMT
501 else:
502 self._fmt = fmt
503
504 def _dispatch(self, msg):
505 for part in msg.walk():
506 maintype = part.get_content_maintype()
507 if maintype == 'text':
508 print(part.get_payload(decode=False), file=self)
509 elif maintype == 'multipart':
510 # Just skip this
511 pass
512 else:
513 print(self._fmt % {
514 'type' : part.get_content_type(),
515 'maintype' : part.get_content_maintype(),
516 'subtype' : part.get_content_subtype(),
517 'filename' : part.get_filename('[no filename]'),
518 'description': part.get('Content-Description',
519 '[no description]'),
520 'encoding' : part.get('Content-Transfer-Encoding',
521 '[no encoding]'),
522 }, file=self)
523
524
525 # Helper used by Generator._make_boundary
526 _width = len(repr(sys.maxsize-1))
527 _fmt = '%%0%dd' % _width
528
529 # Backward compatibility
530 _make_boundary = Generator._make_boundary