Debian Codegraph @

Lib/email/_header_value_parser.py

raw
1 """Header value parser implementing various email-related RFC parsing rules.
2
3 The parsing methods defined in this module implement various email related
4 parsing rules. Principal among them is RFC 5322, which is the followon
5 to RFC 2822 and primarily a clarification of the former. It also implements
6 RFC 2047 encoded word decoding.
7
8 RFC 5322 goes to considerable trouble to maintain backward compatibility with
9 RFC 822 in the parse phase, while cleaning up the structure on the generation
10 phase. This parser supports correct RFC 5322 generation by tagging white space
11 as folding white space only when folding is allowed in the non-obsolete rule
12 sets. Actually, the parser is even more generous when accepting input than RFC
13 5322 mandates, following the spirit of Postel's Law, which RFC 5322 encourages.
14 Where possible deviations from the standard are annotated on the 'defects'
15 attribute of tokens that deviate.
16
17 The general structure of the parser follows RFC 5322, and uses its terminology
18 where there is a direct correspondence. Where the implementation requires a
19 somewhat different structure than that used by the formal grammar, new terms
20 that mimic the closest existing terms are used. Thus, it really helps to have
21 a copy of RFC 5322 handy when studying this code.
22
23 Input to the parser is a string that has already been unfolded according to
24 RFC 5322 rules. According to the RFC this unfolding is the very first step, and
25 this parser leaves the unfolding step to a higher level message parser, which
26 will have already detected the line breaks that need unfolding while
27 determining the beginning and end of each header.
28
29 The output of the parser is a TokenList object, which is a list subclass. A
30 TokenList is a recursive data structure. The terminal nodes of the structure
31 are Terminal objects, which are subclasses of str. These do not correspond
32 directly to terminal objects in the formal grammar, but are instead more
33 practical higher level combinations of true terminals.
34
35 All TokenList and Terminal objects have a 'value' attribute, which produces the
36 semantically meaningful value of that part of the parse subtree. The value of
37 all whitespace tokens (no matter how many sub-tokens they may contain) is a
38 single space, as per the RFC rules. This includes 'CFWS', which is herein
39 included in the general class of whitespace tokens. There is one exception to
40 the rule that whitespace tokens are collapsed into single spaces in values: in
41 the value of a 'bare-quoted-string' (a quoted-string with no leading or
42 trailing whitespace), any whitespace that appeared between the quotation marks
43 is preserved in the returned value. Note that in all Terminal strings quoted
44 pairs are turned into their unquoted values.
45
46 All TokenList and Terminal objects also have a string value, which attempts to
47 be a "canonical" representation of the RFC-compliant form of the substring that
48 produced the parsed subtree, including minimal use of quoted pair quoting.
49 Whitespace runs are not collapsed.
50
51 Comment tokens also have a 'content' attribute providing the string found
52 between the parens (including any nested comments) with whitespace preserved.
53
54 All TokenList and Terminal objects have a 'defects' attribute which is a
55 possibly empty list all of the defects found while creating the token. Defects
56 may appear on any token in the tree, and a composite list of all defects in the
57 subtree is available through the 'all_defects' attribute of any node. (For
58 Terminal notes x.defects == x.all_defects.)
59
60 Each object in a parse tree is called a 'token', and each has a 'token_type'
61 attribute that gives the name from the RFC 5322 grammar that it represents.
62 Not all RFC 5322 nodes are produced, and there is one non-RFC 5322 node that
63 may be produced: 'ptext'. A 'ptext' is a string of printable ascii characters.
64 It is returned in place of lists of (ctext/quoted-pair) and
65 (qtext/quoted-pair).
66
67 XXX: provide complete list of token types.
68 """
69
70 import re
71 import sys
72 import urllib # For urllib.parse.unquote
73 from string import hexdigits
74 from operator import itemgetter
75 from email import _encoded_words as _ew
76 from email import errors
77 from email import utils
78
79 #
80 # Useful constants and functions
81 #
82
83 _WSP = ' \t'
84 WSP = set(_WSP)
85 CFWS_LEADER = WSP | set('(')
86 SPECIALS = set(r'()<>@,:;.\"[]')
87 ATOM_ENDS = SPECIALS | WSP
88 DOT_ATOM_ENDS = ATOM_ENDS - set('.')
89 # '.', '"', and '(' do not end phrases in order to support obs-phrase
90 PHRASE_ENDS = SPECIALS - set('."(')
91 TSPECIALS = (SPECIALS | set('/?=')) - set('.')
92 TOKEN_ENDS = TSPECIALS | WSP
93 ASPECIALS = TSPECIALS | set("*'%")
94 ATTRIBUTE_ENDS = ASPECIALS | WSP
95 EXTENDED_ATTRIBUTE_ENDS = ATTRIBUTE_ENDS - set('%')
96 NLSET = {'\n', '\r'}
97 SPECIALSNL = SPECIALS | NLSET
98
99
100 def make_quoted_pairs(value):
101 """Escape dquote and backslash for use within a quoted-string."""
102 return str(value).replace('\\', '\\\\').replace('"', '\\"')
103
104
105 def make_parenthesis_pairs(value):
106 """Escape parenthesis and backslash for use within a comment."""
107 return str(value).replace('\\', '\\\\') \
108 .replace('(', '\\(').replace(')', '\\)')
109
110
111 def quote_string(value):
112 escaped = make_quoted_pairs(value)
113 return f'"{escaped}"'
114
115
116 # Match a RFC 2047 word, looks like =?utf-8?q?someword?=
117 rfc2047_matcher = re.compile(r'''
118 =\? # literal =?
119 [^?]* # charset
120 \? # literal ?
121 [qQbB] # literal 'q' or 'b', case insensitive
122 \? # literal ?
123 .*? # encoded word
124 \?= # literal ?=
125 ''', re.VERBOSE | re.MULTILINE)
126
127
128 #
129 # TokenList and its subclasses
130 #
131
132 class TokenList(list):
133
134 token_type = None
135 syntactic_break = True
136 ew_combine_allowed = True
137
138 def __init__(self, *args, **kw):
139 super().__init__(*args, **kw)
140 self.defects = []
141
142 def __str__(self):
143 return ''.join(str(x) for x in self)
144
145 def __repr__(self):
146 return '{}({})'.format(self.__class__.__name__,
147 super().__repr__())
148
149 @property
150 def value(self):
151 return ''.join(x.value for x in self if x.value)
152
153 @property
154 def all_defects(self):
155 return sum((x.all_defects for x in self), self.defects)
156
157 def startswith_fws(self):
158 return self[0].startswith_fws()
159
160 @property
161 def as_ew_allowed(self):
162 """True if all top level tokens of this part may be RFC2047 encoded."""
163 return all(part.as_ew_allowed for part in self)
164
165 @property
166 def comments(self):
167 comments = []
168 for token in self:
169 comments.extend(token.comments)
170 return comments
171
172 def fold(self, *, policy):
173 return _refold_parse_tree(self, policy=policy)
174
175 def pprint(self, indent=''):
176 print(self.ppstr(indent=indent))
177
178 def ppstr(self, indent=''):
179 return '\n'.join(self._pp(indent=indent))
180
181 def _pp(self, indent=''):
182 yield '{}{}/{}('.format(
183 indent,
184 self.__class__.__name__,
185 self.token_type)
186 for token in self:
187 if not hasattr(token, '_pp'):
188 yield (indent + ' !! invalid element in token '
189 'list: {!r}'.format(token))
190 else:
191 yield from token._pp(indent+' ')
192 if self.defects:
193 extra = ' Defects: {}'.format(self.defects)
194 else:
195 extra = ''
196 yield '{}){}'.format(indent, extra)
197
198
199 class WhiteSpaceTokenList(TokenList):
200
201 @property
202 def value(self):
203 return ' '
204
205 @property
206 def comments(self):
207 return [x.content for x in self if x.token_type=='comment']
208
209
210 class UnstructuredTokenList(TokenList):
211 token_type = 'unstructured'
212
213
214 class Phrase(TokenList):
215 token_type = 'phrase'
216
217 class Word(TokenList):
218 token_type = 'word'
219
220
221 class CFWSList(WhiteSpaceTokenList):
222 token_type = 'cfws'
223
224
225 class Atom(TokenList):
226 token_type = 'atom'
227
228
229 class Token(TokenList):
230 token_type = 'token'
231 encode_as_ew = False
232
233
234 class EncodedWord(TokenList):
235 token_type = 'encoded-word'
236 cte = None
237 charset = None
238 lang = None
239
240
241 class QuotedString(TokenList):
242
243 token_type = 'quoted-string'
244
245 @property
246 def content(self):
247 for x in self:
248 if x.token_type == 'bare-quoted-string':
249 return x.value
250
251 @property
252 def quoted_value(self):
253 res = []
254 for x in self:
255 if x.token_type == 'bare-quoted-string':
256 res.append(str(x))
257 else:
258 res.append(x.value)
259 return ''.join(res)
260
261 @property
262 def stripped_value(self):
263 for token in self:
264 if token.token_type == 'bare-quoted-string':
265 return token.value
266
267
268 class BareQuotedString(QuotedString):
269
270 token_type = 'bare-quoted-string'
271
272 def __str__(self):
273 return quote_string(''.join(str(x) for x in self))
274
275 @property
276 def value(self):
277 return ''.join(str(x) for x in self)
278
279
280 class Comment(WhiteSpaceTokenList):
281
282 token_type = 'comment'
283
284 def __str__(self):
285 return ''.join(sum([
286 ["("],
287 [self.quote(x) for x in self],
288 [")"],
289 ], []))
290
291 def quote(self, value):
292 if value.token_type == 'comment':
293 return str(value)
294 return str(value).replace('\\', '\\\\').replace(
295 '(', r'\(').replace(
296 ')', r'\)')
297
298 @property
299 def content(self):
300 return ''.join(str(x) for x in self)
301
302 @property
303 def comments(self):
304 return [self.content]
305
306 class AddressList(TokenList):
307
308 token_type = 'address-list'
309
310 @property
311 def addresses(self):
312 return [x for x in self if x.token_type=='address']
313
314 @property
315 def mailboxes(self):
316 return sum((x.mailboxes
317 for x in self if x.token_type=='address'), [])
318
319 @property
320 def all_mailboxes(self):
321 return sum((x.all_mailboxes
322 for x in self if x.token_type=='address'), [])
323
324
325 class Address(TokenList):
326
327 token_type = 'address'
328
329 @property
330 def display_name(self):
331 if self[0].token_type == 'group':
332 return self[0].display_name
333
334 @property
335 def mailboxes(self):
336 if self[0].token_type == 'mailbox':
337 return [self[0]]
338 elif self[0].token_type == 'invalid-mailbox':
339 return []
340 return self[0].mailboxes
341
342 @property
343 def all_mailboxes(self):
344 if self[0].token_type == 'mailbox':
345 return [self[0]]
346 elif self[0].token_type == 'invalid-mailbox':
347 return [self[0]]
348 return self[0].all_mailboxes
349
350 class MailboxList(TokenList):
351
352 token_type = 'mailbox-list'
353
354 @property
355 def mailboxes(self):
356 return [x for x in self if x.token_type=='mailbox']
357
358 @property
359 def all_mailboxes(self):
360 return [x for x in self
361 if x.token_type in ('mailbox', 'invalid-mailbox')]
362
363
364 class GroupList(TokenList):
365
366 token_type = 'group-list'
367
368 @property
369 def mailboxes(self):
370 if not self or self[0].token_type != 'mailbox-list':
371 return []
372 return self[0].mailboxes
373
374 @property
375 def all_mailboxes(self):
376 if not self or self[0].token_type != 'mailbox-list':
377 return []
378 return self[0].all_mailboxes
379
380
381 class Group(TokenList):
382
383 token_type = "group"
384
385 @property
386 def mailboxes(self):
387 if self[2].token_type != 'group-list':
388 return []
389 return self[2].mailboxes
390
391 @property
392 def all_mailboxes(self):
393 if self[2].token_type != 'group-list':
394 return []
395 return self[2].all_mailboxes
396
397 @property
398 def display_name(self):
399 return self[0].display_name
400
401
402 class NameAddr(TokenList):
403
404 token_type = 'name-addr'
405
406 @property
407 def display_name(self):
408 if len(self) == 1:
409 return None
410 return self[0].display_name
411
412 @property
413 def local_part(self):
414 return self[-1].local_part
415
416 @property
417 def domain(self):
418 return self[-1].domain
419
420 @property
421 def route(self):
422 return self[-1].route
423
424 @property
425 def addr_spec(self):
426 return self[-1].addr_spec
427
428
429 class AngleAddr(TokenList):
430
431 token_type = 'angle-addr'
432
433 @property
434 def local_part(self):
435 for x in self:
436 if x.token_type == 'addr-spec':
437 return x.local_part
438
439 @property
440 def domain(self):
441 for x in self:
442 if x.token_type == 'addr-spec':
443 return x.domain
444
445 @property
446 def route(self):
447 for x in self:
448 if x.token_type == 'obs-route':
449 return x.domains
450
451 @property
452 def addr_spec(self):
453 for x in self:
454 if x.token_type == 'addr-spec':
455 if x.local_part:
456 return x.addr_spec
457 else:
458 return quote_string(x.local_part) + x.addr_spec
459 else:
460 return '<>'
461
462
463 class ObsRoute(TokenList):
464
465 token_type = 'obs-route'
466
467 @property
468 def domains(self):
469 return [x.domain for x in self if x.token_type == 'domain']
470
471
472 class Mailbox(TokenList):
473
474 token_type = 'mailbox'
475
476 @property
477 def display_name(self):
478 if self[0].token_type == 'name-addr':
479 return self[0].display_name
480
481 @property
482 def local_part(self):
483 return self[0].local_part
484
485 @property
486 def domain(self):
487 return self[0].domain
488
489 @property
490 def route(self):
491 if self[0].token_type == 'name-addr':
492 return self[0].route
493
494 @property
495 def addr_spec(self):
496 return self[0].addr_spec
497
498
499 class InvalidMailbox(TokenList):
500
501 token_type = 'invalid-mailbox'
502
503 @property
504 def display_name(self):
505 return None
506
507 local_part = domain = route = addr_spec = display_name
508
509
510 class Domain(TokenList):
511
512 token_type = 'domain'
513 as_ew_allowed = False
514
515 @property
516 def domain(self):
517 return ''.join(super().value.split())
518
519
520 class DotAtom(TokenList):
521 token_type = 'dot-atom'
522
523
524 class DotAtomText(TokenList):
525 token_type = 'dot-atom-text'
526 as_ew_allowed = True
527
528
529 class NoFoldLiteral(TokenList):
530 token_type = 'no-fold-literal'
531 as_ew_allowed = False
532
533
534 class AddrSpec(TokenList):
535
536 token_type = 'addr-spec'
537 as_ew_allowed = False
538
539 @property
540 def local_part(self):
541 return self[0].local_part
542
543 @property
544 def domain(self):
545 if len(self) < 3:
546 return None
547 return self[-1].domain
548
549 @property
550 def value(self):
551 if len(self) < 3:
552 return self[0].value
553 return self[0].value.rstrip()+self[1].value+self[2].value.lstrip()
554
555 @property
556 def addr_spec(self):
557 nameset = set(self.local_part)
558 if len(nameset) > len(nameset-DOT_ATOM_ENDS):
559 lp = quote_string(self.local_part)
560 else:
561 lp = self.local_part
562 if self.domain is not None:
563 return lp + '@' + self.domain
564 return lp
565
566
567 class ObsLocalPart(TokenList):
568
569 token_type = 'obs-local-part'
570 as_ew_allowed = False
571
572
573 class DisplayName(Phrase):
574
575 token_type = 'display-name'
576 ew_combine_allowed = False
577
578 @property
579 def display_name(self):
580 res = TokenList(self)
581 if len(res) == 0:
582 return res.value
583 if res[0].token_type == 'cfws':
584 res.pop(0)
585 else:
586 if (isinstance(res[0], TokenList) and
587 res[0][0].token_type == 'cfws'):
588 res[0] = TokenList(res[0][1:])
589 if res[-1].token_type == 'cfws':
590 res.pop()
591 else:
592 if (isinstance(res[-1], TokenList) and
593 res[-1][-1].token_type == 'cfws'):
594 res[-1] = TokenList(res[-1][:-1])
595 return res.value
596
597 @property
598 def value(self):
599 quote = False
600 if self.defects:
601 quote = True
602 else:
603 for x in self:
604 if x.token_type == 'quoted-string':
605 quote = True
606 if len(self) != 0 and quote:
607 pre = post = ''
608 if (self[0].token_type == 'cfws' or
609 isinstance(self[0], TokenList) and
610 self[0][0].token_type == 'cfws'):
611 pre = ' '
612 if (self[-1].token_type == 'cfws' or
613 isinstance(self[-1], TokenList) and
614 self[-1][-1].token_type == 'cfws'):
615 post = ' '
616 return pre+quote_string(self.display_name)+post
617 else:
618 return super().value
619
620
621 class LocalPart(TokenList):
622
623 token_type = 'local-part'
624 as_ew_allowed = False
625
626 @property
627 def value(self):
628 if self[0].token_type == "quoted-string":
629 return self[0].quoted_value
630 else:
631 return self[0].value
632
633 @property
634 def local_part(self):
635 # Strip whitespace from front, back, and around dots.
636 res = [DOT]
637 last = DOT
638 last_is_tl = False
639 for tok in self[0] + [DOT]:
640 if tok.token_type == 'cfws':
641 continue
642 if (last_is_tl and tok.token_type == 'dot' and last and
643 last[-1].token_type == 'cfws'):
644 res[-1] = TokenList(last[:-1])
645 is_tl = isinstance(tok, TokenList)
646 if (is_tl and last.token_type == 'dot' and tok and
647 tok[0].token_type == 'cfws'):
648 res.append(TokenList(tok[1:]))
649 else:
650 res.append(tok)
651 last = res[-1]
652 last_is_tl = is_tl
653 res = TokenList(res[1:-1])
654 return res.value
655
656
657 class DomainLiteral(TokenList):
658
659 token_type = 'domain-literal'
660 as_ew_allowed = False
661
662 @property
663 def domain(self):
664 return ''.join(super().value.split())
665
666 @property
667 def ip(self):
668 for x in self:
669 if x.token_type == 'ptext':
670 return x.value
671
672
673 class MIMEVersion(TokenList):
674
675 token_type = 'mime-version'
676 major = None
677 minor = None
678
679
680 class Parameter(TokenList):
681
682 token_type = 'parameter'
683 sectioned = False
684 extended = False
685 charset = 'us-ascii'
686
687 @property
688 def section_number(self):
689 # Because the first token, the attribute (name) eats CFWS, the second
690 # token is always the section if there is one.
691 return self[1].number if self.sectioned else 0
692
693 @property
694 def param_value(self):
695 # This is part of the "handle quoted extended parameters" hack.
696 for token in self:
697 if token.token_type == 'value':
698 return token.stripped_value
699 if token.token_type == 'quoted-string':
700 for token in token:
701 if token.token_type == 'bare-quoted-string':
702 for token in token:
703 if token.token_type == 'value':
704 return token.stripped_value
705 return ''
706
707
708 class InvalidParameter(Parameter):
709
710 token_type = 'invalid-parameter'
711
712
713 class Attribute(TokenList):
714
715 token_type = 'attribute'
716
717 @property
718 def stripped_value(self):
719 for token in self:
720 if token.token_type.endswith('attrtext'):
721 return token.value
722
723 class Section(TokenList):
724
725 token_type = 'section'
726 number = None
727
728
729 class Value(TokenList):
730
731 token_type = 'value'
732
733 @property
734 def stripped_value(self):
735 token = self[0]
736 if token.token_type == 'cfws':
737 token = self[1]
738 if token.token_type.endswith(
739 ('quoted-string', 'attribute', 'extended-attribute')):
740 return token.stripped_value
741 return self.value
742
743
744 class MimeParameters(TokenList):
745
746 token_type = 'mime-parameters'
747 syntactic_break = False
748
749 @property
750 def params(self):
751 # The RFC specifically states that the ordering of parameters is not
752 # guaranteed and may be reordered by the transport layer. So we have
753 # to assume the RFC 2231 pieces can come in any order. However, we
754 # output them in the order that we first see a given name, which gives
755 # us a stable __str__.
756 params = {} # Using order preserving dict from Python 3.7+
757 for token in self:
758 if not token.token_type.endswith('parameter'):
759 continue
760 if token[0].token_type != 'attribute':
761 continue
762 name = token[0].value.strip()
763 if name not in params:
764 params[name] = []
765 params[name].append((token.section_number, token))
766 for name, parts in params.items():
767 parts = sorted(parts, key=itemgetter(0))
768 first_param = parts[0][1]
769 charset = first_param.charset
770 # Our arbitrary error recovery is to ignore duplicate parameters,
771 # to use appearance order if there are duplicate rfc 2231 parts,
772 # and to ignore gaps. This mimics the error recovery of get_param.
773 if not first_param.extended and len(parts) > 1:
774 if parts[1][0] == 0:
775 parts[1][1].defects.append(errors.InvalidHeaderDefect(
776 'duplicate parameter name; duplicate(s) ignored'))
777 parts = parts[:1]
778 # Else assume the *0* was missing...note that this is different
779 # from get_param, but we registered a defect for this earlier.
780 value_parts = []
781 i = 0
782 for section_number, param in parts:
783 if section_number != i:
784 # We could get fancier here and look for a complete
785 # duplicate extended parameter and ignore the second one
786 # seen. But we're not doing that. The old code didn't.
787 if not param.extended:
788 param.defects.append(errors.InvalidHeaderDefect(
789 'duplicate parameter name; duplicate ignored'))
790 continue
791 else:
792 param.defects.append(errors.InvalidHeaderDefect(
793 "inconsistent RFC2231 parameter numbering"))
794 i += 1
795 value = param.param_value
796 if param.extended:
797 try:
798 value = urllib.parse.unquote_to_bytes(value)
799 except UnicodeEncodeError:
800 # source had surrogate escaped bytes. What we do now
801 # is a bit of an open question. I'm not sure this is
802 # the best choice, but it is what the old algorithm did
803 value = urllib.parse.unquote(value, encoding='latin-1')
804 else:
805 try:
806 value = value.decode(charset, 'surrogateescape')
807 except (LookupError, UnicodeEncodeError):
808 # XXX: there should really be a custom defect for
809 # unknown character set to make it easy to find,
810 # because otherwise unknown charset is a silent
811 # failure.
812 value = value.decode('us-ascii', 'surrogateescape')
813 if utils._has_surrogates(value):
814 param.defects.append(errors.UndecodableBytesDefect())
815 value_parts.append(value)
816 value = ''.join(value_parts)
817 yield name, value
818
819 def __str__(self):
820 params = []
821 for name, value in self.params:
822 if value:
823 params.append('{}={}'.format(name, quote_string(value)))
824 else:
825 params.append(name)
826 params = '; '.join(params)
827 return ' ' + params if params else ''
828
829
830 class ParameterizedHeaderValue(TokenList):
831
832 # Set this false so that the value doesn't wind up on a new line even
833 # if it and the parameters would fit there but not on the first line.
834 syntactic_break = False
835
836 @property
837 def params(self):
838 for token in reversed(self):
839 if token.token_type == 'mime-parameters':
840 return token.params
841 return {}
842
843
844 class ContentType(ParameterizedHeaderValue):
845 token_type = 'content-type'
846 as_ew_allowed = False
847 maintype = 'text'
848 subtype = 'plain'
849
850
851 class ContentDisposition(ParameterizedHeaderValue):
852 token_type = 'content-disposition'
853 as_ew_allowed = False
854 content_disposition = None
855
856
857 class ContentTransferEncoding(TokenList):
858 token_type = 'content-transfer-encoding'
859 as_ew_allowed = False
860 cte = '7bit'
861
862
863 class HeaderLabel(TokenList):
864 token_type = 'header-label'
865 as_ew_allowed = False
866
867
868 class MsgID(TokenList):
869 token_type = 'msg-id'
870 as_ew_allowed = False
871
872 def fold(self, policy):
873 # message-id tokens may not be folded.
874 return str(self) + policy.linesep
875
876
877 class MessageID(MsgID):
878 token_type = 'message-id'
879
880
881 class InvalidMessageID(MessageID):
882 token_type = 'invalid-message-id'
883
884 class MessageIDList(TokenList):
885 token_type = 'message-id-list'
886
887 @property
888 def message_ids(self):
889 return [x for x in self if x.token_type=='msg-id']
890
891 class Header(TokenList):
892 token_type = 'header'
893
894
895 #
896 # Terminal classes and instances
897 #
898
899 class Terminal(str):
900
901 as_ew_allowed = True
902 ew_combine_allowed = True
903 syntactic_break = True
904
905 def __new__(cls, value, token_type):
906 self = super().__new__(cls, value)
907 self.token_type = token_type
908 self.defects = []
909 return self
910
911 def __repr__(self):
912 return "{}({})".format(self.__class__.__name__, super().__repr__())
913
914 def pprint(self):
915 print(self.__class__.__name__ + '/' + self.token_type)
916
917 @property
918 def all_defects(self):
919 return list(self.defects)
920
921 def _pp(self, indent=''):
922 return ["{}{}/{}({}){}".format(
923 indent,
924 self.__class__.__name__,
925 self.token_type,
926 super().__repr__(),
927 '' if not self.defects else ' {}'.format(self.defects),
928 )]
929
930 def pop_trailing_ws(self):
931 # This terminates the recursion.
932 return None
933
934 @property
935 def comments(self):
936 return []
937
938 def __getnewargs__(self):
939 return(str(self), self.token_type)
940
941
942 class WhiteSpaceTerminal(Terminal):
943
944 @property
945 def value(self):
946 return ' '
947
948 def startswith_fws(self):
949 return self and self[0] in WSP
950
951
952 class ValueTerminal(Terminal):
953
954 @property
955 def value(self):
956 return self
957
958 def startswith_fws(self):
959 return False
960
961
962 class EWWhiteSpaceTerminal(WhiteSpaceTerminal):
963
964 @property
965 def value(self):
966 return ''
967
968 def __str__(self):
969 return ''
970
971
972 class _InvalidEwError(errors.HeaderParseError):
973 """Invalid encoded word found while parsing headers."""
974
975
976 # XXX these need to become classes and used as instances so
977 # that a program can't change them in a parse tree and screw
978 # up other parse trees. Maybe should have tests for that, too.
979 DOT = ValueTerminal('.', 'dot')
980 ListSeparator = ValueTerminal(',', 'list-separator')
981 ListSeparator.as_ew_allowed = False
982 ListSeparator.syntactic_break = False
983 RouteComponentMarker = ValueTerminal('@', 'route-component-marker')
984
985 #
986 # Parser
987 #
988
989 # Parse strings according to RFC822/2047/2822/5322 rules.
990 #
991 # This is a stateless parser. Each get_XXX function accepts a string and
992 # returns either a Terminal or a TokenList representing the RFC object named
993 # by the method and a string containing the remaining unparsed characters
994 # from the input. Thus a parser method consumes the next syntactic construct
995 # of a given type and returns a token representing the construct plus the
996 # unparsed remainder of the input string.
997 #
998 # For example, if the first element of a structured header is a 'phrase',
999 # then:
1000 #
1001 # phrase, value = get_phrase(value)
1002 #
1003 # returns the complete phrase from the start of the string value, plus any
1004 # characters left in the string after the phrase is removed.
1005
1006 _wsp_splitter = re.compile(r'([{}]+)'.format(''.join(WSP))).split
1007 _non_atom_end_matcher = re.compile(r"[^{}]+".format(
1008 re.escape(''.join(ATOM_ENDS)))).match
1009 _non_printable_finder = re.compile(r"[\x00-\x20\x7F]").findall
1010 _non_token_end_matcher = re.compile(r"[^{}]+".format(
1011 re.escape(''.join(TOKEN_ENDS)))).match
1012 _non_attribute_end_matcher = re.compile(r"[^{}]+".format(
1013 re.escape(''.join(ATTRIBUTE_ENDS)))).match
1014 _non_extended_attribute_end_matcher = re.compile(r"[^{}]+".format(
1015 re.escape(''.join(EXTENDED_ATTRIBUTE_ENDS)))).match
1016
1017 def _validate_xtext(xtext):
1018 """If input token contains ASCII non-printables, register a defect."""
1019
1020 non_printables = _non_printable_finder(xtext)
1021 if non_printables:
1022 xtext.defects.append(errors.NonPrintableDefect(non_printables))
1023 if utils._has_surrogates(xtext):
1024 xtext.defects.append(errors.UndecodableBytesDefect(
1025 "Non-ASCII characters found in header token"))
1026
1027 def _get_ptext_to_endchars(value, endchars):
1028 """Scan printables/quoted-pairs until endchars and return unquoted ptext.
1029
1030 This function turns a run of qcontent, ccontent-without-comments, or
1031 dtext-with-quoted-printables into a single string by unquoting any
1032 quoted printables. It returns the string, the remaining value, and
1033 a flag that is True iff there were any quoted printables decoded.
1034
1035 """
1036 if not value:
1037 return '', '', False
1038 fragment, *remainder = _wsp_splitter(value, 1)
1039 vchars = []
1040 escape = False
1041 had_qp = False
1042 for pos in range(len(fragment)):
1043 if fragment[pos] == '\\':
1044 if escape:
1045 escape = False
1046 had_qp = True
1047 else:
1048 escape = True
1049 continue
1050 if escape:
1051 escape = False
1052 elif fragment[pos] in endchars:
1053 break
1054 vchars.append(fragment[pos])
1055 else:
1056 pos = pos + 1
1057 return ''.join(vchars), ''.join([fragment[pos:]] + remainder), had_qp
1058
1059 def get_fws(value):
1060 """FWS = 1*WSP
1061
1062 This isn't the RFC definition. We're using fws to represent tokens where
1063 folding can be done, but when we are parsing the *un*folding has already
1064 been done so we don't need to watch out for CRLF.
1065
1066 """
1067 newvalue = value.lstrip()
1068 fws = WhiteSpaceTerminal(value[:len(value)-len(newvalue)], 'fws')
1069 return fws, newvalue
1070
1071 def get_encoded_word(value, terminal_type='vtext'):
1072 """ encoded-word = "=?" charset "?" encoding "?" encoded-text "?="
1073
1074 """
1075 ew = EncodedWord()
1076 if not value.startswith('=?'):
1077 raise errors.HeaderParseError(
1078 "expected encoded word but found {}".format(value))
1079 tok, *remainder = value[2:].split('?=', 1)
1080 if tok == value[2:]:
1081 raise errors.HeaderParseError(
1082 "expected encoded word but found {}".format(value))
1083 remstr = ''.join(remainder)
1084 if (len(remstr) > 1 and
1085 remstr[0] in hexdigits and
1086 remstr[1] in hexdigits and
1087 tok.count('?') < 2):
1088 # The ? after the CTE was followed by an encoded word escape (=XX).
1089 rest, *remainder = remstr.split('?=', 1)
1090 tok = tok + '?=' + rest
1091 if len(tok.split()) > 1:
1092 ew.defects.append(errors.InvalidHeaderDefect(
1093 "whitespace inside encoded word"))
1094 ew.cte = value
1095 value = ''.join(remainder)
1096 try:
1097 text, charset, lang, defects = _ew.decode('=?' + tok + '?=')
1098 except (ValueError, KeyError):
1099 raise _InvalidEwError(
1100 "encoded word format invalid: '{}'".format(ew.cte))
1101 ew.charset = charset
1102 ew.lang = lang
1103 ew.defects.extend(defects)
1104 while text:
1105 if text[0] in WSP:
1106 token, text = get_fws(text)
1107 ew.append(token)
1108 continue
1109 chars, *remainder = _wsp_splitter(text, 1)
1110 vtext = ValueTerminal(chars, terminal_type)
1111 _validate_xtext(vtext)
1112 ew.append(vtext)
1113 text = ''.join(remainder)
1114 # Encoded words should be followed by a WS
1115 if value and value[0] not in WSP:
1116 ew.defects.append(errors.InvalidHeaderDefect(
1117 "missing trailing whitespace after encoded-word"))
1118 return ew, value
1119
1120 def get_unstructured(value):
1121 """unstructured = (*([FWS] vchar) *WSP) / obs-unstruct
1122 obs-unstruct = *((*LF *CR *(obs-utext) *LF *CR)) / FWS)
1123 obs-utext = %d0 / obs-NO-WS-CTL / LF / CR
1124
1125 obs-NO-WS-CTL is control characters except WSP/CR/LF.
1126
1127 So, basically, we have printable runs, plus control characters or nulls in
1128 the obsolete syntax, separated by whitespace. Since RFC 2047 uses the
1129 obsolete syntax in its specification, but requires whitespace on either
1130 side of the encoded words, I can see no reason to need to separate the
1131 non-printable-non-whitespace from the printable runs if they occur, so we
1132 parse this into xtext tokens separated by WSP tokens.
1133
1134 Because an 'unstructured' value must by definition constitute the entire
1135 value, this 'get' routine does not return a remaining value, only the
1136 parsed TokenList.
1137
1138 """
1139 # XXX: but what about bare CR and LF? They might signal the start or
1140 # end of an encoded word. YAGNI for now, since our current parsers
1141 # will never send us strings with bare CR or LF.
1142
1143 unstructured = UnstructuredTokenList()
1144 while value:
1145 if value[0] in WSP:
1146 token, value = get_fws(value)
1147 unstructured.append(token)
1148 continue
1149 valid_ew = True
1150 if value.startswith('=?'):
1151 try:
1152 token, value = get_encoded_word(value, 'utext')
1153 except _InvalidEwError:
1154 valid_ew = False
1155 except errors.HeaderParseError:
1156 # XXX: Need to figure out how to register defects when
1157 # appropriate here.
1158 pass
1159 else:
1160 have_ws = True
1161 if len(unstructured) > 0:
1162 if unstructured[-1].token_type != 'fws':
1163 unstructured.defects.append(errors.InvalidHeaderDefect(
1164 "missing whitespace before encoded word"))
1165 have_ws = False
1166 if have_ws and len(unstructured) > 1:
1167 if unstructured[-2].token_type == 'encoded-word':
1168 unstructured[-1] = EWWhiteSpaceTerminal(
1169 unstructured[-1], 'fws')
1170 unstructured.append(token)
1171 continue
1172 tok, *remainder = _wsp_splitter(value, 1)
1173 # Split in the middle of an atom if there is a rfc2047 encoded word
1174 # which does not have WSP on both sides. The defect will be registered
1175 # the next time through the loop.
1176 # This needs to only be performed when the encoded word is valid;
1177 # otherwise, performing it on an invalid encoded word can cause
1178 # the parser to go in an infinite loop.
1179 if valid_ew and rfc2047_matcher.search(tok):
1180 tok, *remainder = value.partition('=?')
1181 vtext = ValueTerminal(tok, 'utext')
1182 _validate_xtext(vtext)
1183 unstructured.append(vtext)
1184 value = ''.join(remainder)
1185 return unstructured
1186
1187 def get_qp_ctext(value):
1188 r"""ctext = <printable ascii except \ ( )>
1189
1190 This is not the RFC ctext, since we are handling nested comments in comment
1191 and unquoting quoted-pairs here. We allow anything except the '()'
1192 characters, but if we find any ASCII other than the RFC defined printable
1193 ASCII, a NonPrintableDefect is added to the token's defects list. Since
1194 quoted pairs are converted to their unquoted values, what is returned is
1195 a 'ptext' token. In this case it is a WhiteSpaceTerminal, so it's value
1196 is ' '.
1197
1198 """
1199 ptext, value, _ = _get_ptext_to_endchars(value, '()')
1200 ptext = WhiteSpaceTerminal(ptext, 'ptext')
1201 _validate_xtext(ptext)
1202 return ptext, value
1203
1204 def get_qcontent(value):
1205 """qcontent = qtext / quoted-pair
1206
1207 We allow anything except the DQUOTE character, but if we find any ASCII
1208 other than the RFC defined printable ASCII, a NonPrintableDefect is
1209 added to the token's defects list. Any quoted pairs are converted to their
1210 unquoted values, so what is returned is a 'ptext' token. In this case it
1211 is a ValueTerminal.
1212
1213 """
1214 ptext, value, _ = _get_ptext_to_endchars(value, '"')
1215 ptext = ValueTerminal(ptext, 'ptext')
1216 _validate_xtext(ptext)
1217 return ptext, value
1218
1219 def get_atext(value):
1220 """atext = <matches _atext_matcher>
1221
1222 We allow any non-ATOM_ENDS in atext, but add an InvalidATextDefect to
1223 the token's defects list if we find non-atext characters.
1224 """
1225 m = _non_atom_end_matcher(value)
1226 if not m:
1227 raise errors.HeaderParseError(
1228 "expected atext but found '{}'".format(value))
1229 atext = m.group()
1230 value = value[len(atext):]
1231 atext = ValueTerminal(atext, 'atext')
1232 _validate_xtext(atext)
1233 return atext, value
1234
1235 def get_bare_quoted_string(value):
1236 """bare-quoted-string = DQUOTE *([FWS] qcontent) [FWS] DQUOTE
1237
1238 A quoted-string without the leading or trailing white space. Its
1239 value is the text between the quote marks, with whitespace
1240 preserved and quoted pairs decoded.
1241 """
1242 if not value or value[0] != '"':
1243 raise errors.HeaderParseError(
1244 "expected '\"' but found '{}'".format(value))
1245 bare_quoted_string = BareQuotedString()
1246 value = value[1:]
1247 if value and value[0] == '"':
1248 return bare_quoted_string, value[1:]
1249 while value and value[0] != '"':
1250 if value[0] in WSP:
1251 token, value = get_fws(value)
1252 elif value[:2] == '=?':
1253 valid_ew = False
1254 try:
1255 token, value = get_encoded_word(value)
1256 bare_quoted_string.defects.append(errors.InvalidHeaderDefect(
1257 "encoded word inside quoted string"))
1258 valid_ew = True
1259 except errors.HeaderParseError:
1260 token, value = get_qcontent(value)
1261 # Collapse the whitespace between two encoded words that occur in a
1262 # bare-quoted-string.
1263 if valid_ew and len(bare_quoted_string) > 1:
1264 if (bare_quoted_string[-1].token_type == 'fws' and
1265 bare_quoted_string[-2].token_type == 'encoded-word'):
1266 bare_quoted_string[-1] = EWWhiteSpaceTerminal(
1267 bare_quoted_string[-1], 'fws')
1268 else:
1269 token, value = get_qcontent(value)
1270 bare_quoted_string.append(token)
1271 if not value:
1272 bare_quoted_string.defects.append(errors.InvalidHeaderDefect(
1273 "end of header inside quoted string"))
1274 return bare_quoted_string, value
1275 return bare_quoted_string, value[1:]
1276
1277 def get_comment(value):
1278 """comment = "(" *([FWS] ccontent) [FWS] ")"
1279 ccontent = ctext / quoted-pair / comment
1280
1281 We handle nested comments here, and quoted-pair in our qp-ctext routine.
1282 """
1283 if value and value[0] != '(':
1284 raise errors.HeaderParseError(
1285 "expected '(' but found '{}'".format(value))
1286 comment = Comment()
1287 value = value[1:]
1288 while value and value[0] != ")":
1289 if value[0] in WSP:
1290 token, value = get_fws(value)
1291 elif value[0] == '(':
1292 token, value = get_comment(value)
1293 else:
1294 token, value = get_qp_ctext(value)
1295 comment.append(token)
1296 if not value:
1297 comment.defects.append(errors.InvalidHeaderDefect(
1298 "end of header inside comment"))
1299 return comment, value
1300 return comment, value[1:]
1301
1302 def get_cfws(value):
1303 """CFWS = (1*([FWS] comment) [FWS]) / FWS
1304
1305 """
1306 cfws = CFWSList()
1307 while value and value[0] in CFWS_LEADER:
1308 if value[0] in WSP:
1309 token, value = get_fws(value)
1310 else:
1311 token, value = get_comment(value)
1312 cfws.append(token)
1313 return cfws, value
1314
1315 def get_quoted_string(value):
1316 """quoted-string = [CFWS] <bare-quoted-string> [CFWS]
1317
1318 'bare-quoted-string' is an intermediate class defined by this
1319 parser and not by the RFC grammar. It is the quoted string
1320 without any attached CFWS.
1321 """
1322 quoted_string = QuotedString()
1323 if value and value[0] in CFWS_LEADER:
1324 token, value = get_cfws(value)
1325 quoted_string.append(token)
1326 token, value = get_bare_quoted_string(value)
1327 quoted_string.append(token)
1328 if value and value[0] in CFWS_LEADER:
1329 token, value = get_cfws(value)
1330 quoted_string.append(token)
1331 return quoted_string, value
1332
1333 def get_atom(value):
1334 """atom = [CFWS] 1*atext [CFWS]
1335
1336 An atom could be an rfc2047 encoded word.
1337 """
1338 atom = Atom()
1339 if value and value[0] in CFWS_LEADER:
1340 token, value = get_cfws(value)
1341 atom.append(token)
1342 if value and value[0] in ATOM_ENDS:
1343 raise errors.HeaderParseError(
1344 "expected atom but found '{}'".format(value))
1345 if value.startswith('=?'):
1346 try:
1347 token, value = get_encoded_word(value)
1348 except errors.HeaderParseError:
1349 # XXX: need to figure out how to register defects when
1350 # appropriate here.
1351 token, value = get_atext(value)
1352 else:
1353 token, value = get_atext(value)
1354 atom.append(token)
1355 if value and value[0] in CFWS_LEADER:
1356 token, value = get_cfws(value)
1357 atom.append(token)
1358 return atom, value
1359
1360 def get_dot_atom_text(value):
1361 """ dot-text = 1*atext *("." 1*atext)
1362
1363 """
1364 dot_atom_text = DotAtomText()
1365 if not value or value[0] in ATOM_ENDS:
1366 raise errors.HeaderParseError("expected atom at a start of "
1367 "dot-atom-text but found '{}'".format(value))
1368 while value and value[0] not in ATOM_ENDS:
1369 token, value = get_atext(value)
1370 dot_atom_text.append(token)
1371 if value and value[0] == '.':
1372 dot_atom_text.append(DOT)
1373 value = value[1:]
1374 if dot_atom_text[-1] is DOT:
1375 raise errors.HeaderParseError("expected atom at end of dot-atom-text "
1376 "but found '{}'".format('.'+value))
1377 return dot_atom_text, value
1378
1379 def get_dot_atom(value):
1380 """ dot-atom = [CFWS] dot-atom-text [CFWS]
1381
1382 Any place we can have a dot atom, we could instead have an rfc2047 encoded
1383 word.
1384 """
1385 dot_atom = DotAtom()
1386 if value[0] in CFWS_LEADER:
1387 token, value = get_cfws(value)
1388 dot_atom.append(token)
1389 if value.startswith('=?'):
1390 try:
1391 token, value = get_encoded_word(value)
1392 except errors.HeaderParseError:
1393 # XXX: need to figure out how to register defects when
1394 # appropriate here.
1395 token, value = get_dot_atom_text(value)
1396 else:
1397 token, value = get_dot_atom_text(value)
1398 dot_atom.append(token)
1399 if value and value[0] in CFWS_LEADER:
1400 token, value = get_cfws(value)
1401 dot_atom.append(token)
1402 return dot_atom, value
1403
1404 def get_word(value):
1405 """word = atom / quoted-string
1406
1407 Either atom or quoted-string may start with CFWS. We have to peel off this
1408 CFWS first to determine which type of word to parse. Afterward we splice
1409 the leading CFWS, if any, into the parsed sub-token.
1410
1411 If neither an atom or a quoted-string is found before the next special, a
1412 HeaderParseError is raised.
1413
1414 The token returned is either an Atom or a QuotedString, as appropriate.
1415 This means the 'word' level of the formal grammar is not represented in the
1416 parse tree; this is because having that extra layer when manipulating the
1417 parse tree is more confusing than it is helpful.
1418
1419 """
1420 if value[0] in CFWS_LEADER:
1421 leader, value = get_cfws(value)
1422 else:
1423 leader = None
1424 if not value:
1425 raise errors.HeaderParseError(
1426 "Expected 'atom' or 'quoted-string' but found nothing.")
1427 if value[0]=='"':
1428 token, value = get_quoted_string(value)
1429 elif value[0] in SPECIALS:
1430 raise errors.HeaderParseError("Expected 'atom' or 'quoted-string' "
1431 "but found '{}'".format(value))
1432 else:
1433 token, value = get_atom(value)
1434 if leader is not None:
1435 token[:0] = [leader]
1436 return token, value
1437
1438 def get_phrase(value):
1439 """ phrase = 1*word / obs-phrase
1440 obs-phrase = word *(word / "." / CFWS)
1441
1442 This means a phrase can be a sequence of words, periods, and CFWS in any
1443 order as long as it starts with at least one word. If anything other than
1444 words is detected, an ObsoleteHeaderDefect is added to the token's defect
1445 list. We also accept a phrase that starts with CFWS followed by a dot;
1446 this is registered as an InvalidHeaderDefect, since it is not supported by
1447 even the obsolete grammar.
1448
1449 """
1450 phrase = Phrase()
1451 try:
1452 token, value = get_word(value)
1453 phrase.append(token)
1454 except errors.HeaderParseError:
1455 phrase.defects.append(errors.InvalidHeaderDefect(
1456 "phrase does not start with word"))
1457 while value and value[0] not in PHRASE_ENDS:
1458 if value[0]=='.':
1459 phrase.append(DOT)
1460 phrase.defects.append(errors.ObsoleteHeaderDefect(
1461 "period in 'phrase'"))
1462 value = value[1:]
1463 else:
1464 try:
1465 token, value = get_word(value)
1466 except errors.HeaderParseError:
1467 if value[0] in CFWS_LEADER:
1468 token, value = get_cfws(value)
1469 phrase.defects.append(errors.ObsoleteHeaderDefect(
1470 "comment found without atom"))
1471 else:
1472 raise
1473 phrase.append(token)
1474 return phrase, value
1475
1476 def get_local_part(value):
1477 """ local-part = dot-atom / quoted-string / obs-local-part
1478
1479 """
1480 local_part = LocalPart()
1481 leader = None
1482 if value and value[0] in CFWS_LEADER:
1483 leader, value = get_cfws(value)
1484 if not value:
1485 raise errors.HeaderParseError(
1486 "expected local-part but found '{}'".format(value))
1487 try:
1488 token, value = get_dot_atom(value)
1489 except errors.HeaderParseError:
1490 try:
1491 token, value = get_word(value)
1492 except errors.HeaderParseError:
1493 if value[0] != '\\' and value[0] in PHRASE_ENDS:
1494 raise
1495 token = TokenList()
1496 if leader is not None:
1497 token[:0] = [leader]
1498 local_part.append(token)
1499 if value and (value[0]=='\\' or value[0] not in PHRASE_ENDS):
1500 obs_local_part, value = get_obs_local_part(str(local_part) + value)
1501 if obs_local_part.token_type == 'invalid-obs-local-part':
1502 local_part.defects.append(errors.InvalidHeaderDefect(
1503 "local-part is not dot-atom, quoted-string, or obs-local-part"))
1504 else:
1505 local_part.defects.append(errors.ObsoleteHeaderDefect(
1506 "local-part is not a dot-atom (contains CFWS)"))
1507 local_part[0] = obs_local_part
1508 try:
1509 local_part.value.encode('ascii')
1510 except UnicodeEncodeError:
1511 local_part.defects.append(errors.NonASCIILocalPartDefect(
1512 "local-part contains non-ASCII characters)"))
1513 return local_part, value
1514
1515 def get_obs_local_part(value):
1516 """ obs-local-part = word *("." word)
1517 """
1518 obs_local_part = ObsLocalPart()
1519 last_non_ws_was_dot = False
1520 while value and (value[0]=='\\' or value[0] not in PHRASE_ENDS):
1521 if value[0] == '.':
1522 if last_non_ws_was_dot:
1523 obs_local_part.defects.append(errors.InvalidHeaderDefect(
1524 "invalid repeated '.'"))
1525 obs_local_part.append(DOT)
1526 last_non_ws_was_dot = True
1527 value = value[1:]
1528 continue
1529 elif value[0]=='\\':
1530 obs_local_part.append(ValueTerminal(value[0],
1531 'misplaced-special'))
1532 value = value[1:]
1533 obs_local_part.defects.append(errors.InvalidHeaderDefect(
1534 "'\\' character outside of quoted-string/ccontent"))
1535 last_non_ws_was_dot = False
1536 continue
1537 if obs_local_part and obs_local_part[-1].token_type != 'dot':
1538 obs_local_part.defects.append(errors.InvalidHeaderDefect(
1539 "missing '.' between words"))
1540 try:
1541 token, value = get_word(value)
1542 last_non_ws_was_dot = False
1543 except errors.HeaderParseError:
1544 if value[0] not in CFWS_LEADER:
1545 raise
1546 token, value = get_cfws(value)
1547 obs_local_part.append(token)
1548 if not obs_local_part:
1549 raise errors.HeaderParseError(
1550 "expected obs-local-part but found '{}'".format(value))
1551 if (obs_local_part[0].token_type == 'dot' or
1552 obs_local_part[0].token_type=='cfws' and
1553 len(obs_local_part) > 1 and
1554 obs_local_part[1].token_type=='dot'):
1555 obs_local_part.defects.append(errors.InvalidHeaderDefect(
1556 "Invalid leading '.' in local part"))
1557 if (obs_local_part[-1].token_type == 'dot' or
1558 obs_local_part[-1].token_type=='cfws' and
1559 len(obs_local_part) > 1 and
1560 obs_local_part[-2].token_type=='dot'):
1561 obs_local_part.defects.append(errors.InvalidHeaderDefect(
1562 "Invalid trailing '.' in local part"))
1563 if obs_local_part.defects:
1564 obs_local_part.token_type = 'invalid-obs-local-part'
1565 return obs_local_part, value
1566
1567 def get_dtext(value):
1568 r""" dtext = <printable ascii except \ [ ]> / obs-dtext
1569 obs-dtext = obs-NO-WS-CTL / quoted-pair
1570
1571 We allow anything except the excluded characters, but if we find any
1572 ASCII other than the RFC defined printable ASCII, a NonPrintableDefect is
1573 added to the token's defects list. Quoted pairs are converted to their
1574 unquoted values, so what is returned is a ptext token, in this case a
1575 ValueTerminal. If there were quoted-printables, an ObsoleteHeaderDefect is
1576 added to the returned token's defect list.
1577
1578 """
1579 ptext, value, had_qp = _get_ptext_to_endchars(value, '[]')
1580 ptext = ValueTerminal(ptext, 'ptext')
1581 if had_qp:
1582 ptext.defects.append(errors.ObsoleteHeaderDefect(
1583 "quoted printable found in domain-literal"))
1584 _validate_xtext(ptext)
1585 return ptext, value
1586
1587 def _check_for_early_dl_end(value, domain_literal):
1588 if value:
1589 return False
1590 domain_literal.defects.append(errors.InvalidHeaderDefect(
1591 "end of input inside domain-literal"))
1592 domain_literal.append(ValueTerminal(']', 'domain-literal-end'))
1593 return True
1594
1595 def get_domain_literal(value):
1596 """ domain-literal = [CFWS] "[" *([FWS] dtext) [FWS] "]" [CFWS]
1597
1598 """
1599 domain_literal = DomainLiteral()
1600 if value[0] in CFWS_LEADER:
1601 token, value = get_cfws(value)
1602 domain_literal.append(token)
1603 if not value:
1604 raise errors.HeaderParseError("expected domain-literal")
1605 if value[0] != '[':
1606 raise errors.HeaderParseError("expected '[' at start of domain-literal "
1607 "but found '{}'".format(value))
1608 value = value[1:]
1609 domain_literal.append(ValueTerminal('[', 'domain-literal-start'))
1610 if _check_for_early_dl_end(value, domain_literal):
1611 return domain_literal, value
1612 if value[0] in WSP:
1613 token, value = get_fws(value)
1614 domain_literal.append(token)
1615 token, value = get_dtext(value)
1616 domain_literal.append(token)
1617 if _check_for_early_dl_end(value, domain_literal):
1618 return domain_literal, value
1619 if value[0] in WSP:
1620 token, value = get_fws(value)
1621 domain_literal.append(token)
1622 if _check_for_early_dl_end(value, domain_literal):
1623 return domain_literal, value
1624 if value[0] != ']':
1625 raise errors.HeaderParseError("expected ']' at end of domain-literal "
1626 "but found '{}'".format(value))
1627 domain_literal.append(ValueTerminal(']', 'domain-literal-end'))
1628 value = value[1:]
1629 if value and value[0] in CFWS_LEADER:
1630 token, value = get_cfws(value)
1631 domain_literal.append(token)
1632 return domain_literal, value
1633
1634 def get_domain(value):
1635 """ domain = dot-atom / domain-literal / obs-domain
1636 obs-domain = atom *("." atom))
1637
1638 """
1639 domain = Domain()
1640 leader = None
1641 if value and value[0] in CFWS_LEADER:
1642 leader, value = get_cfws(value)
1643 if not value:
1644 raise errors.HeaderParseError(
1645 "expected domain but found '{}'".format(value))
1646 if value[0] == '[':
1647 token, value = get_domain_literal(value)
1648 if leader is not None:
1649 token[:0] = [leader]
1650 domain.append(token)
1651 return domain, value
1652 try:
1653 token, value = get_dot_atom(value)
1654 except errors.HeaderParseError:
1655 token, value = get_atom(value)
1656 if value and value[0] == '@':
1657 raise errors.HeaderParseError('Invalid Domain')
1658 if leader is not None:
1659 token[:0] = [leader]
1660 domain.append(token)
1661 if value and value[0] == '.':
1662 domain.defects.append(errors.ObsoleteHeaderDefect(
1663 "domain is not a dot-atom (contains CFWS)"))
1664 if domain[0].token_type == 'dot-atom':
1665 domain[:] = domain[0]
1666 while value and value[0] == '.':
1667 domain.append(DOT)
1668 token, value = get_atom(value[1:])
1669 domain.append(token)
1670 return domain, value
1671
1672 def get_addr_spec(value):
1673 """ addr-spec = local-part "@" domain
1674
1675 """
1676 addr_spec = AddrSpec()
1677 token, value = get_local_part(value)
1678 addr_spec.append(token)
1679 if not value or value[0] != '@':
1680 addr_spec.defects.append(errors.InvalidHeaderDefect(
1681 "addr-spec local part with no domain"))
1682 return addr_spec, value
1683 addr_spec.append(ValueTerminal('@', 'address-at-symbol'))
1684 token, value = get_domain(value[1:])
1685 addr_spec.append(token)
1686 return addr_spec, value
1687
1688 def get_obs_route(value):
1689 """ obs-route = obs-domain-list ":"
1690 obs-domain-list = *(CFWS / ",") "@" domain *("," [CFWS] ["@" domain])
1691
1692 Returns an obs-route token with the appropriate sub-tokens (that is,
1693 there is no obs-domain-list in the parse tree).
1694 """
1695 obs_route = ObsRoute()
1696 while value and (value[0]==',' or value[0] in CFWS_LEADER):
1697 if value[0] in CFWS_LEADER:
1698 token, value = get_cfws(value)
1699 obs_route.append(token)
1700 elif value[0] == ',':
1701 obs_route.append(ListSeparator)
1702 value = value[1:]
1703 if not value or value[0] != '@':
1704 raise errors.HeaderParseError(
1705 "expected obs-route domain but found '{}'".format(value))
1706 obs_route.append(RouteComponentMarker)
1707 token, value = get_domain(value[1:])
1708 obs_route.append(token)
1709 while value and value[0]==',':
1710 obs_route.append(ListSeparator)
1711 value = value[1:]
1712 if not value:
1713 break
1714 if value[0] in CFWS_LEADER:
1715 token, value = get_cfws(value)
1716 obs_route.append(token)
1717 if not value:
1718 break
1719 if value[0] == '@':
1720 obs_route.append(RouteComponentMarker)
1721 token, value = get_domain(value[1:])
1722 obs_route.append(token)
1723 if not value:
1724 raise errors.HeaderParseError("end of header while parsing obs-route")
1725 if value[0] != ':':
1726 raise errors.HeaderParseError( "expected ':' marking end of "
1727 "obs-route but found '{}'".format(value))
1728 obs_route.append(ValueTerminal(':', 'end-of-obs-route-marker'))
1729 return obs_route, value[1:]
1730
1731 def get_angle_addr(value):
1732 """ angle-addr = [CFWS] "<" addr-spec ">" [CFWS] / obs-angle-addr
1733 obs-angle-addr = [CFWS] "<" obs-route addr-spec ">" [CFWS]
1734
1735 """
1736 angle_addr = AngleAddr()
1737 if value and value[0] in CFWS_LEADER:
1738 token, value = get_cfws(value)
1739 angle_addr.append(token)
1740 if not value or value[0] != '<':
1741 raise errors.HeaderParseError(
1742 "expected angle-addr but found '{}'".format(value))
1743 angle_addr.append(ValueTerminal('<', 'angle-addr-start'))
1744 value = value[1:]
1745 # Although it is not legal per RFC5322, SMTP uses '<>' in certain
1746 # circumstances.
1747 if value and value[0] == '>':
1748 angle_addr.append(ValueTerminal('>', 'angle-addr-end'))
1749 angle_addr.defects.append(errors.InvalidHeaderDefect(
1750 "null addr-spec in angle-addr"))
1751 value = value[1:]
1752 return angle_addr, value
1753 try:
1754 token, value = get_addr_spec(value)
1755 except errors.HeaderParseError:
1756 try:
1757 token, value = get_obs_route(value)
1758 angle_addr.defects.append(errors.ObsoleteHeaderDefect(
1759 "obsolete route specification in angle-addr"))
1760 except errors.HeaderParseError:
1761 raise errors.HeaderParseError(
1762 "expected addr-spec or obs-route but found '{}'".format(value))
1763 angle_addr.append(token)
1764 token, value = get_addr_spec(value)
1765 angle_addr.append(token)
1766 if value and value[0] == '>':
1767 value = value[1:]
1768 else:
1769 angle_addr.defects.append(errors.InvalidHeaderDefect(
1770 "missing trailing '>' on angle-addr"))
1771 angle_addr.append(ValueTerminal('>', 'angle-addr-end'))
1772 if value and value[0] in CFWS_LEADER:
1773 token, value = get_cfws(value)
1774 angle_addr.append(token)
1775 return angle_addr, value
1776
1777 def get_display_name(value):
1778 """ display-name = phrase
1779
1780 Because this is simply a name-rule, we don't return a display-name
1781 token containing a phrase, but rather a display-name token with
1782 the content of the phrase.
1783
1784 """
1785 display_name = DisplayName()
1786 token, value = get_phrase(value)
1787 display_name.extend(token[:])
1788 display_name.defects = token.defects[:]
1789 return display_name, value
1790
1791
1792 def get_name_addr(value):
1793 """ name-addr = [display-name] angle-addr
1794
1795 """
1796 name_addr = NameAddr()
1797 # Both the optional display name and the angle-addr can start with cfws.
1798 leader = None
1799 if not value:
1800 raise errors.HeaderParseError(
1801 "expected name-addr but found '{}'".format(value))
1802 if value[0] in CFWS_LEADER:
1803 leader, value = get_cfws(value)
1804 if not value:
1805 raise errors.HeaderParseError(
1806 "expected name-addr but found '{}'".format(leader))
1807 if value[0] != '<':
1808 if value[0] in PHRASE_ENDS:
1809 raise errors.HeaderParseError(
1810 "expected name-addr but found '{}'".format(value))
1811 token, value = get_display_name(value)
1812 if not value:
1813 raise errors.HeaderParseError(
1814 "expected name-addr but found '{}'".format(token))
1815 if leader is not None:
1816 if isinstance(token[0], TokenList):
1817 token[0][:0] = [leader]
1818 else:
1819 token[:0] = [leader]
1820 leader = None
1821 name_addr.append(token)
1822 token, value = get_angle_addr(value)
1823 if leader is not None:
1824 token[:0] = [leader]
1825 name_addr.append(token)
1826 return name_addr, value
1827
1828 def get_mailbox(value):
1829 """ mailbox = name-addr / addr-spec
1830
1831 """
1832 # The only way to figure out if we are dealing with a name-addr or an
1833 # addr-spec is to try parsing each one.
1834 mailbox = Mailbox()
1835 try:
1836 token, value = get_name_addr(value)
1837 except errors.HeaderParseError:
1838 try:
1839 token, value = get_addr_spec(value)
1840 except errors.HeaderParseError:
1841 raise errors.HeaderParseError(
1842 "expected mailbox but found '{}'".format(value))
1843 if any(isinstance(x, errors.InvalidHeaderDefect)
1844 for x in token.all_defects):
1845 mailbox.token_type = 'invalid-mailbox'
1846 mailbox.append(token)
1847 return mailbox, value
1848
1849 def get_invalid_mailbox(value, endchars):
1850 """ Read everything up to one of the chars in endchars.
1851
1852 This is outside the formal grammar. The InvalidMailbox TokenList that is
1853 returned acts like a Mailbox, but the data attributes are None.
1854
1855 """
1856 invalid_mailbox = InvalidMailbox()
1857 while value and value[0] not in endchars:
1858 if value[0] in PHRASE_ENDS:
1859 invalid_mailbox.append(ValueTerminal(value[0],
1860 'misplaced-special'))
1861 value = value[1:]
1862 else:
1863 token, value = get_phrase(value)
1864 invalid_mailbox.append(token)
1865 return invalid_mailbox, value
1866
1867 def get_mailbox_list(value):
1868 """ mailbox-list = (mailbox *("," mailbox)) / obs-mbox-list
1869 obs-mbox-list = *([CFWS] ",") mailbox *("," [mailbox / CFWS])
1870
1871 For this routine we go outside the formal grammar in order to improve error
1872 handling. We recognize the end of the mailbox list only at the end of the
1873 value or at a ';' (the group terminator). This is so that we can turn
1874 invalid mailboxes into InvalidMailbox tokens and continue parsing any
1875 remaining valid mailboxes. We also allow all mailbox entries to be null,
1876 and this condition is handled appropriately at a higher level.
1877
1878 """
1879 mailbox_list = MailboxList()
1880 while value and value[0] != ';':
1881 try:
1882 token, value = get_mailbox(value)
1883 mailbox_list.append(token)
1884 except errors.HeaderParseError:
1885 leader = None
1886 if value[0] in CFWS_LEADER:
1887 leader, value = get_cfws(value)
1888 if not value or value[0] in ',;':
1889 mailbox_list.append(leader)
1890 mailbox_list.defects.append(errors.ObsoleteHeaderDefect(
1891 "empty element in mailbox-list"))
1892 else:
1893 token, value = get_invalid_mailbox(value, ',;')
1894 if leader is not None:
1895 token[:0] = [leader]
1896 mailbox_list.append(token)
1897 mailbox_list.defects.append(errors.InvalidHeaderDefect(
1898 "invalid mailbox in mailbox-list"))
1899 elif value[0] == ',':
1900 mailbox_list.defects.append(errors.ObsoleteHeaderDefect(
1901 "empty element in mailbox-list"))
1902 else:
1903 token, value = get_invalid_mailbox(value, ',;')
1904 if leader is not None:
1905 token[:0] = [leader]
1906 mailbox_list.append(token)
1907 mailbox_list.defects.append(errors.InvalidHeaderDefect(
1908 "invalid mailbox in mailbox-list"))
1909 if value and value[0] not in ',;':
1910 # Crap after mailbox; treat it as an invalid mailbox.
1911 # The mailbox info will still be available.
1912 mailbox = mailbox_list[-1]
1913 mailbox.token_type = 'invalid-mailbox'
1914 token, value = get_invalid_mailbox(value, ',;')
1915 mailbox.extend(token)
1916 mailbox_list.defects.append(errors.InvalidHeaderDefect(
1917 "invalid mailbox in mailbox-list"))
1918 if value and value[0] == ',':
1919 mailbox_list.append(ListSeparator)
1920 value = value[1:]
1921 return mailbox_list, value
1922
1923
1924 def get_group_list(value):
1925 """ group-list = mailbox-list / CFWS / obs-group-list
1926 obs-group-list = 1*([CFWS] ",") [CFWS]
1927
1928 """
1929 group_list = GroupList()
1930 if not value:
1931 group_list.defects.append(errors.InvalidHeaderDefect(
1932 "end of header before group-list"))
1933 return group_list, value
1934 leader = None
1935 if value and value[0] in CFWS_LEADER:
1936 leader, value = get_cfws(value)
1937 if not value:
1938 # This should never happen in email parsing, since CFWS-only is a
1939 # legal alternative to group-list in a group, which is the only
1940 # place group-list appears.
1941 group_list.defects.append(errors.InvalidHeaderDefect(
1942 "end of header in group-list"))
1943 group_list.append(leader)
1944 return group_list, value
1945 if value[0] == ';':
1946 group_list.append(leader)
1947 return group_list, value
1948 token, value = get_mailbox_list(value)
1949 if len(token.all_mailboxes)==0:
1950 if leader is not None:
1951 group_list.append(leader)
1952 group_list.extend(token)
1953 group_list.defects.append(errors.ObsoleteHeaderDefect(
1954 "group-list with empty entries"))
1955 return group_list, value
1956 if leader is not None:
1957 token[:0] = [leader]
1958 group_list.append(token)
1959 return group_list, value
1960
1961 def get_group(value):
1962 """ group = display-name ":" [group-list] ";" [CFWS]
1963
1964 """
1965 group = Group()
1966 token, value = get_display_name(value)
1967 if not value or value[0] != ':':
1968 raise errors.HeaderParseError("expected ':' at end of group "
1969 "display name but found '{}'".format(value))
1970 group.append(token)
1971 group.append(ValueTerminal(':', 'group-display-name-terminator'))
1972 value = value[1:]
1973 if value and value[0] == ';':
1974 group.append(ValueTerminal(';', 'group-terminator'))
1975 return group, value[1:]
1976 token, value = get_group_list(value)
1977 group.append(token)
1978 if not value:
1979 group.defects.append(errors.InvalidHeaderDefect(
1980 "end of header in group"))
1981 elif value[0] != ';':
1982 raise errors.HeaderParseError(
1983 "expected ';' at end of group but found {}".format(value))
1984 group.append(ValueTerminal(';', 'group-terminator'))
1985 value = value[1:]
1986 if value and value[0] in CFWS_LEADER:
1987 token, value = get_cfws(value)
1988 group.append(token)
1989 return group, value
1990
1991 def get_address(value):
1992 """ address = mailbox / group
1993
1994 Note that counter-intuitively, an address can be either a single address or
1995 a list of addresses (a group). This is why the returned Address object has
1996 a 'mailboxes' attribute which treats a single address as a list of length
1997 one. When you need to differentiate between to two cases, extract the single
1998 element, which is either a mailbox or a group token.
1999
2000 """
2001 # The formal grammar isn't very helpful when parsing an address. mailbox
2002 # and group, especially when allowing for obsolete forms, start off very
2003 # similarly. It is only when you reach one of @, <, or : that you know
2004 # what you've got. So, we try each one in turn, starting with the more
2005 # likely of the two. We could perhaps make this more efficient by looking
2006 # for a phrase and then branching based on the next character, but that
2007 # would be a premature optimization.
2008 address = Address()
2009 try:
2010 token, value = get_group(value)
2011 except errors.HeaderParseError:
2012 try:
2013 token, value = get_mailbox(value)
2014 except errors.HeaderParseError:
2015 raise errors.HeaderParseError(
2016 "expected address but found '{}'".format(value))
2017 address.append(token)
2018 return address, value
2019
2020 def get_address_list(value):
2021 """ address_list = (address *("," address)) / obs-addr-list
2022 obs-addr-list = *([CFWS] ",") address *("," [address / CFWS])
2023
2024 We depart from the formal grammar here by continuing to parse until the end
2025 of the input, assuming the input to be entirely composed of an
2026 address-list. This is always true in email parsing, and allows us
2027 to skip invalid addresses to parse additional valid ones.
2028
2029 """
2030 address_list = AddressList()
2031 while value:
2032 try:
2033 token, value = get_address(value)
2034 address_list.append(token)
2035 except errors.HeaderParseError:
2036 leader = None
2037 if value[0] in CFWS_LEADER:
2038 leader, value = get_cfws(value)
2039 if not value or value[0] == ',':
2040 address_list.append(leader)
2041 address_list.defects.append(errors.ObsoleteHeaderDefect(
2042 "address-list entry with no content"))
2043 else:
2044 token, value = get_invalid_mailbox(value, ',')
2045 if leader is not None:
2046 token[:0] = [leader]
2047 address_list.append(Address([token]))
2048 address_list.defects.append(errors.InvalidHeaderDefect(
2049 "invalid address in address-list"))
2050 elif value[0] == ',':
2051 address_list.defects.append(errors.ObsoleteHeaderDefect(
2052 "empty element in address-list"))
2053 else:
2054 token, value = get_invalid_mailbox(value, ',')
2055 if leader is not None:
2056 token[:0] = [leader]
2057 address_list.append(Address([token]))
2058 address_list.defects.append(errors.InvalidHeaderDefect(
2059 "invalid address in address-list"))
2060 if value and value[0] != ',':
2061 # Crap after address: add it to the address list
2062 # as an invalid mailbox
2063 token, value = get_invalid_mailbox(value, ',')
2064 address_list.append(Address([token]))
2065 address_list.defects.append(errors.InvalidHeaderDefect(
2066 "invalid address in address-list"))
2067 if value: # Must be a , at this point.
2068 address_list.append(ListSeparator)
2069 value = value[1:]
2070 return address_list, value
2071
2072
2073 def get_no_fold_literal(value):
2074 """ no-fold-literal = "[" *dtext "]"
2075 """
2076 no_fold_literal = NoFoldLiteral()
2077 if not value:
2078 raise errors.HeaderParseError(
2079 "expected no-fold-literal but found '{}'".format(value))
2080 if value[0] != '[':
2081 raise errors.HeaderParseError(
2082 "expected '[' at the start of no-fold-literal "
2083 "but found '{}'".format(value))
2084 no_fold_literal.append(ValueTerminal('[', 'no-fold-literal-start'))
2085 value = value[1:]
2086 token, value = get_dtext(value)
2087 no_fold_literal.append(token)
2088 if not value or value[0] != ']':
2089 raise errors.HeaderParseError(
2090 "expected ']' at the end of no-fold-literal "
2091 "but found '{}'".format(value))
2092 no_fold_literal.append(ValueTerminal(']', 'no-fold-literal-end'))
2093 return no_fold_literal, value[1:]
2094
2095 def get_msg_id(value):
2096 """msg-id = [CFWS] "<" id-left '@' id-right ">" [CFWS]
2097 id-left = dot-atom-text / obs-id-left
2098 id-right = dot-atom-text / no-fold-literal / obs-id-right
2099 no-fold-literal = "[" *dtext "]"
2100 """
2101 msg_id = MsgID()
2102 if value and value[0] in CFWS_LEADER:
2103 token, value = get_cfws(value)
2104 msg_id.append(token)
2105 if not value or value[0] != '<':
2106 raise errors.HeaderParseError(
2107 "expected msg-id but found '{}'".format(value))
2108 msg_id.append(ValueTerminal('<', 'msg-id-start'))
2109 value = value[1:]
2110 # Parse id-left.
2111 try:
2112 token, value = get_dot_atom_text(value)
2113 except errors.HeaderParseError:
2114 try:
2115 # obs-id-left is same as local-part of add-spec.
2116 token, value = get_obs_local_part(value)
2117 msg_id.defects.append(errors.ObsoleteHeaderDefect(
2118 "obsolete id-left in msg-id"))
2119 except errors.HeaderParseError:
2120 raise errors.HeaderParseError(
2121 "expected dot-atom-text or obs-id-left"
2122 " but found '{}'".format(value))
2123 msg_id.append(token)
2124 if not value or value[0] != '@':
2125 msg_id.defects.append(errors.InvalidHeaderDefect(
2126 "msg-id with no id-right"))
2127 # Even though there is no id-right, if the local part
2128 # ends with `>` let's just parse it too and return
2129 # along with the defect.
2130 if value and value[0] == '>':
2131 msg_id.append(ValueTerminal('>', 'msg-id-end'))
2132 value = value[1:]
2133 return msg_id, value
2134 msg_id.append(ValueTerminal('@', 'address-at-symbol'))
2135 value = value[1:]
2136 # Parse id-right.
2137 try:
2138 token, value = get_dot_atom_text(value)
2139 except errors.HeaderParseError:
2140 try:
2141 token, value = get_no_fold_literal(value)
2142 except errors.HeaderParseError:
2143 try:
2144 token, value = get_domain(value)
2145 msg_id.defects.append(errors.ObsoleteHeaderDefect(
2146 "obsolete id-right in msg-id"))
2147 except errors.HeaderParseError:
2148 raise errors.HeaderParseError(
2149 "expected dot-atom-text, no-fold-literal or obs-id-right"
2150 " but found '{}'".format(value))
2151 msg_id.append(token)
2152 if value and value[0] == '>':
2153 value = value[1:]
2154 else:
2155 msg_id.defects.append(errors.InvalidHeaderDefect(
2156 "missing trailing '>' on msg-id"))
2157 msg_id.append(ValueTerminal('>', 'msg-id-end'))
2158 if value and value[0] in CFWS_LEADER:
2159 token, value = get_cfws(value)
2160 msg_id.append(token)
2161 return msg_id, value
2162
2163
2164 def parse_message_id(value):
2165 """message-id = "Message-ID:" msg-id CRLF
2166 """
2167 message_id = MessageID()
2168 try:
2169 token, value = get_msg_id(value)
2170 message_id.append(token)
2171 except errors.HeaderParseError as ex:
2172 token = get_unstructured(value)
2173 message_id = InvalidMessageID(token)
2174 message_id.defects.append(
2175 errors.InvalidHeaderDefect("Invalid msg-id: {!r}".format(ex)))
2176 else:
2177 # Value after parsing a valid msg_id should be None.
2178 if value:
2179 message_id.defects.append(errors.InvalidHeaderDefect(
2180 "Unexpected {!r}".format(value)))
2181
2182 return message_id
2183
2184 def parse_message_ids(value):
2185 """in-reply-to = "In-Reply-To:" 1*msg-id CRLF
2186 references = "References:" 1*msg-id CRLF
2187 """
2188 message_id_list = MessageIDList()
2189 while value:
2190 if value[0] == ',':
2191 # message id list separated with commas - this is invalid,
2192 # but happens rather frequently in the wild
2193 message_id_list.defects.append(
2194 errors.InvalidHeaderDefect("comma in msg-id list"))
2195 message_id_list.append(
2196 WhiteSpaceTerminal(' ', 'invalid-comma-replacement'))
2197 value = value[1:]
2198 continue
2199 try:
2200 token, value = get_msg_id(value)
2201 message_id_list.append(token)
2202 except errors.HeaderParseError as ex:
2203 token = get_unstructured(value)
2204 message_id_list.append(InvalidMessageID(token))
2205 message_id_list.defects.append(
2206 errors.InvalidHeaderDefect("Invalid msg-id: {!r}".format(ex)))
2207 break
2208 return message_id_list
2209
2210 #
2211 # XXX: As I begin to add additional header parsers, I'm realizing we probably
2212 # have two level of parser routines: the get_XXX methods that get a token in
2213 # the grammar, and parse_XXX methods that parse an entire field value. So
2214 # get_address_list above should really be a parse_ method, as probably should
2215 # be get_unstructured.
2216 #
2217
2218 def parse_mime_version(value):
2219 """ mime-version = [CFWS] 1*digit [CFWS] "." [CFWS] 1*digit [CFWS]
2220
2221 """
2222 # The [CFWS] is implicit in the RFC 2045 BNF.
2223 # XXX: This routine is a bit verbose, should factor out a get_int method.
2224 mime_version = MIMEVersion()
2225 if not value:
2226 mime_version.defects.append(errors.HeaderMissingRequiredValue(
2227 "Missing MIME version number (eg: 1.0)"))
2228 return mime_version
2229 if value[0] in CFWS_LEADER:
2230 token, value = get_cfws(value)
2231 mime_version.append(token)
2232 if not value:
2233 mime_version.defects.append(errors.HeaderMissingRequiredValue(
2234 "Expected MIME version number but found only CFWS"))
2235 digits = ''
2236 while value and value[0] != '.' and value[0] not in CFWS_LEADER:
2237 digits += value[0]
2238 value = value[1:]
2239 if not digits.isdigit():
2240 mime_version.defects.append(errors.InvalidHeaderDefect(
2241 "Expected MIME major version number but found {!r}".format(digits)))
2242 mime_version.append(ValueTerminal(digits, 'xtext'))
2243 else:
2244 mime_version.major = int(digits)
2245 mime_version.append(ValueTerminal(digits, 'digits'))
2246 if value and value[0] in CFWS_LEADER:
2247 token, value = get_cfws(value)
2248 mime_version.append(token)
2249 if not value or value[0] != '.':
2250 if mime_version.major is not None:
2251 mime_version.defects.append(errors.InvalidHeaderDefect(
2252 "Incomplete MIME version; found only major number"))
2253 if value:
2254 mime_version.append(ValueTerminal(value, 'xtext'))
2255 return mime_version
2256 mime_version.append(ValueTerminal('.', 'version-separator'))
2257 value = value[1:]
2258 if value and value[0] in CFWS_LEADER:
2259 token, value = get_cfws(value)
2260 mime_version.append(token)
2261 if not value:
2262 if mime_version.major is not None:
2263 mime_version.defects.append(errors.InvalidHeaderDefect(
2264 "Incomplete MIME version; found only major number"))
2265 return mime_version
2266 digits = ''
2267 while value and value[0] not in CFWS_LEADER:
2268 digits += value[0]
2269 value = value[1:]
2270 if not digits.isdigit():
2271 mime_version.defects.append(errors.InvalidHeaderDefect(
2272 "Expected MIME minor version number but found {!r}".format(digits)))
2273 mime_version.append(ValueTerminal(digits, 'xtext'))
2274 else:
2275 mime_version.minor = int(digits)
2276 mime_version.append(ValueTerminal(digits, 'digits'))
2277 if value and value[0] in CFWS_LEADER:
2278 token, value = get_cfws(value)
2279 mime_version.append(token)
2280 if value:
2281 mime_version.defects.append(errors.InvalidHeaderDefect(
2282 "Excess non-CFWS text after MIME version"))
2283 mime_version.append(ValueTerminal(value, 'xtext'))
2284 return mime_version
2285
2286 def get_invalid_parameter(value):
2287 """ Read everything up to the next ';'.
2288
2289 This is outside the formal grammar. The InvalidParameter TokenList that is
2290 returned acts like a Parameter, but the data attributes are None.
2291
2292 """
2293 invalid_parameter = InvalidParameter()
2294 while value and value[0] != ';':
2295 if value[0] in PHRASE_ENDS:
2296 invalid_parameter.append(ValueTerminal(value[0],
2297 'misplaced-special'))
2298 value = value[1:]
2299 else:
2300 token, value = get_phrase(value)
2301 invalid_parameter.append(token)
2302 return invalid_parameter, value
2303
2304 def get_ttext(value):
2305 """ttext = <matches _ttext_matcher>
2306
2307 We allow any non-TOKEN_ENDS in ttext, but add defects to the token's
2308 defects list if we find non-ttext characters. We also register defects for
2309 *any* non-printables even though the RFC doesn't exclude all of them,
2310 because we follow the spirit of RFC 5322.
2311
2312 """
2313 m = _non_token_end_matcher(value)
2314 if not m:
2315 raise errors.HeaderParseError(
2316 "expected ttext but found '{}'".format(value))
2317 ttext = m.group()
2318 value = value[len(ttext):]
2319 ttext = ValueTerminal(ttext, 'ttext')
2320 _validate_xtext(ttext)
2321 return ttext, value
2322
2323 def get_token(value):
2324 """token = [CFWS] 1*ttext [CFWS]
2325
2326 The RFC equivalent of ttext is any US-ASCII chars except space, ctls, or
2327 tspecials. We also exclude tabs even though the RFC doesn't.
2328
2329 The RFC implies the CFWS but is not explicit about it in the BNF.
2330
2331 """
2332 mtoken = Token()
2333 if value and value[0] in CFWS_LEADER:
2334 token, value = get_cfws(value)
2335 mtoken.append(token)
2336 if value and value[0] in TOKEN_ENDS:
2337 raise errors.HeaderParseError(
2338 "expected token but found '{}'".format(value))
2339 token, value = get_ttext(value)
2340 mtoken.append(token)
2341 if value and value[0] in CFWS_LEADER:
2342 token, value = get_cfws(value)
2343 mtoken.append(token)
2344 return mtoken, value
2345
2346 def get_attrtext(value):
2347 """attrtext = 1*(any non-ATTRIBUTE_ENDS character)
2348
2349 We allow any non-ATTRIBUTE_ENDS in attrtext, but add defects to the
2350 token's defects list if we find non-attrtext characters. We also register
2351 defects for *any* non-printables even though the RFC doesn't exclude all of
2352 them, because we follow the spirit of RFC 5322.
2353
2354 """
2355 m = _non_attribute_end_matcher(value)
2356 if not m:
2357 raise errors.HeaderParseError(
2358 "expected attrtext but found {!r}".format(value))
2359 attrtext = m.group()
2360 value = value[len(attrtext):]
2361 attrtext = ValueTerminal(attrtext, 'attrtext')
2362 _validate_xtext(attrtext)
2363 return attrtext, value
2364
2365 def get_attribute(value):
2366 """ [CFWS] 1*attrtext [CFWS]
2367
2368 This version of the BNF makes the CFWS explicit, and as usual we use a
2369 value terminal for the actual run of characters. The RFC equivalent of
2370 attrtext is the token characters, with the subtraction of '*', "'", and '%'.
2371 We include tab in the excluded set just as we do for token.
2372
2373 """
2374 attribute = Attribute()
2375 if value and value[0] in CFWS_LEADER:
2376 token, value = get_cfws(value)
2377 attribute.append(token)
2378 if value and value[0] in ATTRIBUTE_ENDS:
2379 raise errors.HeaderParseError(
2380 "expected token but found '{}'".format(value))
2381 token, value = get_attrtext(value)
2382 attribute.append(token)
2383 if value and value[0] in CFWS_LEADER:
2384 token, value = get_cfws(value)
2385 attribute.append(token)
2386 return attribute, value
2387
2388 def get_extended_attrtext(value):
2389 """attrtext = 1*(any non-ATTRIBUTE_ENDS character plus '%')
2390
2391 This is a special parsing routine so that we get a value that
2392 includes % escapes as a single string (which we decode as a single
2393 string later).
2394
2395 """
2396 m = _non_extended_attribute_end_matcher(value)
2397 if not m:
2398 raise errors.HeaderParseError(
2399 "expected extended attrtext but found {!r}".format(value))
2400 attrtext = m.group()
2401 value = value[len(attrtext):]
2402 attrtext = ValueTerminal(attrtext, 'extended-attrtext')
2403 _validate_xtext(attrtext)
2404 return attrtext, value
2405
2406 def get_extended_attribute(value):
2407 """ [CFWS] 1*extended_attrtext [CFWS]
2408
2409 This is like the non-extended version except we allow % characters, so that
2410 we can pick up an encoded value as a single string.
2411
2412 """
2413 # XXX: should we have an ExtendedAttribute TokenList?
2414 attribute = Attribute()
2415 if value and value[0] in CFWS_LEADER:
2416 token, value = get_cfws(value)
2417 attribute.append(token)
2418 if value and value[0] in EXTENDED_ATTRIBUTE_ENDS:
2419 raise errors.HeaderParseError(
2420 "expected token but found '{}'".format(value))
2421 token, value = get_extended_attrtext(value)
2422 attribute.append(token)
2423 if value and value[0] in CFWS_LEADER:
2424 token, value = get_cfws(value)
2425 attribute.append(token)
2426 return attribute, value
2427
2428 def get_section(value):
2429 """ '*' digits
2430
2431 The formal BNF is more complicated because leading 0s are not allowed. We
2432 check for that and add a defect. We also assume no CFWS is allowed between
2433 the '*' and the digits, though the RFC is not crystal clear on that.
2434 The caller should already have dealt with leading CFWS.
2435
2436 """
2437 section = Section()
2438 if not value or value[0] != '*':
2439 raise errors.HeaderParseError("Expected section but found {}".format(
2440 value))
2441 section.append(ValueTerminal('*', 'section-marker'))
2442 value = value[1:]
2443 if not value or not value[0].isdigit():
2444 raise errors.HeaderParseError("Expected section number but "
2445 "found {}".format(value))
2446 digits = ''
2447 while value and value[0].isdigit():
2448 digits += value[0]
2449 value = value[1:]
2450 if digits[0] == '0' and digits != '0':
2451 section.defects.append(errors.InvalidHeaderDefect(
2452 "section number has an invalid leading 0"))
2453 section.number = int(digits)
2454 section.append(ValueTerminal(digits, 'digits'))
2455 return section, value
2456
2457
2458 def get_value(value):
2459 """ quoted-string / attribute
2460
2461 """
2462 v = Value()
2463 if not value:
2464 raise errors.HeaderParseError("Expected value but found end of string")
2465 leader = None
2466 if value[0] in CFWS_LEADER:
2467 leader, value = get_cfws(value)
2468 if not value:
2469 raise errors.HeaderParseError("Expected value but found "
2470 "only {}".format(leader))
2471 if value[0] == '"':
2472 token, value = get_quoted_string(value)
2473 else:
2474 token, value = get_extended_attribute(value)
2475 if leader is not None:
2476 token[:0] = [leader]
2477 v.append(token)
2478 return v, value
2479
2480 def get_parameter(value):
2481 """ attribute [section] ["*"] [CFWS] "=" value
2482
2483 The CFWS is implied by the RFC but not made explicit in the BNF. This
2484 simplified form of the BNF from the RFC is made to conform with the RFC BNF
2485 through some extra checks. We do it this way because it makes both error
2486 recovery and working with the resulting parse tree easier.
2487 """
2488 # It is possible CFWS would also be implicitly allowed between the section
2489 # and the 'extended-attribute' marker (the '*') , but we've never seen that
2490 # in the wild and we will therefore ignore the possibility.
2491 param = Parameter()
2492 token, value = get_attribute(value)
2493 param.append(token)
2494 if not value or value[0] == ';':
2495 param.defects.append(errors.InvalidHeaderDefect("Parameter contains "
2496 "name ({}) but no value".format(token)))
2497 return param, value
2498 if value[0] == '*':
2499 try:
2500 token, value = get_section(value)
2501 param.sectioned = True
2502 param.append(token)
2503 except errors.HeaderParseError:
2504 pass
2505 if not value:
2506 raise errors.HeaderParseError("Incomplete parameter")
2507 if value[0] == '*':
2508 param.append(ValueTerminal('*', 'extended-parameter-marker'))
2509 value = value[1:]
2510 param.extended = True
2511 if value[0] != '=':
2512 raise errors.HeaderParseError("Parameter not followed by '='")
2513 param.append(ValueTerminal('=', 'parameter-separator'))
2514 value = value[1:]
2515 if value and value[0] in CFWS_LEADER:
2516 token, value = get_cfws(value)
2517 param.append(token)
2518 remainder = None
2519 appendto = param
2520 if param.extended and value and value[0] == '"':
2521 # Now for some serious hackery to handle the common invalid case of
2522 # double quotes around an extended value. We also accept (with defect)
2523 # a value marked as encoded that isn't really.
2524 qstring, remainder = get_quoted_string(value)
2525 inner_value = qstring.stripped_value
2526 semi_valid = False
2527 if param.section_number == 0:
2528 if inner_value and inner_value[0] == "'":
2529 semi_valid = True
2530 else:
2531 token, rest = get_attrtext(inner_value)
2532 if rest and rest[0] == "'":
2533 semi_valid = True
2534 else:
2535 try:
2536 token, rest = get_extended_attrtext(inner_value)
2537 except:
2538 pass
2539 else:
2540 if not rest:
2541 semi_valid = True
2542 if semi_valid:
2543 param.defects.append(errors.InvalidHeaderDefect(
2544 "Quoted string value for extended parameter is invalid"))
2545 param.append(qstring)
2546 for t in qstring:
2547 if t.token_type == 'bare-quoted-string':
2548 t[:] = []
2549 appendto = t
2550 break
2551 value = inner_value
2552 else:
2553 remainder = None
2554 param.defects.append(errors.InvalidHeaderDefect(
2555 "Parameter marked as extended but appears to have a "
2556 "quoted string value that is non-encoded"))
2557 if value and value[0] == "'":
2558 token = None
2559 else:
2560 token, value = get_value(value)
2561 if not param.extended or param.section_number > 0:
2562 if not value or value[0] != "'":
2563 appendto.append(token)
2564 if remainder is not None:
2565 assert not value, value
2566 value = remainder
2567 return param, value
2568 param.defects.append(errors.InvalidHeaderDefect(
2569 "Apparent initial-extended-value but attribute "
2570 "was not marked as extended or was not initial section"))
2571 if not value:
2572 # Assume the charset/lang is missing and the token is the value.
2573 param.defects.append(errors.InvalidHeaderDefect(
2574 "Missing required charset/lang delimiters"))
2575 appendto.append(token)
2576 if remainder is None:
2577 return param, value
2578 else:
2579 if token is not None:
2580 for t in token:
2581 if t.token_type == 'extended-attrtext':
2582 break
2583 t.token_type == 'attrtext'
2584 appendto.append(t)
2585 param.charset = t.value
2586 if value[0] != "'":
2587 raise errors.HeaderParseError("Expected RFC2231 char/lang encoding "
2588 "delimiter, but found {!r}".format(value))
2589 appendto.append(ValueTerminal("'", 'RFC2231-delimiter'))
2590 value = value[1:]
2591 if value and value[0] != "'":
2592 token, value = get_attrtext(value)
2593 appendto.append(token)
2594 param.lang = token.value
2595 if not value or value[0] != "'":
2596 raise errors.HeaderParseError("Expected RFC2231 char/lang encoding "
2597 "delimiter, but found {}".format(value))
2598 appendto.append(ValueTerminal("'", 'RFC2231-delimiter'))
2599 value = value[1:]
2600 if remainder is not None:
2601 # Treat the rest of value as bare quoted string content.
2602 v = Value()
2603 while value:
2604 if value[0] in WSP:
2605 token, value = get_fws(value)
2606 elif value[0] == '"':
2607 token = ValueTerminal('"', 'DQUOTE')
2608 value = value[1:]
2609 else:
2610 token, value = get_qcontent(value)
2611 v.append(token)
2612 token = v
2613 else:
2614 token, value = get_value(value)
2615 appendto.append(token)
2616 if remainder is not None:
2617 assert not value, value
2618 value = remainder
2619 return param, value
2620
2621 def parse_mime_parameters(value):
2622 """ parameter *( ";" parameter )
2623
2624 That BNF is meant to indicate this routine should only be called after
2625 finding and handling the leading ';'. There is no corresponding rule in
2626 the formal RFC grammar, but it is more convenient for us for the set of
2627 parameters to be treated as its own TokenList.
2628
2629 This is 'parse' routine because it consumes the remaining value, but it
2630 would never be called to parse a full header. Instead it is called to
2631 parse everything after the non-parameter value of a specific MIME header.
2632
2633 """
2634 mime_parameters = MimeParameters()
2635 while value:
2636 try:
2637 token, value = get_parameter(value)
2638 mime_parameters.append(token)
2639 except errors.HeaderParseError:
2640 leader = None
2641 if value[0] in CFWS_LEADER:
2642 leader, value = get_cfws(value)
2643 if not value:
2644 mime_parameters.append(leader)
2645 return mime_parameters
2646 if value[0] == ';':
2647 if leader is not None:
2648 mime_parameters.append(leader)
2649 mime_parameters.defects.append(errors.InvalidHeaderDefect(
2650 "parameter entry with no content"))
2651 else:
2652 token, value = get_invalid_parameter(value)
2653 if leader:
2654 token[:0] = [leader]
2655 mime_parameters.append(token)
2656 mime_parameters.defects.append(errors.InvalidHeaderDefect(
2657 "invalid parameter {!r}".format(token)))
2658 if value and value[0] != ';':
2659 # Junk after the otherwise valid parameter. Mark it as
2660 # invalid, but it will have a value.
2661 param = mime_parameters[-1]
2662 param.token_type = 'invalid-parameter'
2663 token, value = get_invalid_parameter(value)
2664 param.extend(token)
2665 mime_parameters.defects.append(errors.InvalidHeaderDefect(
2666 "parameter with invalid trailing text {!r}".format(token)))
2667 if value:
2668 # Must be a ';' at this point.
2669 mime_parameters.append(ValueTerminal(';', 'parameter-separator'))
2670 value = value[1:]
2671 return mime_parameters
2672
2673 def _find_mime_parameters(tokenlist, value):
2674 """Do our best to find the parameters in an invalid MIME header
2675
2676 """
2677 while value and value[0] != ';':
2678 if value[0] in PHRASE_ENDS:
2679 tokenlist.append(ValueTerminal(value[0], 'misplaced-special'))
2680 value = value[1:]
2681 else:
2682 token, value = get_phrase(value)
2683 tokenlist.append(token)
2684 if not value:
2685 return
2686 tokenlist.append(ValueTerminal(';', 'parameter-separator'))
2687 tokenlist.append(parse_mime_parameters(value[1:]))
2688
2689 def parse_content_type_header(value):
2690 """ maintype "/" subtype *( ";" parameter )
2691
2692 The maintype and substype are tokens. Theoretically they could
2693 be checked against the official IANA list + x-token, but we
2694 don't do that.
2695 """
2696 ctype = ContentType()
2697 if not value:
2698 ctype.defects.append(errors.HeaderMissingRequiredValue(
2699 "Missing content type specification"))
2700 return ctype
2701 try:
2702 token, value = get_token(value)
2703 except errors.HeaderParseError:
2704 ctype.defects.append(errors.InvalidHeaderDefect(
2705 "Expected content maintype but found {!r}".format(value)))
2706 _find_mime_parameters(ctype, value)
2707 return ctype
2708 ctype.append(token)
2709 # XXX: If we really want to follow the formal grammar we should make
2710 # mantype and subtype specialized TokenLists here. Probably not worth it.
2711 if not value or value[0] != '/':
2712 ctype.defects.append(errors.InvalidHeaderDefect(
2713 "Invalid content type"))
2714 if value:
2715 _find_mime_parameters(ctype, value)
2716 return ctype
2717 ctype.maintype = token.value.strip().lower()
2718 ctype.append(ValueTerminal('/', 'content-type-separator'))
2719 value = value[1:]
2720 try:
2721 token, value = get_token(value)
2722 except errors.HeaderParseError:
2723 ctype.defects.append(errors.InvalidHeaderDefect(
2724 "Expected content subtype but found {!r}".format(value)))
2725 _find_mime_parameters(ctype, value)
2726 return ctype
2727 ctype.append(token)
2728 ctype.subtype = token.value.strip().lower()
2729 if not value:
2730 return ctype
2731 if value[0] != ';':
2732 ctype.defects.append(errors.InvalidHeaderDefect(
2733 "Only parameters are valid after content type, but "
2734 "found {!r}".format(value)))
2735 # The RFC requires that a syntactically invalid content-type be treated
2736 # as text/plain. Perhaps we should postel this, but we should probably
2737 # only do that if we were checking the subtype value against IANA.
2738 del ctype.maintype, ctype.subtype
2739 _find_mime_parameters(ctype, value)
2740 return ctype
2741 ctype.append(ValueTerminal(';', 'parameter-separator'))
2742 ctype.append(parse_mime_parameters(value[1:]))
2743 return ctype
2744
2745 def parse_content_disposition_header(value):
2746 """ disposition-type *( ";" parameter )
2747
2748 """
2749 disp_header = ContentDisposition()
2750 if not value:
2751 disp_header.defects.append(errors.HeaderMissingRequiredValue(
2752 "Missing content disposition"))
2753 return disp_header
2754 try:
2755 token, value = get_token(value)
2756 except errors.HeaderParseError:
2757 disp_header.defects.append(errors.InvalidHeaderDefect(
2758 "Expected content disposition but found {!r}".format(value)))
2759 _find_mime_parameters(disp_header, value)
2760 return disp_header
2761 disp_header.append(token)
2762 disp_header.content_disposition = token.value.strip().lower()
2763 if not value:
2764 return disp_header
2765 if value[0] != ';':
2766 disp_header.defects.append(errors.InvalidHeaderDefect(
2767 "Only parameters are valid after content disposition, but "
2768 "found {!r}".format(value)))
2769 _find_mime_parameters(disp_header, value)
2770 return disp_header
2771 disp_header.append(ValueTerminal(';', 'parameter-separator'))
2772 disp_header.append(parse_mime_parameters(value[1:]))
2773 return disp_header
2774
2775 def parse_content_transfer_encoding_header(value):
2776 """ mechanism
2777
2778 """
2779 # We should probably validate the values, since the list is fixed.
2780 cte_header = ContentTransferEncoding()
2781 if not value:
2782 cte_header.defects.append(errors.HeaderMissingRequiredValue(
2783 "Missing content transfer encoding"))
2784 return cte_header
2785 try:
2786 token, value = get_token(value)
2787 except errors.HeaderParseError:
2788 cte_header.defects.append(errors.InvalidHeaderDefect(
2789 "Expected content transfer encoding but found {!r}".format(value)))
2790 else:
2791 cte_header.append(token)
2792 cte_header.cte = token.value.strip().lower()
2793 if not value:
2794 return cte_header
2795 while value:
2796 cte_header.defects.append(errors.InvalidHeaderDefect(
2797 "Extra text after content transfer encoding"))
2798 if value[0] in PHRASE_ENDS:
2799 cte_header.append(ValueTerminal(value[0], 'misplaced-special'))
2800 value = value[1:]
2801 else:
2802 token, value = get_phrase(value)
2803 cte_header.append(token)
2804 return cte_header
2805
2806
2807 #
2808 # Header folding
2809 #
2810 # Header folding is complex, with lots of rules and corner cases. The
2811 # following code does its best to obey the rules and handle the corner
2812 # cases, but you can be sure there are few bugs:)
2813 #
2814 # This folder generally canonicalizes as it goes, preferring the stringified
2815 # version of each token. The tokens contain information that supports the
2816 # folder, including which tokens can be encoded in which ways.
2817 #
2818 # Folded text is accumulated in a simple list of strings ('lines'), each
2819 # one of which should be less than policy.max_line_length ('maxlen').
2820 #
2821
2822 def _steal_trailing_WSP_if_exists(lines):
2823 wsp = ''
2824 if lines and lines[-1] and lines[-1][-1] in WSP:
2825 wsp = lines[-1][-1]
2826 lines[-1] = lines[-1][:-1]
2827 # gh-142006: if the line is now empty, remove it entirely.
2828 if not lines[-1]:
2829 lines.pop()
2830 return wsp
2831
2832
2833 def _refold_parse_tree(parse_tree, *, policy):
2834 """Return string of contents of parse_tree folded according to RFC rules.
2835
2836 """
2837 # max_line_length 0/None means no limit, ie: infinitely long.
2838 maxlen = policy.max_line_length or sys.maxsize
2839 encoding = 'utf-8' if policy.utf8 else 'us-ascii'
2840 lines = [''] # Folded lines to be output
2841 last_word_is_ew = False
2842 last_ew = None # if there is an encoded word in the last line of lines,
2843 # points to the encoded word's first character
2844 last_charset = None
2845 wrap_as_ew_blocked = 0
2846 want_encoding = False # This is set to True if we need to encode this part
2847 end_ew_not_allowed = Terminal('', 'wrap_as_ew_blocked')
2848 parts = list(parse_tree)
2849 while parts:
2850 part = parts.pop(0)
2851 if part is end_ew_not_allowed:
2852 wrap_as_ew_blocked -= 1
2853 continue
2854 tstr = str(part)
2855 if not want_encoding:
2856 if part.token_type in ('ptext', 'vtext'):
2857 # Encode if tstr contains special characters.
2858 want_encoding = not SPECIALSNL.isdisjoint(tstr)
2859 else:
2860 # Encode if tstr contains newlines.
2861 want_encoding = not NLSET.isdisjoint(tstr)
2862 try:
2863 tstr.encode(encoding)
2864 charset = encoding
2865 except UnicodeEncodeError:
2866 if any(isinstance(x, errors.UndecodableBytesDefect)
2867 for x in part.all_defects):
2868 charset = 'unknown-8bit'
2869 else:
2870 # If policy.utf8 is false this should really be taken from a
2871 # 'charset' property on the policy.
2872 charset = 'utf-8'
2873 want_encoding = True
2874
2875 if part.token_type == 'mime-parameters':
2876 # Mime parameter folding (using RFC2231) is extra special.
2877 _fold_mime_parameters(part, lines, maxlen, encoding)
2878 last_word_is_ew = False
2879 continue
2880
2881 if want_encoding and not wrap_as_ew_blocked:
2882 if not part.as_ew_allowed:
2883 want_encoding = False
2884 last_ew = None
2885 if part.syntactic_break:
2886 encoded_part = part.fold(policy=policy)[:-len(policy.linesep)]
2887 if policy.linesep not in encoded_part:
2888 # It fits on a single line
2889 if len(encoded_part) > maxlen - len(lines[-1]):
2890 # But not on this one, so start a new one.
2891 newline = _steal_trailing_WSP_if_exists(lines)
2892 # XXX what if encoded_part has no leading FWS?
2893 lines.append(newline)
2894 lines[-1] += encoded_part
2895 last_word_is_ew = False
2896 continue
2897 # Either this is not a major syntactic break, so we don't
2898 # want it on a line by itself even if it fits, or it
2899 # doesn't fit on a line by itself. Either way, fall through
2900 # to unpacking the subparts and wrapping them.
2901 if not hasattr(part, 'encode'):
2902 # It's not a Terminal, do each piece individually.
2903 parts = list(part) + parts
2904 want_encoding = False
2905 continue
2906 elif part.as_ew_allowed:
2907 # It's a terminal, wrap it as an encoded word, possibly
2908 # combining it with previously encoded words if allowed.
2909 if (last_ew is not None and
2910 charset != last_charset and
2911 (last_charset == 'unknown-8bit' or
2912 last_charset == 'utf-8' and charset != 'us-ascii')):
2913 last_ew = None
2914 last_ew = _fold_as_ew(
2915 tstr,
2916 lines,
2917 maxlen,
2918 last_ew,
2919 part.ew_combine_allowed,
2920 charset,
2921 last_word_is_ew,
2922 )
2923 last_word_is_ew = True
2924 last_charset = charset
2925 want_encoding = False
2926 continue
2927 else:
2928 # It's a terminal which should be kept non-encoded
2929 # (e.g. a ListSeparator).
2930 last_ew = None
2931 want_encoding = False
2932 # fall through
2933
2934 if len(tstr) <= maxlen - len(lines[-1]):
2935 lines[-1] += tstr
2936 last_word_is_ew = last_word_is_ew and not bool(tstr.strip(_WSP))
2937 continue
2938
2939 # This part is too long to fit. The RFC wants us to break at
2940 # "major syntactic breaks", so unless we don't consider this
2941 # to be one, check if it will fit on the next line by itself.
2942 if (part.syntactic_break and
2943 len(tstr) + 1 <= maxlen):
2944 newline = _steal_trailing_WSP_if_exists(lines)
2945 if newline or part.startswith_fws():
2946 lines.append(newline + tstr)
2947 last_word_is_ew = (last_word_is_ew
2948 and not bool(lines[-1].strip(_WSP)))
2949 last_ew = None
2950 continue
2951 if not hasattr(part, 'encode'):
2952 # It's not a terminal, try folding the subparts.
2953 newparts = list(part)
2954 if part.token_type == 'bare-quoted-string':
2955 # To fold a quoted string we need to create a list of terminal
2956 # tokens that will render the leading and trailing quotes
2957 # and use quoted pairs in the value as appropriate.
2958 newparts = (
2959 [ValueTerminal('"', 'ptext')] +
2960 [ValueTerminal(make_quoted_pairs(p), 'ptext')
2961 for p in newparts] +
2962 [ValueTerminal('"', 'ptext')])
2963 if part.token_type == 'comment':
2964 newparts = (
2965 [ValueTerminal('(', 'ptext')] +
2966 [ValueTerminal(make_parenthesis_pairs(p), 'ptext')
2967 if p.token_type == 'ptext' else p
2968 for p in newparts] +
2969 [ValueTerminal(')', 'ptext')])
2970 if not part.as_ew_allowed:
2971 wrap_as_ew_blocked += 1
2972 newparts.append(end_ew_not_allowed)
2973 parts = newparts + parts
2974 continue
2975 if part.as_ew_allowed and not wrap_as_ew_blocked:
2976 # It doesn't need CTE encoding, but encode it anyway so we can
2977 # wrap it.
2978 parts.insert(0, part)
2979 want_encoding = True
2980 continue
2981 # We can't figure out how to wrap, it, so give up.
2982 newline = _steal_trailing_WSP_if_exists(lines)
2983 if newline or part.startswith_fws():
2984 lines.append(newline + tstr)
2985 else:
2986 # We can't fold it onto the next line either...
2987 lines[-1] += tstr
2988 last_word_is_ew = last_word_is_ew and not bool(tstr.strip(_WSP))
2989
2990 return policy.linesep.join(lines) + policy.linesep
2991
2992 def _fold_as_ew(to_encode, lines, maxlen, last_ew, ew_combine_allowed, charset, last_word_is_ew):
2993 """Fold string to_encode into lines as encoded word, combining if allowed.
2994 Return the new value for last_ew, or None if ew_combine_allowed is False.
2995
2996 If there is already an encoded word in the last line of lines (indicated by
2997 a non-None value for last_ew) and ew_combine_allowed is true, decode the
2998 existing ew, combine it with to_encode, and re-encode. Otherwise, encode
2999 to_encode. In either case, split to_encode as necessary so that the
3000 encoded segments fit within maxlen.
3001
3002 """
3003 if last_ew is not None and ew_combine_allowed:
3004 to_encode = str(
3005 get_unstructured(lines[-1][last_ew:] + to_encode))
3006 lines[-1] = lines[-1][:last_ew]
3007 elif last_word_is_ew:
3008 # If we are following up an encoded word with another encoded word,
3009 # any white space between the two will be ignored when decoded.
3010 # Therefore, we encode all to-be-displayed whitespace in the second
3011 # encoded word.
3012 len_without_wsp = len(lines[-1].rstrip(_WSP))
3013 leading_whitespace = lines[-1][len_without_wsp:]
3014 lines[-1] = (lines[-1][:len_without_wsp]
3015 + (' ' if leading_whitespace else ''))
3016 to_encode = leading_whitespace + to_encode
3017 elif to_encode[0] in WSP:
3018 # We're joining this to non-encoded text, so don't encode
3019 # the leading blank.
3020 leading_wsp = to_encode[0]
3021 to_encode = to_encode[1:]
3022 if (len(lines[-1]) == maxlen):
3023 lines.append(_steal_trailing_WSP_if_exists(lines))
3024 lines[-1] += leading_wsp
3025
3026 trailing_wsp = ''
3027 if to_encode[-1] in WSP:
3028 # Likewise for the trailing space.
3029 trailing_wsp = to_encode[-1]
3030 to_encode = to_encode[:-1]
3031 new_last_ew = len(lines[-1]) if last_ew is None else last_ew
3032
3033 encode_as = 'utf-8' if charset == 'us-ascii' else charset
3034
3035 # The RFC2047 chrome takes up 7 characters plus the length
3036 # of the charset name.
3037 chrome_len = len(encode_as) + 7
3038
3039 if (chrome_len + 1) >= maxlen:
3040 raise errors.HeaderParseError(
3041 "max_line_length is too small to fit an encoded word")
3042
3043 while to_encode:
3044 remaining_space = maxlen - len(lines[-1])
3045 text_space = remaining_space - chrome_len
3046 if text_space <= 0:
3047 newline = _steal_trailing_WSP_if_exists(lines)
3048 lines.append(newline or ' ')
3049 new_last_ew = len(lines[-1])
3050 continue
3051
3052 to_encode_word = to_encode[:text_space]
3053 encoded_word = _ew.encode(to_encode_word, charset=encode_as)
3054 excess = len(encoded_word) - remaining_space
3055 while excess > 0:
3056 # Since the chunk to encode is guaranteed to fit into less than 100 characters,
3057 # shrinking it by one at a time shouldn't take long.
3058 to_encode_word = to_encode_word[:-1]
3059 encoded_word = _ew.encode(to_encode_word, charset=encode_as)
3060 excess = len(encoded_word) - remaining_space
3061 lines[-1] += encoded_word
3062 to_encode = to_encode[len(to_encode_word):]
3063
3064 if to_encode:
3065 lines.append(' ')
3066 new_last_ew = len(lines[-1])
3067 lines[-1] += trailing_wsp
3068 return new_last_ew if ew_combine_allowed else None
3069
3070 def _fold_mime_parameters(part, lines, maxlen, encoding):
3071 """Fold TokenList 'part' into the 'lines' list as mime parameters.
3072
3073 Using the decoded list of parameters and values, format them according to
3074 the RFC rules, including using RFC2231 encoding if the value cannot be
3075 expressed in 'encoding' and/or the parameter+value is too long to fit
3076 within 'maxlen'.
3077
3078 """
3079 # Special case for RFC2231 encoding: start from decoded values and use
3080 # RFC2231 encoding iff needed.
3081 #
3082 # Note that the 1 and 2s being added to the length calculations are
3083 # accounting for the possibly-needed spaces and semicolons we'll be adding.
3084 #
3085 for name, value in part.params:
3086 # XXX What if this ';' puts us over maxlen the first time through the
3087 # loop? We should split the header value onto a newline in that case,
3088 # but to do that we need to recognize the need earlier or reparse the
3089 # header, so I'm going to ignore that bug for now. It'll only put us
3090 # one character over.
3091 if not lines[-1].rstrip().endswith(';'):
3092 lines[-1] += ';'
3093 charset = encoding
3094 error_handler = 'strict'
3095 try:
3096 value.encode(encoding)
3097 encoding_required = False
3098 except UnicodeEncodeError:
3099 encoding_required = True
3100 if utils._has_surrogates(value):
3101 charset = 'unknown-8bit'
3102 error_handler = 'surrogateescape'
3103 else:
3104 charset = 'utf-8'
3105 if encoding_required:
3106 encoded_value = urllib.parse.quote(
3107 value, safe='', errors=error_handler)
3108 tstr = "{}*={}''{}".format(name, charset, encoded_value)
3109 else:
3110 tstr = '{}={}'.format(name, quote_string(value))
3111 if len(lines[-1]) + len(tstr) + 1 < maxlen:
3112 lines[-1] = lines[-1] + ' ' + tstr
3113 continue
3114 elif len(tstr) + 2 <= maxlen:
3115 lines.append(' ' + tstr)
3116 continue
3117 # We need multiple sections. We are allowed to mix encoded and
3118 # non-encoded sections, but we aren't going to. We'll encode them all.
3119 section = 0
3120 extra_chrome = charset + "''"
3121 while value:
3122 chrome_len = len(name) + len(str(section)) + 3 + len(extra_chrome)
3123 if maxlen <= chrome_len + 3:
3124 # We need room for the leading blank, the trailing semicolon,
3125 # and at least one character of the value. If we don't
3126 # have that, we'd be stuck, so in that case fall back to
3127 # the RFC standard width.
3128 maxlen = 78
3129 splitpoint = maxchars = maxlen - chrome_len - 2
3130 while True:
3131 partial = value[:splitpoint]
3132 encoded_value = urllib.parse.quote(
3133 partial, safe='', errors=error_handler)
3134 if len(encoded_value) <= maxchars:
3135 break
3136 splitpoint -= 1
3137 lines.append(" {}*{}*={}{}".format(
3138 name, section, extra_chrome, encoded_value))
3139 extra_chrome = ''
3140 section += 1
3141 value = value[splitpoint:]
3142 if value:
3143 lines[-1] += ';'