Debian Codegraph @

Lib/email/_header_value_parser.py

raw
1 """Header value parser implementing various email-related RFC parsing rules.
2
3 The parsing methods defined in this module implement various email related
4 parsing rules. Principal among them is RFC 5322, which is the followon
5 to RFC 2822 and primarily a clarification of the former. It also implements
6 RFC 2047 encoded word decoding.
7
8 RFC 5322 goes to considerable trouble to maintain backward compatibility with
9 RFC 822 in the parse phase, while cleaning up the structure on the generation
10 phase. This parser supports correct RFC 5322 generation by tagging white space
11 as folding white space only when folding is allowed in the non-obsolete rule
12 sets. Actually, the parser is even more generous when accepting input than RFC
13 5322 mandates, following the spirit of Postel's Law, which RFC 5322 encourages.
14 Where possible deviations from the standard are annotated on the 'defects'
15 attribute of tokens that deviate.
16
17 The general structure of the parser follows RFC 5322, and uses its terminology
18 where there is a direct correspondence. Where the implementation requires a
19 somewhat different structure than that used by the formal grammar, new terms
20 that mimic the closest existing terms are used. Thus, it really helps to have
21 a copy of RFC 5322 handy when studying this code.
22
23 Input to the parser is a string that has already been unfolded according to
24 RFC 5322 rules. According to the RFC this unfolding is the very first step, and
25 this parser leaves the unfolding step to a higher level message parser, which
26 will have already detected the line breaks that need unfolding while
27 determining the beginning and end of each header.
28
29 The output of the parser is a TokenList object, which is a list subclass. A
30 TokenList is a recursive data structure. The terminal nodes of the structure
31 are Terminal objects, which are subclasses of str. These do not correspond
32 directly to terminal objects in the formal grammar, but are instead more
33 practical higher level combinations of true terminals.
34
35 All TokenList and Terminal objects have a 'value' attribute, which produces the
36 semantically meaningful value of that part of the parse subtree. The value of
37 all whitespace tokens (no matter how many sub-tokens they may contain) is a
38 single space, as per the RFC rules. This includes 'CFWS', which is herein
39 included in the general class of whitespace tokens. There is one exception to
40 the rule that whitespace tokens are collapsed into single spaces in values: in
41 the value of a 'bare-quoted-string' (a quoted-string with no leading or
42 trailing whitespace), any whitespace that appeared between the quotation marks
43 is preserved in the returned value. Note that in all Terminal strings quoted
44 pairs are turned into their unquoted values.
45
46 All TokenList and Terminal objects also have a string value, which attempts to
47 be a "canonical" representation of the RFC-compliant form of the substring that
48 produced the parsed subtree, including minimal use of quoted pair quoting.
49 Whitespace runs are not collapsed.
50
51 Comment tokens also have a 'content' attribute providing the string found
52 between the parens (including any nested comments) with whitespace preserved.
53
54 All TokenList and Terminal objects have a 'defects' attribute which is a
55 possibly empty list all of the defects found while creating the token. Defects
56 may appear on any token in the tree, and a composite list of all defects in the
57 subtree is available through the 'all_defects' attribute of any node. (For
58 Terminal notes x.defects == x.all_defects.)
59
60 Each object in a parse tree is called a 'token', and each has a 'token_type'
61 attribute that gives the name from the RFC 5322 grammar that it represents.
62 Not all RFC 5322 nodes are produced, and there is one non-RFC 5322 node that
63 may be produced: 'ptext'. A 'ptext' is a string of printable ascii characters.
64 It is returned in place of lists of (ctext/quoted-pair) and
65 (qtext/quoted-pair).
66
67 XXX: provide complete list of token types.
68 """
69
70 import re
71 import sys
72 import urllib # For urllib.parse.unquote
73 from string import hexdigits
74 from operator import itemgetter
75 from email import _encoded_words as _ew
76 from email import errors
77 from email import utils
78
79 #
80 # Useful constants and functions
81 #
82
83 _WSP = ' \t'
84 WSP = set(_WSP)
85 CFWS_LEADER = WSP | set('(')
86 SPECIALS = set(r'()<>@,:;.\"[]')
87 ATOM_ENDS = SPECIALS | WSP
88 DOT_ATOM_ENDS = ATOM_ENDS - set('.')
89 # '.', '"', and '(' do not end phrases in order to support obs-phrase
90 PHRASE_ENDS = SPECIALS - set('."(')
91 TSPECIALS = (SPECIALS | set('/?=')) - set('.')
92 TOKEN_ENDS = TSPECIALS | WSP
93 ASPECIALS = TSPECIALS | set("*'%")
94 ATTRIBUTE_ENDS = ASPECIALS | WSP
95 EXTENDED_ATTRIBUTE_ENDS = ATTRIBUTE_ENDS - set('%')
96 NLSET = {'\n', '\r'}
97 SPECIALSNL = SPECIALS | NLSET
98
99
100 def make_quoted_pairs(value):
101 """Escape dquote and backslash for use within a quoted-string."""
102 return str(value).replace('\\', '\\\\').replace('"', '\\"')
103
104
105 def make_parenthesis_pairs(value):
106 """Escape parenthesis and backslash for use within a comment."""
107 return str(value).replace('\\', '\\\\') \
108 .replace('(', '\\(').replace(')', '\\)')
109
110
111 def quote_string(value):
112 escaped = make_quoted_pairs(value)
113 return f'"{escaped}"'
114
115
116 # Match a RFC 2047 word, looks like =?utf-8?q?someword?=
117 rfc2047_matcher = re.compile(r'''
118 =\? # literal =?
119 [^?]* # charset
120 \? # literal ?
121 [qQbB] # literal 'q' or 'b', case insensitive
122 \? # literal ?
123 .*? # encoded word
124 \?= # literal ?=
125 ''', re.VERBOSE | re.MULTILINE)
126
127
128 #
129 # TokenList and its subclasses
130 #
131
132 class TokenList(list):
133
134 token_type = None
135 syntactic_break = True
136 ew_combine_allowed = True
137
138 def __init__(self, *args, **kw):
139 super().__init__(*args, **kw)
140 self.defects = []
141
142 def __str__(self):
143 return ''.join(str(x) for x in self)
144
145 def __repr__(self):
146 return '{}({})'.format(self.__class__.__name__,
147 super().__repr__())
148
149 @property
150 def value(self):
151 return ''.join(x.value for x in self if x.value)
152
153 @property
154 def all_defects(self):
155 return sum((x.all_defects for x in self), self.defects)
156
157 def startswith_fws(self):
158 return self[0].startswith_fws()
159
160 @property
161 def as_ew_allowed(self):
162 """True if all top level tokens of this part may be RFC2047 encoded."""
163 return all(part.as_ew_allowed for part in self)
164
165 @property
166 def comments(self):
167 comments = []
168 for token in self:
169 comments.extend(token.comments)
170 return comments
171
172 def fold(self, *, policy):
173 return _refold_parse_tree(self, policy=policy)
174
175 def pprint(self, indent=''):
176 print(self.ppstr(indent=indent))
177
178 def ppstr(self, indent=''):
179 return '\n'.join(self._pp(indent=indent))
180
181 def _pp(self, indent=''):
182 yield '{}{}/{}('.format(
183 indent,
184 self.__class__.__name__,
185 self.token_type)
186 for token in self:
187 if not hasattr(token, '_pp'):
188 yield (indent + ' !! invalid element in token '
189 'list: {!r}'.format(token))
190 else:
191 yield from token._pp(indent+' ')
192 if self.defects:
193 extra = ' Defects: {}'.format(self.defects)
194 else:
195 extra = ''
196 yield '{}){}'.format(indent, extra)
197
198
199 class WhiteSpaceTokenList(TokenList):
200
201 @property
202 def value(self):
203 return ' '
204
205 @property
206 def comments(self):
207 return [x.content for x in self if x.token_type=='comment']
208
209
210 class UnstructuredTokenList(TokenList):
211 token_type = 'unstructured'
212
213
214 class Phrase(TokenList):
215 token_type = 'phrase'
216
217 class Word(TokenList):
218 token_type = 'word'
219
220
221 class CFWSList(WhiteSpaceTokenList):
222 token_type = 'cfws'
223
224
225 class Atom(TokenList):
226 token_type = 'atom'
227
228
229 class Token(TokenList):
230 token_type = 'token'
231 encode_as_ew = False
232
233
234 class EncodedWord(TokenList):
235 token_type = 'encoded-word'
236 cte = None
237 charset = None
238 lang = None
239
240
241 class QuotedString(TokenList):
242
243 token_type = 'quoted-string'
244
245 @property
246 def content(self):
247 for x in self:
248 if x.token_type == 'bare-quoted-string':
249 return x.value
250
251 @property
252 def quoted_value(self):
253 res = []
254 for x in self:
255 if x.token_type == 'bare-quoted-string':
256 res.append(str(x))
257 else:
258 res.append(x.value)
259 return ''.join(res)
260
261 @property
262 def stripped_value(self):
263 for token in self:
264 if token.token_type == 'bare-quoted-string':
265 return token.value
266
267
268 class BareQuotedString(QuotedString):
269
270 token_type = 'bare-quoted-string'
271
272 def __str__(self):
273 return quote_string(''.join(str(x) for x in self))
274
275 @property
276 def value(self):
277 return ''.join(str(x) for x in self)
278
279
280 class Comment(WhiteSpaceTokenList):
281
282 token_type = 'comment'
283
284 def __str__(self):
285 return ''.join(sum([
286 ["("],
287 [self.quote(x) for x in self],
288 [")"],
289 ], []))
290
291 def quote(self, value):
292 if value.token_type == 'comment':
293 return str(value)
294 return str(value).replace('\\', '\\\\').replace(
295 '(', r'\(').replace(
296 ')', r'\)')
297
298 @property
299 def content(self):
300 return ''.join(str(x) for x in self)
301
302 @property
303 def comments(self):
304 return [self.content]
305
306 class AddressList(TokenList):
307
308 token_type = 'address-list'
309
310 @property
311 def addresses(self):
312 return [x for x in self if x.token_type=='address']
313
314 @property
315 def mailboxes(self):
316 return sum((x.mailboxes
317 for x in self if x.token_type=='address'), [])
318
319 @property
320 def all_mailboxes(self):
321 return sum((x.all_mailboxes
322 for x in self if x.token_type=='address'), [])
323
324
325 class Address(TokenList):
326
327 token_type = 'address'
328
329 @property
330 def display_name(self):
331 if self[0].token_type == 'group':
332 return self[0].display_name
333
334 @property
335 def mailboxes(self):
336 if self[0].token_type == 'mailbox':
337 return [self[0]]
338 elif self[0].token_type == 'invalid-mailbox':
339 return []
340 return self[0].mailboxes
341
342 @property
343 def all_mailboxes(self):
344 if self[0].token_type == 'mailbox':
345 return [self[0]]
346 elif self[0].token_type == 'invalid-mailbox':
347 return [self[0]]
348 return self[0].all_mailboxes
349
350 class MailboxList(TokenList):
351
352 token_type = 'mailbox-list'
353
354 @property
355 def mailboxes(self):
356 return [x for x in self if x.token_type=='mailbox']
357
358 @property
359 def all_mailboxes(self):
360 return [x for x in self
361 if x.token_type in ('mailbox', 'invalid-mailbox')]
362
363
364 class GroupList(TokenList):
365
366 token_type = 'group-list'
367
368 @property
369 def mailboxes(self):
370 if not self or self[0].token_type != 'mailbox-list':
371 return []
372 return self[0].mailboxes
373
374 @property
375 def all_mailboxes(self):
376 if not self or self[0].token_type != 'mailbox-list':
377 return []
378 return self[0].all_mailboxes
379
380
381 class Group(TokenList):
382
383 token_type = "group"
384
385 @property
386 def mailboxes(self):
387 if self[2].token_type != 'group-list':
388 return []
389 return self[2].mailboxes
390
391 @property
392 def all_mailboxes(self):
393 if self[2].token_type != 'group-list':
394 return []
395 return self[2].all_mailboxes
396
397 @property
398 def display_name(self):
399 return self[0].display_name
400
401
402 class NameAddr(TokenList):
403
404 token_type = 'name-addr'
405
406 @property
407 def display_name(self):
408 if len(self) == 1:
409 return None
410 return self[0].display_name
411
412 @property
413 def local_part(self):
414 return self[-1].local_part
415
416 @property
417 def domain(self):
418 return self[-1].domain
419
420 @property
421 def route(self):
422 return self[-1].route
423
424 @property
425 def addr_spec(self):
426 return self[-1].addr_spec
427
428
429 class AngleAddr(TokenList):
430
431 token_type = 'angle-addr'
432
433 @property
434 def local_part(self):
435 for x in self:
436 if x.token_type == 'addr-spec':
437 return x.local_part
438
439 @property
440 def domain(self):
441 for x in self:
442 if x.token_type == 'addr-spec':
443 return x.domain
444
445 @property
446 def route(self):
447 for x in self:
448 if x.token_type == 'obs-route':
449 return x.domains
450
451 @property
452 def addr_spec(self):
453 for x in self:
454 if x.token_type == 'addr-spec':
455 if x.local_part:
456 return x.addr_spec
457 else:
458 return quote_string(x.local_part) + x.addr_spec
459 else:
460 return '<>'
461
462
463 class ObsRoute(TokenList):
464
465 token_type = 'obs-route'
466
467 @property
468 def domains(self):
469 return [x.domain for x in self if x.token_type == 'domain']
470
471
472 class Mailbox(TokenList):
473
474 token_type = 'mailbox'
475
476 @property
477 def display_name(self):
478 if self[0].token_type == 'name-addr':
479 return self[0].display_name
480
481 @property
482 def local_part(self):
483 return self[0].local_part
484
485 @property
486 def domain(self):
487 return self[0].domain
488
489 @property
490 def route(self):
491 if self[0].token_type == 'name-addr':
492 return self[0].route
493
494 @property
495 def addr_spec(self):
496 return self[0].addr_spec
497
498
499 class InvalidMailbox(TokenList):
500
501 token_type = 'invalid-mailbox'
502
503 @property
504 def display_name(self):
505 return None
506
507 local_part = domain = route = addr_spec = display_name
508
509
510 class Domain(TokenList):
511
512 token_type = 'domain'
513 as_ew_allowed = False
514
515 @property
516 def domain(self):
517 return ''.join(super().value.split())
518
519
520 class DotAtom(TokenList):
521 token_type = 'dot-atom'
522
523
524 class DotAtomText(TokenList):
525 token_type = 'dot-atom-text'
526 as_ew_allowed = True
527
528
529 class NoFoldLiteral(TokenList):
530 token_type = 'no-fold-literal'
531 as_ew_allowed = False
532
533
534 class AddrSpec(TokenList):
535
536 token_type = 'addr-spec'
537 as_ew_allowed = False
538
539 @property
540 def local_part(self):
541 return self[0].local_part
542
543 @property
544 def domain(self):
545 if len(self) < 3:
546 return None
547 return self[-1].domain
548
549 @property
550 def value(self):
551 if len(self) < 3:
552 return self[0].value
553 return self[0].value.rstrip()+self[1].value+self[2].value.lstrip()
554
555 @property
556 def addr_spec(self):
557 nameset = set(self.local_part)
558 if len(nameset) > len(nameset-DOT_ATOM_ENDS):
559 lp = quote_string(self.local_part)
560 else:
561 lp = self.local_part
562 if self.domain is not None:
563 return lp + '@' + self.domain
564 return lp
565
566
567 class ObsLocalPart(TokenList):
568
569 token_type = 'obs-local-part'
570 as_ew_allowed = False
571
572
573 class DisplayName(Phrase):
574
575 token_type = 'display-name'
576 ew_combine_allowed = False
577
578 @property
579 def display_name(self):
580 res = TokenList(self)
581 if len(res) == 0:
582 return res.value
583 if res[0].token_type == 'cfws':
584 res.pop(0)
585 else:
586 if (isinstance(res[0], TokenList) and
587 res[0][0].token_type == 'cfws'):
588 res[0] = TokenList(res[0][1:])
589 if res[-1].token_type == 'cfws':
590 res.pop()
591 else:
592 if (isinstance(res[-1], TokenList) and
593 res[-1][-1].token_type == 'cfws'):
594 res[-1] = TokenList(res[-1][:-1])
595 return res.value
596
597 @property
598 def value(self):
599 quote = False
600 if self.defects:
601 quote = True
602 else:
603 for x in self:
604 if x.token_type == 'quoted-string':
605 quote = True
606 if len(self) != 0 and quote:
607 pre = post = ''
608 if (self[0].token_type == 'cfws' or
609 isinstance(self[0], TokenList) and
610 self[0][0].token_type == 'cfws'):
611 pre = ' '
612 if (self[-1].token_type == 'cfws' or
613 isinstance(self[-1], TokenList) and
614 self[-1][-1].token_type == 'cfws'):
615 post = ' '
616 return pre+quote_string(self.display_name)+post
617 else:
618 return super().value
619
620
621 class LocalPart(TokenList):
622
623 token_type = 'local-part'
624 as_ew_allowed = False
625
626 @property
627 def value(self):
628 if self[0].token_type == "quoted-string":
629 return self[0].quoted_value
630 else:
631 return self[0].value
632
633 @property
634 def local_part(self):
635 # Strip whitespace from front, back, and around dots.
636 res = [DOT]
637 last = DOT
638 last_is_tl = False
639 for tok in self[0] + [DOT]:
640 if tok.token_type == 'cfws':
641 continue
642 if (last_is_tl and tok.token_type == 'dot' and last and
643 last[-1].token_type == 'cfws'):
644 res[-1] = TokenList(last[:-1])
645 is_tl = isinstance(tok, TokenList)
646 if (is_tl and last.token_type == 'dot' and tok and
647 tok[0].token_type == 'cfws'):
648 res.append(TokenList(tok[1:]))
649 else:
650 res.append(tok)
651 last = res[-1]
652 last_is_tl = is_tl
653 res = TokenList(res[1:-1])
654 return res.value
655
656
657 class DomainLiteral(TokenList):
658
659 token_type = 'domain-literal'
660 as_ew_allowed = False
661
662 @property
663 def domain(self):
664 return ''.join(super().value.split())
665
666 @property
667 def ip(self):
668 for x in self:
669 if x.token_type == 'ptext':
670 return x.value
671
672
673 class MIMEVersion(TokenList):
674
675 token_type = 'mime-version'
676 major = None
677 minor = None
678
679
680 class Parameter(TokenList):
681
682 token_type = 'parameter'
683 sectioned = False
684 extended = False
685 charset = 'us-ascii'
686
687 @property
688 def section_number(self):
689 # Because the first token, the attribute (name) eats CFWS, the second
690 # token is always the section if there is one.
691 return self[1].number if self.sectioned else 0
692
693 @property
694 def param_value(self):
695 # This is part of the "handle quoted extended parameters" hack.
696 for token in self:
697 if token.token_type == 'value':
698 return token.stripped_value
699 if token.token_type == 'quoted-string':
700 for token in token:
701 if token.token_type == 'bare-quoted-string':
702 for token in token:
703 if token.token_type == 'value':
704 return token.stripped_value
705 return ''
706
707
708 class InvalidParameter(Parameter):
709
710 token_type = 'invalid-parameter'
711
712
713 class Attribute(TokenList):
714
715 token_type = 'attribute'
716
717 @property
718 def stripped_value(self):
719 for token in self:
720 if token.token_type.endswith('attrtext'):
721 return token.value
722
723 class Section(TokenList):
724
725 token_type = 'section'
726 number = None
727
728
729 class Value(TokenList):
730
731 token_type = 'value'
732
733 @property
734 def stripped_value(self):
735 token = self[0]
736 if token.token_type == 'cfws':
737 token = self[1]
738 if token.token_type.endswith(
739 ('quoted-string', 'attribute', 'extended-attribute')):
740 return token.stripped_value
741 return self.value
742
743
744 class MimeParameters(TokenList):
745
746 token_type = 'mime-parameters'
747 syntactic_break = False
748
749 @property
750 def params(self):
751 # The RFC specifically states that the ordering of parameters is not
752 # guaranteed and may be reordered by the transport layer. So we have
753 # to assume the RFC 2231 pieces can come in any order. However, we
754 # output them in the order that we first see a given name, which gives
755 # us a stable __str__.
756 params = {} # Using order preserving dict from Python 3.7+
757 for token in self:
758 if not token.token_type.endswith('parameter'):
759 continue
760 if token[0].token_type != 'attribute':
761 continue
762 name = token[0].value.strip()
763 if name not in params:
764 params[name] = []
765 params[name].append((token.section_number, token))
766 for name, parts in params.items():
767 parts = sorted(parts, key=itemgetter(0))
768 first_param = parts[0][1]
769 charset = first_param.charset
770 # Our arbitrary error recovery is to ignore duplicate parameters,
771 # to use appearance order if there are duplicate rfc 2231 parts,
772 # and to ignore gaps. This mimics the error recovery of get_param.
773 if not first_param.extended and len(parts) > 1:
774 if parts[1][0] == 0:
775 parts[1][1].defects.append(errors.InvalidHeaderDefect(
776 'duplicate parameter name; duplicate(s) ignored'))
777 parts = parts[:1]
778 # Else assume the *0* was missing...note that this is different
779 # from get_param, but we registered a defect for this earlier.
780 value_parts = []
781 i = 0
782 for section_number, param in parts:
783 if section_number != i:
784 # We could get fancier here and look for a complete
785 # duplicate extended parameter and ignore the second one
786 # seen. But we're not doing that. The old code didn't.
787 if not param.extended:
788 param.defects.append(errors.InvalidHeaderDefect(
789 'duplicate parameter name; duplicate ignored'))
790 continue
791 else:
792 param.defects.append(errors.InvalidHeaderDefect(
793 "inconsistent RFC2231 parameter numbering"))
794 i += 1
795 value = param.param_value
796 if param.extended:
797 try:
798 value = urllib.parse.unquote_to_bytes(value)
799 except UnicodeEncodeError:
800 # source had surrogate escaped bytes. What we do now
801 # is a bit of an open question. I'm not sure this is
802 # the best choice, but it is what the old algorithm did
803 value = urllib.parse.unquote(value, encoding='latin-1')
804 else:
805 try:
806 value = value.decode(charset, 'surrogateescape')
807 except (LookupError, UnicodeEncodeError):
808 # XXX: there should really be a custom defect for
809 # unknown character set to make it easy to find,
810 # because otherwise unknown charset is a silent
811 # failure.
812 value = value.decode('us-ascii', 'surrogateescape')
813 if utils._has_surrogates(value):
814 param.defects.append(errors.UndecodableBytesDefect())
815 value_parts.append(value)
816 value = ''.join(value_parts)
817 yield name, value
818
819 def __str__(self):
820 params = []
821 for name, value in self.params:
822 if value:
823 params.append('{}={}'.format(name, quote_string(value)))
824 else:
825 params.append(name)
826 params = '; '.join(params)
827 return ' ' + params if params else ''
828
829
830 class ParameterizedHeaderValue(TokenList):
831
832 # Set this false so that the value doesn't wind up on a new line even
833 # if it and the parameters would fit there but not on the first line.
834 syntactic_break = False
835
836 @property
837 def params(self):
838 for token in reversed(self):
839 if token.token_type == 'mime-parameters':
840 return token.params
841 return {}
842
843
844 class ContentType(ParameterizedHeaderValue):
845 token_type = 'content-type'
846 as_ew_allowed = False
847 maintype = 'text'
848 subtype = 'plain'
849
850
851 class ContentDisposition(ParameterizedHeaderValue):
852 token_type = 'content-disposition'
853 as_ew_allowed = False
854 content_disposition = None
855
856
857 class ContentTransferEncoding(TokenList):
858 token_type = 'content-transfer-encoding'
859 as_ew_allowed = False
860 cte = '7bit'
861
862
863 class HeaderLabel(TokenList):
864 token_type = 'header-label'
865 as_ew_allowed = False
866
867
868 class MsgID(TokenList):
869 token_type = 'msg-id'
870 as_ew_allowed = False
871
872 def fold(self, policy):
873 # message-id tokens may not be folded.
874 return str(self) + policy.linesep
875
876
877 class MessageID(MsgID):
878 token_type = 'message-id'
879
880
881 class InvalidMessageID(MessageID):
882 token_type = 'invalid-message-id'
883
884 class MessageIDList(TokenList):
885 token_type = 'message-id-list'
886
887 @property
888 def message_ids(self):
889 return [x for x in self if x.token_type=='msg-id']
890
891 class Header(TokenList):
892 token_type = 'header'
893
894
895 #
896 # Terminal classes and instances
897 #
898
899 class Terminal(str):
900
901 as_ew_allowed = True
902 ew_combine_allowed = True
903 syntactic_break = True
904
905 def __new__(cls, value, token_type):
906 self = super().__new__(cls, value)
907 self.token_type = token_type
908 self.defects = []
909 return self
910
911 def __repr__(self):
912 return "{}({})".format(self.__class__.__name__, super().__repr__())
913
914 def pprint(self):
915 print(self.__class__.__name__ + '/' + self.token_type)
916
917 @property
918 def all_defects(self):
919 return list(self.defects)
920
921 def _pp(self, indent=''):
922 return ["{}{}/{}({}){}".format(
923 indent,
924 self.__class__.__name__,
925 self.token_type,
926 super().__repr__(),
927 '' if not self.defects else ' {}'.format(self.defects),
928 )]
929
930 def pop_trailing_ws(self):
931 # This terminates the recursion.
932 return None
933
934 @property
935 def comments(self):
936 return []
937
938 def __getnewargs__(self):
939 return(str(self), self.token_type)
940
941
942 class WhiteSpaceTerminal(Terminal):
943
944 @property
945 def value(self):
946 return ' '
947
948 def startswith_fws(self):
949 return self and self[0] in WSP
950
951
952 class ValueTerminal(Terminal):
953
954 @property
955 def value(self):
956 return self
957
958 def startswith_fws(self):
959 return False
960
961
962 class EWWhiteSpaceTerminal(WhiteSpaceTerminal):
963
964 @property
965 def value(self):
966 return ''
967
968 def __str__(self):
969 return ''
970
971
972 class _InvalidEwError(errors.HeaderParseError):
973 """Invalid encoded word found while parsing headers."""
974
975
976 # XXX these need to become classes and used as instances so
977 # that a program can't change them in a parse tree and screw
978 # up other parse trees. Maybe should have tests for that, too.
979 DOT = ValueTerminal('.', 'dot')
980 ListSeparator = ValueTerminal(',', 'list-separator')
981 ListSeparator.as_ew_allowed = False
982 ListSeparator.syntactic_break = False
983 RouteComponentMarker = ValueTerminal('@', 'route-component-marker')
984
985 #
986 # Parser
987 #
988
989 # Parse strings according to RFC822/2047/2822/5322 rules.
990 #
991 # This is a stateless parser. Each get_XXX function accepts a string and
992 # returns either a Terminal or a TokenList representing the RFC object named
993 # by the method and a string containing the remaining unparsed characters
994 # from the input. Thus a parser method consumes the next syntactic construct
995 # of a given type and returns a token representing the construct plus the
996 # unparsed remainder of the input string.
997 #
998 # For example, if the first element of a structured header is a 'phrase',
999 # then:
1000 #
1001 # phrase, value = get_phrase(value)
1002 #
1003 # returns the complete phrase from the start of the string value, plus any
1004 # characters left in the string after the phrase is removed.
1005
1006 _wsp_splitter = re.compile(r'([{}]+)'.format(''.join(WSP))).split
1007 _non_atom_end_matcher = re.compile(r"[^{}]+".format(
1008 re.escape(''.join(ATOM_ENDS)))).match
1009 _non_printable_finder = re.compile(r"[\x00-\x20\x7F]").findall
1010 _non_token_end_matcher = re.compile(r"[^{}]+".format(
1011 re.escape(''.join(TOKEN_ENDS)))).match
1012 _non_attribute_end_matcher = re.compile(r"[^{}]+".format(
1013 re.escape(''.join(ATTRIBUTE_ENDS)))).match
1014 _non_extended_attribute_end_matcher = re.compile(r"[^{}]+".format(
1015 re.escape(''.join(EXTENDED_ATTRIBUTE_ENDS)))).match
1016
1017 def _validate_xtext(xtext):
1018 """If input token contains ASCII non-printables, register a defect."""
1019
1020 non_printables = _non_printable_finder(xtext)
1021 if non_printables:
1022 xtext.defects.append(errors.NonPrintableDefect(non_printables))
1023 if utils._has_surrogates(xtext):
1024 xtext.defects.append(errors.UndecodableBytesDefect(
1025 "Non-ASCII characters found in header token"))
1026
1027 def _get_ptext_to_endchars(value, endchars):
1028 """Scan printables/quoted-pairs until endchars and return unquoted ptext.
1029
1030 This function turns a run of qcontent, ccontent-without-comments, or
1031 dtext-with-quoted-printables into a single string by unquoting any
1032 quoted printables. It returns the string, the remaining value, and
1033 a flag that is True iff there were any quoted printables decoded.
1034
1035 """
1036 if not value:
1037 return '', '', False
1038 fragment, *remainder = _wsp_splitter(value, 1)
1039 vchars = []
1040 escape = False
1041 had_qp = False
1042 for pos in range(len(fragment)):
1043 if fragment[pos] == '\\':
1044 if escape:
1045 escape = False
1046 had_qp = True
1047 else:
1048 escape = True
1049 continue
1050 if escape:
1051 escape = False
1052 elif fragment[pos] in endchars:
1053 break
1054 vchars.append(fragment[pos])
1055 else:
1056 pos = pos + 1
1057 return ''.join(vchars), ''.join([fragment[pos:]] + remainder), had_qp
1058
1059 def get_fws(value):
1060 """FWS = 1*WSP
1061
1062 This isn't the RFC definition. We're using fws to represent tokens where
1063 folding can be done, but when we are parsing the *un*folding has already
1064 been done so we don't need to watch out for CRLF.
1065
1066 """
1067 newvalue = value.lstrip()
1068 fws = WhiteSpaceTerminal(value[:len(value)-len(newvalue)], 'fws')
1069 return fws, newvalue
1070
1071 def get_encoded_word(value, terminal_type='vtext'):
1072 """ encoded-word = "=?" charset "?" encoding "?" encoded-text "?="
1073
1074 """
1075 ew = EncodedWord()
1076 if not value.startswith('=?'):
1077 raise errors.HeaderParseError(
1078 "expected encoded word but found {}".format(value))
1079 tok, *remainder = value[2:].split('?=', 1)
1080 if tok == value[2:]:
1081 raise errors.HeaderParseError(
1082 "expected encoded word but found {}".format(value))
1083 remstr = ''.join(remainder)
1084 if (len(remstr) > 1 and
1085 remstr[0] in hexdigits and
1086 remstr[1] in hexdigits and
1087 tok.count('?') < 2):
1088 # The ? after the CTE was followed by an encoded word escape (=XX).
1089 rest, *remainder = remstr.split('?=', 1)
1090 tok = tok + '?=' + rest
1091 if len(tok.split()) > 1:
1092 ew.defects.append(errors.InvalidHeaderDefect(
1093 "whitespace inside encoded word"))
1094 ew.cte = value
1095 value = ''.join(remainder)
1096 try:
1097 text, charset, lang, defects = _ew.decode('=?' + tok + '?=')
1098 except (ValueError, KeyError):
1099 raise _InvalidEwError(
1100 "encoded word format invalid: '{}'".format(ew.cte))
1101 ew.charset = charset
1102 ew.lang = lang
1103 ew.defects.extend(defects)
1104 while text:
1105 if text[0] in WSP:
1106 token, text = get_fws(text)
1107 ew.append(token)
1108 continue
1109 chars, *remainder = _wsp_splitter(text, 1)
1110 vtext = ValueTerminal(chars, terminal_type)
1111 _validate_xtext(vtext)
1112 ew.append(vtext)
1113 text = ''.join(remainder)
1114 # Encoded words should be followed by a WS
1115 if value and value[0] not in WSP:
1116 ew.defects.append(errors.InvalidHeaderDefect(
1117 "missing trailing whitespace after encoded-word"))
1118 return ew, value
1119
1120 def get_unstructured(value):
1121 """unstructured = (*([FWS] vchar) *WSP) / obs-unstruct
1122 obs-unstruct = *((*LF *CR *(obs-utext) *LF *CR)) / FWS)
1123 obs-utext = %d0 / obs-NO-WS-CTL / LF / CR
1124
1125 obs-NO-WS-CTL is control characters except WSP/CR/LF.
1126
1127 So, basically, we have printable runs, plus control characters or nulls in
1128 the obsolete syntax, separated by whitespace. Since RFC 2047 uses the
1129 obsolete syntax in its specification, but requires whitespace on either
1130 side of the encoded words, I can see no reason to need to separate the
1131 non-printable-non-whitespace from the printable runs if they occur, so we
1132 parse this into xtext tokens separated by WSP tokens.
1133
1134 Because an 'unstructured' value must by definition constitute the entire
1135 value, this 'get' routine does not return a remaining value, only the
1136 parsed TokenList.
1137
1138 """
1139 # XXX: but what about bare CR and LF? They might signal the start or
1140 # end of an encoded word. YAGNI for now, since our current parsers
1141 # will never send us strings with bare CR or LF.
1142
1143 unstructured = UnstructuredTokenList()
1144 while value:
1145 if value[0] in WSP:
1146 token, value = get_fws(value)
1147 unstructured.append(token)
1148 continue
1149 valid_ew = True
1150 if value.startswith('=?'):
1151 try:
1152 token, value = get_encoded_word(value, 'utext')
1153 except _InvalidEwError:
1154 valid_ew = False
1155 except errors.HeaderParseError:
1156 # XXX: Need to figure out how to register defects when
1157 # appropriate here.
1158 pass
1159 else:
1160 have_ws = True
1161 if len(unstructured) > 0:
1162 if unstructured[-1].token_type != 'fws':
1163 unstructured.defects.append(errors.InvalidHeaderDefect(
1164 "missing whitespace before encoded word"))
1165 have_ws = False
1166 if have_ws and len(unstructured) > 1:
1167 if unstructured[-2].token_type == 'encoded-word':
1168 unstructured[-1] = EWWhiteSpaceTerminal(
1169 unstructured[-1], 'fws')
1170 unstructured.append(token)
1171 continue
1172 tok, *remainder = _wsp_splitter(value, 1)
1173 # Split in the middle of an atom if there is a rfc2047 encoded word
1174 # which does not have WSP on both sides. The defect will be registered
1175 # the next time through the loop.
1176 # This needs to only be performed when the encoded word is valid;
1177 # otherwise, performing it on an invalid encoded word can cause
1178 # the parser to go in an infinite loop.
1179 if valid_ew and rfc2047_matcher.search(tok):
1180 tok, *remainder = value.partition('=?')
1181 vtext = ValueTerminal(tok, 'utext')
1182 _validate_xtext(vtext)
1183 unstructured.append(vtext)
1184 value = ''.join(remainder)
1185 return unstructured
1186
1187 def get_qp_ctext(value):
1188 r"""ctext = <printable ascii except \ ( )>
1189
1190 This is not the RFC ctext, since we are handling nested comments in comment
1191 and unquoting quoted-pairs here. We allow anything except the '()'
1192 characters, but if we find any ASCII other than the RFC defined printable
1193 ASCII, a NonPrintableDefect is added to the token's defects list. Since
1194 quoted pairs are converted to their unquoted values, what is returned is
1195 a 'ptext' token. In this case it is a WhiteSpaceTerminal, so it's value
1196 is ' '.
1197
1198 """
1199 ptext, value, _ = _get_ptext_to_endchars(value, '()')
1200 ptext = WhiteSpaceTerminal(ptext, 'ptext')
1201 _validate_xtext(ptext)
1202 return ptext, value
1203
1204 def get_qcontent(value):
1205 """qcontent = qtext / quoted-pair
1206
1207 We allow anything except the DQUOTE character, but if we find any ASCII
1208 other than the RFC defined printable ASCII, a NonPrintableDefect is
1209 added to the token's defects list. Any quoted pairs are converted to their
1210 unquoted values, so what is returned is a 'ptext' token. In this case it
1211 is a ValueTerminal.
1212
1213 """
1214 ptext, value, _ = _get_ptext_to_endchars(value, '"')
1215 ptext = ValueTerminal(ptext, 'ptext')
1216 _validate_xtext(ptext)
1217 return ptext, value
1218
1219 def get_atext(value):
1220 """atext = <matches _atext_matcher>
1221
1222 We allow any non-ATOM_ENDS in atext, but add an InvalidATextDefect to
1223 the token's defects list if we find non-atext characters.
1224 """
1225 m = _non_atom_end_matcher(value)
1226 if not m:
1227 raise errors.HeaderParseError(
1228 "expected atext but found '{}'".format(value))
1229 atext = m.group()
1230 value = value[len(atext):]
1231 atext = ValueTerminal(atext, 'atext')
1232 _validate_xtext(atext)
1233 return atext, value
1234
1235 def get_bare_quoted_string(value):
1236 """bare-quoted-string = DQUOTE *([FWS] qcontent) [FWS] DQUOTE
1237
1238 A quoted-string without the leading or trailing white space. Its
1239 value is the text between the quote marks, with whitespace
1240 preserved and quoted pairs decoded.
1241 """
1242 if not value or value[0] != '"':
1243 raise errors.HeaderParseError(
1244 "expected '\"' but found '{}'".format(value))
1245 bare_quoted_string = BareQuotedString()
1246 value = value[1:]
1247 if value and value[0] == '"':
1248 return bare_quoted_string, value[1:]
1249 while value and value[0] != '"':
1250 if value[0] in WSP:
1251 token, value = get_fws(value)
1252 elif value[:2] == '=?':
1253 valid_ew = False
1254 try:
1255 token, value = get_encoded_word(value)
1256 bare_quoted_string.defects.append(errors.InvalidHeaderDefect(
1257 "encoded word inside quoted string"))
1258 valid_ew = True
1259 except errors.HeaderParseError:
1260 token, value = get_qcontent(value)
1261 # Collapse the whitespace between two encoded words that occur in a
1262 # bare-quoted-string.
1263 if valid_ew and len(bare_quoted_string) > 1:
1264 if (bare_quoted_string[-1].token_type == 'fws' and
1265 bare_quoted_string[-2].token_type == 'encoded-word'):
1266 bare_quoted_string[-1] = EWWhiteSpaceTerminal(
1267 bare_quoted_string[-1], 'fws')
1268 else:
1269 token, value = get_qcontent(value)
1270 bare_quoted_string.append(token)
1271 if not value:
1272 bare_quoted_string.defects.append(errors.InvalidHeaderDefect(
1273 "end of header inside quoted string"))
1274 return bare_quoted_string, value
1275 return bare_quoted_string, value[1:]
1276
1277 def get_comment(value):
1278 """comment = "(" *([FWS] ccontent) [FWS] ")"
1279 ccontent = ctext / quoted-pair / comment
1280
1281 We handle nested comments here, and quoted-pair in our qp-ctext routine.
1282 """
1283 if value and value[0] != '(':
1284 raise errors.HeaderParseError(
1285 "expected '(' but found '{}'".format(value))
1286 comment = Comment()
1287 value = value[1:]
1288 while value and value[0] != ")":
1289 if value[0] in WSP:
1290 token, value = get_fws(value)
1291 elif value[0] == '(':
1292 token, value = get_comment(value)
1293 else:
1294 token, value = get_qp_ctext(value)
1295 comment.append(token)
1296 if not value:
1297 comment.defects.append(errors.InvalidHeaderDefect(
1298 "end of header inside comment"))
1299 return comment, value
1300 return comment, value[1:]
1301
1302 def get_cfws(value):
1303 """CFWS = (1*([FWS] comment) [FWS]) / FWS
1304
1305 """
1306 cfws = CFWSList()
1307 while value and value[0] in CFWS_LEADER:
1308 if value[0] in WSP:
1309 token, value = get_fws(value)
1310 else:
1311 token, value = get_comment(value)
1312 cfws.append(token)
1313 return cfws, value
1314
1315 def get_quoted_string(value):
1316 """quoted-string = [CFWS] <bare-quoted-string> [CFWS]
1317
1318 'bare-quoted-string' is an intermediate class defined by this
1319 parser and not by the RFC grammar. It is the quoted string
1320 without any attached CFWS.
1321 """
1322 quoted_string = QuotedString()
1323 if value and value[0] in CFWS_LEADER:
1324 token, value = get_cfws(value)
1325 quoted_string.append(token)
1326 token, value = get_bare_quoted_string(value)
1327 quoted_string.append(token)
1328 if value and value[0] in CFWS_LEADER:
1329 token, value = get_cfws(value)
1330 quoted_string.append(token)
1331 return quoted_string, value
1332
1333 def get_atom(value):
1334 """atom = [CFWS] 1*atext [CFWS]
1335
1336 An atom could be an rfc2047 encoded word.
1337 """
1338 atom = Atom()
1339 if value and value[0] in CFWS_LEADER:
1340 token, value = get_cfws(value)
1341 atom.append(token)
1342 if value and value[0] in ATOM_ENDS:
1343 raise errors.HeaderParseError(
1344 "expected atom but found '{}'".format(value))
1345 if value.startswith('=?'):
1346 try:
1347 token, value = get_encoded_word(value)
1348 except errors.HeaderParseError:
1349 # XXX: need to figure out how to register defects when
1350 # appropriate here.
1351 token, value = get_atext(value)
1352 else:
1353 token, value = get_atext(value)
1354 atom.append(token)
1355 if value and value[0] in CFWS_LEADER:
1356 token, value = get_cfws(value)
1357 atom.append(token)
1358 return atom, value
1359
1360 def get_dot_atom_text(value):
1361 """ dot-text = 1*atext *("." 1*atext)
1362
1363 """
1364 dot_atom_text = DotAtomText()
1365 if not value or value[0] in ATOM_ENDS:
1366 raise errors.HeaderParseError("expected atom at a start of "
1367 "dot-atom-text but found '{}'".format(value))
1368 while value and value[0] not in ATOM_ENDS:
1369 token, value = get_atext(value)
1370 dot_atom_text.append(token)
1371 if value and value[0] == '.':
1372 dot_atom_text.append(DOT)
1373 value = value[1:]
1374 if dot_atom_text[-1] is DOT:
1375 raise errors.HeaderParseError("expected atom at end of dot-atom-text "
1376 "but found '{}'".format('.'+value))
1377 return dot_atom_text, value
1378
1379 def get_dot_atom(value):
1380 """ dot-atom = [CFWS] dot-atom-text [CFWS]
1381
1382 Any place we can have a dot atom, we could instead have an rfc2047 encoded
1383 word.
1384 """
1385 dot_atom = DotAtom()
1386 if value[0] in CFWS_LEADER:
1387 token, value = get_cfws(value)
1388 dot_atom.append(token)
1389 if value.startswith('=?'):
1390 try:
1391 token, value = get_encoded_word(value)
1392 except errors.HeaderParseError:
1393 # XXX: need to figure out how to register defects when
1394 # appropriate here.
1395 token, value = get_dot_atom_text(value)
1396 else:
1397 token, value = get_dot_atom_text(value)
1398 dot_atom.append(token)
1399 if value and value[0] in CFWS_LEADER:
1400 token, value = get_cfws(value)
1401 dot_atom.append(token)
1402 return dot_atom, value
1403
1404 def get_word(value):
1405 """word = atom / quoted-string
1406
1407 Either atom or quoted-string may start with CFWS. We have to peel off this
1408 CFWS first to determine which type of word to parse. Afterward we splice
1409 the leading CFWS, if any, into the parsed sub-token.
1410
1411 If neither an atom or a quoted-string is found before the next special, a
1412 HeaderParseError is raised.
1413
1414 The token returned is either an Atom or a QuotedString, as appropriate.
1415 This means the 'word' level of the formal grammar is not represented in the
1416 parse tree; this is because having that extra layer when manipulating the
1417 parse tree is more confusing than it is helpful.
1418
1419 """
1420 if value[0] in CFWS_LEADER:
1421 leader, value = get_cfws(value)
1422 else:
1423 leader = None
1424 if not value:
1425 raise errors.HeaderParseError(
1426 "Expected 'atom' or 'quoted-string' but found nothing.")
1427 if value[0]=='"':
1428 token, value = get_quoted_string(value)
1429 elif value[0] in SPECIALS:
1430 raise errors.HeaderParseError("Expected 'atom' or 'quoted-string' "
1431 "but found '{}'".format(value))
1432 else:
1433 token, value = get_atom(value)
1434 if leader is not None:
1435 token[:0] = [leader]
1436 return token, value
1437
1438 def get_phrase(value):
1439 """ phrase = 1*word / obs-phrase
1440 obs-phrase = word *(word / "." / CFWS)
1441
1442 This means a phrase can be a sequence of words, periods, and CFWS in any
1443 order as long as it starts with at least one word. If anything other than
1444 words is detected, an ObsoleteHeaderDefect is added to the token's defect
1445 list. We also accept a phrase that starts with CFWS followed by a dot;
1446 this is registered as an InvalidHeaderDefect, since it is not supported by
1447 even the obsolete grammar.
1448
1449 """
1450 phrase = Phrase()
1451 try:
1452 token, value = get_word(value)
1453 phrase.append(token)
1454 except errors.HeaderParseError:
1455 phrase.defects.append(errors.InvalidHeaderDefect(
1456 "phrase does not start with word"))
1457 while value and value[0] not in PHRASE_ENDS:
1458 if value[0]=='.':
1459 phrase.append(DOT)
1460 phrase.defects.append(errors.ObsoleteHeaderDefect(
1461 "period in 'phrase'"))
1462 value = value[1:]
1463 else:
1464 try:
1465 token, value = get_word(value)
1466 if (token[0].token_type == 'encoded-word'
1467 and phrase
1468 and phrase[-1].token_type == 'atom'
1469 and len(phrase[-1]) > 1
1470 and phrase[-1][-2].token_type == 'encoded-word'
1471 and phrase[-1][-1].token_type == 'cfws'
1472 and not phrase[-1][-1].comments
1473 ):
1474 # linear ws between ews needs special handing...
1475 phrase[-1][-1] = EWWhiteSpaceTerminal(phrase[-1], 'fws')
1476 except errors.HeaderParseError:
1477 if value[0] in CFWS_LEADER:
1478 token, value = get_cfws(value)
1479 phrase.defects.append(errors.ObsoleteHeaderDefect(
1480 "comment found without atom"))
1481 else:
1482 raise
1483 phrase.append(token)
1484 return phrase, value
1485
1486 def get_local_part(value):
1487 """ local-part = dot-atom / quoted-string / obs-local-part
1488
1489 """
1490 local_part = LocalPart()
1491 leader = None
1492 if value and value[0] in CFWS_LEADER:
1493 leader, value = get_cfws(value)
1494 if not value:
1495 raise errors.HeaderParseError(
1496 "expected local-part but found '{}'".format(value))
1497 try:
1498 token, value = get_dot_atom(value)
1499 except errors.HeaderParseError:
1500 try:
1501 token, value = get_word(value)
1502 except errors.HeaderParseError:
1503 if value[0] != '\\' and value[0] in PHRASE_ENDS:
1504 raise
1505 token = TokenList()
1506 if leader is not None:
1507 token[:0] = [leader]
1508 local_part.append(token)
1509 if value and (value[0]=='\\' or value[0] not in PHRASE_ENDS):
1510 obs_local_part, value = get_obs_local_part(str(local_part) + value)
1511 if obs_local_part.token_type == 'invalid-obs-local-part':
1512 local_part.defects.append(errors.InvalidHeaderDefect(
1513 "local-part is not dot-atom, quoted-string, or obs-local-part"))
1514 else:
1515 local_part.defects.append(errors.ObsoleteHeaderDefect(
1516 "local-part is not a dot-atom (contains CFWS)"))
1517 local_part[0] = obs_local_part
1518 try:
1519 local_part.value.encode('ascii')
1520 except UnicodeEncodeError:
1521 local_part.defects.append(errors.NonASCIILocalPartDefect(
1522 "local-part contains non-ASCII characters)"))
1523 return local_part, value
1524
1525 def get_obs_local_part(value):
1526 """ obs-local-part = word *("." word)
1527 """
1528 obs_local_part = ObsLocalPart()
1529 last_non_ws_was_dot = False
1530 while value and (value[0]=='\\' or value[0] not in PHRASE_ENDS):
1531 if value[0] == '.':
1532 if last_non_ws_was_dot:
1533 obs_local_part.defects.append(errors.InvalidHeaderDefect(
1534 "invalid repeated '.'"))
1535 obs_local_part.append(DOT)
1536 last_non_ws_was_dot = True
1537 value = value[1:]
1538 continue
1539 elif value[0]=='\\':
1540 obs_local_part.append(ValueTerminal(value[0],
1541 'misplaced-special'))
1542 value = value[1:]
1543 obs_local_part.defects.append(errors.InvalidHeaderDefect(
1544 "'\\' character outside of quoted-string/ccontent"))
1545 last_non_ws_was_dot = False
1546 continue
1547 if obs_local_part and obs_local_part[-1].token_type != 'dot':
1548 obs_local_part.defects.append(errors.InvalidHeaderDefect(
1549 "missing '.' between words"))
1550 try:
1551 token, value = get_word(value)
1552 last_non_ws_was_dot = False
1553 except errors.HeaderParseError:
1554 if value[0] not in CFWS_LEADER:
1555 raise
1556 token, value = get_cfws(value)
1557 obs_local_part.append(token)
1558 if not obs_local_part:
1559 raise errors.HeaderParseError(
1560 "expected obs-local-part but found '{}'".format(value))
1561 if (obs_local_part[0].token_type == 'dot' or
1562 obs_local_part[0].token_type=='cfws' and
1563 len(obs_local_part) > 1 and
1564 obs_local_part[1].token_type=='dot'):
1565 obs_local_part.defects.append(errors.InvalidHeaderDefect(
1566 "Invalid leading '.' in local part"))
1567 if (obs_local_part[-1].token_type == 'dot' or
1568 obs_local_part[-1].token_type=='cfws' and
1569 len(obs_local_part) > 1 and
1570 obs_local_part[-2].token_type=='dot'):
1571 obs_local_part.defects.append(errors.InvalidHeaderDefect(
1572 "Invalid trailing '.' in local part"))
1573 if obs_local_part.defects:
1574 obs_local_part.token_type = 'invalid-obs-local-part'
1575 return obs_local_part, value
1576
1577 def get_dtext(value):
1578 r""" dtext = <printable ascii except \ [ ]> / obs-dtext
1579 obs-dtext = obs-NO-WS-CTL / quoted-pair
1580
1581 We allow anything except the excluded characters, but if we find any
1582 ASCII other than the RFC defined printable ASCII, a NonPrintableDefect is
1583 added to the token's defects list. Quoted pairs are converted to their
1584 unquoted values, so what is returned is a ptext token, in this case a
1585 ValueTerminal. If there were quoted-printables, an ObsoleteHeaderDefect is
1586 added to the returned token's defect list.
1587
1588 """
1589 ptext, value, had_qp = _get_ptext_to_endchars(value, '[]')
1590 ptext = ValueTerminal(ptext, 'ptext')
1591 if had_qp:
1592 ptext.defects.append(errors.ObsoleteHeaderDefect(
1593 "quoted printable found in domain-literal"))
1594 _validate_xtext(ptext)
1595 return ptext, value
1596
1597 def _check_for_early_dl_end(value, domain_literal):
1598 if value:
1599 return False
1600 domain_literal.defects.append(errors.InvalidHeaderDefect(
1601 "end of input inside domain-literal"))
1602 domain_literal.append(ValueTerminal(']', 'domain-literal-end'))
1603 return True
1604
1605 def get_domain_literal(value):
1606 """ domain-literal = [CFWS] "[" *([FWS] dtext) [FWS] "]" [CFWS]
1607
1608 """
1609 domain_literal = DomainLiteral()
1610 if value[0] in CFWS_LEADER:
1611 token, value = get_cfws(value)
1612 domain_literal.append(token)
1613 if not value:
1614 raise errors.HeaderParseError("expected domain-literal")
1615 if value[0] != '[':
1616 raise errors.HeaderParseError("expected '[' at start of domain-literal "
1617 "but found '{}'".format(value))
1618 value = value[1:]
1619 domain_literal.append(ValueTerminal('[', 'domain-literal-start'))
1620 if _check_for_early_dl_end(value, domain_literal):
1621 return domain_literal, value
1622 if value[0] in WSP:
1623 token, value = get_fws(value)
1624 domain_literal.append(token)
1625 token, value = get_dtext(value)
1626 domain_literal.append(token)
1627 if _check_for_early_dl_end(value, domain_literal):
1628 return domain_literal, value
1629 if value[0] in WSP:
1630 token, value = get_fws(value)
1631 domain_literal.append(token)
1632 if _check_for_early_dl_end(value, domain_literal):
1633 return domain_literal, value
1634 if value[0] != ']':
1635 raise errors.HeaderParseError("expected ']' at end of domain-literal "
1636 "but found '{}'".format(value))
1637 domain_literal.append(ValueTerminal(']', 'domain-literal-end'))
1638 value = value[1:]
1639 if value and value[0] in CFWS_LEADER:
1640 token, value = get_cfws(value)
1641 domain_literal.append(token)
1642 return domain_literal, value
1643
1644 def get_domain(value):
1645 """ domain = dot-atom / domain-literal / obs-domain
1646 obs-domain = atom *("." atom))
1647
1648 """
1649 domain = Domain()
1650 leader = None
1651 if value and value[0] in CFWS_LEADER:
1652 leader, value = get_cfws(value)
1653 if not value:
1654 raise errors.HeaderParseError(
1655 "expected domain but found '{}'".format(value))
1656 if value[0] == '[':
1657 token, value = get_domain_literal(value)
1658 if leader is not None:
1659 token[:0] = [leader]
1660 domain.append(token)
1661 return domain, value
1662 try:
1663 token, value = get_dot_atom(value)
1664 except errors.HeaderParseError:
1665 token, value = get_atom(value)
1666 if value and value[0] == '@':
1667 raise errors.HeaderParseError('Invalid Domain')
1668 if leader is not None:
1669 token[:0] = [leader]
1670 domain.append(token)
1671 if value and value[0] == '.':
1672 domain.defects.append(errors.ObsoleteHeaderDefect(
1673 "domain is not a dot-atom (contains CFWS)"))
1674 if domain[0].token_type == 'dot-atom':
1675 domain[:] = domain[0]
1676 while value and value[0] == '.':
1677 domain.append(DOT)
1678 token, value = get_atom(value[1:])
1679 domain.append(token)
1680 return domain, value
1681
1682 def get_addr_spec(value):
1683 """ addr-spec = local-part "@" domain
1684
1685 """
1686 addr_spec = AddrSpec()
1687 token, value = get_local_part(value)
1688 addr_spec.append(token)
1689 if not value or value[0] != '@':
1690 addr_spec.defects.append(errors.InvalidHeaderDefect(
1691 "addr-spec local part with no domain"))
1692 return addr_spec, value
1693 addr_spec.append(ValueTerminal('@', 'address-at-symbol'))
1694 token, value = get_domain(value[1:])
1695 addr_spec.append(token)
1696 return addr_spec, value
1697
1698 def get_obs_route(value):
1699 """ obs-route = obs-domain-list ":"
1700 obs-domain-list = *(CFWS / ",") "@" domain *("," [CFWS] ["@" domain])
1701
1702 Returns an obs-route token with the appropriate sub-tokens (that is,
1703 there is no obs-domain-list in the parse tree).
1704 """
1705 obs_route = ObsRoute()
1706 while value and (value[0]==',' or value[0] in CFWS_LEADER):
1707 if value[0] in CFWS_LEADER:
1708 token, value = get_cfws(value)
1709 obs_route.append(token)
1710 elif value[0] == ',':
1711 obs_route.append(ListSeparator)
1712 value = value[1:]
1713 if not value or value[0] != '@':
1714 raise errors.HeaderParseError(
1715 "expected obs-route domain but found '{}'".format(value))
1716 obs_route.append(RouteComponentMarker)
1717 token, value = get_domain(value[1:])
1718 obs_route.append(token)
1719 while value and value[0]==',':
1720 obs_route.append(ListSeparator)
1721 value = value[1:]
1722 if not value:
1723 break
1724 if value[0] in CFWS_LEADER:
1725 token, value = get_cfws(value)
1726 obs_route.append(token)
1727 if not value:
1728 break
1729 if value[0] == '@':
1730 obs_route.append(RouteComponentMarker)
1731 token, value = get_domain(value[1:])
1732 obs_route.append(token)
1733 if not value:
1734 raise errors.HeaderParseError("end of header while parsing obs-route")
1735 if value[0] != ':':
1736 raise errors.HeaderParseError( "expected ':' marking end of "
1737 "obs-route but found '{}'".format(value))
1738 obs_route.append(ValueTerminal(':', 'end-of-obs-route-marker'))
1739 return obs_route, value[1:]
1740
1741 def get_angle_addr(value):
1742 """ angle-addr = [CFWS] "<" addr-spec ">" [CFWS] / obs-angle-addr
1743 obs-angle-addr = [CFWS] "<" obs-route addr-spec ">" [CFWS]
1744
1745 """
1746 angle_addr = AngleAddr()
1747 if value and value[0] in CFWS_LEADER:
1748 token, value = get_cfws(value)
1749 angle_addr.append(token)
1750 if not value or value[0] != '<':
1751 raise errors.HeaderParseError(
1752 "expected angle-addr but found '{}'".format(value))
1753 angle_addr.append(ValueTerminal('<', 'angle-addr-start'))
1754 value = value[1:]
1755 # Although it is not legal per RFC5322, SMTP uses '<>' in certain
1756 # circumstances.
1757 if value and value[0] == '>':
1758 angle_addr.append(ValueTerminal('>', 'angle-addr-end'))
1759 angle_addr.defects.append(errors.InvalidHeaderDefect(
1760 "null addr-spec in angle-addr"))
1761 value = value[1:]
1762 return angle_addr, value
1763 try:
1764 token, value = get_addr_spec(value)
1765 except errors.HeaderParseError:
1766 try:
1767 token, value = get_obs_route(value)
1768 angle_addr.defects.append(errors.ObsoleteHeaderDefect(
1769 "obsolete route specification in angle-addr"))
1770 except errors.HeaderParseError:
1771 raise errors.HeaderParseError(
1772 "expected addr-spec or obs-route but found '{}'".format(value))
1773 angle_addr.append(token)
1774 token, value = get_addr_spec(value)
1775 angle_addr.append(token)
1776 if value and value[0] == '>':
1777 value = value[1:]
1778 else:
1779 angle_addr.defects.append(errors.InvalidHeaderDefect(
1780 "missing trailing '>' on angle-addr"))
1781 angle_addr.append(ValueTerminal('>', 'angle-addr-end'))
1782 if value and value[0] in CFWS_LEADER:
1783 token, value = get_cfws(value)
1784 angle_addr.append(token)
1785 return angle_addr, value
1786
1787 def get_display_name(value):
1788 """ display-name = phrase
1789
1790 Because this is simply a name-rule, we don't return a display-name
1791 token containing a phrase, but rather a display-name token with
1792 the content of the phrase.
1793
1794 """
1795 display_name = DisplayName()
1796 token, value = get_phrase(value)
1797 display_name.extend(token[:])
1798 display_name.defects = token.defects[:]
1799 return display_name, value
1800
1801
1802 def get_name_addr(value):
1803 """ name-addr = [display-name] angle-addr
1804
1805 """
1806 name_addr = NameAddr()
1807 # Both the optional display name and the angle-addr can start with cfws.
1808 leader = None
1809 if not value:
1810 raise errors.HeaderParseError(
1811 "expected name-addr but found '{}'".format(value))
1812 if value[0] in CFWS_LEADER:
1813 leader, value = get_cfws(value)
1814 if not value:
1815 raise errors.HeaderParseError(
1816 "expected name-addr but found '{}'".format(leader))
1817 if value[0] != '<':
1818 if value[0] in PHRASE_ENDS:
1819 raise errors.HeaderParseError(
1820 "expected name-addr but found '{}'".format(value))
1821 token, value = get_display_name(value)
1822 if not value:
1823 raise errors.HeaderParseError(
1824 "expected name-addr but found '{}'".format(token))
1825 if leader is not None:
1826 if isinstance(token[0], TokenList):
1827 token[0][:0] = [leader]
1828 else:
1829 token[:0] = [leader]
1830 leader = None
1831 name_addr.append(token)
1832 token, value = get_angle_addr(value)
1833 if leader is not None:
1834 token[:0] = [leader]
1835 name_addr.append(token)
1836 return name_addr, value
1837
1838 def get_mailbox(value):
1839 """ mailbox = name-addr / addr-spec
1840
1841 """
1842 # The only way to figure out if we are dealing with a name-addr or an
1843 # addr-spec is to try parsing each one.
1844 mailbox = Mailbox()
1845 try:
1846 token, value = get_name_addr(value)
1847 except errors.HeaderParseError:
1848 try:
1849 token, value = get_addr_spec(value)
1850 except errors.HeaderParseError:
1851 raise errors.HeaderParseError(
1852 "expected mailbox but found '{}'".format(value))
1853 if any(isinstance(x, errors.InvalidHeaderDefect)
1854 for x in token.all_defects):
1855 mailbox.token_type = 'invalid-mailbox'
1856 mailbox.append(token)
1857 return mailbox, value
1858
1859 def get_invalid_mailbox(value, endchars):
1860 """ Read everything up to one of the chars in endchars.
1861
1862 This is outside the formal grammar. The InvalidMailbox TokenList that is
1863 returned acts like a Mailbox, but the data attributes are None.
1864
1865 """
1866 invalid_mailbox = InvalidMailbox()
1867 while value and value[0] not in endchars:
1868 if value[0] in PHRASE_ENDS:
1869 invalid_mailbox.append(ValueTerminal(value[0],
1870 'misplaced-special'))
1871 value = value[1:]
1872 else:
1873 token, value = get_phrase(value)
1874 invalid_mailbox.append(token)
1875 return invalid_mailbox, value
1876
1877 def get_mailbox_list(value):
1878 """ mailbox-list = (mailbox *("," mailbox)) / obs-mbox-list
1879 obs-mbox-list = *([CFWS] ",") mailbox *("," [mailbox / CFWS])
1880
1881 For this routine we go outside the formal grammar in order to improve error
1882 handling. We recognize the end of the mailbox list only at the end of the
1883 value or at a ';' (the group terminator). This is so that we can turn
1884 invalid mailboxes into InvalidMailbox tokens and continue parsing any
1885 remaining valid mailboxes. We also allow all mailbox entries to be null,
1886 and this condition is handled appropriately at a higher level.
1887
1888 """
1889 mailbox_list = MailboxList()
1890 while value and value[0] != ';':
1891 try:
1892 token, value = get_mailbox(value)
1893 mailbox_list.append(token)
1894 except errors.HeaderParseError:
1895 leader = None
1896 if value[0] in CFWS_LEADER:
1897 leader, value = get_cfws(value)
1898 if not value or value[0] in ',;':
1899 mailbox_list.append(leader)
1900 mailbox_list.defects.append(errors.ObsoleteHeaderDefect(
1901 "empty element in mailbox-list"))
1902 else:
1903 token, value = get_invalid_mailbox(value, ',;')
1904 if leader is not None:
1905 token[:0] = [leader]
1906 mailbox_list.append(token)
1907 mailbox_list.defects.append(errors.InvalidHeaderDefect(
1908 "invalid mailbox in mailbox-list"))
1909 elif value[0] == ',':
1910 mailbox_list.defects.append(errors.ObsoleteHeaderDefect(
1911 "empty element in mailbox-list"))
1912 else:
1913 token, value = get_invalid_mailbox(value, ',;')
1914 if leader is not None:
1915 token[:0] = [leader]
1916 mailbox_list.append(token)
1917 mailbox_list.defects.append(errors.InvalidHeaderDefect(
1918 "invalid mailbox in mailbox-list"))
1919 if value and value[0] not in ',;':
1920 # Crap after mailbox; treat it as an invalid mailbox.
1921 # The mailbox info will still be available.
1922 mailbox = mailbox_list[-1]
1923 mailbox.token_type = 'invalid-mailbox'
1924 token, value = get_invalid_mailbox(value, ',;')
1925 mailbox.extend(token)
1926 mailbox_list.defects.append(errors.InvalidHeaderDefect(
1927 "invalid mailbox in mailbox-list"))
1928 if value and value[0] == ',':
1929 mailbox_list.append(ListSeparator)
1930 value = value[1:]
1931 return mailbox_list, value
1932
1933
1934 def get_group_list(value):
1935 """ group-list = mailbox-list / CFWS / obs-group-list
1936 obs-group-list = 1*([CFWS] ",") [CFWS]
1937
1938 """
1939 group_list = GroupList()
1940 if not value:
1941 group_list.defects.append(errors.InvalidHeaderDefect(
1942 "end of header before group-list"))
1943 return group_list, value
1944 leader = None
1945 if value and value[0] in CFWS_LEADER:
1946 leader, value = get_cfws(value)
1947 if not value:
1948 # This should never happen in email parsing, since CFWS-only is a
1949 # legal alternative to group-list in a group, which is the only
1950 # place group-list appears.
1951 group_list.defects.append(errors.InvalidHeaderDefect(
1952 "end of header in group-list"))
1953 group_list.append(leader)
1954 return group_list, value
1955 if value[0] == ';':
1956 group_list.append(leader)
1957 return group_list, value
1958 token, value = get_mailbox_list(value)
1959 if len(token.all_mailboxes)==0:
1960 if leader is not None:
1961 group_list.append(leader)
1962 group_list.extend(token)
1963 group_list.defects.append(errors.ObsoleteHeaderDefect(
1964 "group-list with empty entries"))
1965 return group_list, value
1966 if leader is not None:
1967 token[:0] = [leader]
1968 group_list.append(token)
1969 return group_list, value
1970
1971 def get_group(value):
1972 """ group = display-name ":" [group-list] ";" [CFWS]
1973
1974 """
1975 group = Group()
1976 token, value = get_display_name(value)
1977 if not value or value[0] != ':':
1978 raise errors.HeaderParseError("expected ':' at end of group "
1979 "display name but found '{}'".format(value))
1980 group.append(token)
1981 group.append(ValueTerminal(':', 'group-display-name-terminator'))
1982 value = value[1:]
1983 if value and value[0] == ';':
1984 group.append(ValueTerminal(';', 'group-terminator'))
1985 return group, value[1:]
1986 token, value = get_group_list(value)
1987 group.append(token)
1988 if not value:
1989 group.defects.append(errors.InvalidHeaderDefect(
1990 "end of header in group"))
1991 elif value[0] != ';':
1992 raise errors.HeaderParseError(
1993 "expected ';' at end of group but found {}".format(value))
1994 group.append(ValueTerminal(';', 'group-terminator'))
1995 value = value[1:]
1996 if value and value[0] in CFWS_LEADER:
1997 token, value = get_cfws(value)
1998 group.append(token)
1999 return group, value
2000
2001 def get_address(value):
2002 """ address = mailbox / group
2003
2004 Note that counter-intuitively, an address can be either a single address or
2005 a list of addresses (a group). This is why the returned Address object has
2006 a 'mailboxes' attribute which treats a single address as a list of length
2007 one. When you need to differentiate between to two cases, extract the single
2008 element, which is either a mailbox or a group token.
2009
2010 """
2011 # The formal grammar isn't very helpful when parsing an address. mailbox
2012 # and group, especially when allowing for obsolete forms, start off very
2013 # similarly. It is only when you reach one of @, <, or : that you know
2014 # what you've got. So, we try each one in turn, starting with the more
2015 # likely of the two. We could perhaps make this more efficient by looking
2016 # for a phrase and then branching based on the next character, but that
2017 # would be a premature optimization.
2018 address = Address()
2019 try:
2020 token, value = get_group(value)
2021 except errors.HeaderParseError:
2022 try:
2023 token, value = get_mailbox(value)
2024 except errors.HeaderParseError:
2025 raise errors.HeaderParseError(
2026 "expected address but found '{}'".format(value))
2027 address.append(token)
2028 return address, value
2029
2030 def get_address_list(value):
2031 """ address_list = (address *("," address)) / obs-addr-list
2032 obs-addr-list = *([CFWS] ",") address *("," [address / CFWS])
2033
2034 We depart from the formal grammar here by continuing to parse until the end
2035 of the input, assuming the input to be entirely composed of an
2036 address-list. This is always true in email parsing, and allows us
2037 to skip invalid addresses to parse additional valid ones.
2038
2039 """
2040 address_list = AddressList()
2041 while value:
2042 try:
2043 token, value = get_address(value)
2044 address_list.append(token)
2045 except errors.HeaderParseError:
2046 leader = None
2047 if value[0] in CFWS_LEADER:
2048 leader, value = get_cfws(value)
2049 if not value or value[0] == ',':
2050 address_list.append(leader)
2051 address_list.defects.append(errors.ObsoleteHeaderDefect(
2052 "address-list entry with no content"))
2053 else:
2054 token, value = get_invalid_mailbox(value, ',')
2055 if leader is not None:
2056 token[:0] = [leader]
2057 address_list.append(Address([token]))
2058 address_list.defects.append(errors.InvalidHeaderDefect(
2059 "invalid address in address-list"))
2060 elif value[0] == ',':
2061 address_list.defects.append(errors.ObsoleteHeaderDefect(
2062 "empty element in address-list"))
2063 else:
2064 token, value = get_invalid_mailbox(value, ',')
2065 if leader is not None:
2066 token[:0] = [leader]
2067 address_list.append(Address([token]))
2068 address_list.defects.append(errors.InvalidHeaderDefect(
2069 "invalid address in address-list"))
2070 if value and value[0] != ',':
2071 # Crap after address: add it to the address list
2072 # as an invalid mailbox
2073 token, value = get_invalid_mailbox(value, ',')
2074 address_list.append(Address([token]))
2075 address_list.defects.append(errors.InvalidHeaderDefect(
2076 "invalid address in address-list"))
2077 if value: # Must be a , at this point.
2078 address_list.append(ListSeparator)
2079 value = value[1:]
2080 return address_list, value
2081
2082
2083 def get_no_fold_literal(value):
2084 """ no-fold-literal = "[" *dtext "]"
2085 """
2086 no_fold_literal = NoFoldLiteral()
2087 if not value:
2088 raise errors.HeaderParseError(
2089 "expected no-fold-literal but found '{}'".format(value))
2090 if value[0] != '[':
2091 raise errors.HeaderParseError(
2092 "expected '[' at the start of no-fold-literal "
2093 "but found '{}'".format(value))
2094 no_fold_literal.append(ValueTerminal('[', 'no-fold-literal-start'))
2095 value = value[1:]
2096 token, value = get_dtext(value)
2097 no_fold_literal.append(token)
2098 if not value or value[0] != ']':
2099 raise errors.HeaderParseError(
2100 "expected ']' at the end of no-fold-literal "
2101 "but found '{}'".format(value))
2102 no_fold_literal.append(ValueTerminal(']', 'no-fold-literal-end'))
2103 return no_fold_literal, value[1:]
2104
2105 def get_msg_id(value):
2106 """msg-id = [CFWS] "<" id-left '@' id-right ">" [CFWS]
2107 id-left = dot-atom-text / obs-id-left
2108 id-right = dot-atom-text / no-fold-literal / obs-id-right
2109 no-fold-literal = "[" *dtext "]"
2110 """
2111 msg_id = MsgID()
2112 if value and value[0] in CFWS_LEADER:
2113 token, value = get_cfws(value)
2114 msg_id.append(token)
2115 if not value or value[0] != '<':
2116 raise errors.HeaderParseError(
2117 "expected msg-id but found '{}'".format(value))
2118 msg_id.append(ValueTerminal('<', 'msg-id-start'))
2119 value = value[1:]
2120 # Parse id-left.
2121 try:
2122 token, value = get_dot_atom_text(value)
2123 except errors.HeaderParseError:
2124 try:
2125 # obs-id-left is same as local-part of add-spec.
2126 token, value = get_obs_local_part(value)
2127 msg_id.defects.append(errors.ObsoleteHeaderDefect(
2128 "obsolete id-left in msg-id"))
2129 except errors.HeaderParseError:
2130 raise errors.HeaderParseError(
2131 "expected dot-atom-text or obs-id-left"
2132 " but found '{}'".format(value))
2133 msg_id.append(token)
2134 if not value or value[0] != '@':
2135 msg_id.defects.append(errors.InvalidHeaderDefect(
2136 "msg-id with no id-right"))
2137 # Even though there is no id-right, if the local part
2138 # ends with `>` let's just parse it too and return
2139 # along with the defect.
2140 if value and value[0] == '>':
2141 msg_id.append(ValueTerminal('>', 'msg-id-end'))
2142 value = value[1:]
2143 return msg_id, value
2144 msg_id.append(ValueTerminal('@', 'address-at-symbol'))
2145 value = value[1:]
2146 # Parse id-right.
2147 try:
2148 token, value = get_dot_atom_text(value)
2149 except errors.HeaderParseError:
2150 try:
2151 token, value = get_no_fold_literal(value)
2152 except errors.HeaderParseError:
2153 try:
2154 token, value = get_domain(value)
2155 msg_id.defects.append(errors.ObsoleteHeaderDefect(
2156 "obsolete id-right in msg-id"))
2157 except errors.HeaderParseError:
2158 raise errors.HeaderParseError(
2159 "expected dot-atom-text, no-fold-literal or obs-id-right"
2160 " but found '{}'".format(value))
2161 msg_id.append(token)
2162 if value and value[0] == '>':
2163 value = value[1:]
2164 else:
2165 msg_id.defects.append(errors.InvalidHeaderDefect(
2166 "missing trailing '>' on msg-id"))
2167 msg_id.append(ValueTerminal('>', 'msg-id-end'))
2168 if value and value[0] in CFWS_LEADER:
2169 token, value = get_cfws(value)
2170 msg_id.append(token)
2171 return msg_id, value
2172
2173
2174 def parse_message_id(value):
2175 """message-id = "Message-ID:" msg-id CRLF
2176 """
2177 message_id = MessageID()
2178 try:
2179 token, value = get_msg_id(value)
2180 message_id.append(token)
2181 except errors.HeaderParseError as ex:
2182 token = get_unstructured(value)
2183 message_id = InvalidMessageID(token)
2184 message_id.defects.append(
2185 errors.InvalidHeaderDefect("Invalid msg-id: {!r}".format(ex)))
2186 else:
2187 # Value after parsing a valid msg_id should be None.
2188 if value:
2189 message_id.defects.append(errors.InvalidHeaderDefect(
2190 "Unexpected {!r}".format(value)))
2191
2192 return message_id
2193
2194 def parse_message_ids(value):
2195 """in-reply-to = "In-Reply-To:" 1*msg-id CRLF
2196 references = "References:" 1*msg-id CRLF
2197 """
2198 message_id_list = MessageIDList()
2199 while value:
2200 if value[0] == ',':
2201 # message id list separated with commas - this is invalid,
2202 # but happens rather frequently in the wild
2203 message_id_list.defects.append(
2204 errors.InvalidHeaderDefect("comma in msg-id list"))
2205 message_id_list.append(
2206 WhiteSpaceTerminal(' ', 'invalid-comma-replacement'))
2207 value = value[1:]
2208 continue
2209 try:
2210 token, value = get_msg_id(value)
2211 message_id_list.append(token)
2212 except errors.HeaderParseError as ex:
2213 token = get_unstructured(value)
2214 message_id_list.append(InvalidMessageID(token))
2215 message_id_list.defects.append(
2216 errors.InvalidHeaderDefect("Invalid msg-id: {!r}".format(ex)))
2217 break
2218 return message_id_list
2219
2220 #
2221 # XXX: As I begin to add additional header parsers, I'm realizing we probably
2222 # have two level of parser routines: the get_XXX methods that get a token in
2223 # the grammar, and parse_XXX methods that parse an entire field value. So
2224 # get_address_list above should really be a parse_ method, as probably should
2225 # be get_unstructured.
2226 #
2227
2228 def parse_mime_version(value):
2229 """ mime-version = [CFWS] 1*digit [CFWS] "." [CFWS] 1*digit [CFWS]
2230
2231 """
2232 # The [CFWS] is implicit in the RFC 2045 BNF.
2233 # XXX: This routine is a bit verbose, should factor out a get_int method.
2234 mime_version = MIMEVersion()
2235 if not value:
2236 mime_version.defects.append(errors.HeaderMissingRequiredValue(
2237 "Missing MIME version number (eg: 1.0)"))
2238 return mime_version
2239 if value[0] in CFWS_LEADER:
2240 token, value = get_cfws(value)
2241 mime_version.append(token)
2242 if not value:
2243 mime_version.defects.append(errors.HeaderMissingRequiredValue(
2244 "Expected MIME version number but found only CFWS"))
2245 digits = ''
2246 while value and value[0] != '.' and value[0] not in CFWS_LEADER:
2247 digits += value[0]
2248 value = value[1:]
2249 if not digits.isdigit():
2250 mime_version.defects.append(errors.InvalidHeaderDefect(
2251 "Expected MIME major version number but found {!r}".format(digits)))
2252 mime_version.append(ValueTerminal(digits, 'xtext'))
2253 else:
2254 mime_version.major = int(digits)
2255 mime_version.append(ValueTerminal(digits, 'digits'))
2256 if value and value[0] in CFWS_LEADER:
2257 token, value = get_cfws(value)
2258 mime_version.append(token)
2259 if not value or value[0] != '.':
2260 if mime_version.major is not None:
2261 mime_version.defects.append(errors.InvalidHeaderDefect(
2262 "Incomplete MIME version; found only major number"))
2263 if value:
2264 mime_version.append(ValueTerminal(value, 'xtext'))
2265 return mime_version
2266 mime_version.append(ValueTerminal('.', 'version-separator'))
2267 value = value[1:]
2268 if value and value[0] in CFWS_LEADER:
2269 token, value = get_cfws(value)
2270 mime_version.append(token)
2271 if not value:
2272 if mime_version.major is not None:
2273 mime_version.defects.append(errors.InvalidHeaderDefect(
2274 "Incomplete MIME version; found only major number"))
2275 return mime_version
2276 digits = ''
2277 while value and value[0] not in CFWS_LEADER:
2278 digits += value[0]
2279 value = value[1:]
2280 if not digits.isdigit():
2281 mime_version.defects.append(errors.InvalidHeaderDefect(
2282 "Expected MIME minor version number but found {!r}".format(digits)))
2283 mime_version.append(ValueTerminal(digits, 'xtext'))
2284 else:
2285 mime_version.minor = int(digits)
2286 mime_version.append(ValueTerminal(digits, 'digits'))
2287 if value and value[0] in CFWS_LEADER:
2288 token, value = get_cfws(value)
2289 mime_version.append(token)
2290 if value:
2291 mime_version.defects.append(errors.InvalidHeaderDefect(
2292 "Excess non-CFWS text after MIME version"))
2293 mime_version.append(ValueTerminal(value, 'xtext'))
2294 return mime_version
2295
2296 def get_invalid_parameter(value):
2297 """ Read everything up to the next ';'.
2298
2299 This is outside the formal grammar. The InvalidParameter TokenList that is
2300 returned acts like a Parameter, but the data attributes are None.
2301
2302 """
2303 invalid_parameter = InvalidParameter()
2304 while value and value[0] != ';':
2305 if value[0] in PHRASE_ENDS:
2306 invalid_parameter.append(ValueTerminal(value[0],
2307 'misplaced-special'))
2308 value = value[1:]
2309 else:
2310 token, value = get_phrase(value)
2311 invalid_parameter.append(token)
2312 return invalid_parameter, value
2313
2314 def get_ttext(value):
2315 """ttext = <matches _ttext_matcher>
2316
2317 We allow any non-TOKEN_ENDS in ttext, but add defects to the token's
2318 defects list if we find non-ttext characters. We also register defects for
2319 *any* non-printables even though the RFC doesn't exclude all of them,
2320 because we follow the spirit of RFC 5322.
2321
2322 """
2323 m = _non_token_end_matcher(value)
2324 if not m:
2325 raise errors.HeaderParseError(
2326 "expected ttext but found '{}'".format(value))
2327 ttext = m.group()
2328 value = value[len(ttext):]
2329 ttext = ValueTerminal(ttext, 'ttext')
2330 _validate_xtext(ttext)
2331 return ttext, value
2332
2333 def get_token(value):
2334 """token = [CFWS] 1*ttext [CFWS]
2335
2336 The RFC equivalent of ttext is any US-ASCII chars except space, ctls, or
2337 tspecials. We also exclude tabs even though the RFC doesn't.
2338
2339 The RFC implies the CFWS but is not explicit about it in the BNF.
2340
2341 """
2342 mtoken = Token()
2343 if value and value[0] in CFWS_LEADER:
2344 token, value = get_cfws(value)
2345 mtoken.append(token)
2346 if value and value[0] in TOKEN_ENDS:
2347 raise errors.HeaderParseError(
2348 "expected token but found '{}'".format(value))
2349 token, value = get_ttext(value)
2350 mtoken.append(token)
2351 if value and value[0] in CFWS_LEADER:
2352 token, value = get_cfws(value)
2353 mtoken.append(token)
2354 return mtoken, value
2355
2356 def get_attrtext(value):
2357 """attrtext = 1*(any non-ATTRIBUTE_ENDS character)
2358
2359 We allow any non-ATTRIBUTE_ENDS in attrtext, but add defects to the
2360 token's defects list if we find non-attrtext characters. We also register
2361 defects for *any* non-printables even though the RFC doesn't exclude all of
2362 them, because we follow the spirit of RFC 5322.
2363
2364 """
2365 m = _non_attribute_end_matcher(value)
2366 if not m:
2367 raise errors.HeaderParseError(
2368 "expected attrtext but found {!r}".format(value))
2369 attrtext = m.group()
2370 value = value[len(attrtext):]
2371 attrtext = ValueTerminal(attrtext, 'attrtext')
2372 _validate_xtext(attrtext)
2373 return attrtext, value
2374
2375 def get_attribute(value):
2376 """ [CFWS] 1*attrtext [CFWS]
2377
2378 This version of the BNF makes the CFWS explicit, and as usual we use a
2379 value terminal for the actual run of characters. The RFC equivalent of
2380 attrtext is the token characters, with the subtraction of '*', "'", and '%'.
2381 We include tab in the excluded set just as we do for token.
2382
2383 """
2384 attribute = Attribute()
2385 if value and value[0] in CFWS_LEADER:
2386 token, value = get_cfws(value)
2387 attribute.append(token)
2388 if value and value[0] in ATTRIBUTE_ENDS:
2389 raise errors.HeaderParseError(
2390 "expected token but found '{}'".format(value))
2391 token, value = get_attrtext(value)
2392 attribute.append(token)
2393 if value and value[0] in CFWS_LEADER:
2394 token, value = get_cfws(value)
2395 attribute.append(token)
2396 return attribute, value
2397
2398 def get_extended_attrtext(value):
2399 """attrtext = 1*(any non-ATTRIBUTE_ENDS character plus '%')
2400
2401 This is a special parsing routine so that we get a value that
2402 includes % escapes as a single string (which we decode as a single
2403 string later).
2404
2405 """
2406 m = _non_extended_attribute_end_matcher(value)
2407 if not m:
2408 raise errors.HeaderParseError(
2409 "expected extended attrtext but found {!r}".format(value))
2410 attrtext = m.group()
2411 value = value[len(attrtext):]
2412 attrtext = ValueTerminal(attrtext, 'extended-attrtext')
2413 _validate_xtext(attrtext)
2414 return attrtext, value
2415
2416 def get_extended_attribute(value):
2417 """ [CFWS] 1*extended_attrtext [CFWS]
2418
2419 This is like the non-extended version except we allow % characters, so that
2420 we can pick up an encoded value as a single string.
2421
2422 """
2423 # XXX: should we have an ExtendedAttribute TokenList?
2424 attribute = Attribute()
2425 if value and value[0] in CFWS_LEADER:
2426 token, value = get_cfws(value)
2427 attribute.append(token)
2428 if value and value[0] in EXTENDED_ATTRIBUTE_ENDS:
2429 raise errors.HeaderParseError(
2430 "expected token but found '{}'".format(value))
2431 token, value = get_extended_attrtext(value)
2432 attribute.append(token)
2433 if value and value[0] in CFWS_LEADER:
2434 token, value = get_cfws(value)
2435 attribute.append(token)
2436 return attribute, value
2437
2438 def get_section(value):
2439 """ '*' digits
2440
2441 The formal BNF is more complicated because leading 0s are not allowed. We
2442 check for that and add a defect. We also assume no CFWS is allowed between
2443 the '*' and the digits, though the RFC is not crystal clear on that.
2444 The caller should already have dealt with leading CFWS.
2445
2446 """
2447 section = Section()
2448 if not value or value[0] != '*':
2449 raise errors.HeaderParseError("Expected section but found {}".format(
2450 value))
2451 section.append(ValueTerminal('*', 'section-marker'))
2452 value = value[1:]
2453 if not value or not value[0].isdigit():
2454 raise errors.HeaderParseError("Expected section number but "
2455 "found {}".format(value))
2456 digits = ''
2457 while value and value[0].isdigit():
2458 digits += value[0]
2459 value = value[1:]
2460 if digits[0] == '0' and digits != '0':
2461 section.defects.append(errors.InvalidHeaderDefect(
2462 "section number has an invalid leading 0"))
2463 section.number = int(digits)
2464 section.append(ValueTerminal(digits, 'digits'))
2465 return section, value
2466
2467
2468 def get_value(value):
2469 """ quoted-string / attribute
2470
2471 """
2472 v = Value()
2473 if not value:
2474 raise errors.HeaderParseError("Expected value but found end of string")
2475 leader = None
2476 if value[0] in CFWS_LEADER:
2477 leader, value = get_cfws(value)
2478 if not value:
2479 raise errors.HeaderParseError("Expected value but found "
2480 "only {}".format(leader))
2481 if value[0] == '"':
2482 token, value = get_quoted_string(value)
2483 else:
2484 token, value = get_extended_attribute(value)
2485 if leader is not None:
2486 token[:0] = [leader]
2487 v.append(token)
2488 return v, value
2489
2490 def get_parameter(value):
2491 """ attribute [section] ["*"] [CFWS] "=" value
2492
2493 The CFWS is implied by the RFC but not made explicit in the BNF. This
2494 simplified form of the BNF from the RFC is made to conform with the RFC BNF
2495 through some extra checks. We do it this way because it makes both error
2496 recovery and working with the resulting parse tree easier.
2497 """
2498 # It is possible CFWS would also be implicitly allowed between the section
2499 # and the 'extended-attribute' marker (the '*') , but we've never seen that
2500 # in the wild and we will therefore ignore the possibility.
2501 param = Parameter()
2502 token, value = get_attribute(value)
2503 param.append(token)
2504 if not value or value[0] == ';':
2505 param.defects.append(errors.InvalidHeaderDefect("Parameter contains "
2506 "name ({}) but no value".format(token)))
2507 return param, value
2508 if value[0] == '*':
2509 try:
2510 token, value = get_section(value)
2511 param.sectioned = True
2512 param.append(token)
2513 except errors.HeaderParseError:
2514 pass
2515 if not value:
2516 raise errors.HeaderParseError("Incomplete parameter")
2517 if value[0] == '*':
2518 param.append(ValueTerminal('*', 'extended-parameter-marker'))
2519 value = value[1:]
2520 param.extended = True
2521 if value[0] != '=':
2522 raise errors.HeaderParseError("Parameter not followed by '='")
2523 param.append(ValueTerminal('=', 'parameter-separator'))
2524 value = value[1:]
2525 if value and value[0] in CFWS_LEADER:
2526 token, value = get_cfws(value)
2527 param.append(token)
2528 remainder = None
2529 appendto = param
2530 if param.extended and value and value[0] == '"':
2531 # Now for some serious hackery to handle the common invalid case of
2532 # double quotes around an extended value. We also accept (with defect)
2533 # a value marked as encoded that isn't really.
2534 qstring, remainder = get_quoted_string(value)
2535 inner_value = qstring.stripped_value
2536 semi_valid = False
2537 if param.section_number == 0:
2538 if inner_value and inner_value[0] == "'":
2539 semi_valid = True
2540 else:
2541 token, rest = get_attrtext(inner_value)
2542 if rest and rest[0] == "'":
2543 semi_valid = True
2544 else:
2545 try:
2546 token, rest = get_extended_attrtext(inner_value)
2547 except:
2548 pass
2549 else:
2550 if not rest:
2551 semi_valid = True
2552 if semi_valid:
2553 param.defects.append(errors.InvalidHeaderDefect(
2554 "Quoted string value for extended parameter is invalid"))
2555 param.append(qstring)
2556 for t in qstring:
2557 if t.token_type == 'bare-quoted-string':
2558 t[:] = []
2559 appendto = t
2560 break
2561 value = inner_value
2562 else:
2563 remainder = None
2564 param.defects.append(errors.InvalidHeaderDefect(
2565 "Parameter marked as extended but appears to have a "
2566 "quoted string value that is non-encoded"))
2567 if value and value[0] == "'":
2568 token = None
2569 else:
2570 token, value = get_value(value)
2571 if not param.extended or param.section_number > 0:
2572 if not value or value[0] != "'":
2573 appendto.append(token)
2574 if remainder is not None:
2575 assert not value, value
2576 value = remainder
2577 return param, value
2578 param.defects.append(errors.InvalidHeaderDefect(
2579 "Apparent initial-extended-value but attribute "
2580 "was not marked as extended or was not initial section"))
2581 if not value:
2582 # Assume the charset/lang is missing and the token is the value.
2583 param.defects.append(errors.InvalidHeaderDefect(
2584 "Missing required charset/lang delimiters"))
2585 appendto.append(token)
2586 if remainder is None:
2587 return param, value
2588 else:
2589 if token is not None:
2590 for t in token:
2591 if t.token_type == 'extended-attrtext':
2592 break
2593 t.token_type == 'attrtext'
2594 appendto.append(t)
2595 param.charset = t.value
2596 if value[0] != "'":
2597 raise errors.HeaderParseError("Expected RFC2231 char/lang encoding "
2598 "delimiter, but found {!r}".format(value))
2599 appendto.append(ValueTerminal("'", 'RFC2231-delimiter'))
2600 value = value[1:]
2601 if value and value[0] != "'":
2602 token, value = get_attrtext(value)
2603 appendto.append(token)
2604 param.lang = token.value
2605 if not value or value[0] != "'":
2606 raise errors.HeaderParseError("Expected RFC2231 char/lang encoding "
2607 "delimiter, but found {}".format(value))
2608 appendto.append(ValueTerminal("'", 'RFC2231-delimiter'))
2609 value = value[1:]
2610 if remainder is not None:
2611 # Treat the rest of value as bare quoted string content.
2612 v = Value()
2613 while value:
2614 if value[0] in WSP:
2615 token, value = get_fws(value)
2616 elif value[0] == '"':
2617 token = ValueTerminal('"', 'DQUOTE')
2618 value = value[1:]
2619 else:
2620 token, value = get_qcontent(value)
2621 v.append(token)
2622 token = v
2623 else:
2624 token, value = get_value(value)
2625 appendto.append(token)
2626 if remainder is not None:
2627 assert not value, value
2628 value = remainder
2629 return param, value
2630
2631 def parse_mime_parameters(value):
2632 """ parameter *( ";" parameter )
2633
2634 That BNF is meant to indicate this routine should only be called after
2635 finding and handling the leading ';'. There is no corresponding rule in
2636 the formal RFC grammar, but it is more convenient for us for the set of
2637 parameters to be treated as its own TokenList.
2638
2639 This is 'parse' routine because it consumes the remaining value, but it
2640 would never be called to parse a full header. Instead it is called to
2641 parse everything after the non-parameter value of a specific MIME header.
2642
2643 """
2644 mime_parameters = MimeParameters()
2645 while value:
2646 try:
2647 token, value = get_parameter(value)
2648 mime_parameters.append(token)
2649 except errors.HeaderParseError:
2650 leader = None
2651 if value[0] in CFWS_LEADER:
2652 leader, value = get_cfws(value)
2653 if not value:
2654 mime_parameters.append(leader)
2655 return mime_parameters
2656 if value[0] == ';':
2657 if leader is not None:
2658 mime_parameters.append(leader)
2659 mime_parameters.defects.append(errors.InvalidHeaderDefect(
2660 "parameter entry with no content"))
2661 else:
2662 token, value = get_invalid_parameter(value)
2663 if leader:
2664 token[:0] = [leader]
2665 mime_parameters.append(token)
2666 mime_parameters.defects.append(errors.InvalidHeaderDefect(
2667 "invalid parameter {!r}".format(token)))
2668 if value and value[0] != ';':
2669 # Junk after the otherwise valid parameter. Mark it as
2670 # invalid, but it will have a value.
2671 param = mime_parameters[-1]
2672 param.token_type = 'invalid-parameter'
2673 token, value = get_invalid_parameter(value)
2674 param.extend(token)
2675 mime_parameters.defects.append(errors.InvalidHeaderDefect(
2676 "parameter with invalid trailing text {!r}".format(token)))
2677 if value:
2678 # Must be a ';' at this point.
2679 mime_parameters.append(ValueTerminal(';', 'parameter-separator'))
2680 value = value[1:]
2681 return mime_parameters
2682
2683 def _find_mime_parameters(tokenlist, value):
2684 """Do our best to find the parameters in an invalid MIME header
2685
2686 """
2687 while value and value[0] != ';':
2688 if value[0] in PHRASE_ENDS:
2689 tokenlist.append(ValueTerminal(value[0], 'misplaced-special'))
2690 value = value[1:]
2691 else:
2692 token, value = get_phrase(value)
2693 tokenlist.append(token)
2694 if not value:
2695 return
2696 tokenlist.append(ValueTerminal(';', 'parameter-separator'))
2697 tokenlist.append(parse_mime_parameters(value[1:]))
2698
2699 def parse_content_type_header(value):
2700 """ maintype "/" subtype *( ";" parameter )
2701
2702 The maintype and substype are tokens. Theoretically they could
2703 be checked against the official IANA list + x-token, but we
2704 don't do that.
2705 """
2706 ctype = ContentType()
2707 if not value:
2708 ctype.defects.append(errors.HeaderMissingRequiredValue(
2709 "Missing content type specification"))
2710 return ctype
2711 try:
2712 token, value = get_token(value)
2713 except errors.HeaderParseError:
2714 ctype.defects.append(errors.InvalidHeaderDefect(
2715 "Expected content maintype but found {!r}".format(value)))
2716 _find_mime_parameters(ctype, value)
2717 return ctype
2718 ctype.append(token)
2719 # XXX: If we really want to follow the formal grammar we should make
2720 # mantype and subtype specialized TokenLists here. Probably not worth it.
2721 if not value or value[0] != '/':
2722 ctype.defects.append(errors.InvalidHeaderDefect(
2723 "Invalid content type"))
2724 if value:
2725 _find_mime_parameters(ctype, value)
2726 return ctype
2727 ctype.maintype = token.value.strip().lower()
2728 ctype.append(ValueTerminal('/', 'content-type-separator'))
2729 value = value[1:]
2730 try:
2731 token, value = get_token(value)
2732 except errors.HeaderParseError:
2733 ctype.defects.append(errors.InvalidHeaderDefect(
2734 "Expected content subtype but found {!r}".format(value)))
2735 _find_mime_parameters(ctype, value)
2736 return ctype
2737 ctype.append(token)
2738 ctype.subtype = token.value.strip().lower()
2739 if not value:
2740 return ctype
2741 if value[0] != ';':
2742 ctype.defects.append(errors.InvalidHeaderDefect(
2743 "Only parameters are valid after content type, but "
2744 "found {!r}".format(value)))
2745 # The RFC requires that a syntactically invalid content-type be treated
2746 # as text/plain. Perhaps we should postel this, but we should probably
2747 # only do that if we were checking the subtype value against IANA.
2748 del ctype.maintype, ctype.subtype
2749 _find_mime_parameters(ctype, value)
2750 return ctype
2751 ctype.append(ValueTerminal(';', 'parameter-separator'))
2752 ctype.append(parse_mime_parameters(value[1:]))
2753 return ctype
2754
2755 def parse_content_disposition_header(value):
2756 """ disposition-type *( ";" parameter )
2757
2758 """
2759 disp_header = ContentDisposition()
2760 if not value:
2761 disp_header.defects.append(errors.HeaderMissingRequiredValue(
2762 "Missing content disposition"))
2763 return disp_header
2764 try:
2765 token, value = get_token(value)
2766 except errors.HeaderParseError:
2767 disp_header.defects.append(errors.InvalidHeaderDefect(
2768 "Expected content disposition but found {!r}".format(value)))
2769 _find_mime_parameters(disp_header, value)
2770 return disp_header
2771 disp_header.append(token)
2772 disp_header.content_disposition = token.value.strip().lower()
2773 if not value:
2774 return disp_header
2775 if value[0] != ';':
2776 disp_header.defects.append(errors.InvalidHeaderDefect(
2777 "Only parameters are valid after content disposition, but "
2778 "found {!r}".format(value)))
2779 _find_mime_parameters(disp_header, value)
2780 return disp_header
2781 disp_header.append(ValueTerminal(';', 'parameter-separator'))
2782 disp_header.append(parse_mime_parameters(value[1:]))
2783 return disp_header
2784
2785 def parse_content_transfer_encoding_header(value):
2786 """ mechanism
2787
2788 """
2789 # We should probably validate the values, since the list is fixed.
2790 cte_header = ContentTransferEncoding()
2791 if not value:
2792 cte_header.defects.append(errors.HeaderMissingRequiredValue(
2793 "Missing content transfer encoding"))
2794 return cte_header
2795 try:
2796 token, value = get_token(value)
2797 except errors.HeaderParseError:
2798 cte_header.defects.append(errors.InvalidHeaderDefect(
2799 "Expected content transfer encoding but found {!r}".format(value)))
2800 else:
2801 cte_header.append(token)
2802 cte_header.cte = token.value.strip().lower()
2803 if not value:
2804 return cte_header
2805 while value:
2806 cte_header.defects.append(errors.InvalidHeaderDefect(
2807 "Extra text after content transfer encoding"))
2808 if value[0] in PHRASE_ENDS:
2809 cte_header.append(ValueTerminal(value[0], 'misplaced-special'))
2810 value = value[1:]
2811 else:
2812 token, value = get_phrase(value)
2813 cte_header.append(token)
2814 return cte_header
2815
2816
2817 #
2818 # Header folding
2819 #
2820 # Header folding is complex, with lots of rules and corner cases. The
2821 # following code does its best to obey the rules and handle the corner
2822 # cases, but you can be sure there are few bugs:)
2823 #
2824 # This folder generally canonicalizes as it goes, preferring the stringified
2825 # version of each token. The tokens contain information that supports the
2826 # folder, including which tokens can be encoded in which ways.
2827 #
2828 # Folded text is accumulated in a simple list of strings ('lines'), each
2829 # one of which should be less than policy.max_line_length ('maxlen').
2830 #
2831
2832 def _steal_trailing_WSP_if_exists(lines):
2833 wsp = ''
2834 if lines and lines[-1] and lines[-1][-1] in WSP:
2835 wsp = lines[-1][-1]
2836 lines[-1] = lines[-1][:-1]
2837 # gh-142006: if the line is now empty, remove it entirely.
2838 if not lines[-1]:
2839 lines.pop()
2840 return wsp
2841
2842
2843 def _refold_parse_tree(parse_tree, *, policy):
2844 """Return string of contents of parse_tree folded according to RFC rules.
2845
2846 """
2847 # max_line_length 0/None means no limit, ie: infinitely long.
2848 maxlen = policy.max_line_length or sys.maxsize
2849 encoding = 'utf-8' if policy.utf8 else 'us-ascii'
2850 lines = [''] # Folded lines to be output
2851 last_word_is_ew = False
2852 last_ew = None # if there is an encoded word in the last line of lines,
2853 # points to the encoded word's first character
2854 last_charset = None
2855 wrap_as_ew_blocked = 0
2856 want_encoding = False # This is set to True if we need to encode this part
2857 end_ew_not_allowed = Terminal('', 'wrap_as_ew_blocked')
2858 parts = list(parse_tree)
2859 while parts:
2860 part = parts.pop(0)
2861 if part is end_ew_not_allowed:
2862 wrap_as_ew_blocked -= 1
2863 continue
2864 tstr = str(part)
2865 if not want_encoding:
2866 if part.token_type in ('ptext', 'vtext'):
2867 # Encode if tstr contains special characters.
2868 want_encoding = not SPECIALSNL.isdisjoint(tstr)
2869 else:
2870 # Encode if tstr contains newlines.
2871 want_encoding = not NLSET.isdisjoint(tstr)
2872 try:
2873 tstr.encode(encoding)
2874 charset = encoding
2875 except UnicodeEncodeError:
2876 if any(isinstance(x, errors.UndecodableBytesDefect)
2877 for x in part.all_defects):
2878 charset = 'unknown-8bit'
2879 else:
2880 # If policy.utf8 is false this should really be taken from a
2881 # 'charset' property on the policy.
2882 charset = 'utf-8'
2883 want_encoding = True
2884
2885 if part.token_type == 'mime-parameters':
2886 # Mime parameter folding (using RFC2231) is extra special.
2887 _fold_mime_parameters(part, lines, maxlen, encoding)
2888 last_word_is_ew = False
2889 continue
2890
2891 if want_encoding and not wrap_as_ew_blocked:
2892 if not part.as_ew_allowed:
2893 want_encoding = False
2894 last_ew = None
2895 if part.syntactic_break:
2896 encoded_part = part.fold(policy=policy)[:-len(policy.linesep)]
2897 if policy.linesep not in encoded_part:
2898 # It fits on a single line
2899 if len(encoded_part) > maxlen - len(lines[-1]):
2900 # But not on this one, so start a new one.
2901 newline = _steal_trailing_WSP_if_exists(lines)
2902 # XXX what if encoded_part has no leading FWS?
2903 lines.append(newline)
2904 lines[-1] += encoded_part
2905 last_word_is_ew = False
2906 continue
2907 # Either this is not a major syntactic break, so we don't
2908 # want it on a line by itself even if it fits, or it
2909 # doesn't fit on a line by itself. Either way, fall through
2910 # to unpacking the subparts and wrapping them.
2911 if not hasattr(part, 'encode'):
2912 # It's not a Terminal, do each piece individually.
2913 parts = list(part) + parts
2914 want_encoding = False
2915 continue
2916 elif part.as_ew_allowed:
2917 # It's a terminal, wrap it as an encoded word, possibly
2918 # combining it with previously encoded words if allowed.
2919 if (last_ew is not None and
2920 charset != last_charset and
2921 (last_charset == 'unknown-8bit' or
2922 last_charset == 'utf-8' and charset != 'us-ascii')):
2923 last_ew = None
2924 last_ew = _fold_as_ew(
2925 tstr,
2926 lines,
2927 maxlen,
2928 last_ew,
2929 part.ew_combine_allowed,
2930 charset,
2931 last_word_is_ew,
2932 )
2933 last_word_is_ew = True
2934 last_charset = charset
2935 want_encoding = False
2936 continue
2937 else:
2938 # It's a terminal which should be kept non-encoded
2939 # (e.g. a ListSeparator).
2940 last_ew = None
2941 want_encoding = False
2942 # fall through
2943
2944 if len(tstr) <= maxlen - len(lines[-1]):
2945 lines[-1] += tstr
2946 last_word_is_ew = last_word_is_ew and not bool(tstr.strip(_WSP))
2947 continue
2948
2949 # This part is too long to fit. The RFC wants us to break at
2950 # "major syntactic breaks", so unless we don't consider this
2951 # to be one, check if it will fit on the next line by itself.
2952 if (part.syntactic_break and
2953 len(tstr) + 1 <= maxlen):
2954 newline = _steal_trailing_WSP_if_exists(lines)
2955 if newline or part.startswith_fws():
2956 lines.append(newline + tstr)
2957 last_word_is_ew = (last_word_is_ew
2958 and not bool(lines[-1].strip(_WSP)))
2959 last_ew = None
2960 continue
2961 if not hasattr(part, 'encode'):
2962 # It's not a terminal, try folding the subparts.
2963 newparts = list(part)
2964 if part.token_type == 'bare-quoted-string':
2965 # To fold a quoted string we need to create a list of terminal
2966 # tokens that will render the leading and trailing quotes
2967 # and use quoted pairs in the value as appropriate.
2968 newparts = (
2969 [ValueTerminal('"', 'ptext')] +
2970 [ValueTerminal(make_quoted_pairs(p), 'ptext')
2971 for p in newparts] +
2972 [ValueTerminal('"', 'ptext')])
2973 if part.token_type == 'comment':
2974 newparts = (
2975 [ValueTerminal('(', 'ptext')] +
2976 [ValueTerminal(make_parenthesis_pairs(p), 'ptext')
2977 if p.token_type == 'ptext' else p
2978 for p in newparts] +
2979 [ValueTerminal(')', 'ptext')])
2980 if not part.as_ew_allowed:
2981 wrap_as_ew_blocked += 1
2982 newparts.append(end_ew_not_allowed)
2983 parts = newparts + parts
2984 continue
2985 if part.as_ew_allowed and not wrap_as_ew_blocked:
2986 # It doesn't need CTE encoding, but encode it anyway so we can
2987 # wrap it.
2988 parts.insert(0, part)
2989 want_encoding = True
2990 continue
2991 # We can't figure out how to wrap, it, so give up.
2992 newline = _steal_trailing_WSP_if_exists(lines)
2993 if newline or part.startswith_fws():
2994 lines.append(newline + tstr)
2995 else:
2996 # We can't fold it onto the next line either...
2997 lines[-1] += tstr
2998 last_word_is_ew = last_word_is_ew and not bool(tstr.strip(_WSP))
2999
3000 return policy.linesep.join(lines) + policy.linesep
3001
3002 def _fold_as_ew(to_encode, lines, maxlen, last_ew, ew_combine_allowed, charset, last_word_is_ew):
3003 """Fold string to_encode into lines as encoded word, combining if allowed.
3004 Return the new value for last_ew, or None if ew_combine_allowed is False.
3005
3006 If there is already an encoded word in the last line of lines (indicated by
3007 a non-None value for last_ew) and ew_combine_allowed is true, decode the
3008 existing ew, combine it with to_encode, and re-encode. Otherwise, encode
3009 to_encode. In either case, split to_encode as necessary so that the
3010 encoded segments fit within maxlen.
3011
3012 """
3013 if last_ew is not None and ew_combine_allowed:
3014 to_encode = str(
3015 get_unstructured(lines[-1][last_ew:] + to_encode))
3016 lines[-1] = lines[-1][:last_ew]
3017 elif last_word_is_ew:
3018 # If we are following up an encoded word with another encoded word,
3019 # any white space between the two will be ignored when decoded.
3020 # Therefore, we encode all to-be-displayed whitespace in the second
3021 # encoded word.
3022 len_without_wsp = len(lines[-1].rstrip(_WSP))
3023 leading_whitespace = lines[-1][len_without_wsp:]
3024 lines[-1] = (lines[-1][:len_without_wsp]
3025 + (' ' if leading_whitespace else ''))
3026 to_encode = leading_whitespace + to_encode
3027 elif to_encode[0] in WSP:
3028 # We're joining this to non-encoded text, so don't encode
3029 # the leading blank.
3030 leading_wsp = to_encode[0]
3031 to_encode = to_encode[1:]
3032 if (len(lines[-1]) == maxlen):
3033 lines.append(_steal_trailing_WSP_if_exists(lines))
3034 lines[-1] += leading_wsp
3035
3036 trailing_wsp = ''
3037 if to_encode[-1] in WSP:
3038 # Likewise for the trailing space.
3039 trailing_wsp = to_encode[-1]
3040 to_encode = to_encode[:-1]
3041 new_last_ew = len(lines[-1]) if last_ew is None else last_ew
3042
3043 encode_as = 'utf-8' if charset == 'us-ascii' else charset
3044
3045 # The RFC2047 chrome takes up 7 characters plus the length
3046 # of the charset name.
3047 chrome_len = len(encode_as) + 7
3048
3049 if (chrome_len + 1) >= maxlen:
3050 raise errors.HeaderParseError(
3051 "max_line_length is too small to fit an encoded word")
3052
3053 while to_encode:
3054 remaining_space = maxlen - len(lines[-1])
3055 text_space = remaining_space - chrome_len
3056 if text_space <= 0:
3057 newline = _steal_trailing_WSP_if_exists(lines)
3058 lines.append(newline or ' ')
3059 new_last_ew = len(lines[-1])
3060 continue
3061
3062 to_encode_word = to_encode[:text_space]
3063 encoded_word = _ew.encode(to_encode_word, charset=encode_as)
3064 excess = len(encoded_word) - remaining_space
3065 while excess > 0:
3066 # Since the chunk to encode is guaranteed to fit into less than 100 characters,
3067 # shrinking it by one at a time shouldn't take long.
3068 to_encode_word = to_encode_word[:-1]
3069 encoded_word = _ew.encode(to_encode_word, charset=encode_as)
3070 excess = len(encoded_word) - remaining_space
3071 lines[-1] += encoded_word
3072 to_encode = to_encode[len(to_encode_word):]
3073
3074 if to_encode:
3075 lines.append(' ')
3076 new_last_ew = len(lines[-1])
3077 lines[-1] += trailing_wsp
3078 return new_last_ew if ew_combine_allowed else None
3079
3080 def _fold_mime_parameters(part, lines, maxlen, encoding):
3081 """Fold TokenList 'part' into the 'lines' list as mime parameters.
3082
3083 Using the decoded list of parameters and values, format them according to
3084 the RFC rules, including using RFC2231 encoding if the value cannot be
3085 expressed in 'encoding' and/or the parameter+value is too long to fit
3086 within 'maxlen'.
3087
3088 """
3089 # Special case for RFC2231 encoding: start from decoded values and use
3090 # RFC2231 encoding iff needed.
3091 #
3092 # Note that the 1 and 2s being added to the length calculations are
3093 # accounting for the possibly-needed spaces and semicolons we'll be adding.
3094 #
3095 for name, value in part.params:
3096 # XXX What if this ';' puts us over maxlen the first time through the
3097 # loop? We should split the header value onto a newline in that case,
3098 # but to do that we need to recognize the need earlier or reparse the
3099 # header, so I'm going to ignore that bug for now. It'll only put us
3100 # one character over.
3101 if not lines[-1].rstrip().endswith(';'):
3102 lines[-1] += ';'
3103 charset = encoding
3104 error_handler = 'strict'
3105 try:
3106 value.encode(encoding)
3107 encoding_required = False
3108 except UnicodeEncodeError:
3109 encoding_required = True
3110 if utils._has_surrogates(value):
3111 charset = 'unknown-8bit'
3112 error_handler = 'surrogateescape'
3113 else:
3114 charset = 'utf-8'
3115 if encoding_required:
3116 encoded_value = urllib.parse.quote(
3117 value, safe='', errors=error_handler)
3118 tstr = "{}*={}''{}".format(name, charset, encoded_value)
3119 else:
3120 tstr = '{}={}'.format(name, quote_string(value))
3121 if len(lines[-1]) + len(tstr) + 1 < maxlen:
3122 lines[-1] = lines[-1] + ' ' + tstr
3123 continue
3124 elif len(tstr) + 2 <= maxlen:
3125 lines.append(' ' + tstr)
3126 continue
3127 # We need multiple sections. We are allowed to mix encoded and
3128 # non-encoded sections, but we aren't going to. We'll encode them all.
3129 section = 0
3130 extra_chrome = charset + "''"
3131 while value:
3132 chrome_len = len(name) + len(str(section)) + 3 + len(extra_chrome)
3133 if maxlen <= chrome_len + 3:
3134 # We need room for the leading blank, the trailing semicolon,
3135 # and at least one character of the value. If we don't
3136 # have that, we'd be stuck, so in that case fall back to
3137 # the RFC standard width.
3138 maxlen = 78
3139 splitpoint = maxchars = maxlen - chrome_len - 2
3140 while True:
3141 partial = value[:splitpoint]
3142 encoded_value = urllib.parse.quote(
3143 partial, safe='', errors=error_handler)
3144 if len(encoded_value) <= maxchars:
3145 break
3146 splitpoint -= 1
3147 lines.append(" {}*{}*={}{}".format(
3148 name, section, extra_chrome, encoded_value))
3149 extra_chrome = ''
3150 section += 1
3151 value = value[splitpoint:]
3152 if value:
3153 lines[-1] += ';'