| 1 |
"""Header value parser implementing various email-related RFC parsing rules. |
| 2 |
|
| 3 |
The parsing methods defined in this module implement various email related |
| 4 |
parsing rules. Principal among them is RFC 5322, which is the followon |
| 5 |
to RFC 2822 and primarily a clarification of the former. It also implements |
| 6 |
RFC 2047 encoded word decoding. |
| 7 |
|
| 8 |
RFC 5322 goes to considerable trouble to maintain backward compatibility with |
| 9 |
RFC 822 in the parse phase, while cleaning up the structure on the generation |
| 10 |
phase. This parser supports correct RFC 5322 generation by tagging white space |
| 11 |
as folding white space only when folding is allowed in the non-obsolete rule |
| 12 |
sets. Actually, the parser is even more generous when accepting input than RFC |
| 13 |
5322 mandates, following the spirit of Postel's Law, which RFC 5322 encourages. |
| 14 |
Where possible deviations from the standard are annotated on the 'defects' |
| 15 |
attribute of tokens that deviate. |
| 16 |
|
| 17 |
The general structure of the parser follows RFC 5322, and uses its terminology |
| 18 |
where there is a direct correspondence. Where the implementation requires a |
| 19 |
somewhat different structure than that used by the formal grammar, new terms |
| 20 |
that mimic the closest existing terms are used. Thus, it really helps to have |
| 21 |
a copy of RFC 5322 handy when studying this code. |
| 22 |
|
| 23 |
Input to the parser is a string that has already been unfolded according to |
| 24 |
RFC 5322 rules. According to the RFC this unfolding is the very first step, and |
| 25 |
this parser leaves the unfolding step to a higher level message parser, which |
| 26 |
will have already detected the line breaks that need unfolding while |
| 27 |
determining the beginning and end of each header. |
| 28 |
|
| 29 |
The output of the parser is a TokenList object, which is a list subclass. A |
| 30 |
TokenList is a recursive data structure. The terminal nodes of the structure |
| 31 |
are Terminal objects, which are subclasses of str. These do not correspond |
| 32 |
directly to terminal objects in the formal grammar, but are instead more |
| 33 |
practical higher level combinations of true terminals. |
| 34 |
|
| 35 |
All TokenList and Terminal objects have a 'value' attribute, which produces the |
| 36 |
semantically meaningful value of that part of the parse subtree. The value of |
| 37 |
all whitespace tokens (no matter how many sub-tokens they may contain) is a |
| 38 |
single space, as per the RFC rules. This includes 'CFWS', which is herein |
| 39 |
included in the general class of whitespace tokens. There is one exception to |
| 40 |
the rule that whitespace tokens are collapsed into single spaces in values: in |
| 41 |
the value of a 'bare-quoted-string' (a quoted-string with no leading or |
| 42 |
trailing whitespace), any whitespace that appeared between the quotation marks |
| 43 |
is preserved in the returned value. Note that in all Terminal strings quoted |
| 44 |
pairs are turned into their unquoted values. |
| 45 |
|
| 46 |
All TokenList and Terminal objects also have a string value, which attempts to |
| 47 |
be a "canonical" representation of the RFC-compliant form of the substring that |
| 48 |
produced the parsed subtree, including minimal use of quoted pair quoting. |
| 49 |
Whitespace runs are not collapsed. |
| 50 |
|
| 51 |
Comment tokens also have a 'content' attribute providing the string found |
| 52 |
between the parens (including any nested comments) with whitespace preserved. |
| 53 |
|
| 54 |
All TokenList and Terminal objects have a 'defects' attribute which is a |
| 55 |
possibly empty list all of the defects found while creating the token. Defects |
| 56 |
may appear on any token in the tree, and a composite list of all defects in the |
| 57 |
subtree is available through the 'all_defects' attribute of any node. (For |
| 58 |
Terminal notes x.defects == x.all_defects.) |
| 59 |
|
| 60 |
Each object in a parse tree is called a 'token', and each has a 'token_type' |
| 61 |
attribute that gives the name from the RFC 5322 grammar that it represents. |
| 62 |
Not all RFC 5322 nodes are produced, and there is one non-RFC 5322 node that |
| 63 |
may be produced: 'ptext'. A 'ptext' is a string of printable ascii characters. |
| 64 |
It is returned in place of lists of (ctext/quoted-pair) and |
| 65 |
(qtext/quoted-pair). |
| 66 |
|
| 67 |
XXX: provide complete list of token types. |
| 68 |
""" |
| 69 |
|
| 70 |
import re |
| 71 |
import sys |
| 72 |
import urllib # For urllib.parse.unquote |
| 73 |
from string import hexdigits |
| 74 |
from operator import itemgetter |
| 75 |
from email import _encoded_words as _ew |
| 76 |
from email import errors |
| 77 |
from email import utils |
| 78 |
|
| 79 |
# |
| 80 |
# Useful constants and functions |
| 81 |
# |
| 82 |
|
| 83 |
_WSP = ' \t' |
| 84 |
WSP = set(_WSP) |
| 85 |
CFWS_LEADER = WSP | set('(') |
| 86 |
SPECIALS = set(r'()<>@,:;.\"[]') |
| 87 |
ATOM_ENDS = SPECIALS | WSP |
| 88 |
DOT_ATOM_ENDS = ATOM_ENDS - set('.') |
| 89 |
# '.', '"', and '(' do not end phrases in order to support obs-phrase |
| 90 |
PHRASE_ENDS = SPECIALS - set('."(') |
| 91 |
TSPECIALS = (SPECIALS | set('/?=')) - set('.') |
| 92 |
TOKEN_ENDS = TSPECIALS | WSP |
| 93 |
ASPECIALS = TSPECIALS | set("*'%") |
| 94 |
ATTRIBUTE_ENDS = ASPECIALS | WSP |
| 95 |
EXTENDED_ATTRIBUTE_ENDS = ATTRIBUTE_ENDS - set('%') |
| 96 |
NLSET = {'\n', '\r'} |
| 97 |
SPECIALSNL = SPECIALS | NLSET |
| 98 |
|
| 99 |
|
| 100 |
def make_quoted_pairs(value): |
| 101 |
"""Escape dquote and backslash for use within a quoted-string.""" |
| 102 |
return str(value).replace('\\', '\\\\').replace('"', '\\"') |
| 103 |
|
| 104 |
|
| 105 |
def make_parenthesis_pairs(value): |
| 106 |
"""Escape parenthesis and backslash for use within a comment.""" |
| 107 |
return str(value).replace('\\', '\\\\') \ |
| 108 |
.replace('(', '\\(').replace(')', '\\)') |
| 109 |
|
| 110 |
|
| 111 |
def quote_string(value): |
| 112 |
escaped = make_quoted_pairs(value) |
| 113 |
return f'"{escaped}"' |
| 114 |
|
| 115 |
|
| 116 |
# Match a RFC 2047 word, looks like =?utf-8?q?someword?= |
| 117 |
rfc2047_matcher = re.compile(r''' |
| 118 |
=\? # literal =? |
| 119 |
[^?]* # charset |
| 120 |
\? # literal ? |
| 121 |
[qQbB] # literal 'q' or 'b', case insensitive |
| 122 |
\? # literal ? |
| 123 |
.*? # encoded word |
| 124 |
\?= # literal ?= |
| 125 |
''', re.VERBOSE | re.MULTILINE) |
| 126 |
|
| 127 |
|
| 128 |
# |
| 129 |
# TokenList and its subclasses |
| 130 |
# |
| 131 |
|
| 132 |
class TokenList(list): |
| 133 |
|
| 134 |
token_type = None |
| 135 |
syntactic_break = True |
| 136 |
ew_combine_allowed = True |
| 137 |
|
| 138 |
def __init__(self, *args, **kw): |
| 139 |
super().__init__(*args, **kw) |
| 140 |
self.defects = [] |
| 141 |
|
| 142 |
def __str__(self): |
| 143 |
return ''.join(str(x) for x in self) |
| 144 |
|
| 145 |
def __repr__(self): |
| 146 |
return '{}({})'.format(self.__class__.__name__, |
| 147 |
super().__repr__()) |
| 148 |
|
| 149 |
@property |
| 150 |
def value(self): |
| 151 |
return ''.join(x.value for x in self if x.value) |
| 152 |
|
| 153 |
@property |
| 154 |
def all_defects(self): |
| 155 |
return sum((x.all_defects for x in self), self.defects) |
| 156 |
|
| 157 |
def startswith_fws(self): |
| 158 |
return self[0].startswith_fws() |
| 159 |
|
| 160 |
@property |
| 161 |
def as_ew_allowed(self): |
| 162 |
"""True if all top level tokens of this part may be RFC2047 encoded.""" |
| 163 |
return all(part.as_ew_allowed for part in self) |
| 164 |
|
| 165 |
@property |
| 166 |
def comments(self): |
| 167 |
comments = [] |
| 168 |
for token in self: |
| 169 |
comments.extend(token.comments) |
| 170 |
return comments |
| 171 |
|
| 172 |
def fold(self, *, policy): |
| 173 |
return _refold_parse_tree(self, policy=policy) |
| 174 |
|
| 175 |
def pprint(self, indent=''): |
| 176 |
print(self.ppstr(indent=indent)) |
| 177 |
|
| 178 |
def ppstr(self, indent=''): |
| 179 |
return '\n'.join(self._pp(indent=indent)) |
| 180 |
|
| 181 |
def _pp(self, indent=''): |
| 182 |
yield '{}{}/{}('.format( |
| 183 |
indent, |
| 184 |
self.__class__.__name__, |
| 185 |
self.token_type) |
| 186 |
for token in self: |
| 187 |
if not hasattr(token, '_pp'): |
| 188 |
yield (indent + ' !! invalid element in token ' |
| 189 |
'list: {!r}'.format(token)) |
| 190 |
else: |
| 191 |
yield from token._pp(indent+' ') |
| 192 |
if self.defects: |
| 193 |
extra = ' Defects: {}'.format(self.defects) |
| 194 |
else: |
| 195 |
extra = '' |
| 196 |
yield '{}){}'.format(indent, extra) |
| 197 |
|
| 198 |
|
| 199 |
class WhiteSpaceTokenList(TokenList): |
| 200 |
|
| 201 |
@property |
| 202 |
def value(self): |
| 203 |
return ' ' |
| 204 |
|
| 205 |
@property |
| 206 |
def comments(self): |
| 207 |
return [x.content for x in self if x.token_type=='comment'] |
| 208 |
|
| 209 |
|
| 210 |
class UnstructuredTokenList(TokenList): |
| 211 |
token_type = 'unstructured' |
| 212 |
|
| 213 |
|
| 214 |
class Phrase(TokenList): |
| 215 |
token_type = 'phrase' |
| 216 |
|
| 217 |
class Word(TokenList): |
| 218 |
token_type = 'word' |
| 219 |
|
| 220 |
|
| 221 |
class CFWSList(WhiteSpaceTokenList): |
| 222 |
token_type = 'cfws' |
| 223 |
|
| 224 |
|
| 225 |
class Atom(TokenList): |
| 226 |
token_type = 'atom' |
| 227 |
|
| 228 |
|
| 229 |
class Token(TokenList): |
| 230 |
token_type = 'token' |
| 231 |
encode_as_ew = False |
| 232 |
|
| 233 |
|
| 234 |
class EncodedWord(TokenList): |
| 235 |
token_type = 'encoded-word' |
| 236 |
cte = None |
| 237 |
charset = None |
| 238 |
lang = None |
| 239 |
|
| 240 |
|
| 241 |
class QuotedString(TokenList): |
| 242 |
|
| 243 |
token_type = 'quoted-string' |
| 244 |
|
| 245 |
@property |
| 246 |
def content(self): |
| 247 |
for x in self: |
| 248 |
if x.token_type == 'bare-quoted-string': |
| 249 |
return x.value |
| 250 |
|
| 251 |
@property |
| 252 |
def quoted_value(self): |
| 253 |
res = [] |
| 254 |
for x in self: |
| 255 |
if x.token_type == 'bare-quoted-string': |
| 256 |
res.append(str(x)) |
| 257 |
else: |
| 258 |
res.append(x.value) |
| 259 |
return ''.join(res) |
| 260 |
|
| 261 |
@property |
| 262 |
def stripped_value(self): |
| 263 |
for token in self: |
| 264 |
if token.token_type == 'bare-quoted-string': |
| 265 |
return token.value |
| 266 |
|
| 267 |
|
| 268 |
class BareQuotedString(QuotedString): |
| 269 |
|
| 270 |
token_type = 'bare-quoted-string' |
| 271 |
|
| 272 |
def __str__(self): |
| 273 |
return quote_string(''.join(str(x) for x in self)) |
| 274 |
|
| 275 |
@property |
| 276 |
def value(self): |
| 277 |
return ''.join(str(x) for x in self) |
| 278 |
|
| 279 |
|
| 280 |
class Comment(WhiteSpaceTokenList): |
| 281 |
|
| 282 |
token_type = 'comment' |
| 283 |
|
| 284 |
def __str__(self): |
| 285 |
return ''.join(sum([ |
| 286 |
["("], |
| 287 |
[self.quote(x) for x in self], |
| 288 |
[")"], |
| 289 |
], [])) |
| 290 |
|
| 291 |
def quote(self, value): |
| 292 |
if value.token_type == 'comment': |
| 293 |
return str(value) |
| 294 |
return str(value).replace('\\', '\\\\').replace( |
| 295 |
'(', r'\(').replace( |
| 296 |
')', r'\)') |
| 297 |
|
| 298 |
@property |
| 299 |
def content(self): |
| 300 |
return ''.join(str(x) for x in self) |
| 301 |
|
| 302 |
@property |
| 303 |
def comments(self): |
| 304 |
return [self.content] |
| 305 |
|
| 306 |
class AddressList(TokenList): |
| 307 |
|
| 308 |
token_type = 'address-list' |
| 309 |
|
| 310 |
@property |
| 311 |
def addresses(self): |
| 312 |
return [x for x in self if x.token_type=='address'] |
| 313 |
|
| 314 |
@property |
| 315 |
def mailboxes(self): |
| 316 |
return sum((x.mailboxes |
| 317 |
for x in self if x.token_type=='address'), []) |
| 318 |
|
| 319 |
@property |
| 320 |
def all_mailboxes(self): |
| 321 |
return sum((x.all_mailboxes |
| 322 |
for x in self if x.token_type=='address'), []) |
| 323 |
|
| 324 |
|
| 325 |
class Address(TokenList): |
| 326 |
|
| 327 |
token_type = 'address' |
| 328 |
|
| 329 |
@property |
| 330 |
def display_name(self): |
| 331 |
if self[0].token_type == 'group': |
| 332 |
return self[0].display_name |
| 333 |
|
| 334 |
@property |
| 335 |
def mailboxes(self): |
| 336 |
if self[0].token_type == 'mailbox': |
| 337 |
return [self[0]] |
| 338 |
elif self[0].token_type == 'invalid-mailbox': |
| 339 |
return [] |
| 340 |
return self[0].mailboxes |
| 341 |
|
| 342 |
@property |
| 343 |
def all_mailboxes(self): |
| 344 |
if self[0].token_type == 'mailbox': |
| 345 |
return [self[0]] |
| 346 |
elif self[0].token_type == 'invalid-mailbox': |
| 347 |
return [self[0]] |
| 348 |
return self[0].all_mailboxes |
| 349 |
|
| 350 |
class MailboxList(TokenList): |
| 351 |
|
| 352 |
token_type = 'mailbox-list' |
| 353 |
|
| 354 |
@property |
| 355 |
def mailboxes(self): |
| 356 |
return [x for x in self if x.token_type=='mailbox'] |
| 357 |
|
| 358 |
@property |
| 359 |
def all_mailboxes(self): |
| 360 |
return [x for x in self |
| 361 |
if x.token_type in ('mailbox', 'invalid-mailbox')] |
| 362 |
|
| 363 |
|
| 364 |
class GroupList(TokenList): |
| 365 |
|
| 366 |
token_type = 'group-list' |
| 367 |
|
| 368 |
@property |
| 369 |
def mailboxes(self): |
| 370 |
if not self or self[0].token_type != 'mailbox-list': |
| 371 |
return [] |
| 372 |
return self[0].mailboxes |
| 373 |
|
| 374 |
@property |
| 375 |
def all_mailboxes(self): |
| 376 |
if not self or self[0].token_type != 'mailbox-list': |
| 377 |
return [] |
| 378 |
return self[0].all_mailboxes |
| 379 |
|
| 380 |
|
| 381 |
class Group(TokenList): |
| 382 |
|
| 383 |
token_type = "group" |
| 384 |
|
| 385 |
@property |
| 386 |
def mailboxes(self): |
| 387 |
if self[2].token_type != 'group-list': |
| 388 |
return [] |
| 389 |
return self[2].mailboxes |
| 390 |
|
| 391 |
@property |
| 392 |
def all_mailboxes(self): |
| 393 |
if self[2].token_type != 'group-list': |
| 394 |
return [] |
| 395 |
return self[2].all_mailboxes |
| 396 |
|
| 397 |
@property |
| 398 |
def display_name(self): |
| 399 |
return self[0].display_name |
| 400 |
|
| 401 |
|
| 402 |
class NameAddr(TokenList): |
| 403 |
|
| 404 |
token_type = 'name-addr' |
| 405 |
|
| 406 |
@property |
| 407 |
def display_name(self): |
| 408 |
if len(self) == 1: |
| 409 |
return None |
| 410 |
return self[0].display_name |
| 411 |
|
| 412 |
@property |
| 413 |
def local_part(self): |
| 414 |
return self[-1].local_part |
| 415 |
|
| 416 |
@property |
| 417 |
def domain(self): |
| 418 |
return self[-1].domain |
| 419 |
|
| 420 |
@property |
| 421 |
def route(self): |
| 422 |
return self[-1].route |
| 423 |
|
| 424 |
@property |
| 425 |
def addr_spec(self): |
| 426 |
return self[-1].addr_spec |
| 427 |
|
| 428 |
|
| 429 |
class AngleAddr(TokenList): |
| 430 |
|
| 431 |
token_type = 'angle-addr' |
| 432 |
|
| 433 |
@property |
| 434 |
def local_part(self): |
| 435 |
for x in self: |
| 436 |
if x.token_type == 'addr-spec': |
| 437 |
return x.local_part |
| 438 |
|
| 439 |
@property |
| 440 |
def domain(self): |
| 441 |
for x in self: |
| 442 |
if x.token_type == 'addr-spec': |
| 443 |
return x.domain |
| 444 |
|
| 445 |
@property |
| 446 |
def route(self): |
| 447 |
for x in self: |
| 448 |
if x.token_type == 'obs-route': |
| 449 |
return x.domains |
| 450 |
|
| 451 |
@property |
| 452 |
def addr_spec(self): |
| 453 |
for x in self: |
| 454 |
if x.token_type == 'addr-spec': |
| 455 |
if x.local_part: |
| 456 |
return x.addr_spec |
| 457 |
else: |
| 458 |
return quote_string(x.local_part) + x.addr_spec |
| 459 |
else: |
| 460 |
return '<>' |
| 461 |
|
| 462 |
|
| 463 |
class ObsRoute(TokenList): |
| 464 |
|
| 465 |
token_type = 'obs-route' |
| 466 |
|
| 467 |
@property |
| 468 |
def domains(self): |
| 469 |
return [x.domain for x in self if x.token_type == 'domain'] |
| 470 |
|
| 471 |
|
| 472 |
class Mailbox(TokenList): |
| 473 |
|
| 474 |
token_type = 'mailbox' |
| 475 |
|
| 476 |
@property |
| 477 |
def display_name(self): |
| 478 |
if self[0].token_type == 'name-addr': |
| 479 |
return self[0].display_name |
| 480 |
|
| 481 |
@property |
| 482 |
def local_part(self): |
| 483 |
return self[0].local_part |
| 484 |
|
| 485 |
@property |
| 486 |
def domain(self): |
| 487 |
return self[0].domain |
| 488 |
|
| 489 |
@property |
| 490 |
def route(self): |
| 491 |
if self[0].token_type == 'name-addr': |
| 492 |
return self[0].route |
| 493 |
|
| 494 |
@property |
| 495 |
def addr_spec(self): |
| 496 |
return self[0].addr_spec |
| 497 |
|
| 498 |
|
| 499 |
class InvalidMailbox(TokenList): |
| 500 |
|
| 501 |
token_type = 'invalid-mailbox' |
| 502 |
|
| 503 |
@property |
| 504 |
def display_name(self): |
| 505 |
return None |
| 506 |
|
| 507 |
local_part = domain = route = addr_spec = display_name |
| 508 |
|
| 509 |
|
| 510 |
class Domain(TokenList): |
| 511 |
|
| 512 |
token_type = 'domain' |
| 513 |
as_ew_allowed = False |
| 514 |
|
| 515 |
@property |
| 516 |
def domain(self): |
| 517 |
return ''.join(super().value.split()) |
| 518 |
|
| 519 |
|
| 520 |
class DotAtom(TokenList): |
| 521 |
token_type = 'dot-atom' |
| 522 |
|
| 523 |
|
| 524 |
class DotAtomText(TokenList): |
| 525 |
token_type = 'dot-atom-text' |
| 526 |
as_ew_allowed = True |
| 527 |
|
| 528 |
|
| 529 |
class NoFoldLiteral(TokenList): |
| 530 |
token_type = 'no-fold-literal' |
| 531 |
as_ew_allowed = False |
| 532 |
|
| 533 |
|
| 534 |
class AddrSpec(TokenList): |
| 535 |
|
| 536 |
token_type = 'addr-spec' |
| 537 |
as_ew_allowed = False |
| 538 |
|
| 539 |
@property |
| 540 |
def local_part(self): |
| 541 |
return self[0].local_part |
| 542 |
|
| 543 |
@property |
| 544 |
def domain(self): |
| 545 |
if len(self) < 3: |
| 546 |
return None |
| 547 |
return self[-1].domain |
| 548 |
|
| 549 |
@property |
| 550 |
def value(self): |
| 551 |
if len(self) < 3: |
| 552 |
return self[0].value |
| 553 |
return self[0].value.rstrip()+self[1].value+self[2].value.lstrip() |
| 554 |
|
| 555 |
@property |
| 556 |
def addr_spec(self): |
| 557 |
nameset = set(self.local_part) |
| 558 |
if len(nameset) > len(nameset-DOT_ATOM_ENDS): |
| 559 |
lp = quote_string(self.local_part) |
| 560 |
else: |
| 561 |
lp = self.local_part |
| 562 |
if self.domain is not None: |
| 563 |
return lp + '@' + self.domain |
| 564 |
return lp |
| 565 |
|
| 566 |
|
| 567 |
class ObsLocalPart(TokenList): |
| 568 |
|
| 569 |
token_type = 'obs-local-part' |
| 570 |
as_ew_allowed = False |
| 571 |
|
| 572 |
|
| 573 |
class DisplayName(Phrase): |
| 574 |
|
| 575 |
token_type = 'display-name' |
| 576 |
ew_combine_allowed = False |
| 577 |
|
| 578 |
@property |
| 579 |
def display_name(self): |
| 580 |
res = TokenList(self) |
| 581 |
if len(res) == 0: |
| 582 |
return res.value |
| 583 |
if res[0].token_type == 'cfws': |
| 584 |
res.pop(0) |
| 585 |
else: |
| 586 |
if (isinstance(res[0], TokenList) and |
| 587 |
res[0][0].token_type == 'cfws'): |
| 588 |
res[0] = TokenList(res[0][1:]) |
| 589 |
if res[-1].token_type == 'cfws': |
| 590 |
res.pop() |
| 591 |
else: |
| 592 |
if (isinstance(res[-1], TokenList) and |
| 593 |
res[-1][-1].token_type == 'cfws'): |
| 594 |
res[-1] = TokenList(res[-1][:-1]) |
| 595 |
return res.value |
| 596 |
|
| 597 |
@property |
| 598 |
def value(self): |
| 599 |
quote = False |
| 600 |
if self.defects: |
| 601 |
quote = True |
| 602 |
else: |
| 603 |
for x in self: |
| 604 |
if x.token_type == 'quoted-string': |
| 605 |
quote = True |
| 606 |
if len(self) != 0 and quote: |
| 607 |
pre = post = '' |
| 608 |
if (self[0].token_type == 'cfws' or |
| 609 |
isinstance(self[0], TokenList) and |
| 610 |
self[0][0].token_type == 'cfws'): |
| 611 |
pre = ' ' |
| 612 |
if (self[-1].token_type == 'cfws' or |
| 613 |
isinstance(self[-1], TokenList) and |
| 614 |
self[-1][-1].token_type == 'cfws'): |
| 615 |
post = ' ' |
| 616 |
return pre+quote_string(self.display_name)+post |
| 617 |
else: |
| 618 |
return super().value |
| 619 |
|
| 620 |
|
| 621 |
class LocalPart(TokenList): |
| 622 |
|
| 623 |
token_type = 'local-part' |
| 624 |
as_ew_allowed = False |
| 625 |
|
| 626 |
@property |
| 627 |
def value(self): |
| 628 |
if self[0].token_type == "quoted-string": |
| 629 |
return self[0].quoted_value |
| 630 |
else: |
| 631 |
return self[0].value |
| 632 |
|
| 633 |
@property |
| 634 |
def local_part(self): |
| 635 |
# Strip whitespace from front, back, and around dots. |
| 636 |
res = [DOT] |
| 637 |
last = DOT |
| 638 |
last_is_tl = False |
| 639 |
for tok in self[0] + [DOT]: |
| 640 |
if tok.token_type == 'cfws': |
| 641 |
continue |
| 642 |
if (last_is_tl and tok.token_type == 'dot' and last and |
| 643 |
last[-1].token_type == 'cfws'): |
| 644 |
res[-1] = TokenList(last[:-1]) |
| 645 |
is_tl = isinstance(tok, TokenList) |
| 646 |
if (is_tl and last.token_type == 'dot' and tok and |
| 647 |
tok[0].token_type == 'cfws'): |
| 648 |
res.append(TokenList(tok[1:])) |
| 649 |
else: |
| 650 |
res.append(tok) |
| 651 |
last = res[-1] |
| 652 |
last_is_tl = is_tl |
| 653 |
res = TokenList(res[1:-1]) |
| 654 |
return res.value |
| 655 |
|
| 656 |
|
| 657 |
class DomainLiteral(TokenList): |
| 658 |
|
| 659 |
token_type = 'domain-literal' |
| 660 |
as_ew_allowed = False |
| 661 |
|
| 662 |
@property |
| 663 |
def domain(self): |
| 664 |
return ''.join(super().value.split()) |
| 665 |
|
| 666 |
@property |
| 667 |
def ip(self): |
| 668 |
for x in self: |
| 669 |
if x.token_type == 'ptext': |
| 670 |
return x.value |
| 671 |
|
| 672 |
|
| 673 |
class MIMEVersion(TokenList): |
| 674 |
|
| 675 |
token_type = 'mime-version' |
| 676 |
major = None |
| 677 |
minor = None |
| 678 |
|
| 679 |
|
| 680 |
class Parameter(TokenList): |
| 681 |
|
| 682 |
token_type = 'parameter' |
| 683 |
sectioned = False |
| 684 |
extended = False |
| 685 |
charset = 'us-ascii' |
| 686 |
|
| 687 |
@property |
| 688 |
def section_number(self): |
| 689 |
# Because the first token, the attribute (name) eats CFWS, the second |
| 690 |
# token is always the section if there is one. |
| 691 |
return self[1].number if self.sectioned else 0 |
| 692 |
|
| 693 |
@property |
| 694 |
def param_value(self): |
| 695 |
# This is part of the "handle quoted extended parameters" hack. |
| 696 |
for token in self: |
| 697 |
if token.token_type == 'value': |
| 698 |
return token.stripped_value |
| 699 |
if token.token_type == 'quoted-string': |
| 700 |
for token in token: |
| 701 |
if token.token_type == 'bare-quoted-string': |
| 702 |
for token in token: |
| 703 |
if token.token_type == 'value': |
| 704 |
return token.stripped_value |
| 705 |
return '' |
| 706 |
|
| 707 |
|
| 708 |
class InvalidParameter(Parameter): |
| 709 |
|
| 710 |
token_type = 'invalid-parameter' |
| 711 |
|
| 712 |
|
| 713 |
class Attribute(TokenList): |
| 714 |
|
| 715 |
token_type = 'attribute' |
| 716 |
|
| 717 |
@property |
| 718 |
def stripped_value(self): |
| 719 |
for token in self: |
| 720 |
if token.token_type.endswith('attrtext'): |
| 721 |
return token.value |
| 722 |
|
| 723 |
class Section(TokenList): |
| 724 |
|
| 725 |
token_type = 'section' |
| 726 |
number = None |
| 727 |
|
| 728 |
|
| 729 |
class Value(TokenList): |
| 730 |
|
| 731 |
token_type = 'value' |
| 732 |
|
| 733 |
@property |
| 734 |
def stripped_value(self): |
| 735 |
token = self[0] |
| 736 |
if token.token_type == 'cfws': |
| 737 |
token = self[1] |
| 738 |
if token.token_type.endswith( |
| 739 |
('quoted-string', 'attribute', 'extended-attribute')): |
| 740 |
return token.stripped_value |
| 741 |
return self.value |
| 742 |
|
| 743 |
|
| 744 |
class MimeParameters(TokenList): |
| 745 |
|
| 746 |
token_type = 'mime-parameters' |
| 747 |
syntactic_break = False |
| 748 |
|
| 749 |
@property |
| 750 |
def params(self): |
| 751 |
# The RFC specifically states that the ordering of parameters is not |
| 752 |
# guaranteed and may be reordered by the transport layer. So we have |
| 753 |
# to assume the RFC 2231 pieces can come in any order. However, we |
| 754 |
# output them in the order that we first see a given name, which gives |
| 755 |
# us a stable __str__. |
| 756 |
params = {} # Using order preserving dict from Python 3.7+ |
| 757 |
for token in self: |
| 758 |
if not token.token_type.endswith('parameter'): |
| 759 |
continue |
| 760 |
if token[0].token_type != 'attribute': |
| 761 |
continue |
| 762 |
name = token[0].value.strip() |
| 763 |
if name not in params: |
| 764 |
params[name] = [] |
| 765 |
params[name].append((token.section_number, token)) |
| 766 |
for name, parts in params.items(): |
| 767 |
parts = sorted(parts, key=itemgetter(0)) |
| 768 |
first_param = parts[0][1] |
| 769 |
charset = first_param.charset |
| 770 |
# Our arbitrary error recovery is to ignore duplicate parameters, |
| 771 |
# to use appearance order if there are duplicate rfc 2231 parts, |
| 772 |
# and to ignore gaps. This mimics the error recovery of get_param. |
| 773 |
if not first_param.extended and len(parts) > 1: |
| 774 |
if parts[1][0] == 0: |
| 775 |
parts[1][1].defects.append(errors.InvalidHeaderDefect( |
| 776 |
'duplicate parameter name; duplicate(s) ignored')) |
| 777 |
parts = parts[:1] |
| 778 |
# Else assume the *0* was missing...note that this is different |
| 779 |
# from get_param, but we registered a defect for this earlier. |
| 780 |
value_parts = [] |
| 781 |
i = 0 |
| 782 |
for section_number, param in parts: |
| 783 |
if section_number != i: |
| 784 |
# We could get fancier here and look for a complete |
| 785 |
# duplicate extended parameter and ignore the second one |
| 786 |
# seen. But we're not doing that. The old code didn't. |
| 787 |
if not param.extended: |
| 788 |
param.defects.append(errors.InvalidHeaderDefect( |
| 789 |
'duplicate parameter name; duplicate ignored')) |
| 790 |
continue |
| 791 |
else: |
| 792 |
param.defects.append(errors.InvalidHeaderDefect( |
| 793 |
"inconsistent RFC2231 parameter numbering")) |
| 794 |
i += 1 |
| 795 |
value = param.param_value |
| 796 |
if param.extended: |
| 797 |
try: |
| 798 |
value = urllib.parse.unquote_to_bytes(value) |
| 799 |
except UnicodeEncodeError: |
| 800 |
# source had surrogate escaped bytes. What we do now |
| 801 |
# is a bit of an open question. I'm not sure this is |
| 802 |
# the best choice, but it is what the old algorithm did |
| 803 |
value = urllib.parse.unquote(value, encoding='latin-1') |
| 804 |
else: |
| 805 |
try: |
| 806 |
value = value.decode(charset, 'surrogateescape') |
| 807 |
except (LookupError, UnicodeEncodeError): |
| 808 |
# XXX: there should really be a custom defect for |
| 809 |
# unknown character set to make it easy to find, |
| 810 |
# because otherwise unknown charset is a silent |
| 811 |
# failure. |
| 812 |
value = value.decode('us-ascii', 'surrogateescape') |
| 813 |
if utils._has_surrogates(value): |
| 814 |
param.defects.append(errors.UndecodableBytesDefect()) |
| 815 |
value_parts.append(value) |
| 816 |
value = ''.join(value_parts) |
| 817 |
yield name, value |
| 818 |
|
| 819 |
def __str__(self): |
| 820 |
params = [] |
| 821 |
for name, value in self.params: |
| 822 |
if value: |
| 823 |
params.append('{}={}'.format(name, quote_string(value))) |
| 824 |
else: |
| 825 |
params.append(name) |
| 826 |
params = '; '.join(params) |
| 827 |
return ' ' + params if params else '' |
| 828 |
|
| 829 |
|
| 830 |
class ParameterizedHeaderValue(TokenList): |
| 831 |
|
| 832 |
# Set this false so that the value doesn't wind up on a new line even |
| 833 |
# if it and the parameters would fit there but not on the first line. |
| 834 |
syntactic_break = False |
| 835 |
|
| 836 |
@property |
| 837 |
def params(self): |
| 838 |
for token in reversed(self): |
| 839 |
if token.token_type == 'mime-parameters': |
| 840 |
return token.params |
| 841 |
return {} |
| 842 |
|
| 843 |
|
| 844 |
class ContentType(ParameterizedHeaderValue): |
| 845 |
token_type = 'content-type' |
| 846 |
as_ew_allowed = False |
| 847 |
maintype = 'text' |
| 848 |
subtype = 'plain' |
| 849 |
|
| 850 |
|
| 851 |
class ContentDisposition(ParameterizedHeaderValue): |
| 852 |
token_type = 'content-disposition' |
| 853 |
as_ew_allowed = False |
| 854 |
content_disposition = None |
| 855 |
|
| 856 |
|
| 857 |
class ContentTransferEncoding(TokenList): |
| 858 |
token_type = 'content-transfer-encoding' |
| 859 |
as_ew_allowed = False |
| 860 |
cte = '7bit' |
| 861 |
|
| 862 |
|
| 863 |
class HeaderLabel(TokenList): |
| 864 |
token_type = 'header-label' |
| 865 |
as_ew_allowed = False |
| 866 |
|
| 867 |
|
| 868 |
class MsgID(TokenList): |
| 869 |
token_type = 'msg-id' |
| 870 |
as_ew_allowed = False |
| 871 |
|
| 872 |
def fold(self, policy): |
| 873 |
# message-id tokens may not be folded. |
| 874 |
return str(self) + policy.linesep |
| 875 |
|
| 876 |
|
| 877 |
class MessageID(MsgID): |
| 878 |
token_type = 'message-id' |
| 879 |
|
| 880 |
|
| 881 |
class InvalidMessageID(MessageID): |
| 882 |
token_type = 'invalid-message-id' |
| 883 |
|
| 884 |
class MessageIDList(TokenList): |
| 885 |
token_type = 'message-id-list' |
| 886 |
|
| 887 |
@property |
| 888 |
def message_ids(self): |
| 889 |
return [x for x in self if x.token_type=='msg-id'] |
| 890 |
|
| 891 |
class Header(TokenList): |
| 892 |
token_type = 'header' |
| 893 |
|
| 894 |
|
| 895 |
# |
| 896 |
# Terminal classes and instances |
| 897 |
# |
| 898 |
|
| 899 |
class Terminal(str): |
| 900 |
|
| 901 |
as_ew_allowed = True |
| 902 |
ew_combine_allowed = True |
| 903 |
syntactic_break = True |
| 904 |
|
| 905 |
def __new__(cls, value, token_type): |
| 906 |
self = super().__new__(cls, value) |
| 907 |
self.token_type = token_type |
| 908 |
self.defects = [] |
| 909 |
return self |
| 910 |
|
| 911 |
def __repr__(self): |
| 912 |
return "{}({})".format(self.__class__.__name__, super().__repr__()) |
| 913 |
|
| 914 |
def pprint(self): |
| 915 |
print(self.__class__.__name__ + '/' + self.token_type) |
| 916 |
|
| 917 |
@property |
| 918 |
def all_defects(self): |
| 919 |
return list(self.defects) |
| 920 |
|
| 921 |
def _pp(self, indent=''): |
| 922 |
return ["{}{}/{}({}){}".format( |
| 923 |
indent, |
| 924 |
self.__class__.__name__, |
| 925 |
self.token_type, |
| 926 |
super().__repr__(), |
| 927 |
'' if not self.defects else ' {}'.format(self.defects), |
| 928 |
)] |
| 929 |
|
| 930 |
def pop_trailing_ws(self): |
| 931 |
# This terminates the recursion. |
| 932 |
return None |
| 933 |
|
| 934 |
@property |
| 935 |
def comments(self): |
| 936 |
return [] |
| 937 |
|
| 938 |
def __getnewargs__(self): |
| 939 |
return(str(self), self.token_type) |
| 940 |
|
| 941 |
|
| 942 |
class WhiteSpaceTerminal(Terminal): |
| 943 |
|
| 944 |
@property |
| 945 |
def value(self): |
| 946 |
return ' ' |
| 947 |
|
| 948 |
def startswith_fws(self): |
| 949 |
return self and self[0] in WSP |
| 950 |
|
| 951 |
|
| 952 |
class ValueTerminal(Terminal): |
| 953 |
|
| 954 |
@property |
| 955 |
def value(self): |
| 956 |
return self |
| 957 |
|
| 958 |
def startswith_fws(self): |
| 959 |
return False |
| 960 |
|
| 961 |
|
| 962 |
class EWWhiteSpaceTerminal(WhiteSpaceTerminal): |
| 963 |
|
| 964 |
@property |
| 965 |
def value(self): |
| 966 |
return '' |
| 967 |
|
| 968 |
def __str__(self): |
| 969 |
return '' |
| 970 |
|
| 971 |
|
| 972 |
class _InvalidEwError(errors.HeaderParseError): |
| 973 |
"""Invalid encoded word found while parsing headers.""" |
| 974 |
|
| 975 |
|
| 976 |
# XXX these need to become classes and used as instances so |
| 977 |
# that a program can't change them in a parse tree and screw |
| 978 |
# up other parse trees. Maybe should have tests for that, too. |
| 979 |
DOT = ValueTerminal('.', 'dot') |
| 980 |
ListSeparator = ValueTerminal(',', 'list-separator') |
| 981 |
ListSeparator.as_ew_allowed = False |
| 982 |
ListSeparator.syntactic_break = False |
| 983 |
RouteComponentMarker = ValueTerminal('@', 'route-component-marker') |
| 984 |
|
| 985 |
# |
| 986 |
# Parser |
| 987 |
# |
| 988 |
|
| 989 |
# Parse strings according to RFC822/2047/2822/5322 rules. |
| 990 |
# |
| 991 |
# This is a stateless parser. Each get_XXX function accepts a string and |
| 992 |
# returns either a Terminal or a TokenList representing the RFC object named |
| 993 |
# by the method and a string containing the remaining unparsed characters |
| 994 |
# from the input. Thus a parser method consumes the next syntactic construct |
| 995 |
# of a given type and returns a token representing the construct plus the |
| 996 |
# unparsed remainder of the input string. |
| 997 |
# |
| 998 |
# For example, if the first element of a structured header is a 'phrase', |
| 999 |
# then: |
| 1000 |
# |
| 1001 |
# phrase, value = get_phrase(value) |
| 1002 |
# |
| 1003 |
# returns the complete phrase from the start of the string value, plus any |
| 1004 |
# characters left in the string after the phrase is removed. |
| 1005 |
|
| 1006 |
_wsp_splitter = re.compile(r'([{}]+)'.format(''.join(WSP))).split |
| 1007 |
_non_atom_end_matcher = re.compile(r"[^{}]+".format( |
| 1008 |
re.escape(''.join(ATOM_ENDS)))).match |
| 1009 |
_non_printable_finder = re.compile(r"[\x00-\x20\x7F]").findall |
| 1010 |
_non_token_end_matcher = re.compile(r"[^{}]+".format( |
| 1011 |
re.escape(''.join(TOKEN_ENDS)))).match |
| 1012 |
_non_attribute_end_matcher = re.compile(r"[^{}]+".format( |
| 1013 |
re.escape(''.join(ATTRIBUTE_ENDS)))).match |
| 1014 |
_non_extended_attribute_end_matcher = re.compile(r"[^{}]+".format( |
| 1015 |
re.escape(''.join(EXTENDED_ATTRIBUTE_ENDS)))).match |
| 1016 |
|
| 1017 |
def _validate_xtext(xtext): |
| 1018 |
"""If input token contains ASCII non-printables, register a defect.""" |
| 1019 |
|
| 1020 |
non_printables = _non_printable_finder(xtext) |
| 1021 |
if non_printables: |
| 1022 |
xtext.defects.append(errors.NonPrintableDefect(non_printables)) |
| 1023 |
if utils._has_surrogates(xtext): |
| 1024 |
xtext.defects.append(errors.UndecodableBytesDefect( |
| 1025 |
"Non-ASCII characters found in header token")) |
| 1026 |
|
| 1027 |
def _get_ptext_to_endchars(value, endchars): |
| 1028 |
"""Scan printables/quoted-pairs until endchars and return unquoted ptext. |
| 1029 |
|
| 1030 |
This function turns a run of qcontent, ccontent-without-comments, or |
| 1031 |
dtext-with-quoted-printables into a single string by unquoting any |
| 1032 |
quoted printables. It returns the string, the remaining value, and |
| 1033 |
a flag that is True iff there were any quoted printables decoded. |
| 1034 |
|
| 1035 |
""" |
| 1036 |
if not value: |
| 1037 |
return '', '', False |
| 1038 |
fragment, *remainder = _wsp_splitter(value, 1) |
| 1039 |
vchars = [] |
| 1040 |
escape = False |
| 1041 |
had_qp = False |
| 1042 |
for pos in range(len(fragment)): |
| 1043 |
if fragment[pos] == '\\': |
| 1044 |
if escape: |
| 1045 |
escape = False |
| 1046 |
had_qp = True |
| 1047 |
else: |
| 1048 |
escape = True |
| 1049 |
continue |
| 1050 |
if escape: |
| 1051 |
escape = False |
| 1052 |
elif fragment[pos] in endchars: |
| 1053 |
break |
| 1054 |
vchars.append(fragment[pos]) |
| 1055 |
else: |
| 1056 |
pos = pos + 1 |
| 1057 |
return ''.join(vchars), ''.join([fragment[pos:]] + remainder), had_qp |
| 1058 |
|
| 1059 |
def get_fws(value): |
| 1060 |
"""FWS = 1*WSP |
| 1061 |
|
| 1062 |
This isn't the RFC definition. We're using fws to represent tokens where |
| 1063 |
folding can be done, but when we are parsing the *un*folding has already |
| 1064 |
been done so we don't need to watch out for CRLF. |
| 1065 |
|
| 1066 |
""" |
| 1067 |
newvalue = value.lstrip() |
| 1068 |
fws = WhiteSpaceTerminal(value[:len(value)-len(newvalue)], 'fws') |
| 1069 |
return fws, newvalue |
| 1070 |
|
| 1071 |
def get_encoded_word(value, terminal_type='vtext'): |
| 1072 |
""" encoded-word = "=?" charset "?" encoding "?" encoded-text "?=" |
| 1073 |
|
| 1074 |
""" |
| 1075 |
ew = EncodedWord() |
| 1076 |
if not value.startswith('=?'): |
| 1077 |
raise errors.HeaderParseError( |
| 1078 |
"expected encoded word but found {}".format(value)) |
| 1079 |
tok, *remainder = value[2:].split('?=', 1) |
| 1080 |
if tok == value[2:]: |
| 1081 |
raise errors.HeaderParseError( |
| 1082 |
"expected encoded word but found {}".format(value)) |
| 1083 |
remstr = ''.join(remainder) |
| 1084 |
if (len(remstr) > 1 and |
| 1085 |
remstr[0] in hexdigits and |
| 1086 |
remstr[1] in hexdigits and |
| 1087 |
tok.count('?') < 2): |
| 1088 |
# The ? after the CTE was followed by an encoded word escape (=XX). |
| 1089 |
rest, *remainder = remstr.split('?=', 1) |
| 1090 |
tok = tok + '?=' + rest |
| 1091 |
if len(tok.split()) > 1: |
| 1092 |
ew.defects.append(errors.InvalidHeaderDefect( |
| 1093 |
"whitespace inside encoded word")) |
| 1094 |
ew.cte = value |
| 1095 |
value = ''.join(remainder) |
| 1096 |
try: |
| 1097 |
text, charset, lang, defects = _ew.decode('=?' + tok + '?=') |
| 1098 |
except (ValueError, KeyError): |
| 1099 |
raise _InvalidEwError( |
| 1100 |
"encoded word format invalid: '{}'".format(ew.cte)) |
| 1101 |
ew.charset = charset |
| 1102 |
ew.lang = lang |
| 1103 |
ew.defects.extend(defects) |
| 1104 |
while text: |
| 1105 |
if text[0] in WSP: |
| 1106 |
token, text = get_fws(text) |
| 1107 |
ew.append(token) |
| 1108 |
continue |
| 1109 |
chars, *remainder = _wsp_splitter(text, 1) |
| 1110 |
vtext = ValueTerminal(chars, terminal_type) |
| 1111 |
_validate_xtext(vtext) |
| 1112 |
ew.append(vtext) |
| 1113 |
text = ''.join(remainder) |
| 1114 |
# Encoded words should be followed by a WS |
| 1115 |
if value and value[0] not in WSP: |
| 1116 |
ew.defects.append(errors.InvalidHeaderDefect( |
| 1117 |
"missing trailing whitespace after encoded-word")) |
| 1118 |
return ew, value |
| 1119 |
|
| 1120 |
def get_unstructured(value): |
| 1121 |
"""unstructured = (*([FWS] vchar) *WSP) / obs-unstruct |
| 1122 |
obs-unstruct = *((*LF *CR *(obs-utext) *LF *CR)) / FWS) |
| 1123 |
obs-utext = %d0 / obs-NO-WS-CTL / LF / CR |
| 1124 |
|
| 1125 |
obs-NO-WS-CTL is control characters except WSP/CR/LF. |
| 1126 |
|
| 1127 |
So, basically, we have printable runs, plus control characters or nulls in |
| 1128 |
the obsolete syntax, separated by whitespace. Since RFC 2047 uses the |
| 1129 |
obsolete syntax in its specification, but requires whitespace on either |
| 1130 |
side of the encoded words, I can see no reason to need to separate the |
| 1131 |
non-printable-non-whitespace from the printable runs if they occur, so we |
| 1132 |
parse this into xtext tokens separated by WSP tokens. |
| 1133 |
|
| 1134 |
Because an 'unstructured' value must by definition constitute the entire |
| 1135 |
value, this 'get' routine does not return a remaining value, only the |
| 1136 |
parsed TokenList. |
| 1137 |
|
| 1138 |
""" |
| 1139 |
# XXX: but what about bare CR and LF? They might signal the start or |
| 1140 |
# end of an encoded word. YAGNI for now, since our current parsers |
| 1141 |
# will never send us strings with bare CR or LF. |
| 1142 |
|
| 1143 |
unstructured = UnstructuredTokenList() |
| 1144 |
while value: |
| 1145 |
if value[0] in WSP: |
| 1146 |
token, value = get_fws(value) |
| 1147 |
unstructured.append(token) |
| 1148 |
continue |
| 1149 |
valid_ew = True |
| 1150 |
if value.startswith('=?'): |
| 1151 |
try: |
| 1152 |
token, value = get_encoded_word(value, 'utext') |
| 1153 |
except _InvalidEwError: |
| 1154 |
valid_ew = False |
| 1155 |
except errors.HeaderParseError: |
| 1156 |
# XXX: Need to figure out how to register defects when |
| 1157 |
# appropriate here. |
| 1158 |
pass |
| 1159 |
else: |
| 1160 |
have_ws = True |
| 1161 |
if len(unstructured) > 0: |
| 1162 |
if unstructured[-1].token_type != 'fws': |
| 1163 |
unstructured.defects.append(errors.InvalidHeaderDefect( |
| 1164 |
"missing whitespace before encoded word")) |
| 1165 |
have_ws = False |
| 1166 |
if have_ws and len(unstructured) > 1: |
| 1167 |
if unstructured[-2].token_type == 'encoded-word': |
| 1168 |
unstructured[-1] = EWWhiteSpaceTerminal( |
| 1169 |
unstructured[-1], 'fws') |
| 1170 |
unstructured.append(token) |
| 1171 |
continue |
| 1172 |
tok, *remainder = _wsp_splitter(value, 1) |
| 1173 |
# Split in the middle of an atom if there is a rfc2047 encoded word |
| 1174 |
# which does not have WSP on both sides. The defect will be registered |
| 1175 |
# the next time through the loop. |
| 1176 |
# This needs to only be performed when the encoded word is valid; |
| 1177 |
# otherwise, performing it on an invalid encoded word can cause |
| 1178 |
# the parser to go in an infinite loop. |
| 1179 |
if valid_ew and rfc2047_matcher.search(tok): |
| 1180 |
tok, *remainder = value.partition('=?') |
| 1181 |
vtext = ValueTerminal(tok, 'utext') |
| 1182 |
_validate_xtext(vtext) |
| 1183 |
unstructured.append(vtext) |
| 1184 |
value = ''.join(remainder) |
| 1185 |
return unstructured |
| 1186 |
|
| 1187 |
def get_qp_ctext(value): |
| 1188 |
r"""ctext = <printable ascii except \ ( )> |
| 1189 |
|
| 1190 |
This is not the RFC ctext, since we are handling nested comments in comment |
| 1191 |
and unquoting quoted-pairs here. We allow anything except the '()' |
| 1192 |
characters, but if we find any ASCII other than the RFC defined printable |
| 1193 |
ASCII, a NonPrintableDefect is added to the token's defects list. Since |
| 1194 |
quoted pairs are converted to their unquoted values, what is returned is |
| 1195 |
a 'ptext' token. In this case it is a WhiteSpaceTerminal, so it's value |
| 1196 |
is ' '. |
| 1197 |
|
| 1198 |
""" |
| 1199 |
ptext, value, _ = _get_ptext_to_endchars(value, '()') |
| 1200 |
ptext = WhiteSpaceTerminal(ptext, 'ptext') |
| 1201 |
_validate_xtext(ptext) |
| 1202 |
return ptext, value |
| 1203 |
|
| 1204 |
def get_qcontent(value): |
| 1205 |
"""qcontent = qtext / quoted-pair |
| 1206 |
|
| 1207 |
We allow anything except the DQUOTE character, but if we find any ASCII |
| 1208 |
other than the RFC defined printable ASCII, a NonPrintableDefect is |
| 1209 |
added to the token's defects list. Any quoted pairs are converted to their |
| 1210 |
unquoted values, so what is returned is a 'ptext' token. In this case it |
| 1211 |
is a ValueTerminal. |
| 1212 |
|
| 1213 |
""" |
| 1214 |
ptext, value, _ = _get_ptext_to_endchars(value, '"') |
| 1215 |
ptext = ValueTerminal(ptext, 'ptext') |
| 1216 |
_validate_xtext(ptext) |
| 1217 |
return ptext, value |
| 1218 |
|
| 1219 |
def get_atext(value): |
| 1220 |
"""atext = <matches _atext_matcher> |
| 1221 |
|
| 1222 |
We allow any non-ATOM_ENDS in atext, but add an InvalidATextDefect to |
| 1223 |
the token's defects list if we find non-atext characters. |
| 1224 |
""" |
| 1225 |
m = _non_atom_end_matcher(value) |
| 1226 |
if not m: |
| 1227 |
raise errors.HeaderParseError( |
| 1228 |
"expected atext but found '{}'".format(value)) |
| 1229 |
atext = m.group() |
| 1230 |
value = value[len(atext):] |
| 1231 |
atext = ValueTerminal(atext, 'atext') |
| 1232 |
_validate_xtext(atext) |
| 1233 |
return atext, value |
| 1234 |
|
| 1235 |
def get_bare_quoted_string(value): |
| 1236 |
"""bare-quoted-string = DQUOTE *([FWS] qcontent) [FWS] DQUOTE |
| 1237 |
|
| 1238 |
A quoted-string without the leading or trailing white space. Its |
| 1239 |
value is the text between the quote marks, with whitespace |
| 1240 |
preserved and quoted pairs decoded. |
| 1241 |
""" |
| 1242 |
if not value or value[0] != '"': |
| 1243 |
raise errors.HeaderParseError( |
| 1244 |
"expected '\"' but found '{}'".format(value)) |
| 1245 |
bare_quoted_string = BareQuotedString() |
| 1246 |
value = value[1:] |
| 1247 |
if value and value[0] == '"': |
| 1248 |
return bare_quoted_string, value[1:] |
| 1249 |
while value and value[0] != '"': |
| 1250 |
if value[0] in WSP: |
| 1251 |
token, value = get_fws(value) |
| 1252 |
elif value[:2] == '=?': |
| 1253 |
valid_ew = False |
| 1254 |
try: |
| 1255 |
token, value = get_encoded_word(value) |
| 1256 |
bare_quoted_string.defects.append(errors.InvalidHeaderDefect( |
| 1257 |
"encoded word inside quoted string")) |
| 1258 |
valid_ew = True |
| 1259 |
except errors.HeaderParseError: |
| 1260 |
token, value = get_qcontent(value) |
| 1261 |
# Collapse the whitespace between two encoded words that occur in a |
| 1262 |
# bare-quoted-string. |
| 1263 |
if valid_ew and len(bare_quoted_string) > 1: |
| 1264 |
if (bare_quoted_string[-1].token_type == 'fws' and |
| 1265 |
bare_quoted_string[-2].token_type == 'encoded-word'): |
| 1266 |
bare_quoted_string[-1] = EWWhiteSpaceTerminal( |
| 1267 |
bare_quoted_string[-1], 'fws') |
| 1268 |
else: |
| 1269 |
token, value = get_qcontent(value) |
| 1270 |
bare_quoted_string.append(token) |
| 1271 |
if not value: |
| 1272 |
bare_quoted_string.defects.append(errors.InvalidHeaderDefect( |
| 1273 |
"end of header inside quoted string")) |
| 1274 |
return bare_quoted_string, value |
| 1275 |
return bare_quoted_string, value[1:] |
| 1276 |
|
| 1277 |
def get_comment(value): |
| 1278 |
"""comment = "(" *([FWS] ccontent) [FWS] ")" |
| 1279 |
ccontent = ctext / quoted-pair / comment |
| 1280 |
|
| 1281 |
We handle nested comments here, and quoted-pair in our qp-ctext routine. |
| 1282 |
""" |
| 1283 |
if value and value[0] != '(': |
| 1284 |
raise errors.HeaderParseError( |
| 1285 |
"expected '(' but found '{}'".format(value)) |
| 1286 |
comment = Comment() |
| 1287 |
value = value[1:] |
| 1288 |
while value and value[0] != ")": |
| 1289 |
if value[0] in WSP: |
| 1290 |
token, value = get_fws(value) |
| 1291 |
elif value[0] == '(': |
| 1292 |
token, value = get_comment(value) |
| 1293 |
else: |
| 1294 |
token, value = get_qp_ctext(value) |
| 1295 |
comment.append(token) |
| 1296 |
if not value: |
| 1297 |
comment.defects.append(errors.InvalidHeaderDefect( |
| 1298 |
"end of header inside comment")) |
| 1299 |
return comment, value |
| 1300 |
return comment, value[1:] |
| 1301 |
|
| 1302 |
def get_cfws(value): |
| 1303 |
"""CFWS = (1*([FWS] comment) [FWS]) / FWS |
| 1304 |
|
| 1305 |
""" |
| 1306 |
cfws = CFWSList() |
| 1307 |
while value and value[0] in CFWS_LEADER: |
| 1308 |
if value[0] in WSP: |
| 1309 |
token, value = get_fws(value) |
| 1310 |
else: |
| 1311 |
token, value = get_comment(value) |
| 1312 |
cfws.append(token) |
| 1313 |
return cfws, value |
| 1314 |
|
| 1315 |
def get_quoted_string(value): |
| 1316 |
"""quoted-string = [CFWS] <bare-quoted-string> [CFWS] |
| 1317 |
|
| 1318 |
'bare-quoted-string' is an intermediate class defined by this |
| 1319 |
parser and not by the RFC grammar. It is the quoted string |
| 1320 |
without any attached CFWS. |
| 1321 |
""" |
| 1322 |
quoted_string = QuotedString() |
| 1323 |
if value and value[0] in CFWS_LEADER: |
| 1324 |
token, value = get_cfws(value) |
| 1325 |
quoted_string.append(token) |
| 1326 |
token, value = get_bare_quoted_string(value) |
| 1327 |
quoted_string.append(token) |
| 1328 |
if value and value[0] in CFWS_LEADER: |
| 1329 |
token, value = get_cfws(value) |
| 1330 |
quoted_string.append(token) |
| 1331 |
return quoted_string, value |
| 1332 |
|
| 1333 |
def get_atom(value): |
| 1334 |
"""atom = [CFWS] 1*atext [CFWS] |
| 1335 |
|
| 1336 |
An atom could be an rfc2047 encoded word. |
| 1337 |
""" |
| 1338 |
atom = Atom() |
| 1339 |
if value and value[0] in CFWS_LEADER: |
| 1340 |
token, value = get_cfws(value) |
| 1341 |
atom.append(token) |
| 1342 |
if value and value[0] in ATOM_ENDS: |
| 1343 |
raise errors.HeaderParseError( |
| 1344 |
"expected atom but found '{}'".format(value)) |
| 1345 |
if value.startswith('=?'): |
| 1346 |
try: |
| 1347 |
token, value = get_encoded_word(value) |
| 1348 |
except errors.HeaderParseError: |
| 1349 |
# XXX: need to figure out how to register defects when |
| 1350 |
# appropriate here. |
| 1351 |
token, value = get_atext(value) |
| 1352 |
else: |
| 1353 |
token, value = get_atext(value) |
| 1354 |
atom.append(token) |
| 1355 |
if value and value[0] in CFWS_LEADER: |
| 1356 |
token, value = get_cfws(value) |
| 1357 |
atom.append(token) |
| 1358 |
return atom, value |
| 1359 |
|
| 1360 |
def get_dot_atom_text(value): |
| 1361 |
""" dot-text = 1*atext *("." 1*atext) |
| 1362 |
|
| 1363 |
""" |
| 1364 |
dot_atom_text = DotAtomText() |
| 1365 |
if not value or value[0] in ATOM_ENDS: |
| 1366 |
raise errors.HeaderParseError("expected atom at a start of " |
| 1367 |
"dot-atom-text but found '{}'".format(value)) |
| 1368 |
while value and value[0] not in ATOM_ENDS: |
| 1369 |
token, value = get_atext(value) |
| 1370 |
dot_atom_text.append(token) |
| 1371 |
if value and value[0] == '.': |
| 1372 |
dot_atom_text.append(DOT) |
| 1373 |
value = value[1:] |
| 1374 |
if dot_atom_text[-1] is DOT: |
| 1375 |
raise errors.HeaderParseError("expected atom at end of dot-atom-text " |
| 1376 |
"but found '{}'".format('.'+value)) |
| 1377 |
return dot_atom_text, value |
| 1378 |
|
| 1379 |
def get_dot_atom(value): |
| 1380 |
""" dot-atom = [CFWS] dot-atom-text [CFWS] |
| 1381 |
|
| 1382 |
Any place we can have a dot atom, we could instead have an rfc2047 encoded |
| 1383 |
word. |
| 1384 |
""" |
| 1385 |
dot_atom = DotAtom() |
| 1386 |
if value[0] in CFWS_LEADER: |
| 1387 |
token, value = get_cfws(value) |
| 1388 |
dot_atom.append(token) |
| 1389 |
if value.startswith('=?'): |
| 1390 |
try: |
| 1391 |
token, value = get_encoded_word(value) |
| 1392 |
except errors.HeaderParseError: |
| 1393 |
# XXX: need to figure out how to register defects when |
| 1394 |
# appropriate here. |
| 1395 |
token, value = get_dot_atom_text(value) |
| 1396 |
else: |
| 1397 |
token, value = get_dot_atom_text(value) |
| 1398 |
dot_atom.append(token) |
| 1399 |
if value and value[0] in CFWS_LEADER: |
| 1400 |
token, value = get_cfws(value) |
| 1401 |
dot_atom.append(token) |
| 1402 |
return dot_atom, value |
| 1403 |
|
| 1404 |
def get_word(value): |
| 1405 |
"""word = atom / quoted-string |
| 1406 |
|
| 1407 |
Either atom or quoted-string may start with CFWS. We have to peel off this |
| 1408 |
CFWS first to determine which type of word to parse. Afterward we splice |
| 1409 |
the leading CFWS, if any, into the parsed sub-token. |
| 1410 |
|
| 1411 |
If neither an atom or a quoted-string is found before the next special, a |
| 1412 |
HeaderParseError is raised. |
| 1413 |
|
| 1414 |
The token returned is either an Atom or a QuotedString, as appropriate. |
| 1415 |
This means the 'word' level of the formal grammar is not represented in the |
| 1416 |
parse tree; this is because having that extra layer when manipulating the |
| 1417 |
parse tree is more confusing than it is helpful. |
| 1418 |
|
| 1419 |
""" |
| 1420 |
if value[0] in CFWS_LEADER: |
| 1421 |
leader, value = get_cfws(value) |
| 1422 |
else: |
| 1423 |
leader = None |
| 1424 |
if not value: |
| 1425 |
raise errors.HeaderParseError( |
| 1426 |
"Expected 'atom' or 'quoted-string' but found nothing.") |
| 1427 |
if value[0]=='"': |
| 1428 |
token, value = get_quoted_string(value) |
| 1429 |
elif value[0] in SPECIALS: |
| 1430 |
raise errors.HeaderParseError("Expected 'atom' or 'quoted-string' " |
| 1431 |
"but found '{}'".format(value)) |
| 1432 |
else: |
| 1433 |
token, value = get_atom(value) |
| 1434 |
if leader is not None: |
| 1435 |
token[:0] = [leader] |
| 1436 |
return token, value |
| 1437 |
|
| 1438 |
def get_phrase(value): |
| 1439 |
""" phrase = 1*word / obs-phrase |
| 1440 |
obs-phrase = word *(word / "." / CFWS) |
| 1441 |
|
| 1442 |
This means a phrase can be a sequence of words, periods, and CFWS in any |
| 1443 |
order as long as it starts with at least one word. If anything other than |
| 1444 |
words is detected, an ObsoleteHeaderDefect is added to the token's defect |
| 1445 |
list. We also accept a phrase that starts with CFWS followed by a dot; |
| 1446 |
this is registered as an InvalidHeaderDefect, since it is not supported by |
| 1447 |
even the obsolete grammar. |
| 1448 |
|
| 1449 |
""" |
| 1450 |
phrase = Phrase() |
| 1451 |
try: |
| 1452 |
token, value = get_word(value) |
| 1453 |
phrase.append(token) |
| 1454 |
except errors.HeaderParseError: |
| 1455 |
phrase.defects.append(errors.InvalidHeaderDefect( |
| 1456 |
"phrase does not start with word")) |
| 1457 |
while value and value[0] not in PHRASE_ENDS: |
| 1458 |
if value[0]=='.': |
| 1459 |
phrase.append(DOT) |
| 1460 |
phrase.defects.append(errors.ObsoleteHeaderDefect( |
| 1461 |
"period in 'phrase'")) |
| 1462 |
value = value[1:] |
| 1463 |
else: |
| 1464 |
try: |
| 1465 |
token, value = get_word(value) |
| 1466 |
if (token[0].token_type == 'encoded-word' |
| 1467 |
and phrase |
| 1468 |
and phrase[-1].token_type == 'atom' |
| 1469 |
and len(phrase[-1]) > 1 |
| 1470 |
and phrase[-1][-2].token_type == 'encoded-word' |
| 1471 |
and phrase[-1][-1].token_type == 'cfws' |
| 1472 |
and not phrase[-1][-1].comments |
| 1473 |
): |
| 1474 |
# linear ws between ews needs special handing... |
| 1475 |
phrase[-1][-1] = EWWhiteSpaceTerminal(phrase[-1], 'fws') |
| 1476 |
except errors.HeaderParseError: |
| 1477 |
if value[0] in CFWS_LEADER: |
| 1478 |
token, value = get_cfws(value) |
| 1479 |
phrase.defects.append(errors.ObsoleteHeaderDefect( |
| 1480 |
"comment found without atom")) |
| 1481 |
else: |
| 1482 |
raise |
| 1483 |
phrase.append(token) |
| 1484 |
return phrase, value |
| 1485 |
|
| 1486 |
def get_local_part(value): |
| 1487 |
""" local-part = dot-atom / quoted-string / obs-local-part |
| 1488 |
|
| 1489 |
""" |
| 1490 |
local_part = LocalPart() |
| 1491 |
leader = None |
| 1492 |
if value and value[0] in CFWS_LEADER: |
| 1493 |
leader, value = get_cfws(value) |
| 1494 |
if not value: |
| 1495 |
raise errors.HeaderParseError( |
| 1496 |
"expected local-part but found '{}'".format(value)) |
| 1497 |
try: |
| 1498 |
token, value = get_dot_atom(value) |
| 1499 |
except errors.HeaderParseError: |
| 1500 |
try: |
| 1501 |
token, value = get_word(value) |
| 1502 |
except errors.HeaderParseError: |
| 1503 |
if value[0] != '\\' and value[0] in PHRASE_ENDS: |
| 1504 |
raise |
| 1505 |
token = TokenList() |
| 1506 |
if leader is not None: |
| 1507 |
token[:0] = [leader] |
| 1508 |
local_part.append(token) |
| 1509 |
if value and (value[0]=='\\' or value[0] not in PHRASE_ENDS): |
| 1510 |
obs_local_part, value = get_obs_local_part(str(local_part) + value) |
| 1511 |
if obs_local_part.token_type == 'invalid-obs-local-part': |
| 1512 |
local_part.defects.append(errors.InvalidHeaderDefect( |
| 1513 |
"local-part is not dot-atom, quoted-string, or obs-local-part")) |
| 1514 |
else: |
| 1515 |
local_part.defects.append(errors.ObsoleteHeaderDefect( |
| 1516 |
"local-part is not a dot-atom (contains CFWS)")) |
| 1517 |
local_part[0] = obs_local_part |
| 1518 |
try: |
| 1519 |
local_part.value.encode('ascii') |
| 1520 |
except UnicodeEncodeError: |
| 1521 |
local_part.defects.append(errors.NonASCIILocalPartDefect( |
| 1522 |
"local-part contains non-ASCII characters)")) |
| 1523 |
return local_part, value |
| 1524 |
|
| 1525 |
def get_obs_local_part(value): |
| 1526 |
""" obs-local-part = word *("." word) |
| 1527 |
""" |
| 1528 |
obs_local_part = ObsLocalPart() |
| 1529 |
last_non_ws_was_dot = False |
| 1530 |
while value and (value[0]=='\\' or value[0] not in PHRASE_ENDS): |
| 1531 |
if value[0] == '.': |
| 1532 |
if last_non_ws_was_dot: |
| 1533 |
obs_local_part.defects.append(errors.InvalidHeaderDefect( |
| 1534 |
"invalid repeated '.'")) |
| 1535 |
obs_local_part.append(DOT) |
| 1536 |
last_non_ws_was_dot = True |
| 1537 |
value = value[1:] |
| 1538 |
continue |
| 1539 |
elif value[0]=='\\': |
| 1540 |
obs_local_part.append(ValueTerminal(value[0], |
| 1541 |
'misplaced-special')) |
| 1542 |
value = value[1:] |
| 1543 |
obs_local_part.defects.append(errors.InvalidHeaderDefect( |
| 1544 |
"'\\' character outside of quoted-string/ccontent")) |
| 1545 |
last_non_ws_was_dot = False |
| 1546 |
continue |
| 1547 |
if obs_local_part and obs_local_part[-1].token_type != 'dot': |
| 1548 |
obs_local_part.defects.append(errors.InvalidHeaderDefect( |
| 1549 |
"missing '.' between words")) |
| 1550 |
try: |
| 1551 |
token, value = get_word(value) |
| 1552 |
last_non_ws_was_dot = False |
| 1553 |
except errors.HeaderParseError: |
| 1554 |
if value[0] not in CFWS_LEADER: |
| 1555 |
raise |
| 1556 |
token, value = get_cfws(value) |
| 1557 |
obs_local_part.append(token) |
| 1558 |
if not obs_local_part: |
| 1559 |
raise errors.HeaderParseError( |
| 1560 |
"expected obs-local-part but found '{}'".format(value)) |
| 1561 |
if (obs_local_part[0].token_type == 'dot' or |
| 1562 |
obs_local_part[0].token_type=='cfws' and |
| 1563 |
len(obs_local_part) > 1 and |
| 1564 |
obs_local_part[1].token_type=='dot'): |
| 1565 |
obs_local_part.defects.append(errors.InvalidHeaderDefect( |
| 1566 |
"Invalid leading '.' in local part")) |
| 1567 |
if (obs_local_part[-1].token_type == 'dot' or |
| 1568 |
obs_local_part[-1].token_type=='cfws' and |
| 1569 |
len(obs_local_part) > 1 and |
| 1570 |
obs_local_part[-2].token_type=='dot'): |
| 1571 |
obs_local_part.defects.append(errors.InvalidHeaderDefect( |
| 1572 |
"Invalid trailing '.' in local part")) |
| 1573 |
if obs_local_part.defects: |
| 1574 |
obs_local_part.token_type = 'invalid-obs-local-part' |
| 1575 |
return obs_local_part, value |
| 1576 |
|
| 1577 |
def get_dtext(value): |
| 1578 |
r""" dtext = <printable ascii except \ [ ]> / obs-dtext |
| 1579 |
obs-dtext = obs-NO-WS-CTL / quoted-pair |
| 1580 |
|
| 1581 |
We allow anything except the excluded characters, but if we find any |
| 1582 |
ASCII other than the RFC defined printable ASCII, a NonPrintableDefect is |
| 1583 |
added to the token's defects list. Quoted pairs are converted to their |
| 1584 |
unquoted values, so what is returned is a ptext token, in this case a |
| 1585 |
ValueTerminal. If there were quoted-printables, an ObsoleteHeaderDefect is |
| 1586 |
added to the returned token's defect list. |
| 1587 |
|
| 1588 |
""" |
| 1589 |
ptext, value, had_qp = _get_ptext_to_endchars(value, '[]') |
| 1590 |
ptext = ValueTerminal(ptext, 'ptext') |
| 1591 |
if had_qp: |
| 1592 |
ptext.defects.append(errors.ObsoleteHeaderDefect( |
| 1593 |
"quoted printable found in domain-literal")) |
| 1594 |
_validate_xtext(ptext) |
| 1595 |
return ptext, value |
| 1596 |
|
| 1597 |
def _check_for_early_dl_end(value, domain_literal): |
| 1598 |
if value: |
| 1599 |
return False |
| 1600 |
domain_literal.defects.append(errors.InvalidHeaderDefect( |
| 1601 |
"end of input inside domain-literal")) |
| 1602 |
domain_literal.append(ValueTerminal(']', 'domain-literal-end')) |
| 1603 |
return True |
| 1604 |
|
| 1605 |
def get_domain_literal(value): |
| 1606 |
""" domain-literal = [CFWS] "[" *([FWS] dtext) [FWS] "]" [CFWS] |
| 1607 |
|
| 1608 |
""" |
| 1609 |
domain_literal = DomainLiteral() |
| 1610 |
if value[0] in CFWS_LEADER: |
| 1611 |
token, value = get_cfws(value) |
| 1612 |
domain_literal.append(token) |
| 1613 |
if not value: |
| 1614 |
raise errors.HeaderParseError("expected domain-literal") |
| 1615 |
if value[0] != '[': |
| 1616 |
raise errors.HeaderParseError("expected '[' at start of domain-literal " |
| 1617 |
"but found '{}'".format(value)) |
| 1618 |
value = value[1:] |
| 1619 |
domain_literal.append(ValueTerminal('[', 'domain-literal-start')) |
| 1620 |
if _check_for_early_dl_end(value, domain_literal): |
| 1621 |
return domain_literal, value |
| 1622 |
if value[0] in WSP: |
| 1623 |
token, value = get_fws(value) |
| 1624 |
domain_literal.append(token) |
| 1625 |
token, value = get_dtext(value) |
| 1626 |
domain_literal.append(token) |
| 1627 |
if _check_for_early_dl_end(value, domain_literal): |
| 1628 |
return domain_literal, value |
| 1629 |
if value[0] in WSP: |
| 1630 |
token, value = get_fws(value) |
| 1631 |
domain_literal.append(token) |
| 1632 |
if _check_for_early_dl_end(value, domain_literal): |
| 1633 |
return domain_literal, value |
| 1634 |
if value[0] != ']': |
| 1635 |
raise errors.HeaderParseError("expected ']' at end of domain-literal " |
| 1636 |
"but found '{}'".format(value)) |
| 1637 |
domain_literal.append(ValueTerminal(']', 'domain-literal-end')) |
| 1638 |
value = value[1:] |
| 1639 |
if value and value[0] in CFWS_LEADER: |
| 1640 |
token, value = get_cfws(value) |
| 1641 |
domain_literal.append(token) |
| 1642 |
return domain_literal, value |
| 1643 |
|
| 1644 |
def get_domain(value): |
| 1645 |
""" domain = dot-atom / domain-literal / obs-domain |
| 1646 |
obs-domain = atom *("." atom)) |
| 1647 |
|
| 1648 |
""" |
| 1649 |
domain = Domain() |
| 1650 |
leader = None |
| 1651 |
if value and value[0] in CFWS_LEADER: |
| 1652 |
leader, value = get_cfws(value) |
| 1653 |
if not value: |
| 1654 |
raise errors.HeaderParseError( |
| 1655 |
"expected domain but found '{}'".format(value)) |
| 1656 |
if value[0] == '[': |
| 1657 |
token, value = get_domain_literal(value) |
| 1658 |
if leader is not None: |
| 1659 |
token[:0] = [leader] |
| 1660 |
domain.append(token) |
| 1661 |
return domain, value |
| 1662 |
try: |
| 1663 |
token, value = get_dot_atom(value) |
| 1664 |
except errors.HeaderParseError: |
| 1665 |
token, value = get_atom(value) |
| 1666 |
if value and value[0] == '@': |
| 1667 |
raise errors.HeaderParseError('Invalid Domain') |
| 1668 |
if leader is not None: |
| 1669 |
token[:0] = [leader] |
| 1670 |
domain.append(token) |
| 1671 |
if value and value[0] == '.': |
| 1672 |
domain.defects.append(errors.ObsoleteHeaderDefect( |
| 1673 |
"domain is not a dot-atom (contains CFWS)")) |
| 1674 |
if domain[0].token_type == 'dot-atom': |
| 1675 |
domain[:] = domain[0] |
| 1676 |
while value and value[0] == '.': |
| 1677 |
domain.append(DOT) |
| 1678 |
token, value = get_atom(value[1:]) |
| 1679 |
domain.append(token) |
| 1680 |
return domain, value |
| 1681 |
|
| 1682 |
def get_addr_spec(value): |
| 1683 |
""" addr-spec = local-part "@" domain |
| 1684 |
|
| 1685 |
""" |
| 1686 |
addr_spec = AddrSpec() |
| 1687 |
token, value = get_local_part(value) |
| 1688 |
addr_spec.append(token) |
| 1689 |
if not value or value[0] != '@': |
| 1690 |
addr_spec.defects.append(errors.InvalidHeaderDefect( |
| 1691 |
"addr-spec local part with no domain")) |
| 1692 |
return addr_spec, value |
| 1693 |
addr_spec.append(ValueTerminal('@', 'address-at-symbol')) |
| 1694 |
token, value = get_domain(value[1:]) |
| 1695 |
addr_spec.append(token) |
| 1696 |
return addr_spec, value |
| 1697 |
|
| 1698 |
def get_obs_route(value): |
| 1699 |
""" obs-route = obs-domain-list ":" |
| 1700 |
obs-domain-list = *(CFWS / ",") "@" domain *("," [CFWS] ["@" domain]) |
| 1701 |
|
| 1702 |
Returns an obs-route token with the appropriate sub-tokens (that is, |
| 1703 |
there is no obs-domain-list in the parse tree). |
| 1704 |
""" |
| 1705 |
obs_route = ObsRoute() |
| 1706 |
while value and (value[0]==',' or value[0] in CFWS_LEADER): |
| 1707 |
if value[0] in CFWS_LEADER: |
| 1708 |
token, value = get_cfws(value) |
| 1709 |
obs_route.append(token) |
| 1710 |
elif value[0] == ',': |
| 1711 |
obs_route.append(ListSeparator) |
| 1712 |
value = value[1:] |
| 1713 |
if not value or value[0] != '@': |
| 1714 |
raise errors.HeaderParseError( |
| 1715 |
"expected obs-route domain but found '{}'".format(value)) |
| 1716 |
obs_route.append(RouteComponentMarker) |
| 1717 |
token, value = get_domain(value[1:]) |
| 1718 |
obs_route.append(token) |
| 1719 |
while value and value[0]==',': |
| 1720 |
obs_route.append(ListSeparator) |
| 1721 |
value = value[1:] |
| 1722 |
if not value: |
| 1723 |
break |
| 1724 |
if value[0] in CFWS_LEADER: |
| 1725 |
token, value = get_cfws(value) |
| 1726 |
obs_route.append(token) |
| 1727 |
if not value: |
| 1728 |
break |
| 1729 |
if value[0] == '@': |
| 1730 |
obs_route.append(RouteComponentMarker) |
| 1731 |
token, value = get_domain(value[1:]) |
| 1732 |
obs_route.append(token) |
| 1733 |
if not value: |
| 1734 |
raise errors.HeaderParseError("end of header while parsing obs-route") |
| 1735 |
if value[0] != ':': |
| 1736 |
raise errors.HeaderParseError( "expected ':' marking end of " |
| 1737 |
"obs-route but found '{}'".format(value)) |
| 1738 |
obs_route.append(ValueTerminal(':', 'end-of-obs-route-marker')) |
| 1739 |
return obs_route, value[1:] |
| 1740 |
|
| 1741 |
def get_angle_addr(value): |
| 1742 |
""" angle-addr = [CFWS] "<" addr-spec ">" [CFWS] / obs-angle-addr |
| 1743 |
obs-angle-addr = [CFWS] "<" obs-route addr-spec ">" [CFWS] |
| 1744 |
|
| 1745 |
""" |
| 1746 |
angle_addr = AngleAddr() |
| 1747 |
if value and value[0] in CFWS_LEADER: |
| 1748 |
token, value = get_cfws(value) |
| 1749 |
angle_addr.append(token) |
| 1750 |
if not value or value[0] != '<': |
| 1751 |
raise errors.HeaderParseError( |
| 1752 |
"expected angle-addr but found '{}'".format(value)) |
| 1753 |
angle_addr.append(ValueTerminal('<', 'angle-addr-start')) |
| 1754 |
value = value[1:] |
| 1755 |
# Although it is not legal per RFC5322, SMTP uses '<>' in certain |
| 1756 |
# circumstances. |
| 1757 |
if value and value[0] == '>': |
| 1758 |
angle_addr.append(ValueTerminal('>', 'angle-addr-end')) |
| 1759 |
angle_addr.defects.append(errors.InvalidHeaderDefect( |
| 1760 |
"null addr-spec in angle-addr")) |
| 1761 |
value = value[1:] |
| 1762 |
return angle_addr, value |
| 1763 |
try: |
| 1764 |
token, value = get_addr_spec(value) |
| 1765 |
except errors.HeaderParseError: |
| 1766 |
try: |
| 1767 |
token, value = get_obs_route(value) |
| 1768 |
angle_addr.defects.append(errors.ObsoleteHeaderDefect( |
| 1769 |
"obsolete route specification in angle-addr")) |
| 1770 |
except errors.HeaderParseError: |
| 1771 |
raise errors.HeaderParseError( |
| 1772 |
"expected addr-spec or obs-route but found '{}'".format(value)) |
| 1773 |
angle_addr.append(token) |
| 1774 |
token, value = get_addr_spec(value) |
| 1775 |
angle_addr.append(token) |
| 1776 |
if value and value[0] == '>': |
| 1777 |
value = value[1:] |
| 1778 |
else: |
| 1779 |
angle_addr.defects.append(errors.InvalidHeaderDefect( |
| 1780 |
"missing trailing '>' on angle-addr")) |
| 1781 |
angle_addr.append(ValueTerminal('>', 'angle-addr-end')) |
| 1782 |
if value and value[0] in CFWS_LEADER: |
| 1783 |
token, value = get_cfws(value) |
| 1784 |
angle_addr.append(token) |
| 1785 |
return angle_addr, value |
| 1786 |
|
| 1787 |
def get_display_name(value): |
| 1788 |
""" display-name = phrase |
| 1789 |
|
| 1790 |
Because this is simply a name-rule, we don't return a display-name |
| 1791 |
token containing a phrase, but rather a display-name token with |
| 1792 |
the content of the phrase. |
| 1793 |
|
| 1794 |
""" |
| 1795 |
display_name = DisplayName() |
| 1796 |
token, value = get_phrase(value) |
| 1797 |
display_name.extend(token[:]) |
| 1798 |
display_name.defects = token.defects[:] |
| 1799 |
return display_name, value |
| 1800 |
|
| 1801 |
|
| 1802 |
def get_name_addr(value): |
| 1803 |
""" name-addr = [display-name] angle-addr |
| 1804 |
|
| 1805 |
""" |
| 1806 |
name_addr = NameAddr() |
| 1807 |
# Both the optional display name and the angle-addr can start with cfws. |
| 1808 |
leader = None |
| 1809 |
if not value: |
| 1810 |
raise errors.HeaderParseError( |
| 1811 |
"expected name-addr but found '{}'".format(value)) |
| 1812 |
if value[0] in CFWS_LEADER: |
| 1813 |
leader, value = get_cfws(value) |
| 1814 |
if not value: |
| 1815 |
raise errors.HeaderParseError( |
| 1816 |
"expected name-addr but found '{}'".format(leader)) |
| 1817 |
if value[0] != '<': |
| 1818 |
if value[0] in PHRASE_ENDS: |
| 1819 |
raise errors.HeaderParseError( |
| 1820 |
"expected name-addr but found '{}'".format(value)) |
| 1821 |
token, value = get_display_name(value) |
| 1822 |
if not value: |
| 1823 |
raise errors.HeaderParseError( |
| 1824 |
"expected name-addr but found '{}'".format(token)) |
| 1825 |
if leader is not None: |
| 1826 |
if isinstance(token[0], TokenList): |
| 1827 |
token[0][:0] = [leader] |
| 1828 |
else: |
| 1829 |
token[:0] = [leader] |
| 1830 |
leader = None |
| 1831 |
name_addr.append(token) |
| 1832 |
token, value = get_angle_addr(value) |
| 1833 |
if leader is not None: |
| 1834 |
token[:0] = [leader] |
| 1835 |
name_addr.append(token) |
| 1836 |
return name_addr, value |
| 1837 |
|
| 1838 |
def get_mailbox(value): |
| 1839 |
""" mailbox = name-addr / addr-spec |
| 1840 |
|
| 1841 |
""" |
| 1842 |
# The only way to figure out if we are dealing with a name-addr or an |
| 1843 |
# addr-spec is to try parsing each one. |
| 1844 |
mailbox = Mailbox() |
| 1845 |
try: |
| 1846 |
token, value = get_name_addr(value) |
| 1847 |
except errors.HeaderParseError: |
| 1848 |
try: |
| 1849 |
token, value = get_addr_spec(value) |
| 1850 |
except errors.HeaderParseError: |
| 1851 |
raise errors.HeaderParseError( |
| 1852 |
"expected mailbox but found '{}'".format(value)) |
| 1853 |
if any(isinstance(x, errors.InvalidHeaderDefect) |
| 1854 |
for x in token.all_defects): |
| 1855 |
mailbox.token_type = 'invalid-mailbox' |
| 1856 |
mailbox.append(token) |
| 1857 |
return mailbox, value |
| 1858 |
|
| 1859 |
def get_invalid_mailbox(value, endchars): |
| 1860 |
""" Read everything up to one of the chars in endchars. |
| 1861 |
|
| 1862 |
This is outside the formal grammar. The InvalidMailbox TokenList that is |
| 1863 |
returned acts like a Mailbox, but the data attributes are None. |
| 1864 |
|
| 1865 |
""" |
| 1866 |
invalid_mailbox = InvalidMailbox() |
| 1867 |
while value and value[0] not in endchars: |
| 1868 |
if value[0] in PHRASE_ENDS: |
| 1869 |
invalid_mailbox.append(ValueTerminal(value[0], |
| 1870 |
'misplaced-special')) |
| 1871 |
value = value[1:] |
| 1872 |
else: |
| 1873 |
token, value = get_phrase(value) |
| 1874 |
invalid_mailbox.append(token) |
| 1875 |
return invalid_mailbox, value |
| 1876 |
|
| 1877 |
def get_mailbox_list(value): |
| 1878 |
""" mailbox-list = (mailbox *("," mailbox)) / obs-mbox-list |
| 1879 |
obs-mbox-list = *([CFWS] ",") mailbox *("," [mailbox / CFWS]) |
| 1880 |
|
| 1881 |
For this routine we go outside the formal grammar in order to improve error |
| 1882 |
handling. We recognize the end of the mailbox list only at the end of the |
| 1883 |
value or at a ';' (the group terminator). This is so that we can turn |
| 1884 |
invalid mailboxes into InvalidMailbox tokens and continue parsing any |
| 1885 |
remaining valid mailboxes. We also allow all mailbox entries to be null, |
| 1886 |
and this condition is handled appropriately at a higher level. |
| 1887 |
|
| 1888 |
""" |
| 1889 |
mailbox_list = MailboxList() |
| 1890 |
while value and value[0] != ';': |
| 1891 |
try: |
| 1892 |
token, value = get_mailbox(value) |
| 1893 |
mailbox_list.append(token) |
| 1894 |
except errors.HeaderParseError: |
| 1895 |
leader = None |
| 1896 |
if value[0] in CFWS_LEADER: |
| 1897 |
leader, value = get_cfws(value) |
| 1898 |
if not value or value[0] in ',;': |
| 1899 |
mailbox_list.append(leader) |
| 1900 |
mailbox_list.defects.append(errors.ObsoleteHeaderDefect( |
| 1901 |
"empty element in mailbox-list")) |
| 1902 |
else: |
| 1903 |
token, value = get_invalid_mailbox(value, ',;') |
| 1904 |
if leader is not None: |
| 1905 |
token[:0] = [leader] |
| 1906 |
mailbox_list.append(token) |
| 1907 |
mailbox_list.defects.append(errors.InvalidHeaderDefect( |
| 1908 |
"invalid mailbox in mailbox-list")) |
| 1909 |
elif value[0] == ',': |
| 1910 |
mailbox_list.defects.append(errors.ObsoleteHeaderDefect( |
| 1911 |
"empty element in mailbox-list")) |
| 1912 |
else: |
| 1913 |
token, value = get_invalid_mailbox(value, ',;') |
| 1914 |
if leader is not None: |
| 1915 |
token[:0] = [leader] |
| 1916 |
mailbox_list.append(token) |
| 1917 |
mailbox_list.defects.append(errors.InvalidHeaderDefect( |
| 1918 |
"invalid mailbox in mailbox-list")) |
| 1919 |
if value and value[0] not in ',;': |
| 1920 |
# Crap after mailbox; treat it as an invalid mailbox. |
| 1921 |
# The mailbox info will still be available. |
| 1922 |
mailbox = mailbox_list[-1] |
| 1923 |
mailbox.token_type = 'invalid-mailbox' |
| 1924 |
token, value = get_invalid_mailbox(value, ',;') |
| 1925 |
mailbox.extend(token) |
| 1926 |
mailbox_list.defects.append(errors.InvalidHeaderDefect( |
| 1927 |
"invalid mailbox in mailbox-list")) |
| 1928 |
if value and value[0] == ',': |
| 1929 |
mailbox_list.append(ListSeparator) |
| 1930 |
value = value[1:] |
| 1931 |
return mailbox_list, value |
| 1932 |
|
| 1933 |
|
| 1934 |
def get_group_list(value): |
| 1935 |
""" group-list = mailbox-list / CFWS / obs-group-list |
| 1936 |
obs-group-list = 1*([CFWS] ",") [CFWS] |
| 1937 |
|
| 1938 |
""" |
| 1939 |
group_list = GroupList() |
| 1940 |
if not value: |
| 1941 |
group_list.defects.append(errors.InvalidHeaderDefect( |
| 1942 |
"end of header before group-list")) |
| 1943 |
return group_list, value |
| 1944 |
leader = None |
| 1945 |
if value and value[0] in CFWS_LEADER: |
| 1946 |
leader, value = get_cfws(value) |
| 1947 |
if not value: |
| 1948 |
# This should never happen in email parsing, since CFWS-only is a |
| 1949 |
# legal alternative to group-list in a group, which is the only |
| 1950 |
# place group-list appears. |
| 1951 |
group_list.defects.append(errors.InvalidHeaderDefect( |
| 1952 |
"end of header in group-list")) |
| 1953 |
group_list.append(leader) |
| 1954 |
return group_list, value |
| 1955 |
if value[0] == ';': |
| 1956 |
group_list.append(leader) |
| 1957 |
return group_list, value |
| 1958 |
token, value = get_mailbox_list(value) |
| 1959 |
if len(token.all_mailboxes)==0: |
| 1960 |
if leader is not None: |
| 1961 |
group_list.append(leader) |
| 1962 |
group_list.extend(token) |
| 1963 |
group_list.defects.append(errors.ObsoleteHeaderDefect( |
| 1964 |
"group-list with empty entries")) |
| 1965 |
return group_list, value |
| 1966 |
if leader is not None: |
| 1967 |
token[:0] = [leader] |
| 1968 |
group_list.append(token) |
| 1969 |
return group_list, value |
| 1970 |
|
| 1971 |
def get_group(value): |
| 1972 |
""" group = display-name ":" [group-list] ";" [CFWS] |
| 1973 |
|
| 1974 |
""" |
| 1975 |
group = Group() |
| 1976 |
token, value = get_display_name(value) |
| 1977 |
if not value or value[0] != ':': |
| 1978 |
raise errors.HeaderParseError("expected ':' at end of group " |
| 1979 |
"display name but found '{}'".format(value)) |
| 1980 |
group.append(token) |
| 1981 |
group.append(ValueTerminal(':', 'group-display-name-terminator')) |
| 1982 |
value = value[1:] |
| 1983 |
if value and value[0] == ';': |
| 1984 |
group.append(ValueTerminal(';', 'group-terminator')) |
| 1985 |
return group, value[1:] |
| 1986 |
token, value = get_group_list(value) |
| 1987 |
group.append(token) |
| 1988 |
if not value: |
| 1989 |
group.defects.append(errors.InvalidHeaderDefect( |
| 1990 |
"end of header in group")) |
| 1991 |
elif value[0] != ';': |
| 1992 |
raise errors.HeaderParseError( |
| 1993 |
"expected ';' at end of group but found {}".format(value)) |
| 1994 |
group.append(ValueTerminal(';', 'group-terminator')) |
| 1995 |
value = value[1:] |
| 1996 |
if value and value[0] in CFWS_LEADER: |
| 1997 |
token, value = get_cfws(value) |
| 1998 |
group.append(token) |
| 1999 |
return group, value |
| 2000 |
|
| 2001 |
def get_address(value): |
| 2002 |
""" address = mailbox / group |
| 2003 |
|
| 2004 |
Note that counter-intuitively, an address can be either a single address or |
| 2005 |
a list of addresses (a group). This is why the returned Address object has |
| 2006 |
a 'mailboxes' attribute which treats a single address as a list of length |
| 2007 |
one. When you need to differentiate between to two cases, extract the single |
| 2008 |
element, which is either a mailbox or a group token. |
| 2009 |
|
| 2010 |
""" |
| 2011 |
# The formal grammar isn't very helpful when parsing an address. mailbox |
| 2012 |
# and group, especially when allowing for obsolete forms, start off very |
| 2013 |
# similarly. It is only when you reach one of @, <, or : that you know |
| 2014 |
# what you've got. So, we try each one in turn, starting with the more |
| 2015 |
# likely of the two. We could perhaps make this more efficient by looking |
| 2016 |
# for a phrase and then branching based on the next character, but that |
| 2017 |
# would be a premature optimization. |
| 2018 |
address = Address() |
| 2019 |
try: |
| 2020 |
token, value = get_group(value) |
| 2021 |
except errors.HeaderParseError: |
| 2022 |
try: |
| 2023 |
token, value = get_mailbox(value) |
| 2024 |
except errors.HeaderParseError: |
| 2025 |
raise errors.HeaderParseError( |
| 2026 |
"expected address but found '{}'".format(value)) |
| 2027 |
address.append(token) |
| 2028 |
return address, value |
| 2029 |
|
| 2030 |
def get_address_list(value): |
| 2031 |
""" address_list = (address *("," address)) / obs-addr-list |
| 2032 |
obs-addr-list = *([CFWS] ",") address *("," [address / CFWS]) |
| 2033 |
|
| 2034 |
We depart from the formal grammar here by continuing to parse until the end |
| 2035 |
of the input, assuming the input to be entirely composed of an |
| 2036 |
address-list. This is always true in email parsing, and allows us |
| 2037 |
to skip invalid addresses to parse additional valid ones. |
| 2038 |
|
| 2039 |
""" |
| 2040 |
address_list = AddressList() |
| 2041 |
while value: |
| 2042 |
try: |
| 2043 |
token, value = get_address(value) |
| 2044 |
address_list.append(token) |
| 2045 |
except errors.HeaderParseError: |
| 2046 |
leader = None |
| 2047 |
if value[0] in CFWS_LEADER: |
| 2048 |
leader, value = get_cfws(value) |
| 2049 |
if not value or value[0] == ',': |
| 2050 |
address_list.append(leader) |
| 2051 |
address_list.defects.append(errors.ObsoleteHeaderDefect( |
| 2052 |
"address-list entry with no content")) |
| 2053 |
else: |
| 2054 |
token, value = get_invalid_mailbox(value, ',') |
| 2055 |
if leader is not None: |
| 2056 |
token[:0] = [leader] |
| 2057 |
address_list.append(Address([token])) |
| 2058 |
address_list.defects.append(errors.InvalidHeaderDefect( |
| 2059 |
"invalid address in address-list")) |
| 2060 |
elif value[0] == ',': |
| 2061 |
address_list.defects.append(errors.ObsoleteHeaderDefect( |
| 2062 |
"empty element in address-list")) |
| 2063 |
else: |
| 2064 |
token, value = get_invalid_mailbox(value, ',') |
| 2065 |
if leader is not None: |
| 2066 |
token[:0] = [leader] |
| 2067 |
address_list.append(Address([token])) |
| 2068 |
address_list.defects.append(errors.InvalidHeaderDefect( |
| 2069 |
"invalid address in address-list")) |
| 2070 |
if value and value[0] != ',': |
| 2071 |
# Crap after address: add it to the address list |
| 2072 |
# as an invalid mailbox |
| 2073 |
token, value = get_invalid_mailbox(value, ',') |
| 2074 |
address_list.append(Address([token])) |
| 2075 |
address_list.defects.append(errors.InvalidHeaderDefect( |
| 2076 |
"invalid address in address-list")) |
| 2077 |
if value: # Must be a , at this point. |
| 2078 |
address_list.append(ListSeparator) |
| 2079 |
value = value[1:] |
| 2080 |
return address_list, value |
| 2081 |
|
| 2082 |
|
| 2083 |
def get_no_fold_literal(value): |
| 2084 |
""" no-fold-literal = "[" *dtext "]" |
| 2085 |
""" |
| 2086 |
no_fold_literal = NoFoldLiteral() |
| 2087 |
if not value: |
| 2088 |
raise errors.HeaderParseError( |
| 2089 |
"expected no-fold-literal but found '{}'".format(value)) |
| 2090 |
if value[0] != '[': |
| 2091 |
raise errors.HeaderParseError( |
| 2092 |
"expected '[' at the start of no-fold-literal " |
| 2093 |
"but found '{}'".format(value)) |
| 2094 |
no_fold_literal.append(ValueTerminal('[', 'no-fold-literal-start')) |
| 2095 |
value = value[1:] |
| 2096 |
token, value = get_dtext(value) |
| 2097 |
no_fold_literal.append(token) |
| 2098 |
if not value or value[0] != ']': |
| 2099 |
raise errors.HeaderParseError( |
| 2100 |
"expected ']' at the end of no-fold-literal " |
| 2101 |
"but found '{}'".format(value)) |
| 2102 |
no_fold_literal.append(ValueTerminal(']', 'no-fold-literal-end')) |
| 2103 |
return no_fold_literal, value[1:] |
| 2104 |
|
| 2105 |
def get_msg_id(value): |
| 2106 |
"""msg-id = [CFWS] "<" id-left '@' id-right ">" [CFWS] |
| 2107 |
id-left = dot-atom-text / obs-id-left |
| 2108 |
id-right = dot-atom-text / no-fold-literal / obs-id-right |
| 2109 |
no-fold-literal = "[" *dtext "]" |
| 2110 |
""" |
| 2111 |
msg_id = MsgID() |
| 2112 |
if value and value[0] in CFWS_LEADER: |
| 2113 |
token, value = get_cfws(value) |
| 2114 |
msg_id.append(token) |
| 2115 |
if not value or value[0] != '<': |
| 2116 |
raise errors.HeaderParseError( |
| 2117 |
"expected msg-id but found '{}'".format(value)) |
| 2118 |
msg_id.append(ValueTerminal('<', 'msg-id-start')) |
| 2119 |
value = value[1:] |
| 2120 |
# Parse id-left. |
| 2121 |
try: |
| 2122 |
token, value = get_dot_atom_text(value) |
| 2123 |
except errors.HeaderParseError: |
| 2124 |
try: |
| 2125 |
# obs-id-left is same as local-part of add-spec. |
| 2126 |
token, value = get_obs_local_part(value) |
| 2127 |
msg_id.defects.append(errors.ObsoleteHeaderDefect( |
| 2128 |
"obsolete id-left in msg-id")) |
| 2129 |
except errors.HeaderParseError: |
| 2130 |
raise errors.HeaderParseError( |
| 2131 |
"expected dot-atom-text or obs-id-left" |
| 2132 |
" but found '{}'".format(value)) |
| 2133 |
msg_id.append(token) |
| 2134 |
if not value or value[0] != '@': |
| 2135 |
msg_id.defects.append(errors.InvalidHeaderDefect( |
| 2136 |
"msg-id with no id-right")) |
| 2137 |
# Even though there is no id-right, if the local part |
| 2138 |
# ends with `>` let's just parse it too and return |
| 2139 |
# along with the defect. |
| 2140 |
if value and value[0] == '>': |
| 2141 |
msg_id.append(ValueTerminal('>', 'msg-id-end')) |
| 2142 |
value = value[1:] |
| 2143 |
return msg_id, value |
| 2144 |
msg_id.append(ValueTerminal('@', 'address-at-symbol')) |
| 2145 |
value = value[1:] |
| 2146 |
# Parse id-right. |
| 2147 |
try: |
| 2148 |
token, value = get_dot_atom_text(value) |
| 2149 |
except errors.HeaderParseError: |
| 2150 |
try: |
| 2151 |
token, value = get_no_fold_literal(value) |
| 2152 |
except errors.HeaderParseError: |
| 2153 |
try: |
| 2154 |
token, value = get_domain(value) |
| 2155 |
msg_id.defects.append(errors.ObsoleteHeaderDefect( |
| 2156 |
"obsolete id-right in msg-id")) |
| 2157 |
except errors.HeaderParseError: |
| 2158 |
raise errors.HeaderParseError( |
| 2159 |
"expected dot-atom-text, no-fold-literal or obs-id-right" |
| 2160 |
" but found '{}'".format(value)) |
| 2161 |
msg_id.append(token) |
| 2162 |
if value and value[0] == '>': |
| 2163 |
value = value[1:] |
| 2164 |
else: |
| 2165 |
msg_id.defects.append(errors.InvalidHeaderDefect( |
| 2166 |
"missing trailing '>' on msg-id")) |
| 2167 |
msg_id.append(ValueTerminal('>', 'msg-id-end')) |
| 2168 |
if value and value[0] in CFWS_LEADER: |
| 2169 |
token, value = get_cfws(value) |
| 2170 |
msg_id.append(token) |
| 2171 |
return msg_id, value |
| 2172 |
|
| 2173 |
|
| 2174 |
def parse_message_id(value): |
| 2175 |
"""message-id = "Message-ID:" msg-id CRLF |
| 2176 |
""" |
| 2177 |
message_id = MessageID() |
| 2178 |
try: |
| 2179 |
token, value = get_msg_id(value) |
| 2180 |
message_id.append(token) |
| 2181 |
except errors.HeaderParseError as ex: |
| 2182 |
token = get_unstructured(value) |
| 2183 |
message_id = InvalidMessageID(token) |
| 2184 |
message_id.defects.append( |
| 2185 |
errors.InvalidHeaderDefect("Invalid msg-id: {!r}".format(ex))) |
| 2186 |
else: |
| 2187 |
# Value after parsing a valid msg_id should be None. |
| 2188 |
if value: |
| 2189 |
message_id.defects.append(errors.InvalidHeaderDefect( |
| 2190 |
"Unexpected {!r}".format(value))) |
| 2191 |
|
| 2192 |
return message_id |
| 2193 |
|
| 2194 |
def parse_message_ids(value): |
| 2195 |
"""in-reply-to = "In-Reply-To:" 1*msg-id CRLF |
| 2196 |
references = "References:" 1*msg-id CRLF |
| 2197 |
""" |
| 2198 |
message_id_list = MessageIDList() |
| 2199 |
while value: |
| 2200 |
if value[0] == ',': |
| 2201 |
# message id list separated with commas - this is invalid, |
| 2202 |
# but happens rather frequently in the wild |
| 2203 |
message_id_list.defects.append( |
| 2204 |
errors.InvalidHeaderDefect("comma in msg-id list")) |
| 2205 |
message_id_list.append( |
| 2206 |
WhiteSpaceTerminal(' ', 'invalid-comma-replacement')) |
| 2207 |
value = value[1:] |
| 2208 |
continue |
| 2209 |
try: |
| 2210 |
token, value = get_msg_id(value) |
| 2211 |
message_id_list.append(token) |
| 2212 |
except errors.HeaderParseError as ex: |
| 2213 |
token = get_unstructured(value) |
| 2214 |
message_id_list.append(InvalidMessageID(token)) |
| 2215 |
message_id_list.defects.append( |
| 2216 |
errors.InvalidHeaderDefect("Invalid msg-id: {!r}".format(ex))) |
| 2217 |
break |
| 2218 |
return message_id_list |
| 2219 |
|
| 2220 |
# |
| 2221 |
# XXX: As I begin to add additional header parsers, I'm realizing we probably |
| 2222 |
# have two level of parser routines: the get_XXX methods that get a token in |
| 2223 |
# the grammar, and parse_XXX methods that parse an entire field value. So |
| 2224 |
# get_address_list above should really be a parse_ method, as probably should |
| 2225 |
# be get_unstructured. |
| 2226 |
# |
| 2227 |
|
| 2228 |
def parse_mime_version(value): |
| 2229 |
""" mime-version = [CFWS] 1*digit [CFWS] "." [CFWS] 1*digit [CFWS] |
| 2230 |
|
| 2231 |
""" |
| 2232 |
# The [CFWS] is implicit in the RFC 2045 BNF. |
| 2233 |
# XXX: This routine is a bit verbose, should factor out a get_int method. |
| 2234 |
mime_version = MIMEVersion() |
| 2235 |
if not value: |
| 2236 |
mime_version.defects.append(errors.HeaderMissingRequiredValue( |
| 2237 |
"Missing MIME version number (eg: 1.0)")) |
| 2238 |
return mime_version |
| 2239 |
if value[0] in CFWS_LEADER: |
| 2240 |
token, value = get_cfws(value) |
| 2241 |
mime_version.append(token) |
| 2242 |
if not value: |
| 2243 |
mime_version.defects.append(errors.HeaderMissingRequiredValue( |
| 2244 |
"Expected MIME version number but found only CFWS")) |
| 2245 |
digits = '' |
| 2246 |
while value and value[0] != '.' and value[0] not in CFWS_LEADER: |
| 2247 |
digits += value[0] |
| 2248 |
value = value[1:] |
| 2249 |
if not digits.isdigit(): |
| 2250 |
mime_version.defects.append(errors.InvalidHeaderDefect( |
| 2251 |
"Expected MIME major version number but found {!r}".format(digits))) |
| 2252 |
mime_version.append(ValueTerminal(digits, 'xtext')) |
| 2253 |
else: |
| 2254 |
mime_version.major = int(digits) |
| 2255 |
mime_version.append(ValueTerminal(digits, 'digits')) |
| 2256 |
if value and value[0] in CFWS_LEADER: |
| 2257 |
token, value = get_cfws(value) |
| 2258 |
mime_version.append(token) |
| 2259 |
if not value or value[0] != '.': |
| 2260 |
if mime_version.major is not None: |
| 2261 |
mime_version.defects.append(errors.InvalidHeaderDefect( |
| 2262 |
"Incomplete MIME version; found only major number")) |
| 2263 |
if value: |
| 2264 |
mime_version.append(ValueTerminal(value, 'xtext')) |
| 2265 |
return mime_version |
| 2266 |
mime_version.append(ValueTerminal('.', 'version-separator')) |
| 2267 |
value = value[1:] |
| 2268 |
if value and value[0] in CFWS_LEADER: |
| 2269 |
token, value = get_cfws(value) |
| 2270 |
mime_version.append(token) |
| 2271 |
if not value: |
| 2272 |
if mime_version.major is not None: |
| 2273 |
mime_version.defects.append(errors.InvalidHeaderDefect( |
| 2274 |
"Incomplete MIME version; found only major number")) |
| 2275 |
return mime_version |
| 2276 |
digits = '' |
| 2277 |
while value and value[0] not in CFWS_LEADER: |
| 2278 |
digits += value[0] |
| 2279 |
value = value[1:] |
| 2280 |
if not digits.isdigit(): |
| 2281 |
mime_version.defects.append(errors.InvalidHeaderDefect( |
| 2282 |
"Expected MIME minor version number but found {!r}".format(digits))) |
| 2283 |
mime_version.append(ValueTerminal(digits, 'xtext')) |
| 2284 |
else: |
| 2285 |
mime_version.minor = int(digits) |
| 2286 |
mime_version.append(ValueTerminal(digits, 'digits')) |
| 2287 |
if value and value[0] in CFWS_LEADER: |
| 2288 |
token, value = get_cfws(value) |
| 2289 |
mime_version.append(token) |
| 2290 |
if value: |
| 2291 |
mime_version.defects.append(errors.InvalidHeaderDefect( |
| 2292 |
"Excess non-CFWS text after MIME version")) |
| 2293 |
mime_version.append(ValueTerminal(value, 'xtext')) |
| 2294 |
return mime_version |
| 2295 |
|
| 2296 |
def get_invalid_parameter(value): |
| 2297 |
""" Read everything up to the next ';'. |
| 2298 |
|
| 2299 |
This is outside the formal grammar. The InvalidParameter TokenList that is |
| 2300 |
returned acts like a Parameter, but the data attributes are None. |
| 2301 |
|
| 2302 |
""" |
| 2303 |
invalid_parameter = InvalidParameter() |
| 2304 |
while value and value[0] != ';': |
| 2305 |
if value[0] in PHRASE_ENDS: |
| 2306 |
invalid_parameter.append(ValueTerminal(value[0], |
| 2307 |
'misplaced-special')) |
| 2308 |
value = value[1:] |
| 2309 |
else: |
| 2310 |
token, value = get_phrase(value) |
| 2311 |
invalid_parameter.append(token) |
| 2312 |
return invalid_parameter, value |
| 2313 |
|
| 2314 |
def get_ttext(value): |
| 2315 |
"""ttext = <matches _ttext_matcher> |
| 2316 |
|
| 2317 |
We allow any non-TOKEN_ENDS in ttext, but add defects to the token's |
| 2318 |
defects list if we find non-ttext characters. We also register defects for |
| 2319 |
*any* non-printables even though the RFC doesn't exclude all of them, |
| 2320 |
because we follow the spirit of RFC 5322. |
| 2321 |
|
| 2322 |
""" |
| 2323 |
m = _non_token_end_matcher(value) |
| 2324 |
if not m: |
| 2325 |
raise errors.HeaderParseError( |
| 2326 |
"expected ttext but found '{}'".format(value)) |
| 2327 |
ttext = m.group() |
| 2328 |
value = value[len(ttext):] |
| 2329 |
ttext = ValueTerminal(ttext, 'ttext') |
| 2330 |
_validate_xtext(ttext) |
| 2331 |
return ttext, value |
| 2332 |
|
| 2333 |
def get_token(value): |
| 2334 |
"""token = [CFWS] 1*ttext [CFWS] |
| 2335 |
|
| 2336 |
The RFC equivalent of ttext is any US-ASCII chars except space, ctls, or |
| 2337 |
tspecials. We also exclude tabs even though the RFC doesn't. |
| 2338 |
|
| 2339 |
The RFC implies the CFWS but is not explicit about it in the BNF. |
| 2340 |
|
| 2341 |
""" |
| 2342 |
mtoken = Token() |
| 2343 |
if value and value[0] in CFWS_LEADER: |
| 2344 |
token, value = get_cfws(value) |
| 2345 |
mtoken.append(token) |
| 2346 |
if value and value[0] in TOKEN_ENDS: |
| 2347 |
raise errors.HeaderParseError( |
| 2348 |
"expected token but found '{}'".format(value)) |
| 2349 |
token, value = get_ttext(value) |
| 2350 |
mtoken.append(token) |
| 2351 |
if value and value[0] in CFWS_LEADER: |
| 2352 |
token, value = get_cfws(value) |
| 2353 |
mtoken.append(token) |
| 2354 |
return mtoken, value |
| 2355 |
|
| 2356 |
def get_attrtext(value): |
| 2357 |
"""attrtext = 1*(any non-ATTRIBUTE_ENDS character) |
| 2358 |
|
| 2359 |
We allow any non-ATTRIBUTE_ENDS in attrtext, but add defects to the |
| 2360 |
token's defects list if we find non-attrtext characters. We also register |
| 2361 |
defects for *any* non-printables even though the RFC doesn't exclude all of |
| 2362 |
them, because we follow the spirit of RFC 5322. |
| 2363 |
|
| 2364 |
""" |
| 2365 |
m = _non_attribute_end_matcher(value) |
| 2366 |
if not m: |
| 2367 |
raise errors.HeaderParseError( |
| 2368 |
"expected attrtext but found {!r}".format(value)) |
| 2369 |
attrtext = m.group() |
| 2370 |
value = value[len(attrtext):] |
| 2371 |
attrtext = ValueTerminal(attrtext, 'attrtext') |
| 2372 |
_validate_xtext(attrtext) |
| 2373 |
return attrtext, value |
| 2374 |
|
| 2375 |
def get_attribute(value): |
| 2376 |
""" [CFWS] 1*attrtext [CFWS] |
| 2377 |
|
| 2378 |
This version of the BNF makes the CFWS explicit, and as usual we use a |
| 2379 |
value terminal for the actual run of characters. The RFC equivalent of |
| 2380 |
attrtext is the token characters, with the subtraction of '*', "'", and '%'. |
| 2381 |
We include tab in the excluded set just as we do for token. |
| 2382 |
|
| 2383 |
""" |
| 2384 |
attribute = Attribute() |
| 2385 |
if value and value[0] in CFWS_LEADER: |
| 2386 |
token, value = get_cfws(value) |
| 2387 |
attribute.append(token) |
| 2388 |
if value and value[0] in ATTRIBUTE_ENDS: |
| 2389 |
raise errors.HeaderParseError( |
| 2390 |
"expected token but found '{}'".format(value)) |
| 2391 |
token, value = get_attrtext(value) |
| 2392 |
attribute.append(token) |
| 2393 |
if value and value[0] in CFWS_LEADER: |
| 2394 |
token, value = get_cfws(value) |
| 2395 |
attribute.append(token) |
| 2396 |
return attribute, value |
| 2397 |
|
| 2398 |
def get_extended_attrtext(value): |
| 2399 |
"""attrtext = 1*(any non-ATTRIBUTE_ENDS character plus '%') |
| 2400 |
|
| 2401 |
This is a special parsing routine so that we get a value that |
| 2402 |
includes % escapes as a single string (which we decode as a single |
| 2403 |
string later). |
| 2404 |
|
| 2405 |
""" |
| 2406 |
m = _non_extended_attribute_end_matcher(value) |
| 2407 |
if not m: |
| 2408 |
raise errors.HeaderParseError( |
| 2409 |
"expected extended attrtext but found {!r}".format(value)) |
| 2410 |
attrtext = m.group() |
| 2411 |
value = value[len(attrtext):] |
| 2412 |
attrtext = ValueTerminal(attrtext, 'extended-attrtext') |
| 2413 |
_validate_xtext(attrtext) |
| 2414 |
return attrtext, value |
| 2415 |
|
| 2416 |
def get_extended_attribute(value): |
| 2417 |
""" [CFWS] 1*extended_attrtext [CFWS] |
| 2418 |
|
| 2419 |
This is like the non-extended version except we allow % characters, so that |
| 2420 |
we can pick up an encoded value as a single string. |
| 2421 |
|
| 2422 |
""" |
| 2423 |
# XXX: should we have an ExtendedAttribute TokenList? |
| 2424 |
attribute = Attribute() |
| 2425 |
if value and value[0] in CFWS_LEADER: |
| 2426 |
token, value = get_cfws(value) |
| 2427 |
attribute.append(token) |
| 2428 |
if value and value[0] in EXTENDED_ATTRIBUTE_ENDS: |
| 2429 |
raise errors.HeaderParseError( |
| 2430 |
"expected token but found '{}'".format(value)) |
| 2431 |
token, value = get_extended_attrtext(value) |
| 2432 |
attribute.append(token) |
| 2433 |
if value and value[0] in CFWS_LEADER: |
| 2434 |
token, value = get_cfws(value) |
| 2435 |
attribute.append(token) |
| 2436 |
return attribute, value |
| 2437 |
|
| 2438 |
def get_section(value): |
| 2439 |
""" '*' digits |
| 2440 |
|
| 2441 |
The formal BNF is more complicated because leading 0s are not allowed. We |
| 2442 |
check for that and add a defect. We also assume no CFWS is allowed between |
| 2443 |
the '*' and the digits, though the RFC is not crystal clear on that. |
| 2444 |
The caller should already have dealt with leading CFWS. |
| 2445 |
|
| 2446 |
""" |
| 2447 |
section = Section() |
| 2448 |
if not value or value[0] != '*': |
| 2449 |
raise errors.HeaderParseError("Expected section but found {}".format( |
| 2450 |
value)) |
| 2451 |
section.append(ValueTerminal('*', 'section-marker')) |
| 2452 |
value = value[1:] |
| 2453 |
if not value or not value[0].isdigit(): |
| 2454 |
raise errors.HeaderParseError("Expected section number but " |
| 2455 |
"found {}".format(value)) |
| 2456 |
digits = '' |
| 2457 |
while value and value[0].isdigit(): |
| 2458 |
digits += value[0] |
| 2459 |
value = value[1:] |
| 2460 |
if digits[0] == '0' and digits != '0': |
| 2461 |
section.defects.append(errors.InvalidHeaderDefect( |
| 2462 |
"section number has an invalid leading 0")) |
| 2463 |
section.number = int(digits) |
| 2464 |
section.append(ValueTerminal(digits, 'digits')) |
| 2465 |
return section, value |
| 2466 |
|
| 2467 |
|
| 2468 |
def get_value(value): |
| 2469 |
""" quoted-string / attribute |
| 2470 |
|
| 2471 |
""" |
| 2472 |
v = Value() |
| 2473 |
if not value: |
| 2474 |
raise errors.HeaderParseError("Expected value but found end of string") |
| 2475 |
leader = None |
| 2476 |
if value[0] in CFWS_LEADER: |
| 2477 |
leader, value = get_cfws(value) |
| 2478 |
if not value: |
| 2479 |
raise errors.HeaderParseError("Expected value but found " |
| 2480 |
"only {}".format(leader)) |
| 2481 |
if value[0] == '"': |
| 2482 |
token, value = get_quoted_string(value) |
| 2483 |
else: |
| 2484 |
token, value = get_extended_attribute(value) |
| 2485 |
if leader is not None: |
| 2486 |
token[:0] = [leader] |
| 2487 |
v.append(token) |
| 2488 |
return v, value |
| 2489 |
|
| 2490 |
def get_parameter(value): |
| 2491 |
""" attribute [section] ["*"] [CFWS] "=" value |
| 2492 |
|
| 2493 |
The CFWS is implied by the RFC but not made explicit in the BNF. This |
| 2494 |
simplified form of the BNF from the RFC is made to conform with the RFC BNF |
| 2495 |
through some extra checks. We do it this way because it makes both error |
| 2496 |
recovery and working with the resulting parse tree easier. |
| 2497 |
""" |
| 2498 |
# It is possible CFWS would also be implicitly allowed between the section |
| 2499 |
# and the 'extended-attribute' marker (the '*') , but we've never seen that |
| 2500 |
# in the wild and we will therefore ignore the possibility. |
| 2501 |
param = Parameter() |
| 2502 |
token, value = get_attribute(value) |
| 2503 |
param.append(token) |
| 2504 |
if not value or value[0] == ';': |
| 2505 |
param.defects.append(errors.InvalidHeaderDefect("Parameter contains " |
| 2506 |
"name ({}) but no value".format(token))) |
| 2507 |
return param, value |
| 2508 |
if value[0] == '*': |
| 2509 |
try: |
| 2510 |
token, value = get_section(value) |
| 2511 |
param.sectioned = True |
| 2512 |
param.append(token) |
| 2513 |
except errors.HeaderParseError: |
| 2514 |
pass |
| 2515 |
if not value: |
| 2516 |
raise errors.HeaderParseError("Incomplete parameter") |
| 2517 |
if value[0] == '*': |
| 2518 |
param.append(ValueTerminal('*', 'extended-parameter-marker')) |
| 2519 |
value = value[1:] |
| 2520 |
param.extended = True |
| 2521 |
if value[0] != '=': |
| 2522 |
raise errors.HeaderParseError("Parameter not followed by '='") |
| 2523 |
param.append(ValueTerminal('=', 'parameter-separator')) |
| 2524 |
value = value[1:] |
| 2525 |
if value and value[0] in CFWS_LEADER: |
| 2526 |
token, value = get_cfws(value) |
| 2527 |
param.append(token) |
| 2528 |
remainder = None |
| 2529 |
appendto = param |
| 2530 |
if param.extended and value and value[0] == '"': |
| 2531 |
# Now for some serious hackery to handle the common invalid case of |
| 2532 |
# double quotes around an extended value. We also accept (with defect) |
| 2533 |
# a value marked as encoded that isn't really. |
| 2534 |
qstring, remainder = get_quoted_string(value) |
| 2535 |
inner_value = qstring.stripped_value |
| 2536 |
semi_valid = False |
| 2537 |
if param.section_number == 0: |
| 2538 |
if inner_value and inner_value[0] == "'": |
| 2539 |
semi_valid = True |
| 2540 |
else: |
| 2541 |
token, rest = get_attrtext(inner_value) |
| 2542 |
if rest and rest[0] == "'": |
| 2543 |
semi_valid = True |
| 2544 |
else: |
| 2545 |
try: |
| 2546 |
token, rest = get_extended_attrtext(inner_value) |
| 2547 |
except: |
| 2548 |
pass |
| 2549 |
else: |
| 2550 |
if not rest: |
| 2551 |
semi_valid = True |
| 2552 |
if semi_valid: |
| 2553 |
param.defects.append(errors.InvalidHeaderDefect( |
| 2554 |
"Quoted string value for extended parameter is invalid")) |
| 2555 |
param.append(qstring) |
| 2556 |
for t in qstring: |
| 2557 |
if t.token_type == 'bare-quoted-string': |
| 2558 |
t[:] = [] |
| 2559 |
appendto = t |
| 2560 |
break |
| 2561 |
value = inner_value |
| 2562 |
else: |
| 2563 |
remainder = None |
| 2564 |
param.defects.append(errors.InvalidHeaderDefect( |
| 2565 |
"Parameter marked as extended but appears to have a " |
| 2566 |
"quoted string value that is non-encoded")) |
| 2567 |
if value and value[0] == "'": |
| 2568 |
token = None |
| 2569 |
else: |
| 2570 |
token, value = get_value(value) |
| 2571 |
if not param.extended or param.section_number > 0: |
| 2572 |
if not value or value[0] != "'": |
| 2573 |
appendto.append(token) |
| 2574 |
if remainder is not None: |
| 2575 |
assert not value, value |
| 2576 |
value = remainder |
| 2577 |
return param, value |
| 2578 |
param.defects.append(errors.InvalidHeaderDefect( |
| 2579 |
"Apparent initial-extended-value but attribute " |
| 2580 |
"was not marked as extended or was not initial section")) |
| 2581 |
if not value: |
| 2582 |
# Assume the charset/lang is missing and the token is the value. |
| 2583 |
param.defects.append(errors.InvalidHeaderDefect( |
| 2584 |
"Missing required charset/lang delimiters")) |
| 2585 |
appendto.append(token) |
| 2586 |
if remainder is None: |
| 2587 |
return param, value |
| 2588 |
else: |
| 2589 |
if token is not None: |
| 2590 |
for t in token: |
| 2591 |
if t.token_type == 'extended-attrtext': |
| 2592 |
break |
| 2593 |
t.token_type == 'attrtext' |
| 2594 |
appendto.append(t) |
| 2595 |
param.charset = t.value |
| 2596 |
if value[0] != "'": |
| 2597 |
raise errors.HeaderParseError("Expected RFC2231 char/lang encoding " |
| 2598 |
"delimiter, but found {!r}".format(value)) |
| 2599 |
appendto.append(ValueTerminal("'", 'RFC2231-delimiter')) |
| 2600 |
value = value[1:] |
| 2601 |
if value and value[0] != "'": |
| 2602 |
token, value = get_attrtext(value) |
| 2603 |
appendto.append(token) |
| 2604 |
param.lang = token.value |
| 2605 |
if not value or value[0] != "'": |
| 2606 |
raise errors.HeaderParseError("Expected RFC2231 char/lang encoding " |
| 2607 |
"delimiter, but found {}".format(value)) |
| 2608 |
appendto.append(ValueTerminal("'", 'RFC2231-delimiter')) |
| 2609 |
value = value[1:] |
| 2610 |
if remainder is not None: |
| 2611 |
# Treat the rest of value as bare quoted string content. |
| 2612 |
v = Value() |
| 2613 |
while value: |
| 2614 |
if value[0] in WSP: |
| 2615 |
token, value = get_fws(value) |
| 2616 |
elif value[0] == '"': |
| 2617 |
token = ValueTerminal('"', 'DQUOTE') |
| 2618 |
value = value[1:] |
| 2619 |
else: |
| 2620 |
token, value = get_qcontent(value) |
| 2621 |
v.append(token) |
| 2622 |
token = v |
| 2623 |
else: |
| 2624 |
token, value = get_value(value) |
| 2625 |
appendto.append(token) |
| 2626 |
if remainder is not None: |
| 2627 |
assert not value, value |
| 2628 |
value = remainder |
| 2629 |
return param, value |
| 2630 |
|
| 2631 |
def parse_mime_parameters(value): |
| 2632 |
""" parameter *( ";" parameter ) |
| 2633 |
|
| 2634 |
That BNF is meant to indicate this routine should only be called after |
| 2635 |
finding and handling the leading ';'. There is no corresponding rule in |
| 2636 |
the formal RFC grammar, but it is more convenient for us for the set of |
| 2637 |
parameters to be treated as its own TokenList. |
| 2638 |
|
| 2639 |
This is 'parse' routine because it consumes the remaining value, but it |
| 2640 |
would never be called to parse a full header. Instead it is called to |
| 2641 |
parse everything after the non-parameter value of a specific MIME header. |
| 2642 |
|
| 2643 |
""" |
| 2644 |
mime_parameters = MimeParameters() |
| 2645 |
while value: |
| 2646 |
try: |
| 2647 |
token, value = get_parameter(value) |
| 2648 |
mime_parameters.append(token) |
| 2649 |
except errors.HeaderParseError: |
| 2650 |
leader = None |
| 2651 |
if value[0] in CFWS_LEADER: |
| 2652 |
leader, value = get_cfws(value) |
| 2653 |
if not value: |
| 2654 |
mime_parameters.append(leader) |
| 2655 |
return mime_parameters |
| 2656 |
if value[0] == ';': |
| 2657 |
if leader is not None: |
| 2658 |
mime_parameters.append(leader) |
| 2659 |
mime_parameters.defects.append(errors.InvalidHeaderDefect( |
| 2660 |
"parameter entry with no content")) |
| 2661 |
else: |
| 2662 |
token, value = get_invalid_parameter(value) |
| 2663 |
if leader: |
| 2664 |
token[:0] = [leader] |
| 2665 |
mime_parameters.append(token) |
| 2666 |
mime_parameters.defects.append(errors.InvalidHeaderDefect( |
| 2667 |
"invalid parameter {!r}".format(token))) |
| 2668 |
if value and value[0] != ';': |
| 2669 |
# Junk after the otherwise valid parameter. Mark it as |
| 2670 |
# invalid, but it will have a value. |
| 2671 |
param = mime_parameters[-1] |
| 2672 |
param.token_type = 'invalid-parameter' |
| 2673 |
token, value = get_invalid_parameter(value) |
| 2674 |
param.extend(token) |
| 2675 |
mime_parameters.defects.append(errors.InvalidHeaderDefect( |
| 2676 |
"parameter with invalid trailing text {!r}".format(token))) |
| 2677 |
if value: |
| 2678 |
# Must be a ';' at this point. |
| 2679 |
mime_parameters.append(ValueTerminal(';', 'parameter-separator')) |
| 2680 |
value = value[1:] |
| 2681 |
return mime_parameters |
| 2682 |
|
| 2683 |
def _find_mime_parameters(tokenlist, value): |
| 2684 |
"""Do our best to find the parameters in an invalid MIME header |
| 2685 |
|
| 2686 |
""" |
| 2687 |
while value and value[0] != ';': |
| 2688 |
if value[0] in PHRASE_ENDS: |
| 2689 |
tokenlist.append(ValueTerminal(value[0], 'misplaced-special')) |
| 2690 |
value = value[1:] |
| 2691 |
else: |
| 2692 |
token, value = get_phrase(value) |
| 2693 |
tokenlist.append(token) |
| 2694 |
if not value: |
| 2695 |
return |
| 2696 |
tokenlist.append(ValueTerminal(';', 'parameter-separator')) |
| 2697 |
tokenlist.append(parse_mime_parameters(value[1:])) |
| 2698 |
|
| 2699 |
def parse_content_type_header(value): |
| 2700 |
""" maintype "/" subtype *( ";" parameter ) |
| 2701 |
|
| 2702 |
The maintype and substype are tokens. Theoretically they could |
| 2703 |
be checked against the official IANA list + x-token, but we |
| 2704 |
don't do that. |
| 2705 |
""" |
| 2706 |
ctype = ContentType() |
| 2707 |
if not value: |
| 2708 |
ctype.defects.append(errors.HeaderMissingRequiredValue( |
| 2709 |
"Missing content type specification")) |
| 2710 |
return ctype |
| 2711 |
try: |
| 2712 |
token, value = get_token(value) |
| 2713 |
except errors.HeaderParseError: |
| 2714 |
ctype.defects.append(errors.InvalidHeaderDefect( |
| 2715 |
"Expected content maintype but found {!r}".format(value))) |
| 2716 |
_find_mime_parameters(ctype, value) |
| 2717 |
return ctype |
| 2718 |
ctype.append(token) |
| 2719 |
# XXX: If we really want to follow the formal grammar we should make |
| 2720 |
# mantype and subtype specialized TokenLists here. Probably not worth it. |
| 2721 |
if not value or value[0] != '/': |
| 2722 |
ctype.defects.append(errors.InvalidHeaderDefect( |
| 2723 |
"Invalid content type")) |
| 2724 |
if value: |
| 2725 |
_find_mime_parameters(ctype, value) |
| 2726 |
return ctype |
| 2727 |
ctype.maintype = token.value.strip().lower() |
| 2728 |
ctype.append(ValueTerminal('/', 'content-type-separator')) |
| 2729 |
value = value[1:] |
| 2730 |
try: |
| 2731 |
token, value = get_token(value) |
| 2732 |
except errors.HeaderParseError: |
| 2733 |
ctype.defects.append(errors.InvalidHeaderDefect( |
| 2734 |
"Expected content subtype but found {!r}".format(value))) |
| 2735 |
_find_mime_parameters(ctype, value) |
| 2736 |
return ctype |
| 2737 |
ctype.append(token) |
| 2738 |
ctype.subtype = token.value.strip().lower() |
| 2739 |
if not value: |
| 2740 |
return ctype |
| 2741 |
if value[0] != ';': |
| 2742 |
ctype.defects.append(errors.InvalidHeaderDefect( |
| 2743 |
"Only parameters are valid after content type, but " |
| 2744 |
"found {!r}".format(value))) |
| 2745 |
# The RFC requires that a syntactically invalid content-type be treated |
| 2746 |
# as text/plain. Perhaps we should postel this, but we should probably |
| 2747 |
# only do that if we were checking the subtype value against IANA. |
| 2748 |
del ctype.maintype, ctype.subtype |
| 2749 |
_find_mime_parameters(ctype, value) |
| 2750 |
return ctype |
| 2751 |
ctype.append(ValueTerminal(';', 'parameter-separator')) |
| 2752 |
ctype.append(parse_mime_parameters(value[1:])) |
| 2753 |
return ctype |
| 2754 |
|
| 2755 |
def parse_content_disposition_header(value): |
| 2756 |
""" disposition-type *( ";" parameter ) |
| 2757 |
|
| 2758 |
""" |
| 2759 |
disp_header = ContentDisposition() |
| 2760 |
if not value: |
| 2761 |
disp_header.defects.append(errors.HeaderMissingRequiredValue( |
| 2762 |
"Missing content disposition")) |
| 2763 |
return disp_header |
| 2764 |
try: |
| 2765 |
token, value = get_token(value) |
| 2766 |
except errors.HeaderParseError: |
| 2767 |
disp_header.defects.append(errors.InvalidHeaderDefect( |
| 2768 |
"Expected content disposition but found {!r}".format(value))) |
| 2769 |
_find_mime_parameters(disp_header, value) |
| 2770 |
return disp_header |
| 2771 |
disp_header.append(token) |
| 2772 |
disp_header.content_disposition = token.value.strip().lower() |
| 2773 |
if not value: |
| 2774 |
return disp_header |
| 2775 |
if value[0] != ';': |
| 2776 |
disp_header.defects.append(errors.InvalidHeaderDefect( |
| 2777 |
"Only parameters are valid after content disposition, but " |
| 2778 |
"found {!r}".format(value))) |
| 2779 |
_find_mime_parameters(disp_header, value) |
| 2780 |
return disp_header |
| 2781 |
disp_header.append(ValueTerminal(';', 'parameter-separator')) |
| 2782 |
disp_header.append(parse_mime_parameters(value[1:])) |
| 2783 |
return disp_header |
| 2784 |
|
| 2785 |
def parse_content_transfer_encoding_header(value): |
| 2786 |
""" mechanism |
| 2787 |
|
| 2788 |
""" |
| 2789 |
# We should probably validate the values, since the list is fixed. |
| 2790 |
cte_header = ContentTransferEncoding() |
| 2791 |
if not value: |
| 2792 |
cte_header.defects.append(errors.HeaderMissingRequiredValue( |
| 2793 |
"Missing content transfer encoding")) |
| 2794 |
return cte_header |
| 2795 |
try: |
| 2796 |
token, value = get_token(value) |
| 2797 |
except errors.HeaderParseError: |
| 2798 |
cte_header.defects.append(errors.InvalidHeaderDefect( |
| 2799 |
"Expected content transfer encoding but found {!r}".format(value))) |
| 2800 |
else: |
| 2801 |
cte_header.append(token) |
| 2802 |
cte_header.cte = token.value.strip().lower() |
| 2803 |
if not value: |
| 2804 |
return cte_header |
| 2805 |
while value: |
| 2806 |
cte_header.defects.append(errors.InvalidHeaderDefect( |
| 2807 |
"Extra text after content transfer encoding")) |
| 2808 |
if value[0] in PHRASE_ENDS: |
| 2809 |
cte_header.append(ValueTerminal(value[0], 'misplaced-special')) |
| 2810 |
value = value[1:] |
| 2811 |
else: |
| 2812 |
token, value = get_phrase(value) |
| 2813 |
cte_header.append(token) |
| 2814 |
return cte_header |
| 2815 |
|
| 2816 |
|
| 2817 |
# |
| 2818 |
# Header folding |
| 2819 |
# |
| 2820 |
# Header folding is complex, with lots of rules and corner cases. The |
| 2821 |
# following code does its best to obey the rules and handle the corner |
| 2822 |
# cases, but you can be sure there are few bugs:) |
| 2823 |
# |
| 2824 |
# This folder generally canonicalizes as it goes, preferring the stringified |
| 2825 |
# version of each token. The tokens contain information that supports the |
| 2826 |
# folder, including which tokens can be encoded in which ways. |
| 2827 |
# |
| 2828 |
# Folded text is accumulated in a simple list of strings ('lines'), each |
| 2829 |
# one of which should be less than policy.max_line_length ('maxlen'). |
| 2830 |
# |
| 2831 |
|
| 2832 |
def _steal_trailing_WSP_if_exists(lines): |
| 2833 |
wsp = '' |
| 2834 |
if lines and lines[-1] and lines[-1][-1] in WSP: |
| 2835 |
wsp = lines[-1][-1] |
| 2836 |
lines[-1] = lines[-1][:-1] |
| 2837 |
# gh-142006: if the line is now empty, remove it entirely. |
| 2838 |
if not lines[-1]: |
| 2839 |
lines.pop() |
| 2840 |
return wsp |
| 2841 |
|
| 2842 |
|
| 2843 |
def _refold_parse_tree(parse_tree, *, policy): |
| 2844 |
"""Return string of contents of parse_tree folded according to RFC rules. |
| 2845 |
|
| 2846 |
""" |
| 2847 |
# max_line_length 0/None means no limit, ie: infinitely long. |
| 2848 |
maxlen = policy.max_line_length or sys.maxsize |
| 2849 |
encoding = 'utf-8' if policy.utf8 else 'us-ascii' |
| 2850 |
lines = [''] # Folded lines to be output |
| 2851 |
last_word_is_ew = False |
| 2852 |
last_ew = None # if there is an encoded word in the last line of lines, |
| 2853 |
# points to the encoded word's first character |
| 2854 |
last_charset = None |
| 2855 |
wrap_as_ew_blocked = 0 |
| 2856 |
want_encoding = False # This is set to True if we need to encode this part |
| 2857 |
end_ew_not_allowed = Terminal('', 'wrap_as_ew_blocked') |
| 2858 |
parts = list(parse_tree) |
| 2859 |
while parts: |
| 2860 |
part = parts.pop(0) |
| 2861 |
if part is end_ew_not_allowed: |
| 2862 |
wrap_as_ew_blocked -= 1 |
| 2863 |
continue |
| 2864 |
tstr = str(part) |
| 2865 |
if not want_encoding: |
| 2866 |
if part.token_type in ('ptext', 'vtext'): |
| 2867 |
# Encode if tstr contains special characters. |
| 2868 |
want_encoding = not SPECIALSNL.isdisjoint(tstr) |
| 2869 |
else: |
| 2870 |
# Encode if tstr contains newlines. |
| 2871 |
want_encoding = not NLSET.isdisjoint(tstr) |
| 2872 |
try: |
| 2873 |
tstr.encode(encoding) |
| 2874 |
charset = encoding |
| 2875 |
except UnicodeEncodeError: |
| 2876 |
if any(isinstance(x, errors.UndecodableBytesDefect) |
| 2877 |
for x in part.all_defects): |
| 2878 |
charset = 'unknown-8bit' |
| 2879 |
else: |
| 2880 |
# If policy.utf8 is false this should really be taken from a |
| 2881 |
# 'charset' property on the policy. |
| 2882 |
charset = 'utf-8' |
| 2883 |
want_encoding = True |
| 2884 |
|
| 2885 |
if part.token_type == 'mime-parameters': |
| 2886 |
# Mime parameter folding (using RFC2231) is extra special. |
| 2887 |
_fold_mime_parameters(part, lines, maxlen, encoding) |
| 2888 |
last_word_is_ew = False |
| 2889 |
continue |
| 2890 |
|
| 2891 |
if want_encoding and not wrap_as_ew_blocked: |
| 2892 |
if not part.as_ew_allowed: |
| 2893 |
want_encoding = False |
| 2894 |
last_ew = None |
| 2895 |
if part.syntactic_break: |
| 2896 |
encoded_part = part.fold(policy=policy)[:-len(policy.linesep)] |
| 2897 |
if policy.linesep not in encoded_part: |
| 2898 |
# It fits on a single line |
| 2899 |
if len(encoded_part) > maxlen - len(lines[-1]): |
| 2900 |
# But not on this one, so start a new one. |
| 2901 |
newline = _steal_trailing_WSP_if_exists(lines) |
| 2902 |
# XXX what if encoded_part has no leading FWS? |
| 2903 |
lines.append(newline) |
| 2904 |
lines[-1] += encoded_part |
| 2905 |
last_word_is_ew = False |
| 2906 |
continue |
| 2907 |
# Either this is not a major syntactic break, so we don't |
| 2908 |
# want it on a line by itself even if it fits, or it |
| 2909 |
# doesn't fit on a line by itself. Either way, fall through |
| 2910 |
# to unpacking the subparts and wrapping them. |
| 2911 |
if not hasattr(part, 'encode'): |
| 2912 |
# It's not a Terminal, do each piece individually. |
| 2913 |
parts = list(part) + parts |
| 2914 |
want_encoding = False |
| 2915 |
continue |
| 2916 |
elif part.as_ew_allowed: |
| 2917 |
# It's a terminal, wrap it as an encoded word, possibly |
| 2918 |
# combining it with previously encoded words if allowed. |
| 2919 |
if (last_ew is not None and |
| 2920 |
charset != last_charset and |
| 2921 |
(last_charset == 'unknown-8bit' or |
| 2922 |
last_charset == 'utf-8' and charset != 'us-ascii')): |
| 2923 |
last_ew = None |
| 2924 |
last_ew = _fold_as_ew( |
| 2925 |
tstr, |
| 2926 |
lines, |
| 2927 |
maxlen, |
| 2928 |
last_ew, |
| 2929 |
part.ew_combine_allowed, |
| 2930 |
charset, |
| 2931 |
last_word_is_ew, |
| 2932 |
) |
| 2933 |
last_word_is_ew = True |
| 2934 |
last_charset = charset |
| 2935 |
want_encoding = False |
| 2936 |
continue |
| 2937 |
else: |
| 2938 |
# It's a terminal which should be kept non-encoded |
| 2939 |
# (e.g. a ListSeparator). |
| 2940 |
last_ew = None |
| 2941 |
want_encoding = False |
| 2942 |
# fall through |
| 2943 |
|
| 2944 |
if len(tstr) <= maxlen - len(lines[-1]): |
| 2945 |
lines[-1] += tstr |
| 2946 |
last_word_is_ew = last_word_is_ew and not bool(tstr.strip(_WSP)) |
| 2947 |
continue |
| 2948 |
|
| 2949 |
# This part is too long to fit. The RFC wants us to break at |
| 2950 |
# "major syntactic breaks", so unless we don't consider this |
| 2951 |
# to be one, check if it will fit on the next line by itself. |
| 2952 |
if (part.syntactic_break and |
| 2953 |
len(tstr) + 1 <= maxlen): |
| 2954 |
newline = _steal_trailing_WSP_if_exists(lines) |
| 2955 |
if newline or part.startswith_fws(): |
| 2956 |
lines.append(newline + tstr) |
| 2957 |
last_word_is_ew = (last_word_is_ew |
| 2958 |
and not bool(lines[-1].strip(_WSP))) |
| 2959 |
last_ew = None |
| 2960 |
continue |
| 2961 |
if not hasattr(part, 'encode'): |
| 2962 |
# It's not a terminal, try folding the subparts. |
| 2963 |
newparts = list(part) |
| 2964 |
if part.token_type == 'bare-quoted-string': |
| 2965 |
# To fold a quoted string we need to create a list of terminal |
| 2966 |
# tokens that will render the leading and trailing quotes |
| 2967 |
# and use quoted pairs in the value as appropriate. |
| 2968 |
newparts = ( |
| 2969 |
[ValueTerminal('"', 'ptext')] + |
| 2970 |
[ValueTerminal(make_quoted_pairs(p), 'ptext') |
| 2971 |
for p in newparts] + |
| 2972 |
[ValueTerminal('"', 'ptext')]) |
| 2973 |
if part.token_type == 'comment': |
| 2974 |
newparts = ( |
| 2975 |
[ValueTerminal('(', 'ptext')] + |
| 2976 |
[ValueTerminal(make_parenthesis_pairs(p), 'ptext') |
| 2977 |
if p.token_type == 'ptext' else p |
| 2978 |
for p in newparts] + |
| 2979 |
[ValueTerminal(')', 'ptext')]) |
| 2980 |
if not part.as_ew_allowed: |
| 2981 |
wrap_as_ew_blocked += 1 |
| 2982 |
newparts.append(end_ew_not_allowed) |
| 2983 |
parts = newparts + parts |
| 2984 |
continue |
| 2985 |
if part.as_ew_allowed and not wrap_as_ew_blocked: |
| 2986 |
# It doesn't need CTE encoding, but encode it anyway so we can |
| 2987 |
# wrap it. |
| 2988 |
parts.insert(0, part) |
| 2989 |
want_encoding = True |
| 2990 |
continue |
| 2991 |
# We can't figure out how to wrap, it, so give up. |
| 2992 |
newline = _steal_trailing_WSP_if_exists(lines) |
| 2993 |
if newline or part.startswith_fws(): |
| 2994 |
lines.append(newline + tstr) |
| 2995 |
else: |
| 2996 |
# We can't fold it onto the next line either... |
| 2997 |
lines[-1] += tstr |
| 2998 |
last_word_is_ew = last_word_is_ew and not bool(tstr.strip(_WSP)) |
| 2999 |
|
| 3000 |
return policy.linesep.join(lines) + policy.linesep |
| 3001 |
|
| 3002 |
def _fold_as_ew(to_encode, lines, maxlen, last_ew, ew_combine_allowed, charset, last_word_is_ew): |
| 3003 |
"""Fold string to_encode into lines as encoded word, combining if allowed. |
| 3004 |
Return the new value for last_ew, or None if ew_combine_allowed is False. |
| 3005 |
|
| 3006 |
If there is already an encoded word in the last line of lines (indicated by |
| 3007 |
a non-None value for last_ew) and ew_combine_allowed is true, decode the |
| 3008 |
existing ew, combine it with to_encode, and re-encode. Otherwise, encode |
| 3009 |
to_encode. In either case, split to_encode as necessary so that the |
| 3010 |
encoded segments fit within maxlen. |
| 3011 |
|
| 3012 |
""" |
| 3013 |
if last_ew is not None and ew_combine_allowed: |
| 3014 |
to_encode = str( |
| 3015 |
get_unstructured(lines[-1][last_ew:] + to_encode)) |
| 3016 |
lines[-1] = lines[-1][:last_ew] |
| 3017 |
elif last_word_is_ew: |
| 3018 |
# If we are following up an encoded word with another encoded word, |
| 3019 |
# any white space between the two will be ignored when decoded. |
| 3020 |
# Therefore, we encode all to-be-displayed whitespace in the second |
| 3021 |
# encoded word. |
| 3022 |
len_without_wsp = len(lines[-1].rstrip(_WSP)) |
| 3023 |
leading_whitespace = lines[-1][len_without_wsp:] |
| 3024 |
lines[-1] = (lines[-1][:len_without_wsp] |
| 3025 |
+ (' ' if leading_whitespace else '')) |
| 3026 |
to_encode = leading_whitespace + to_encode |
| 3027 |
elif to_encode[0] in WSP: |
| 3028 |
# We're joining this to non-encoded text, so don't encode |
| 3029 |
# the leading blank. |
| 3030 |
leading_wsp = to_encode[0] |
| 3031 |
to_encode = to_encode[1:] |
| 3032 |
if (len(lines[-1]) == maxlen): |
| 3033 |
lines.append(_steal_trailing_WSP_if_exists(lines)) |
| 3034 |
lines[-1] += leading_wsp |
| 3035 |
|
| 3036 |
trailing_wsp = '' |
| 3037 |
if to_encode[-1] in WSP: |
| 3038 |
# Likewise for the trailing space. |
| 3039 |
trailing_wsp = to_encode[-1] |
| 3040 |
to_encode = to_encode[:-1] |
| 3041 |
new_last_ew = len(lines[-1]) if last_ew is None else last_ew |
| 3042 |
|
| 3043 |
encode_as = 'utf-8' if charset == 'us-ascii' else charset |
| 3044 |
|
| 3045 |
# The RFC2047 chrome takes up 7 characters plus the length |
| 3046 |
# of the charset name. |
| 3047 |
chrome_len = len(encode_as) + 7 |
| 3048 |
|
| 3049 |
if (chrome_len + 1) >= maxlen: |
| 3050 |
raise errors.HeaderParseError( |
| 3051 |
"max_line_length is too small to fit an encoded word") |
| 3052 |
|
| 3053 |
while to_encode: |
| 3054 |
remaining_space = maxlen - len(lines[-1]) |
| 3055 |
text_space = remaining_space - chrome_len |
| 3056 |
if text_space <= 0: |
| 3057 |
newline = _steal_trailing_WSP_if_exists(lines) |
| 3058 |
lines.append(newline or ' ') |
| 3059 |
new_last_ew = len(lines[-1]) |
| 3060 |
continue |
| 3061 |
|
| 3062 |
to_encode_word = to_encode[:text_space] |
| 3063 |
encoded_word = _ew.encode(to_encode_word, charset=encode_as) |
| 3064 |
excess = len(encoded_word) - remaining_space |
| 3065 |
while excess > 0: |
| 3066 |
# Since the chunk to encode is guaranteed to fit into less than 100 characters, |
| 3067 |
# shrinking it by one at a time shouldn't take long. |
| 3068 |
to_encode_word = to_encode_word[:-1] |
| 3069 |
encoded_word = _ew.encode(to_encode_word, charset=encode_as) |
| 3070 |
excess = len(encoded_word) - remaining_space |
| 3071 |
lines[-1] += encoded_word |
| 3072 |
to_encode = to_encode[len(to_encode_word):] |
| 3073 |
|
| 3074 |
if to_encode: |
| 3075 |
lines.append(' ') |
| 3076 |
new_last_ew = len(lines[-1]) |
| 3077 |
lines[-1] += trailing_wsp |
| 3078 |
return new_last_ew if ew_combine_allowed else None |
| 3079 |
|
| 3080 |
def _fold_mime_parameters(part, lines, maxlen, encoding): |
| 3081 |
"""Fold TokenList 'part' into the 'lines' list as mime parameters. |
| 3082 |
|
| 3083 |
Using the decoded list of parameters and values, format them according to |
| 3084 |
the RFC rules, including using RFC2231 encoding if the value cannot be |
| 3085 |
expressed in 'encoding' and/or the parameter+value is too long to fit |
| 3086 |
within 'maxlen'. |
| 3087 |
|
| 3088 |
""" |
| 3089 |
# Special case for RFC2231 encoding: start from decoded values and use |
| 3090 |
# RFC2231 encoding iff needed. |
| 3091 |
# |
| 3092 |
# Note that the 1 and 2s being added to the length calculations are |
| 3093 |
# accounting for the possibly-needed spaces and semicolons we'll be adding. |
| 3094 |
# |
| 3095 |
for name, value in part.params: |
| 3096 |
# XXX What if this ';' puts us over maxlen the first time through the |
| 3097 |
# loop? We should split the header value onto a newline in that case, |
| 3098 |
# but to do that we need to recognize the need earlier or reparse the |
| 3099 |
# header, so I'm going to ignore that bug for now. It'll only put us |
| 3100 |
# one character over. |
| 3101 |
if not lines[-1].rstrip().endswith(';'): |
| 3102 |
lines[-1] += ';' |
| 3103 |
charset = encoding |
| 3104 |
error_handler = 'strict' |
| 3105 |
try: |
| 3106 |
value.encode(encoding) |
| 3107 |
encoding_required = False |
| 3108 |
except UnicodeEncodeError: |
| 3109 |
encoding_required = True |
| 3110 |
if utils._has_surrogates(value): |
| 3111 |
charset = 'unknown-8bit' |
| 3112 |
error_handler = 'surrogateescape' |
| 3113 |
else: |
| 3114 |
charset = 'utf-8' |
| 3115 |
if encoding_required: |
| 3116 |
encoded_value = urllib.parse.quote( |
| 3117 |
value, safe='', errors=error_handler) |
| 3118 |
tstr = "{}*={}''{}".format(name, charset, encoded_value) |
| 3119 |
else: |
| 3120 |
tstr = '{}={}'.format(name, quote_string(value)) |
| 3121 |
if len(lines[-1]) + len(tstr) + 1 < maxlen: |
| 3122 |
lines[-1] = lines[-1] + ' ' + tstr |
| 3123 |
continue |
| 3124 |
elif len(tstr) + 2 <= maxlen: |
| 3125 |
lines.append(' ' + tstr) |
| 3126 |
continue |
| 3127 |
# We need multiple sections. We are allowed to mix encoded and |
| 3128 |
# non-encoded sections, but we aren't going to. We'll encode them all. |
| 3129 |
section = 0 |
| 3130 |
extra_chrome = charset + "''" |
| 3131 |
while value: |
| 3132 |
chrome_len = len(name) + len(str(section)) + 3 + len(extra_chrome) |
| 3133 |
if maxlen <= chrome_len + 3: |
| 3134 |
# We need room for the leading blank, the trailing semicolon, |
| 3135 |
# and at least one character of the value. If we don't |
| 3136 |
# have that, we'd be stuck, so in that case fall back to |
| 3137 |
# the RFC standard width. |
| 3138 |
maxlen = 78 |
| 3139 |
splitpoint = maxchars = maxlen - chrome_len - 2 |
| 3140 |
while True: |
| 3141 |
partial = value[:splitpoint] |
| 3142 |
encoded_value = urllib.parse.quote( |
| 3143 |
partial, safe='', errors=error_handler) |
| 3144 |
if len(encoded_value) <= maxchars: |
| 3145 |
break |
| 3146 |
splitpoint -= 1 |
| 3147 |
lines.append(" {}*{}*={}{}".format( |
| 3148 |
name, section, extra_chrome, encoded_value)) |
| 3149 |
extra_chrome = '' |
| 3150 |
section += 1 |
| 3151 |
value = value[splitpoint:] |
| 3152 |
if value: |
| 3153 |
lines[-1] += ';' |