| 1 |
""" Routines for manipulating RFC2047 encoded words. |
| 2 |
|
| 3 |
This is currently a package-private API, but will be considered for promotion |
| 4 |
to a public API if there is demand. |
| 5 |
|
| 6 |
""" |
| 7 |
|
| 8 |
# An ecoded word looks like this: |
| 9 |
# |
| 10 |
# =?charset[*lang]?cte?encoded_string?= |
| 11 |
# |
| 12 |
# for more information about charset see the charset module. Here it is one |
| 13 |
# of the preferred MIME charset names (hopefully; you never know when parsing). |
| 14 |
# cte (Content Transfer Encoding) is either 'q' or 'b' (ignoring case). In |
| 15 |
# theory other letters could be used for other encodings, but in practice this |
| 16 |
# (almost?) never happens. There could be a public API for adding entries |
| 17 |
# to the CTE tables, but YAGNI for now. 'q' is Quoted Printable, 'b' is |
| 18 |
# Base64. The meaning of encoded_string should be obvious. 'lang' is optional |
| 19 |
# as indicated by the brackets (they are not part of the syntax) but is almost |
| 20 |
# never encountered in practice. |
| 21 |
# |
| 22 |
# The general interface for a CTE decoder is that it takes the encoded_string |
| 23 |
# as its argument, and returns a tuple (cte_decoded_string, defects). The |
| 24 |
# cte_decoded_string is the original binary that was encoded using the |
| 25 |
# specified cte. 'defects' is a list of MessageDefect instances indicating any |
| 26 |
# problems encountered during conversion. 'charset' and 'lang' are the |
| 27 |
# corresponding strings extracted from the EW, case preserved. |
| 28 |
# |
| 29 |
# The general interface for a CTE encoder is that it takes a binary sequence |
| 30 |
# as input and returns the cte_encoded_string, which is an ascii-only string. |
| 31 |
# |
| 32 |
# Each decoder must also supply a length function that takes the binary |
| 33 |
# sequence as its argument and returns the length of the resulting encoded |
| 34 |
# string. |
| 35 |
# |
| 36 |
# The main API functions for the module are decode, which calls the decoder |
| 37 |
# referenced by the cte specifier, and encode, which adds the appropriate |
| 38 |
# RFC 2047 "chrome" to the encoded string, and can optionally automatically |
| 39 |
# select the shortest possible encoding. See their docstrings below for |
| 40 |
# details. |
| 41 |
|
| 42 |
import re |
| 43 |
import base64 |
| 44 |
import binascii |
| 45 |
import functools |
| 46 |
from string import ascii_letters, digits |
| 47 |
from email import errors |
| 48 |
|
| 49 |
__all__ = ['decode_q', |
| 50 |
'encode_q', |
| 51 |
'decode_b', |
| 52 |
'encode_b', |
| 53 |
'len_q', |
| 54 |
'len_b', |
| 55 |
'decode', |
| 56 |
'encode', |
| 57 |
] |
| 58 |
|
| 59 |
# |
| 60 |
# Quoted Printable |
| 61 |
# |
| 62 |
|
| 63 |
# regex based decoder. |
| 64 |
_q_byte_subber = functools.partial(re.compile(br'=([a-fA-F0-9]{2})').sub, |
| 65 |
lambda m: bytes.fromhex(m.group(1).decode())) |
| 66 |
|
| 67 |
def decode_q(encoded): |
| 68 |
encoded = encoded.replace(b'_', b' ') |
| 69 |
return _q_byte_subber(encoded), [] |
| 70 |
|
| 71 |
|
| 72 |
# dict mapping bytes to their encoded form |
| 73 |
class _QByteMap(dict): |
| 74 |
|
| 75 |
safe = b'-!*+/' + ascii_letters.encode('ascii') + digits.encode('ascii') |
| 76 |
|
| 77 |
def __missing__(self, key): |
| 78 |
if key in self.safe: |
| 79 |
self[key] = chr(key) |
| 80 |
else: |
| 81 |
self[key] = "={:02X}".format(key) |
| 82 |
return self[key] |
| 83 |
|
| 84 |
_q_byte_map = _QByteMap() |
| 85 |
|
| 86 |
# In headers spaces are mapped to '_'. |
| 87 |
_q_byte_map[ord(' ')] = '_' |
| 88 |
|
| 89 |
def encode_q(bstring): |
| 90 |
return ''.join(_q_byte_map[x] for x in bstring) |
| 91 |
|
| 92 |
def len_q(bstring): |
| 93 |
return sum(len(_q_byte_map[x]) for x in bstring) |
| 94 |
|
| 95 |
|
| 96 |
# |
| 97 |
# Base64 |
| 98 |
# |
| 99 |
|
| 100 |
def decode_b(encoded): |
| 101 |
# First try encoding with validate=True, fixing the padding if needed. |
| 102 |
# This will succeed only if encoded includes no invalid characters. |
| 103 |
pad_err = len(encoded) % 4 |
| 104 |
missing_padding = b'==='[:4-pad_err] if pad_err else b'' |
| 105 |
try: |
| 106 |
return ( |
| 107 |
base64.b64decode(encoded + missing_padding, validate=True), |
| 108 |
[errors.InvalidBase64PaddingDefect()] if pad_err else [], |
| 109 |
) |
| 110 |
except binascii.Error: |
| 111 |
# Since we had correct padding, this is likely an invalid char error. |
| 112 |
# |
| 113 |
# The non-alphabet characters are ignored as far as padding |
| 114 |
# goes, but we don't know how many there are. So try without adding |
| 115 |
# padding to see if it works. |
| 116 |
try: |
| 117 |
return ( |
| 118 |
base64.b64decode(encoded, validate=False), |
| 119 |
[errors.InvalidBase64CharactersDefect()], |
| 120 |
) |
| 121 |
except binascii.Error: |
| 122 |
# Add as much padding as could possibly be necessary (extra padding |
| 123 |
# is ignored). |
| 124 |
try: |
| 125 |
return ( |
| 126 |
base64.b64decode(encoded + b'==', validate=False), |
| 127 |
[errors.InvalidBase64CharactersDefect(), |
| 128 |
errors.InvalidBase64PaddingDefect()], |
| 129 |
) |
| 130 |
except binascii.Error: |
| 131 |
# This only happens when the encoded string's length is 1 more |
| 132 |
# than a multiple of 4, which is invalid. |
| 133 |
# |
| 134 |
# bpo-27397: Just return the encoded string since there's no |
| 135 |
# way to decode. |
| 136 |
return encoded, [errors.InvalidBase64LengthDefect()] |
| 137 |
|
| 138 |
def encode_b(bstring): |
| 139 |
return base64.b64encode(bstring).decode('ascii') |
| 140 |
|
| 141 |
def len_b(bstring): |
| 142 |
groups_of_3, leftover = divmod(len(bstring), 3) |
| 143 |
# 4 bytes out for each 3 bytes (or nonzero fraction thereof) in. |
| 144 |
return groups_of_3 * 4 + (4 if leftover else 0) |
| 145 |
|
| 146 |
|
| 147 |
_cte_decoders = { |
| 148 |
'q': decode_q, |
| 149 |
'b': decode_b, |
| 150 |
} |
| 151 |
|
| 152 |
def decode(ew): |
| 153 |
"""Decode encoded word and return (string, charset, lang, defects) tuple. |
| 154 |
|
| 155 |
An RFC 2047/2243 encoded word has the form: |
| 156 |
|
| 157 |
=?charset*lang?cte?encoded_string?= |
| 158 |
|
| 159 |
where '*lang' may be omitted but the other parts may not be. |
| 160 |
|
| 161 |
This function expects exactly such a string (that is, it does not check the |
| 162 |
syntax and may raise errors if the string is not well formed), and returns |
| 163 |
the encoded_string decoded first from its Content Transfer Encoding and |
| 164 |
then from the resulting bytes into unicode using the specified charset. If |
| 165 |
the cte-decoded string does not successfully decode using the specified |
| 166 |
character set, a defect is added to the defects list and the unknown octets |
| 167 |
are replaced by the unicode 'unknown' character \\uFDFF. |
| 168 |
|
| 169 |
The specified charset and language are returned. The default for language, |
| 170 |
which is rarely if ever encountered, is the empty string. |
| 171 |
|
| 172 |
""" |
| 173 |
_, charset, cte, cte_string, _ = ew.split('?') |
| 174 |
charset, _, lang = charset.partition('*') |
| 175 |
cte = cte.lower() |
| 176 |
# Recover the original bytes and do CTE decoding. |
| 177 |
bstring = cte_string.encode('ascii', 'surrogateescape') |
| 178 |
bstring, defects = _cte_decoders[cte](bstring) |
| 179 |
# Turn the CTE decoded bytes into unicode. |
| 180 |
try: |
| 181 |
string = bstring.decode(charset) |
| 182 |
except UnicodeDecodeError: |
| 183 |
defects.append(errors.UndecodableBytesDefect("Encoded word " |
| 184 |
f"contains bytes not decodable using {charset!r} charset")) |
| 185 |
string = bstring.decode(charset, 'surrogateescape') |
| 186 |
except (LookupError, UnicodeEncodeError): |
| 187 |
string = bstring.decode('ascii', 'surrogateescape') |
| 188 |
if charset.lower() != 'unknown-8bit': |
| 189 |
defects.append(errors.CharsetError(f"Unknown charset {charset!r} " |
| 190 |
f"in encoded word; decoded as unknown bytes")) |
| 191 |
return string, charset, lang, defects |
| 192 |
|
| 193 |
|
| 194 |
_cte_encoders = { |
| 195 |
'q': encode_q, |
| 196 |
'b': encode_b, |
| 197 |
} |
| 198 |
|
| 199 |
_cte_encode_length = { |
| 200 |
'q': len_q, |
| 201 |
'b': len_b, |
| 202 |
} |
| 203 |
|
| 204 |
def encode(string, charset='utf-8', encoding=None, lang=''): |
| 205 |
"""Encode string using the CTE encoding that produces the shorter result. |
| 206 |
|
| 207 |
Produces an RFC 2047/2243 encoded word of the form: |
| 208 |
|
| 209 |
=?charset*lang?cte?encoded_string?= |
| 210 |
|
| 211 |
where '*lang' is omitted unless the 'lang' parameter is given a value. |
| 212 |
Optional argument charset (defaults to utf-8) specifies the charset to use |
| 213 |
to encode the string to binary before CTE encoding it. Optional argument |
| 214 |
'encoding' is the cte specifier for the encoding that should be used ('q' |
| 215 |
or 'b'); if it is None (the default) the encoding which produces the |
| 216 |
shortest encoded sequence is used, except that 'q' is preferred if it is up |
| 217 |
to five characters longer. Optional argument 'lang' (default '') gives the |
| 218 |
RFC 2243 language string to specify in the encoded word. |
| 219 |
|
| 220 |
""" |
| 221 |
if charset == 'unknown-8bit': |
| 222 |
bstring = string.encode('utf-8', 'surrogateescape') |
| 223 |
else: |
| 224 |
bstring = string.encode(charset) |
| 225 |
if encoding is None: |
| 226 |
qlen = _cte_encode_length['q'](bstring) |
| 227 |
blen = _cte_encode_length['b'](bstring) |
| 228 |
# Bias toward q. 5 is arbitrary. |
| 229 |
encoding = 'q' if qlen - blen < 5 else 'b' |
| 230 |
encoded = _cte_encoders[encoding](bstring) |
| 231 |
if lang: |
| 232 |
lang = '*' + lang |
| 233 |
return "=?{}{}?{}?{}?=".format(charset, lang, encoding, encoded) |