| 1 |
# Copyright (C) 2001 Python Software Foundation |
| 2 |
# Author: Ben Gertzfield, Barry Warsaw |
| 3 |
# Contact: [email protected] |
| 4 |
|
| 5 |
__all__ = [ |
| 6 |
'Charset', |
| 7 |
'add_alias', |
| 8 |
'add_charset', |
| 9 |
'add_codec', |
| 10 |
] |
| 11 |
|
| 12 |
from functools import partial |
| 13 |
|
| 14 |
import email.base64mime |
| 15 |
import email.quoprimime |
| 16 |
|
| 17 |
from email import errors |
| 18 |
from email.encoders import encode_7or8bit |
| 19 |
|
| 20 |
|
| 21 |
# Flags for types of header encodings |
| 22 |
QP = 1 # Quoted-Printable |
| 23 |
BASE64 = 2 # Base64 |
| 24 |
SHORTEST = 3 # the shorter of QP and base64, but only for headers |
| 25 |
|
| 26 |
# In "=?charset?q?hello_world?=", the =?, ?q?, and ?= add up to 7 |
| 27 |
RFC2047_CHROME_LEN = 7 |
| 28 |
|
| 29 |
DEFAULT_CHARSET = 'us-ascii' |
| 30 |
UNKNOWN8BIT = 'unknown-8bit' |
| 31 |
EMPTYSTRING = '' |
| 32 |
|
| 33 |
|
| 34 |
# Defaults |
| 35 |
CHARSETS = { |
| 36 |
# input header enc body enc output conv |
| 37 |
'iso-8859-1': (QP, QP, None), |
| 38 |
'iso-8859-2': (QP, QP, None), |
| 39 |
'iso-8859-3': (QP, QP, None), |
| 40 |
'iso-8859-4': (QP, QP, None), |
| 41 |
# iso-8859-5 is Cyrillic, and not especially used |
| 42 |
# iso-8859-6 is Arabic, also not particularly used |
| 43 |
# iso-8859-7 is Greek, QP will not make it readable |
| 44 |
# iso-8859-8 is Hebrew, QP will not make it readable |
| 45 |
'iso-8859-9': (QP, QP, None), |
| 46 |
'iso-8859-10': (QP, QP, None), |
| 47 |
# iso-8859-11 is Thai, QP will not make it readable |
| 48 |
'iso-8859-13': (QP, QP, None), |
| 49 |
'iso-8859-14': (QP, QP, None), |
| 50 |
'iso-8859-15': (QP, QP, None), |
| 51 |
'iso-8859-16': (QP, QP, None), |
| 52 |
'windows-1252':(QP, QP, None), |
| 53 |
'viscii': (QP, QP, None), |
| 54 |
'us-ascii': (None, None, None), |
| 55 |
'big5': (BASE64, BASE64, None), |
| 56 |
'gb2312': (BASE64, BASE64, None), |
| 57 |
'euc-jp': (BASE64, None, 'iso-2022-jp'), |
| 58 |
'shift_jis': (BASE64, None, 'iso-2022-jp'), |
| 59 |
'iso-2022-jp': (BASE64, None, None), |
| 60 |
'koi8-r': (BASE64, BASE64, None), |
| 61 |
'utf-8': (SHORTEST, BASE64, 'utf-8'), |
| 62 |
} |
| 63 |
|
| 64 |
# Aliases for other commonly-used names for character sets. Map |
| 65 |
# them to the real ones used in email. |
| 66 |
ALIASES = { |
| 67 |
'latin_1': 'iso-8859-1', |
| 68 |
'latin-1': 'iso-8859-1', |
| 69 |
'latin_2': 'iso-8859-2', |
| 70 |
'latin-2': 'iso-8859-2', |
| 71 |
'latin_3': 'iso-8859-3', |
| 72 |
'latin-3': 'iso-8859-3', |
| 73 |
'latin_4': 'iso-8859-4', |
| 74 |
'latin-4': 'iso-8859-4', |
| 75 |
'latin_5': 'iso-8859-9', |
| 76 |
'latin-5': 'iso-8859-9', |
| 77 |
'latin_6': 'iso-8859-10', |
| 78 |
'latin-6': 'iso-8859-10', |
| 79 |
'latin_7': 'iso-8859-13', |
| 80 |
'latin-7': 'iso-8859-13', |
| 81 |
'latin_8': 'iso-8859-14', |
| 82 |
'latin-8': 'iso-8859-14', |
| 83 |
'latin_9': 'iso-8859-15', |
| 84 |
'latin-9': 'iso-8859-15', |
| 85 |
'latin_10':'iso-8859-16', |
| 86 |
'latin-10':'iso-8859-16', |
| 87 |
'cp949': 'ks_c_5601-1987', |
| 88 |
'euc_jp': 'euc-jp', |
| 89 |
'euc_kr': 'euc-kr', |
| 90 |
'ascii': 'us-ascii', |
| 91 |
} |
| 92 |
|
| 93 |
|
| 94 |
# Map charsets to their Unicode codec strings. |
| 95 |
CODEC_MAP = { |
| 96 |
# Hack: We don't want *any* conversion for stuff marked us-ascii, as all |
| 97 |
# sorts of garbage might be sent to us in the guise of 7-bit us-ascii. |
| 98 |
# Let that stuff pass through without conversion to/from Unicode. |
| 99 |
'us-ascii': None, |
| 100 |
} |
| 101 |
|
| 102 |
|
| 103 |
# Convenience functions for extending the above mappings |
| 104 |
def add_charset(charset, header_enc=None, body_enc=None, output_charset=None): |
| 105 |
"""Add character set properties to the global registry. |
| 106 |
|
| 107 |
charset is the input character set, and must be the canonical name of a |
| 108 |
character set. |
| 109 |
|
| 110 |
Optional header_enc and body_enc is either charset.QP for |
| 111 |
quoted-printable, charset.BASE64 for base64 encoding, charset.SHORTEST for |
| 112 |
the shortest of qp or base64 encoding, or None for no encoding. SHORTEST |
| 113 |
is only valid for header_enc. It describes how message headers and |
| 114 |
message bodies in the input charset are to be encoded. Default is no |
| 115 |
encoding. |
| 116 |
|
| 117 |
Optional output_charset is the character set that the output should be |
| 118 |
in. Conversions will proceed from input charset, to Unicode, to the |
| 119 |
output charset when the method Charset.convert() is called. The default |
| 120 |
is to output in the same character set as the input. |
| 121 |
|
| 122 |
Both input_charset and output_charset must have Unicode codec entries in |
| 123 |
the module's charset-to-codec mapping; use add_codec(charset, codecname) |
| 124 |
to add codecs the module does not know about. See the codecs module's |
| 125 |
documentation for more information. |
| 126 |
""" |
| 127 |
if body_enc == SHORTEST: |
| 128 |
raise ValueError('SHORTEST not allowed for body_enc') |
| 129 |
CHARSETS[charset] = (header_enc, body_enc, output_charset) |
| 130 |
|
| 131 |
|
| 132 |
def add_alias(alias, canonical): |
| 133 |
"""Add a character set alias. |
| 134 |
|
| 135 |
alias is the alias name, e.g. latin-1 |
| 136 |
canonical is the character set's canonical name, e.g. iso-8859-1 |
| 137 |
""" |
| 138 |
ALIASES[alias] = canonical |
| 139 |
|
| 140 |
|
| 141 |
def add_codec(charset, codecname): |
| 142 |
"""Add a codec that map characters in the given charset to/from Unicode. |
| 143 |
|
| 144 |
charset is the canonical name of a character set. codecname is the name |
| 145 |
of a Python codec, as appropriate for the second argument to the unicode() |
| 146 |
built-in, or to the encode() method of a Unicode string. |
| 147 |
""" |
| 148 |
CODEC_MAP[charset] = codecname |
| 149 |
|
| 150 |
|
| 151 |
# Convenience function for encoding strings, taking into account |
| 152 |
# that they might be unknown-8bit (ie: have surrogate-escaped bytes) |
| 153 |
def _encode(string, codec): |
| 154 |
if codec == UNKNOWN8BIT: |
| 155 |
return string.encode('ascii', 'surrogateescape') |
| 156 |
else: |
| 157 |
return string.encode(codec) |
| 158 |
|
| 159 |
|
| 160 |
class Charset: |
| 161 |
"""Map character sets to their email properties. |
| 162 |
|
| 163 |
This class provides information about the requirements imposed on email |
| 164 |
for a specific character set. It also provides convenience routines for |
| 165 |
converting between character sets, given the availability of the |
| 166 |
applicable codecs. Given a character set, it will do its best to provide |
| 167 |
information on how to use that character set in an email in an |
| 168 |
RFC-compliant way. |
| 169 |
|
| 170 |
Certain character sets must be encoded with quoted-printable or base64 |
| 171 |
when used in email headers or bodies. Certain character sets must be |
| 172 |
converted outright, and are not allowed in email. Instances of this |
| 173 |
module expose the following information about a character set: |
| 174 |
|
| 175 |
input_charset: The initial character set specified. Common aliases |
| 176 |
are converted to their 'official' email names (e.g. latin_1 |
| 177 |
is converted to iso-8859-1). Defaults to 7-bit us-ascii. |
| 178 |
|
| 179 |
header_encoding: If the character set must be encoded before it can be |
| 180 |
used in an email header, this attribute will be set to |
| 181 |
charset.QP (for quoted-printable), charset.BASE64 (for |
| 182 |
base64 encoding), or charset.SHORTEST for the shortest of |
| 183 |
QP or BASE64 encoding. Otherwise, it will be None. |
| 184 |
|
| 185 |
body_encoding: Same as header_encoding, but describes the encoding for the |
| 186 |
mail message's body, which indeed may be different than the |
| 187 |
header encoding. charset.SHORTEST is not allowed for |
| 188 |
body_encoding. |
| 189 |
|
| 190 |
output_charset: Some character sets must be converted before they can be |
| 191 |
used in email headers or bodies. If the input_charset is |
| 192 |
one of them, this attribute will contain the name of the |
| 193 |
charset output will be converted to. Otherwise, it will |
| 194 |
be None. |
| 195 |
|
| 196 |
input_codec: The name of the Python codec used to convert the |
| 197 |
input_charset to Unicode. If no conversion codec is |
| 198 |
necessary, this attribute will be None. |
| 199 |
|
| 200 |
output_codec: The name of the Python codec used to convert Unicode |
| 201 |
to the output_charset. If no conversion codec is necessary, |
| 202 |
this attribute will have the same value as the input_codec. |
| 203 |
""" |
| 204 |
def __init__(self, input_charset=DEFAULT_CHARSET): |
| 205 |
# RFC 2046, $4.1.2 says charsets are not case sensitive. We coerce to |
| 206 |
# unicode because its .lower() is locale insensitive. If the argument |
| 207 |
# is already a unicode, we leave it at that, but ensure that the |
| 208 |
# charset is ASCII, as the standard (RFC XXX) requires. |
| 209 |
try: |
| 210 |
if isinstance(input_charset, str): |
| 211 |
input_charset.encode('ascii') |
| 212 |
else: |
| 213 |
input_charset = str(input_charset, 'ascii') |
| 214 |
except UnicodeError: |
| 215 |
raise errors.CharsetError(input_charset) |
| 216 |
input_charset = input_charset.lower() |
| 217 |
# Set the input charset after filtering through the aliases |
| 218 |
self.input_charset = ALIASES.get(input_charset, input_charset) |
| 219 |
# We can try to guess which encoding and conversion to use by the |
| 220 |
# charset_map dictionary. Try that first, but let the user override |
| 221 |
# it. |
| 222 |
henc, benc, conv = CHARSETS.get(self.input_charset, |
| 223 |
(SHORTEST, BASE64, None)) |
| 224 |
if not conv: |
| 225 |
conv = self.input_charset |
| 226 |
# Set the attributes, allowing the arguments to override the default. |
| 227 |
self.header_encoding = henc |
| 228 |
self.body_encoding = benc |
| 229 |
self.output_charset = ALIASES.get(conv, conv) |
| 230 |
# Now set the codecs. If one isn't defined for input_charset, |
| 231 |
# guess and try a Unicode codec with the same name as input_codec. |
| 232 |
self.input_codec = CODEC_MAP.get(self.input_charset, |
| 233 |
self.input_charset) |
| 234 |
self.output_codec = CODEC_MAP.get(self.output_charset, |
| 235 |
self.output_charset) |
| 236 |
|
| 237 |
def __repr__(self): |
| 238 |
return self.input_charset.lower() |
| 239 |
|
| 240 |
def __eq__(self, other): |
| 241 |
return str(self) == str(other).lower() |
| 242 |
|
| 243 |
def get_body_encoding(self): |
| 244 |
"""Return the content-transfer-encoding used for body encoding. |
| 245 |
|
| 246 |
This is either the string 'quoted-printable' or 'base64' depending on |
| 247 |
the encoding used, or it is a function in which case you should call |
| 248 |
the function with a single argument, the Message object being |
| 249 |
encoded. The function should then set the Content-Transfer-Encoding |
| 250 |
header itself to whatever is appropriate. |
| 251 |
|
| 252 |
Returns "quoted-printable" if self.body_encoding is QP. |
| 253 |
Returns "base64" if self.body_encoding is BASE64. |
| 254 |
Returns conversion function otherwise. |
| 255 |
""" |
| 256 |
assert self.body_encoding != SHORTEST |
| 257 |
if self.body_encoding == QP: |
| 258 |
return 'quoted-printable' |
| 259 |
elif self.body_encoding == BASE64: |
| 260 |
return 'base64' |
| 261 |
else: |
| 262 |
return encode_7or8bit |
| 263 |
|
| 264 |
def get_output_charset(self): |
| 265 |
"""Return the output character set. |
| 266 |
|
| 267 |
This is self.output_charset if that is not None, otherwise it is |
| 268 |
self.input_charset. |
| 269 |
""" |
| 270 |
return self.output_charset or self.input_charset |
| 271 |
|
| 272 |
def header_encode(self, string): |
| 273 |
"""Header-encode a string by converting it first to bytes. |
| 274 |
|
| 275 |
The type of encoding (base64 or quoted-printable) will be based on |
| 276 |
this charset's `header_encoding`. |
| 277 |
|
| 278 |
:param string: A unicode string for the header. It must be possible |
| 279 |
to encode this string to bytes using the character set's |
| 280 |
output codec. |
| 281 |
:return: The encoded string, with RFC 2047 chrome. |
| 282 |
""" |
| 283 |
codec = self.output_codec or 'us-ascii' |
| 284 |
header_bytes = _encode(string, codec) |
| 285 |
# 7bit/8bit encodings return the string unchanged (modulo conversions) |
| 286 |
encoder_module = self._get_encoder(header_bytes) |
| 287 |
if encoder_module is None: |
| 288 |
return string |
| 289 |
return encoder_module.header_encode(header_bytes, codec) |
| 290 |
|
| 291 |
def header_encode_lines(self, string, maxlengths): |
| 292 |
"""Header-encode a string by converting it first to bytes. |
| 293 |
|
| 294 |
This is similar to `header_encode()` except that the string is fit |
| 295 |
into maximum line lengths as given by the argument. |
| 296 |
|
| 297 |
:param string: A unicode string for the header. It must be possible |
| 298 |
to encode this string to bytes using the character set's |
| 299 |
output codec. |
| 300 |
:param maxlengths: Maximum line length iterator. Each element |
| 301 |
returned from this iterator will provide the next maximum line |
| 302 |
length. This parameter is used as an argument to built-in next() |
| 303 |
and should never be exhausted. The maximum line lengths should |
| 304 |
not count the RFC 2047 chrome. These line lengths are only a |
| 305 |
hint; the splitter does the best it can. |
| 306 |
:return: Lines of encoded strings, each with RFC 2047 chrome. |
| 307 |
""" |
| 308 |
# See which encoding we should use. |
| 309 |
codec = self.output_codec or 'us-ascii' |
| 310 |
header_bytes = _encode(string, codec) |
| 311 |
encoder_module = self._get_encoder(header_bytes) |
| 312 |
encoder = partial(encoder_module.header_encode, charset=codec) |
| 313 |
# Calculate the number of characters that the RFC 2047 chrome will |
| 314 |
# contribute to each line. |
| 315 |
charset = self.get_output_charset() |
| 316 |
extra = len(charset) + RFC2047_CHROME_LEN |
| 317 |
# Now comes the hard part. We must encode bytes but we can't split on |
| 318 |
# bytes because some character sets are variable length and each |
| 319 |
# encoded word must stand on its own. So the problem is you have to |
| 320 |
# encode to bytes to figure out this word's length, but you must split |
| 321 |
# on characters. This causes two problems: first, we don't know how |
| 322 |
# many octets a specific substring of unicode characters will get |
| 323 |
# encoded to, and second, we don't know how many ASCII characters |
| 324 |
# those octets will get encoded to. Unless we try it. Which seems |
| 325 |
# inefficient. In the interest of being correct rather than fast (and |
| 326 |
# in the hope that there will be few encoded headers in any such |
| 327 |
# message), brute force it. :( |
| 328 |
lines = [] |
| 329 |
current_line = [] |
| 330 |
maxlen = next(maxlengths) - extra |
| 331 |
for character in string: |
| 332 |
current_line.append(character) |
| 333 |
this_line = EMPTYSTRING.join(current_line) |
| 334 |
length = encoder_module.header_length(_encode(this_line, charset)) |
| 335 |
if length > maxlen: |
| 336 |
# This last character doesn't fit so pop it off. |
| 337 |
current_line.pop() |
| 338 |
# Does nothing fit on the first line? |
| 339 |
if not lines and not current_line: |
| 340 |
lines.append(None) |
| 341 |
else: |
| 342 |
joined_line = EMPTYSTRING.join(current_line) |
| 343 |
header_bytes = _encode(joined_line, codec) |
| 344 |
lines.append(encoder(header_bytes)) |
| 345 |
current_line = [character] |
| 346 |
maxlen = next(maxlengths) - extra |
| 347 |
joined_line = EMPTYSTRING.join(current_line) |
| 348 |
header_bytes = _encode(joined_line, codec) |
| 349 |
lines.append(encoder(header_bytes)) |
| 350 |
return lines |
| 351 |
|
| 352 |
def _get_encoder(self, header_bytes): |
| 353 |
if self.header_encoding == BASE64: |
| 354 |
return email.base64mime |
| 355 |
elif self.header_encoding == QP: |
| 356 |
return email.quoprimime |
| 357 |
elif self.header_encoding == SHORTEST: |
| 358 |
len64 = email.base64mime.header_length(header_bytes) |
| 359 |
lenqp = email.quoprimime.header_length(header_bytes) |
| 360 |
if len64 < lenqp: |
| 361 |
return email.base64mime |
| 362 |
else: |
| 363 |
return email.quoprimime |
| 364 |
else: |
| 365 |
return None |
| 366 |
|
| 367 |
def body_encode(self, string): |
| 368 |
"""Body-encode a string by converting it first to bytes. |
| 369 |
|
| 370 |
The type of encoding (base64 or quoted-printable) will be based on |
| 371 |
self.body_encoding. If body_encoding is None, we assume the |
| 372 |
output charset is a 7bit encoding, so re-encoding the decoded |
| 373 |
string using the ascii codec produces the correct string version |
| 374 |
of the content. |
| 375 |
""" |
| 376 |
if not string: |
| 377 |
return string |
| 378 |
if self.body_encoding is BASE64: |
| 379 |
if isinstance(string, str): |
| 380 |
string = string.encode(self.output_charset) |
| 381 |
return email.base64mime.body_encode(string) |
| 382 |
elif self.body_encoding is QP: |
| 383 |
# quopromime.body_encode takes a string, but operates on it as if |
| 384 |
# it were a list of byte codes. For a (minimal) history on why |
| 385 |
# this is so, see changeset 0cf700464177. To correctly encode a |
| 386 |
# character set, then, we must turn it into pseudo bytes via the |
| 387 |
# latin1 charset, which will encode any byte as a single code point |
| 388 |
# between 0 and 255, which is what body_encode is expecting. |
| 389 |
if isinstance(string, str): |
| 390 |
string = string.encode(self.output_charset) |
| 391 |
string = string.decode('latin1') |
| 392 |
return email.quoprimime.body_encode(string) |
| 393 |
else: |
| 394 |
if isinstance(string, str): |
| 395 |
string = string.encode(self.output_charset).decode('ascii') |
| 396 |
return string |