Debian Codegraph @

Lib/email/charset.py

raw
1 # Copyright (C) 2001 Python Software Foundation
2 # Author: Ben Gertzfield, Barry Warsaw
3 # Contact: [email protected]
4
5 __all__ = [
6 'Charset',
7 'add_alias',
8 'add_charset',
9 'add_codec',
10 ]
11
12 from functools import partial
13
14 import email.base64mime
15 import email.quoprimime
16
17 from email import errors
18 from email.encoders import encode_7or8bit
19
20
21 # Flags for types of header encodings
22 QP = 1 # Quoted-Printable
23 BASE64 = 2 # Base64
24 SHORTEST = 3 # the shorter of QP and base64, but only for headers
25
26 # In "=?charset?q?hello_world?=", the =?, ?q?, and ?= add up to 7
27 RFC2047_CHROME_LEN = 7
28
29 DEFAULT_CHARSET = 'us-ascii'
30 UNKNOWN8BIT = 'unknown-8bit'
31 EMPTYSTRING = ''
32
33
34 # Defaults
35 CHARSETS = {
36 # input header enc body enc output conv
37 'iso-8859-1': (QP, QP, None),
38 'iso-8859-2': (QP, QP, None),
39 'iso-8859-3': (QP, QP, None),
40 'iso-8859-4': (QP, QP, None),
41 # iso-8859-5 is Cyrillic, and not especially used
42 # iso-8859-6 is Arabic, also not particularly used
43 # iso-8859-7 is Greek, QP will not make it readable
44 # iso-8859-8 is Hebrew, QP will not make it readable
45 'iso-8859-9': (QP, QP, None),
46 'iso-8859-10': (QP, QP, None),
47 # iso-8859-11 is Thai, QP will not make it readable
48 'iso-8859-13': (QP, QP, None),
49 'iso-8859-14': (QP, QP, None),
50 'iso-8859-15': (QP, QP, None),
51 'iso-8859-16': (QP, QP, None),
52 'windows-1252':(QP, QP, None),
53 'viscii': (QP, QP, None),
54 'us-ascii': (None, None, None),
55 'big5': (BASE64, BASE64, None),
56 'gb2312': (BASE64, BASE64, None),
57 'euc-jp': (BASE64, None, 'iso-2022-jp'),
58 'shift_jis': (BASE64, None, 'iso-2022-jp'),
59 'iso-2022-jp': (BASE64, None, None),
60 'koi8-r': (BASE64, BASE64, None),
61 'utf-8': (SHORTEST, BASE64, 'utf-8'),
62 }
63
64 # Aliases for other commonly-used names for character sets. Map
65 # them to the real ones used in email.
66 ALIASES = {
67 'latin_1': 'iso-8859-1',
68 'latin-1': 'iso-8859-1',
69 'latin_2': 'iso-8859-2',
70 'latin-2': 'iso-8859-2',
71 'latin_3': 'iso-8859-3',
72 'latin-3': 'iso-8859-3',
73 'latin_4': 'iso-8859-4',
74 'latin-4': 'iso-8859-4',
75 'latin_5': 'iso-8859-9',
76 'latin-5': 'iso-8859-9',
77 'latin_6': 'iso-8859-10',
78 'latin-6': 'iso-8859-10',
79 'latin_7': 'iso-8859-13',
80 'latin-7': 'iso-8859-13',
81 'latin_8': 'iso-8859-14',
82 'latin-8': 'iso-8859-14',
83 'latin_9': 'iso-8859-15',
84 'latin-9': 'iso-8859-15',
85 'latin_10':'iso-8859-16',
86 'latin-10':'iso-8859-16',
87 'cp949': 'ks_c_5601-1987',
88 'euc_jp': 'euc-jp',
89 'euc_kr': 'euc-kr',
90 'ascii': 'us-ascii',
91 }
92
93
94 # Map charsets to their Unicode codec strings.
95 CODEC_MAP = {
96 # Hack: We don't want *any* conversion for stuff marked us-ascii, as all
97 # sorts of garbage might be sent to us in the guise of 7-bit us-ascii.
98 # Let that stuff pass through without conversion to/from Unicode.
99 'us-ascii': None,
100 }
101
102
103 # Convenience functions for extending the above mappings
104 def add_charset(charset, header_enc=None, body_enc=None, output_charset=None):
105 """Add character set properties to the global registry.
106
107 charset is the input character set, and must be the canonical name of a
108 character set.
109
110 Optional header_enc and body_enc is either charset.QP for
111 quoted-printable, charset.BASE64 for base64 encoding, charset.SHORTEST for
112 the shortest of qp or base64 encoding, or None for no encoding. SHORTEST
113 is only valid for header_enc. It describes how message headers and
114 message bodies in the input charset are to be encoded. Default is no
115 encoding.
116
117 Optional output_charset is the character set that the output should be
118 in. Conversions will proceed from input charset, to Unicode, to the
119 output charset when the method Charset.convert() is called. The default
120 is to output in the same character set as the input.
121
122 Both input_charset and output_charset must have Unicode codec entries in
123 the module's charset-to-codec mapping; use add_codec(charset, codecname)
124 to add codecs the module does not know about. See the codecs module's
125 documentation for more information.
126 """
127 if body_enc == SHORTEST:
128 raise ValueError('SHORTEST not allowed for body_enc')
129 CHARSETS[charset] = (header_enc, body_enc, output_charset)
130
131
132 def add_alias(alias, canonical):
133 """Add a character set alias.
134
135 alias is the alias name, e.g. latin-1
136 canonical is the character set's canonical name, e.g. iso-8859-1
137 """
138 ALIASES[alias] = canonical
139
140
141 def add_codec(charset, codecname):
142 """Add a codec that map characters in the given charset to/from Unicode.
143
144 charset is the canonical name of a character set. codecname is the name
145 of a Python codec, as appropriate for the second argument to the unicode()
146 built-in, or to the encode() method of a Unicode string.
147 """
148 CODEC_MAP[charset] = codecname
149
150
151 # Convenience function for encoding strings, taking into account
152 # that they might be unknown-8bit (ie: have surrogate-escaped bytes)
153 def _encode(string, codec):
154 if codec == UNKNOWN8BIT:
155 return string.encode('ascii', 'surrogateescape')
156 else:
157 return string.encode(codec)
158
159
160 class Charset:
161 """Map character sets to their email properties.
162
163 This class provides information about the requirements imposed on email
164 for a specific character set. It also provides convenience routines for
165 converting between character sets, given the availability of the
166 applicable codecs. Given a character set, it will do its best to provide
167 information on how to use that character set in an email in an
168 RFC-compliant way.
169
170 Certain character sets must be encoded with quoted-printable or base64
171 when used in email headers or bodies. Certain character sets must be
172 converted outright, and are not allowed in email. Instances of this
173 module expose the following information about a character set:
174
175 input_charset: The initial character set specified. Common aliases
176 are converted to their 'official' email names (e.g. latin_1
177 is converted to iso-8859-1). Defaults to 7-bit us-ascii.
178
179 header_encoding: If the character set must be encoded before it can be
180 used in an email header, this attribute will be set to
181 charset.QP (for quoted-printable), charset.BASE64 (for
182 base64 encoding), or charset.SHORTEST for the shortest of
183 QP or BASE64 encoding. Otherwise, it will be None.
184
185 body_encoding: Same as header_encoding, but describes the encoding for the
186 mail message's body, which indeed may be different than the
187 header encoding. charset.SHORTEST is not allowed for
188 body_encoding.
189
190 output_charset: Some character sets must be converted before they can be
191 used in email headers or bodies. If the input_charset is
192 one of them, this attribute will contain the name of the
193 charset output will be converted to. Otherwise, it will
194 be None.
195
196 input_codec: The name of the Python codec used to convert the
197 input_charset to Unicode. If no conversion codec is
198 necessary, this attribute will be None.
199
200 output_codec: The name of the Python codec used to convert Unicode
201 to the output_charset. If no conversion codec is necessary,
202 this attribute will have the same value as the input_codec.
203 """
204 def __init__(self, input_charset=DEFAULT_CHARSET):
205 # RFC 2046, $4.1.2 says charsets are not case sensitive. We coerce to
206 # unicode because its .lower() is locale insensitive. If the argument
207 # is already a unicode, we leave it at that, but ensure that the
208 # charset is ASCII, as the standard (RFC XXX) requires.
209 try:
210 if isinstance(input_charset, str):
211 input_charset.encode('ascii')
212 else:
213 input_charset = str(input_charset, 'ascii')
214 except UnicodeError:
215 raise errors.CharsetError(input_charset)
216 input_charset = input_charset.lower()
217 # Set the input charset after filtering through the aliases
218 self.input_charset = ALIASES.get(input_charset, input_charset)
219 # We can try to guess which encoding and conversion to use by the
220 # charset_map dictionary. Try that first, but let the user override
221 # it.
222 henc, benc, conv = CHARSETS.get(self.input_charset,
223 (SHORTEST, BASE64, None))
224 if not conv:
225 conv = self.input_charset
226 # Set the attributes, allowing the arguments to override the default.
227 self.header_encoding = henc
228 self.body_encoding = benc
229 self.output_charset = ALIASES.get(conv, conv)
230 # Now set the codecs. If one isn't defined for input_charset,
231 # guess and try a Unicode codec with the same name as input_codec.
232 self.input_codec = CODEC_MAP.get(self.input_charset,
233 self.input_charset)
234 self.output_codec = CODEC_MAP.get(self.output_charset,
235 self.output_charset)
236
237 def __repr__(self):
238 return self.input_charset.lower()
239
240 def __eq__(self, other):
241 return str(self) == str(other).lower()
242
243 def get_body_encoding(self):
244 """Return the content-transfer-encoding used for body encoding.
245
246 This is either the string 'quoted-printable' or 'base64' depending on
247 the encoding used, or it is a function in which case you should call
248 the function with a single argument, the Message object being
249 encoded. The function should then set the Content-Transfer-Encoding
250 header itself to whatever is appropriate.
251
252 Returns "quoted-printable" if self.body_encoding is QP.
253 Returns "base64" if self.body_encoding is BASE64.
254 Returns conversion function otherwise.
255 """
256 assert self.body_encoding != SHORTEST
257 if self.body_encoding == QP:
258 return 'quoted-printable'
259 elif self.body_encoding == BASE64:
260 return 'base64'
261 else:
262 return encode_7or8bit
263
264 def get_output_charset(self):
265 """Return the output character set.
266
267 This is self.output_charset if that is not None, otherwise it is
268 self.input_charset.
269 """
270 return self.output_charset or self.input_charset
271
272 def header_encode(self, string):
273 """Header-encode a string by converting it first to bytes.
274
275 The type of encoding (base64 or quoted-printable) will be based on
276 this charset's `header_encoding`.
277
278 :param string: A unicode string for the header. It must be possible
279 to encode this string to bytes using the character set's
280 output codec.
281 :return: The encoded string, with RFC 2047 chrome.
282 """
283 codec = self.output_codec or 'us-ascii'
284 header_bytes = _encode(string, codec)
285 # 7bit/8bit encodings return the string unchanged (modulo conversions)
286 encoder_module = self._get_encoder(header_bytes)
287 if encoder_module is None:
288 return string
289 return encoder_module.header_encode(header_bytes, codec)
290
291 def header_encode_lines(self, string, maxlengths):
292 """Header-encode a string by converting it first to bytes.
293
294 This is similar to `header_encode()` except that the string is fit
295 into maximum line lengths as given by the argument.
296
297 :param string: A unicode string for the header. It must be possible
298 to encode this string to bytes using the character set's
299 output codec.
300 :param maxlengths: Maximum line length iterator. Each element
301 returned from this iterator will provide the next maximum line
302 length. This parameter is used as an argument to built-in next()
303 and should never be exhausted. The maximum line lengths should
304 not count the RFC 2047 chrome. These line lengths are only a
305 hint; the splitter does the best it can.
306 :return: Lines of encoded strings, each with RFC 2047 chrome.
307 """
308 # See which encoding we should use.
309 codec = self.output_codec or 'us-ascii'
310 header_bytes = _encode(string, codec)
311 encoder_module = self._get_encoder(header_bytes)
312 encoder = partial(encoder_module.header_encode, charset=codec)
313 # Calculate the number of characters that the RFC 2047 chrome will
314 # contribute to each line.
315 charset = self.get_output_charset()
316 extra = len(charset) + RFC2047_CHROME_LEN
317 # Now comes the hard part. We must encode bytes but we can't split on
318 # bytes because some character sets are variable length and each
319 # encoded word must stand on its own. So the problem is you have to
320 # encode to bytes to figure out this word's length, but you must split
321 # on characters. This causes two problems: first, we don't know how
322 # many octets a specific substring of unicode characters will get
323 # encoded to, and second, we don't know how many ASCII characters
324 # those octets will get encoded to. Unless we try it. Which seems
325 # inefficient. In the interest of being correct rather than fast (and
326 # in the hope that there will be few encoded headers in any such
327 # message), brute force it. :(
328 lines = []
329 current_line = []
330 maxlen = next(maxlengths) - extra
331 for character in string:
332 current_line.append(character)
333 this_line = EMPTYSTRING.join(current_line)
334 length = encoder_module.header_length(_encode(this_line, charset))
335 if length > maxlen:
336 # This last character doesn't fit so pop it off.
337 current_line.pop()
338 # Does nothing fit on the first line?
339 if not lines and not current_line:
340 lines.append(None)
341 else:
342 joined_line = EMPTYSTRING.join(current_line)
343 header_bytes = _encode(joined_line, codec)
344 lines.append(encoder(header_bytes))
345 current_line = [character]
346 maxlen = next(maxlengths) - extra
347 joined_line = EMPTYSTRING.join(current_line)
348 header_bytes = _encode(joined_line, codec)
349 lines.append(encoder(header_bytes))
350 return lines
351
352 def _get_encoder(self, header_bytes):
353 if self.header_encoding == BASE64:
354 return email.base64mime
355 elif self.header_encoding == QP:
356 return email.quoprimime
357 elif self.header_encoding == SHORTEST:
358 len64 = email.base64mime.header_length(header_bytes)
359 lenqp = email.quoprimime.header_length(header_bytes)
360 if len64 < lenqp:
361 return email.base64mime
362 else:
363 return email.quoprimime
364 else:
365 return None
366
367 def body_encode(self, string):
368 """Body-encode a string by converting it first to bytes.
369
370 The type of encoding (base64 or quoted-printable) will be based on
371 self.body_encoding. If body_encoding is None, we assume the
372 output charset is a 7bit encoding, so re-encoding the decoded
373 string using the ascii codec produces the correct string version
374 of the content.
375 """
376 if not string:
377 return string
378 if self.body_encoding is BASE64:
379 if isinstance(string, str):
380 string = string.encode(self.output_charset)
381 return email.base64mime.body_encode(string)
382 elif self.body_encoding is QP:
383 # quopromime.body_encode takes a string, but operates on it as if
384 # it were a list of byte codes. For a (minimal) history on why
385 # this is so, see changeset 0cf700464177. To correctly encode a
386 # character set, then, we must turn it into pseudo bytes via the
387 # latin1 charset, which will encode any byte as a single code point
388 # between 0 and 255, which is what body_encode is expecting.
389 if isinstance(string, str):
390 string = string.encode(self.output_charset)
391 string = string.decode('latin1')
392 return email.quoprimime.body_encode(string)
393 else:
394 if isinstance(string, str):
395 string = string.encode(self.output_charset).decode('ascii')
396 return string